def loadMsgs(self,labels):
# For each label, return a list of message Ids
self.labels = labels
ts = (parser.parse(self.db.getTimestamp()) + datetime.timedelta(days=-1))
for item in self.labels:
print "Loading " + item['name'] + "..."
idList = self.getMsgIds(item['id'], 'after: ' + ts.strftime('%Y/%m/%d'))
# For each id, add 'get' to batch
n = 0
self.curLab = item['name']
self.batch = BatchHttpRequest(callback=self.insertMsg)
for num in idList:
self.batch.add(self.messages.get(userId='me',id=num),request_id=num)
n = n + 1
# After 1000 or at the end of the list, execute batch and commit
if ((n % 1000) == 0) | (n == len(idList)):
print str(n) + " / " + str(len(idList))
try:
self.batch.execute(http=self.http)
except:
print "Unexpected error:", sys.exc_info()[0]
raise
self.db.commit()
self.batch = BatchHttpRequest(callback=self.insertMsg)
print "Finished " + item['name'] + "!"
def main(argv):
# Authenticate and construct service.
service, flags = sample_tools.init(
argv, 'content', 'v2', __doc__, __file__, parents=[argparser])
merchant_id = flags.merchant_id
product_ids = flags.product_ids
batch = BatchHttpRequest(callback=product_updated)
for product_id in product_ids:
new_status = {
'availability': 'out of stock',
'price': {'value': 3.14, 'currency': 'USD'}}
# Add product update to the batch.
batch.add(service.inventory().set(
merchantId=merchant_id,
storeCode=product_id.split(':')[0],
productId=product_id,
body=new_status))
try:
batch.execute()
except client.AccessTokenRefreshError:
print ('The credentials have been revoked or expired, please re-run the '
'application to re-authorize')
def test_http_errors_passed_to_callback(self):
batch = BatchHttpRequest()
callbacks = Callbacks()
cred_1 = MockCredentials('Foo')
cred_2 = MockCredentials('Bar')
http = HttpMockSequence([
({'status': '200',
'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'},
BATCH_RESPONSE_WITH_401),
({'status': '200',
'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'},
BATCH_RESPONSE_WITH_401),
])
creds_http_1 = HttpMockSequence([])
cred_1.authorize(creds_http_1)
creds_http_2 = HttpMockSequence([])
cred_2.authorize(creds_http_2)
self.request1.http = creds_http_1
self.request2.http = creds_http_2
batch.add(self.request1, callback=callbacks.f)
batch.add(self.request2, callback=callbacks.f)
batch.execute(http=http)
self.assertEqual(None, callbacks.responses['1'])
self.assertEqual(401, callbacks.exceptions['1'].resp.status)
self.assertEqual(
'Authorization Required', callbacks.exceptions['1'].resp.reason)
self.assertEqual({u'baz': u'qux'}, callbacks.responses['2'])
self.assertEqual(None, callbacks.exceptions['2'])
def _insert_item_all_users(self):
"""Insert a timeline item to all authorized users."""
logging.info('Inserting timeline item to all users')
users = Credentials.all()
total_users = users.count()
if total_users > 10:
return 'Total user count is %d. Aborting broadcast to save your quota' % (
total_users)
body = {
'text': 'Hello Everyone!',
'notification': {'level': 'DEFAULT'}
}
batch_responses = _BatchCallback()
batch = BatchHttpRequest(callback=batch_responses.callback)
for user in users:
creds = StorageByKeyName(
Credentials, user.key().name(), 'credentials').get()
mirror_service = util.create_service('mirror', 'v1', creds)
batch.add(
mirror_service.timeline().insert(body=body),
request_id=user.key().name())
batch.execute(httplib2.Http())
return 'Successfully sent cards to %d users (%d failed).' % (
batch_responses.success, batch_responses.failure)
def main():
"""Main entrypoint."""
client = create_client()
files = fetch_all_metadata(client)
dupes = find_dupes(files)
print '{} duplicates found. '.format(len(dupes))
if len(dupes) == 0:
print 'We are done.'
return
print 'Please check them.'
total = 0
for dupeset in dupes:
print '--'
for dupe in dupeset:
print dupe['alternateLink'], dupe['title']
for dupe in dupeset[1:]:
total += int(dupe['quotaBytesUsed'])
print '--'
print '{} Gigabytes wasted.'.format(total / ONE_GIG)
conf = raw_input('Great. Now trash the extras? (y/n) ')
if conf.strip() == 'y':
print 'Trashing.'
batch = BatchHttpRequest()
for dupeset in dupes:
for dupe in dupeset[1:]:
batch.add(client.files().trash(fileId=dupe['id']))
batch.execute()
print 'We are done. Check the trash for your files.'
else:
print 'Not touching anything.'
def get_messages(self, max_results=10, request_format=None,
label_ids=[], page_token=None):
response = self._get_message_ids(max_results, label_ids, page_token)
if not response:
return []
if not request_format:
request_format = 'metadata'
messages = []
def on_get_message(request_id, response, exception):
if exception is not None:
return
messages.append(response)
batch = BatchHttpRequest(callback=on_get_message)
try:
for message in response['messages']:
# message_ids.append(message['id'])
batch.add(self._users.messages().get(id=message['id'],
userId='me',
format=request_format))
batch.execute(http=self._http)
except KeyError:
return messages
return messages
def get(self):
video_url = self.request.get("url")
"""Render the main page."""
logging.info('Inserting timeline item to all users')
users = Credentials.all()
total_users = users.count()
if total_users > 10:
return 'Total user count is %d. Aborting broadcast to save your quota' % (
total_users)
body = {
'notification': {'level': 'DEFAULT'},
'text': video_url,
}
if 'youtube' in video_url:
body['menuItems'] = [{'action' : 'PLAY_VIDEO', 'payload' : video_url}]
batch_responses = _BatchCallback()
batch = BatchHttpRequest(callback=batch_responses.callback)
for user in users:
creds = StorageByKeyName(
Credentials, user.key().name(), 'credentials').get()
mirror_service = util.create_service('mirror', 'v1', creds)
timeline = retrieve_all_timeline_items(mirror_service)
batch.add(
mirror_service.timeline().insert(body=body),
request_id=user.key().name())
batch.execute(httplib2.Http())
self._render_template('')
def main():
"""Main entrypoint."""
client = create_client()
files = fetch_all_metadata(client)
dupes = find_dupes(files)
print '{} duplicates found. '.format(len(dupes))
if len(dupes) == 0:
print 'We are done.'
return
print 'Please check them.'
total = 0
for dupeset in dupes:
print '--'
for dupe in dupeset:
print dupe['alternateLink'], dupe['title']
for dupe in dupeset[1:]:
total += int(dupe['quotaBytesUsed'])
print '--'
print '{} Gigabytes wasted.'.format(total / ONE_GIG)
conf = raw_input('Great. Now trash the extras? (y/n) ')
if conf.strip() == 'y':
print 'Trashing.'
batch = BatchHttpRequest()
for dupeset in dupes:
for dupe in dupeset[1:]:
batch.add(client.files().trash(fileId=dupe['id']))
if len(batch._order) == 1000: # batch maxes out at 1k
batch.execute()
batch = BatchHttpRequest()
batch.execute()
print 'We are done. Check the trash for your files.'
else:
print 'Not touching anything.'
def delete_playlists(youtube, args):
playlists_req = youtube.playlists().list(
part = "id,snippet",
mine = True,
maxResults = MAX_RESULTS,
)
def delete_playlist(request_id, response, exception):
playlist_id = request_id
if exception:
raise exception
else:
sys.stderr.write(u"Deleted {id}\n".format(id = playlist_id))
delete = False
batch_req = BatchHttpRequest(callback = delete_playlist)
while playlists_req:
playlists = playlists_req.execute()
for playlist in playlists["items"]:
if re.match(args.pattern, playlist["snippet"]["title"]):
if args.pretend:
sys.stderr.write(u"Deleting {}\n".format(playlist["snippet"]["title"]))
else:
delete_req = youtube.playlists().delete(id = playlist["id"])
batch_req.add(delete_req, request_id = playlist["id"])
delete = True
playlists_req = youtube.playlists().list_next(playlists_req, playlists)
if delete:
batch_req.execute()
def batchInsertProducts(self,product_qset):
GoogleProduct = get_model('gmerchant','GoogleProduct')
def product_inserted(unused_request_id, response, exception):
if exception is not None:
# Do something with the exception.
print 'There was an error: ' + str(exception)
else:
offer_id = smart_text(response['offerId'].encode('ascii', 'ignore'))
gp = GoogleProduct.objects.get(google_shopping_id=offer_id)
if not gp.google_shopping_created:
gp.google_shopping_created = datetime.now()
else:
gp.google_shopping_updated = datetime.now()
gp.save()
print ('Product with offerId "%s" and title "%s" was created.' %
(offer_id, smart_text(response['title'].encode('ascii', 'ignore'))))
for block in chunks(product_qset,BATCH_SIZE):
#Build a new batch request for this block of products
batch = BatchHttpRequest(callback=product_inserted)
for i in block:
product = self.buildProduct(i)
# Add product to the batch.
batch.add(self.service.products().insert(merchantId=self.merchant_id,
body=product))
try:
#Let's send this batch off to the Goog.
batch.execute()
except client.AccessTokenRefreshError:
warn_exp_token()
def share(self, users, share_type='writer', send_notifications=False, email_message=None):
"""
Share a document with a given list of users.
"""
if type(users) is str:
users = [users]
def batch_callback(request_id, response, exception):
print("Response for request_id (%s):" % request_id)
print(response)
# Potentially log or re-raise exceptions
if exception:
raise exception
batch_request = BatchHttpRequest(callback=batch_callback)
for count, user in enumerate(users):
batch_entry = self.drive.service.permissions().insert(fileId=self._id, sendNotificationEmails=send_notifications, emailMessage=email_message,
body={
'value': user,
'type': 'user',
'role': share_type
})
batch_request.add(batch_entry, request_id="batch"+str(count))
batch_request.execute()
def test_deserialize_response(self):
batch = BatchHttpRequest()
resp, content = batch._deserialize_response(RESPONSE)
self.assertEquals(resp.status, 200)
self.assertEquals(resp.reason, 'OK')
self.assertEquals(resp.version, 11)
self.assertEquals(content, '{"answer": 42}')
def test_deserialize_response(self):
batch = BatchHttpRequest()
resp, content = batch._deserialize_response(RESPONSE)
self.assertEqual(200, resp.status)
self.assertEqual('OK', resp.reason)
self.assertEqual(11, resp.version)
self.assertEqual('{"answer": 42}', content)
def test_deserialize_response(self):
batch = BatchHttpRequest()
resp, content = batch._deserialize_response(RESPONSE)
self.assertEquals(resp.status, 200)
self.assertEquals(resp.reason, 'OK')
self.assertEquals(resp.version, 11)
self.assertEquals(content, '{"answer": 42}')
def test_deserialize_response(self):
batch = BatchHttpRequest()
resp, content = batch._deserialize_response(RESPONSE)
self.assertEqual(200, resp.status)
self.assertEqual('OK', resp.reason)
self.assertEqual(11, resp.version)
self.assertEqual('{"answer": 42}', content)
def post(self):
domain = self.request.get("domain")
logging.info("Execing cleanup task for domain (%s)" % domain)
http = httplib2.Http()
httplib2.debuglevel = 4
credentials = get_credentials(settings.RESELLER_ADMIN)
credentials.authorize(http)
service = build("reseller", settings.RESELLER_API_VERSION, http=http)
response = service.customers().get(
customerId=domain).execute(num_retries=5)
def delete_sub_callback(request_id, response, exception):
# just log the exception.
logging.exception(exception)
pass
if not response.get("alternateEmail"):
logging.info("Skipping cleanup, customer not resold..")
exit()
response = service.subscriptions().list(
customerId=domain,
maxResults=100).execute(num_retries=5)
# resort the subscriptions and bump GAFB subs to the bottom
subs = sorted(
response['subscriptions'],
cmp=lambda a, b: int(a['skuId'] == ResellerSKU.GoogleApps) - 1)
batch = BatchHttpRequest(callback=delete_sub_callback)
logging.info("Purging %d subs" % len(subs))
for s in subs:
if s['status'] in [ResellerDeletionType.Cancel,
ResellerDeletionType.Suspend,
ResellerDeletionType.Downgrade]:
logging.info("Skipping subscription, in deleted state")
continue
# Google-Drive-storage / Google-Vault must be cancelled.
deletionType = ResellerDeletionType.Cancel
# GAfB cannot be 'cancelled', and must be 'suspended'
if s['skuId'] == ResellerSKU.GoogleApps:
deletionType = ResellerDeletionType.Suspend
request = service.subscriptions().delete(
customerId=domain,
subscriptionId=s['subscriptionId'],
deletionType=deletionType)
batch.add(request)
batch.execute(http=http)
def lambda_handler(event, context):
ORG_LIST = [ 1935, 1940, 1936, 1944, 1937, 1938, 1943, 2103, 1941, 1939, 1945, 1942 ]
calendarService = getGoogleService()
print("Retrieving list of Google Calendars")
googleCalendarList = getCalendarList(calendarService)
print("Processing RSS events for Org. IDs: %s" % ', '.join([str(o) for o in ORG_LIST]))
for org in ORG_LIST:
print("Retrieving RSS feed at \"%s\"" % createRssEventUrl(org))
rssEvents = getRssEvents(org)
print("Found %s RSS feed events" % len(rssEvents['events']))
# Find last-dated item in RSS feed
minEventDatetime, maxEventDatetime = findMinMaxRssDatetime(rssEvents['events'])
# Retrieve existing calendars. Create if not found.
targetCalendar = None
for googleCalendar in googleCalendarList.itervalues():
if rssEvents['summary'] == googleCalendar['summary']:
print("Found existing google calendar \"%s\" - \"%s\"" % (googleCalendar['id'], googleCalendar['summary']))
targetCalendar = googleCalendar
break
if targetCalendar is None:
print("Creating new google calendar \"%s\"" % rssEvents['summary'])
targetCalendar = createCalendar(calendarService, rssEvents['summary'])
print("Created new calendar \"%s\" - \"%s\"" % (targetCalendar['id'], targetCalendar['summary']))
# Read and cache google events up to last-dated RSS item
print("Retrieving all events in calendar \"%s\" from %s to %s" % (targetCalendar['summary'], minEventDatetime.isoformat('T'), maxEventDatetime.isoformat('T')))
calEvents = getCalendarEvents(calendarService, targetCalendar['id'], minEventDatetime, maxEventDatetime)
print("Found %s Google Calendar events" % len(calEvents))
# Compare RSS item to google cached events: if missing, create
batchCount = 0
createCount = 0
batchRequest = BatchHttpRequest()
for rssEvent in rssEvents['events'].itervalues():
if isValidEvent(rssEvent):
if compareOrCreateEvent(calendarService, targetCalendar['id'], batchRequest, rssEvent, calEvents):
batchCount += 1
if batchCount == 100:
createCount += batchCount
print("Calling batch request to create %s events" % batchCount)
googleApiCall(lambda: batchRequest.execute())
batchCount = 0
batchRequest = BatchHttpRequest()
if batchCount:
createCount += batchCount
print("Calling batch request to create %s events" % batchCount)
googleApiCall(lambda: batchRequest.execute())
if createCount + len(calEvents) != len(rssEvents['events']):
print("WARNING: Mismatched event counts: %s RSS events, %s existing Google Calendar Events, %s created Google Calendar Events" % (
len(rssEvents['events']), len(calEvents), createCount))
print("Done processesing all RSS feeds and Google Calendar events")
print("Exiting")
def batch_remove(objects, service, http):
def cb(req_id, response, exception):
if exception:
print req_id, exception
batch = BatchHttpRequest()
for obj in objects:
batch.add(service.objects().delete(bucket=BUCKET, object=obj), callback=cb)
return batch.execute(http=http)
def unread_subjects(self):
batch = BatchHttpRequest()
messages = self.unread_ids()
self.subjects = []
if messages:
for msg in messages:
batch.add(self.service.users().messages().get(userId='me', id=msg['id']), callback=self.subject_callback)
batch.execute()
return self.subjects
def main(argv):
lService, lFlags = sample_tools.init(
argv,
'webmasters',
'v3',
__doc__,
__file__,
parents=[argparser],
scope='https://www.googleapis.com/auth/webmasters.readonly')
lLines = [line.rstrip('\n') for line in open(lFlags.url_file)]
wipe_data()
lPos = 0
lBatch = BatchHttpRequest()
for lURL in lLines:
# TODO: Is it possible to minimize the request count (send one get two)?
lRequest = {
'startDate':
lFlags.start_date,
'endDate':
lFlags.end_date,
'dimensions': ['page'],
'dimensionFilterGroups': [{
'filters': [{
'dimension': 'page',
'expression': lURL
}]
}],
}
# TODO: Test with arg (maybe split '?')
#lURL.split("//")[-1].split("/")[0]
theSiteURL = (lURL.split("//")[0] + "//" +
(lURL.split("//")[-1].split("/")[0]))
#theSiteURL = lURL
print "Adding " + lURL
lBatch.add(
lService.searchanalytics().query(siteUrl=theSiteURL,
body=lRequest), HandleRequest)
lPos += 1
# Try 10 QPS and 20 QPS -- 10 should work... Analytics is 5? Webmasters is 10? Search Analytics is 3?
if lPos == 5: # 5 queries per second is a Google imposed limit
lBatch.execute()
time.sleep(1) # If it runs too fast Google will deny the request.
lBatch = BatchHttpRequest()
lPos = 0
if lPos:
lBatch.execute()
def post(self):
urlfetch.set_default_fetch_deadline(60)
http = decorator.http()
calendar_id = self.request.POST.get("calendar_id")
widget = '<iframe src="https://www.google.com/calendar/embed?height=600&wkst=1&bgcolor=%23FFFFFF&src='+calendar_id.replace("@","%40")+'&color=%23B1365F&ctz=Asia%2FCalcutta" style=" border-width:0 " width="800" height="600" frameborder="0" scrolling="no"></iframe>'
batch = BatchHttpRequest()
for event in events:
batch.add(service.events().insert(calendarId=calendar_id, body=event))
batch.execute(http=http)
self.response.write("Updated calendar: <br/>\n"+widget)
def getGmailInbox(self):
#request = self.client.users().messages().list(userId='me')
request = self.client.users().messages().list(userId='me', includeSpamTrash='true')
while request != None:
messages_doc= request.execute()
batch = BatchHttpRequest(callback = self.saveEmails)
for msg_id in messages_doc['messages']:
batch.add(self.client.users().messages().get(userId = 'me', id = msg_id['id']))
batch.execute()
request = self.client.users().messages().list_next(request, messages_doc)
def _process_batch(updates):
if not updates:
return
def callback(request_id, response, exception):
if exception is not None:
raise exception
batch = BatchHttpRequest(callback=callback)
[batch.add(update) for update in updates]
batch.execute()
def _process_batch(updates):
if not updates:
return
def callback(request_id, response, exception):
if exception is not None:
raise exception
batch = BatchHttpRequest(callback=callback)
[batch.add(update) for update in updates]
batch.execute()
def executeMultipleRequests(responseOrder, makeRequest):
responses = [None for i in xrange(len(responseOrder))]
def batchCallback(request_id, response, exception):
if exception:
return
responses[responseOrder.index(request_id)] = response
batch = BatchHttpRequest()
for i in xrange(len(responseOrder)):
batch.add(makeRequest(i), request_id = responseOrder[i], callback = batchCallback)
batch.execute()
return responses
def test_execute_global_callback(self):
callbacks = Callbacks()
batch = BatchHttpRequest(callback=callbacks.f)
batch.add(self.request1)
batch.add(self.request2)
http = HttpMockSequence(
[({"status": "200", "content-type": 'multipart/mixed; boundary="batch_foobarbaz"'}, BATCH_RESPONSE)]
)
batch.execute(http)
self.assertEqual({"foo": 42}, callbacks.responses["1"])
self.assertEqual({"baz": "qux"}, callbacks.responses["2"])
def test_serialize_request_no_body(self):
batch = BatchHttpRequest()
request = HttpRequest(
None,
None,
'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
method='POST',
body='',
headers={'content-type': 'application/json'},
methodId=None,
resumable=None)
s = batch._serialize_request(request).splitlines()
self.assertEquals(s, NO_BODY_EXPECTED.splitlines())
def test_serialize_request_no_body(self):
batch = BatchHttpRequest()
request = HttpRequest(
None,
None,
'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
method='POST',
body='',
headers={'content-type': 'application/json'},
methodId=None,
resumable=None)
s = batch._serialize_request(request).splitlines()
self.assertEqual(NO_BODY_EXPECTED.splitlines(), s)
def main(argv):
global lLines
global lFlags
lService, lFlags = sample_tools.init(
argv,
'webmasters',
'v3',
__doc__,
__file__,
parents=[argparser],
scope='https://www.googleapis.com/auth/webmasters.readonly')
lLines = [line.rstrip('\n') for line in open(lFlags.url_file)]
lPos = 0
lBatch = BatchHttpRequest()
for lURL in lLines:
lRequest = {
'startDate':
lFlags.start_date,
'endDate':
lFlags.end_date,
'dimensions': ['query'],
'dimensionFilterGroups': [{
'filters': [{
'dimension': 'page',
'expression': lURL
}]
}]
}
theSiteURL = (lURL.split("//")[0] + "//" +
(lURL.split("//")[-1].split("/")[0]))
lBatch.add(
lService.searchanalytics().query(siteUrl=theSiteURL,
body=lRequest), HandleRequest)
lPos += 1
if lPos == 5:
lBatch.execute()
time.sleep(
0.5) # If it runs too fast Google will deny the request.
lBatch = BatchHttpRequest()
lPos = 0
if lPos:
lBatch.execute()
def test_execute_global_callback(self):
callbacks = Callbacks()
batch = BatchHttpRequest(callback=callbacks.f)
batch.add(self.request1)
batch.add(self.request2)
http = HttpMockSequence([
({'status': '200',
'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'},
BATCH_RESPONSE),
])
batch.execute(http=http)
self.assertEqual({'foo': 42}, callbacks.responses['1'])
self.assertEqual({'baz': 'qux'}, callbacks.responses['2'])
def test_serialize_request_no_body(self):
batch = BatchHttpRequest()
request = HttpRequest(
None,
None,
"https://www.googleapis.com/someapi/v1/collection/?foo=bar",
method="POST",
body="",
headers={"content-type": "application/json"},
methodId=None,
resumable=None,
)
s = batch._serialize_request(request).splitlines()
self.assertEqual(NO_BODY_EXPECTED.splitlines(), s)
def runWorker(self, service=None, userId=None):
service = self.service
if not self.threads:
return
batch = BatchHttpRequest()
count = 0
while self.threads and count < self.batch_request_limit:
count += 1
thread = self.threads.pop(0)
batch.add(callback=self.cb, request_id=thread,
request=service.users().threads().modify(
userId=self.userId, id=thread, body=self.payload))
batch.execute()
def GetMessages(historyItems):
messageIds = list()
for historyItem in historyItems:
for message in historyItem['messages']:
messageIds.append(message['id'])
if len(messageIds) > 0:
batch = BatchHttpRequest()
for msgId in messageIds:
batch.add(gmail_service.users().messages().get(id=msgId, userId='me', format='full'), callback=PrintMessage)
batch.execute(http=http)
def get_list(wf, http, service):
# Retrieve a page of threads
threads = service.users().threads().list(
userId='me', labelIds=['INBOX'], maxResults=100).execute()
batch = BatchHttpRequest()
if 'threads' in threads and len(threads['threads']) > 0:
fields = 'messages/id,messages/threadId,messages/labelIds,messages/snippet,messages/payload/headers'
for thread in threads['threads']:
batch.add(service.users().threads().get(
userId='me', id=thread['id'], fields=fields), callback=list_threads)
batch.execute(http=http)
return EMAIL_LIST
def test_add_fail_for_resumable(self):
batch = BatchHttpRequest()
upload = MediaFileUpload(
datafile('small.png'), chunksize=500, resumable=True)
self.request1.resumable = upload
self.assertRaises(BatchError, batch.add, self.request1, request_id='1')
def test_serialize_request_media_body(self):
batch = BatchHttpRequest()
f = open(datafile('small.png'))
body = f.read()
f.close()
request = HttpRequest(
None,
None,
'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
method='POST',
body=body,
headers={'content-type': 'application/json'},
methodId=None,
resumable=None)
# Just testing it shouldn't raise an exception.
s = batch._serialize_request(request).splitlines()
def delete_playlists(youtube, args):
playlists_req = youtube.playlists().list(
part="id,snippet",
mine=True,
maxResults=MAX_RESULTS,
)
def delete_playlist(request_id, response, exception):
playlist_id = request_id
if exception:
raise exception
else:
sys.stderr.write(u"Deleted {id}\n".format(id=playlist_id))
delete = False
batch_req = BatchHttpRequest(callback=delete_playlist)
while playlists_req:
playlists = playlists_req.execute()
for playlist in playlists["items"]:
if re.match(args.pattern, playlist["snippet"]["title"]):
if args.pretend:
sys.stderr.write(u"Deleting {}\n".format(
playlist["snippet"]["title"]))
else:
delete_req = youtube.playlists().delete(id=playlist["id"])
batch_req.add(delete_req, request_id=playlist["id"])
delete = True
playlists_req = youtube.playlists().list_next(playlists_req, playlists)
if delete:
batch_req.execute()
def task_():
if request.method == "POST":
env = os.getenv('SERVER_SOFTWARE')
if (env and env.startswith('Google App Engine/')):
db = MySQLdb.connect(
unix_socket='/cloudsql/peppy-linker-102423:daniel-george',
user='******',
db='sheepdog')
cursor = db.cursor()
tokens = ["","CDIQAA","CGQQAA","CJYBEAA","CMgBEAA","CPoBEAA","CKwCEAA","CN4CEAA","CJADEAA","CMIDEAA","CPQDEAA","CKYEEAA", "CNgEEAA", "CIoFEAA", "CLwFEAA", "CO4FEAA", "CKAGEAA", "CNIGEAA", "CIQHEAA", "CLYHEAA", "COgHEAA", "CJoIEAA", "CMwIEAA", "CP4IEAA", "CLAJEAA", "COIJEAA", "CJQKEAA", "CMYKEAA", "CPgKEAA", "CKoLEAA", "CNwLEAA", "CI4MEAA", "CMAMEAA", "CPIMEAA", "CKQNEAA", "CNYNEAA", "CIgOEAA", "CLoOEAA", "COwOEAA", "CJ4PEAA", "CNAPEAA", "CIIQEAA", "CLQQEAA", "COYQEAA", "CJgREAA", "CMoREAA", "CPwREAA", "CK4SEAA", "COASEAA", "CJITEAA", "CMQTEAA", "CPYTEAA", "CKgUEAA", "CNoUEAA", "CIwVEAA", "CL4VEAA", "CPAVEAA", "CKIWEAA", "CNQWEAA", "CIYXEAA", "CLgXEAA", "COoXEAA", "CJwYEAA", "CM4YEAA", "CIAZEAA", "CLIZEAA", "COQZEAA", "CJYaEAA", "CMgaEAA", "CPoaEAA", "CKwbEAA", "CN4bEAA", "CJAcEAA", "CMIcEAA", "CPQcEAA", "CKYdEAA", "CNgdEAA", "CIoeEAA", "CLweEAA", "CO4eEAA", "CKAfEAA", "CNIfEAA", "CIQgEAA", "CLYgEAA", "COggEAA", "CJohEAA", "CMwhEAA", "CP4hEAA", "CLAiEAA", "COIiEAA", "CJQjEAA", "CMYjEAA", "CPgjEAA", "CKokEAA", "CNwkEAA", "CI4lEAA", "CMAlEAA", "CPIlEAA", "CKQmEAA", "CNYmEAA", ]
batch = BatchHttpRequest()
user = request.form.get('user')
for token in tokens[:20]:
try:
playlistitems_list_request = youtube.playlistItems().list(
playlistId=user,
part="snippet",
pageToken=token,
maxResults=50
)
except NameError:
pass
def list1(request_id,response,exception):
for playlist_item in response["items"]:
video_id = playlist_item["snippet"]["resourceId"]["videoId"]
cursor.execute("""INSERT INTO sheepdog.videoIds (videoId) VALUES (%s);""", [video_id])
db.commit()
batch.add(playlistitems_list_request, callback=list1)
batch.execute(http=http)
return 'string'
def print_popular_videos_with_analytics():
for table in unique_link_table:
num_batch = 0
batch = BatchHttpRequest()
for video_id in sorted(unique_link_table[table], key=unique_link_table[table].get, reverse=True):
if num_batch >= 1000:
break
if len(list_popular_music_videos) >= 20:
break
batch.add(youtube.videos().list(id=video_id, part="snippet"), callback=process_result)
num_batch += 1
try:
print "not missing http"
batch.execute()
except ValueError:
print "missing http"
if len(list_popular_music_videos) >= 20:
for video in list_popular_music_videos[:50]:
if len(video["items"]) > 0:
list_popular_music_ids.append((video["items"][0]["id"], video["items"][0]["snippet"]["title"]))
write_popular_to_database(table, list_popular_music_ids)
del list_popular_music_ids[:]
def share(self,
users,
share_type='writer',
send_notifications=False,
email_message=None):
"""
Share a document with a given list of users.
"""
if type(users) is str:
users = [users]
def batch_callback(request_id, response, exception):
print("Response for request_id (%s):" % request_id)
print(response)
# Potentially log or re-raise exceptions
if exception:
raise exception
batch_request = BatchHttpRequest(callback=batch_callback)
for count, user in enumerate(users):
batch_entry = self.drive.service.permissions().insert(
fileId=self._id,
sendNotificationEmails=send_notifications,
emailMessage=email_message,
body={
'value': user,
'type': 'user',
'role': share_type
})
batch_request.add(batch_entry, request_id="batch" + str(count))
batch_request.execute()
def main(argv):
# Authenticate and construct service.
service, config, flags = shopping_common.init(argv,
__doc__,
parents=[argparser])
merchant_id = config['merchantId']
product_ids = flags.product_ids
batch = BatchHttpRequest(callback=product_updated)
for product_id in product_ids:
new_status = {
'availability': 'out of stock',
'price': {
'value': 3.14,
'currency': 'USD'
}
}
# Add product update to the batch.
batch.add(service.inventory().set(merchantId=merchant_id,
storeCode=product_id.split(':')[0],
productId=product_id,
body=new_status))
try:
batch.execute()
except client.AccessTokenRefreshError:
print(
'The credentials have been revoked or expired, please re-run the '
'application to re-authorize')
def main(argv):
# Authenticate and construct service.
service, flags = sample_tools.init(argv,
'content',
'v2',
__doc__,
__file__,
parents=[argparser])
merchant_id = flags.merchant_id
batch = BatchHttpRequest(callback=account_inserted)
for _ in range(BATCH_SIZE):
name = 'account%s' % shopping_common.get_unique_id()
account = {
'name': name,
'websiteUrl': 'https://%s.example.com/' % (name, )
}
# Add account to the batch.
batch.add(service.accounts().insert(merchantId=merchant_id,
body=account))
try:
batch.execute()
except client.AccessTokenRefreshError:
print(
'The credentials have been revoked or expired, please re-run the '
'application to re-authorize')
def del_users(service, service2, delete):
account_summaries = service.management().accountSummaries().list().execute(
)
for account in account_summaries.get('items', []):
account_id = account.get('id')
account_links = service2.management().accountUserLinks().list(
accountId=account_id).execute()
batch = BatchHttpRequest(callback=call_back)
for user in account_links.get('items', []):
users_ref = user.get('userRef')
user_mail = users_ref.get('email')
users_id = user.get('id')
for x in delete:
if x == user_mail:
print x
print account_id
delete_account = service2.management().accountUserLinks(
).delete(accountId=account_id, linkId=users_id)
batch.add(delete_account)
batch.execute()
time.sleep(1)
def _insert_item_all_users(self):
"""Insert a timeline item to all authorized users."""
logging.info('Inserting timeline item to all users')
users = Credentials.all()
total_users = users.count()
if total_users > 10:
return 'Total user count is %d. Aborting broadcast to save your quota' % (
total_users)
body = {
'text': 'Hello Everyone!',
'notification': {
'level': 'DEFAULT'
}
}
batch_responses = _BatchCallback()
batch = BatchHttpRequest(callback=batch_responses.callback)
for user in users:
creds = StorageByKeyName(Credentials,
user.key().name(), 'credentials').get()
mirror_service = util.create_service('mirror', 'v1', creds)
batch.add(mirror_service.timeline().insert(body=body),
request_id=user.key().name())
batch.execute(httplib2.Http())
return 'Successfully sent cards to %d users (%d failed).' % (
batch_responses.success, batch_responses.failure)
def test_http_errors_passed_to_callback(self):
batch = BatchHttpRequest()
callbacks = Callbacks()
cred_1 = MockCredentials('Foo')
cred_2 = MockCredentials('Bar')
http = HttpMockSequence([
({'status': '200',
'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'},
BATCH_RESPONSE_WITH_401),
({'status': '200',
'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'},
BATCH_RESPONSE_WITH_401),
])
creds_http_1 = HttpMockSequence([])
cred_1.authorize(creds_http_1)
creds_http_2 = HttpMockSequence([])
cred_2.authorize(creds_http_2)
self.request1.http = creds_http_1
self.request2.http = creds_http_2
batch.add(self.request1, callback=callbacks.f)
batch.add(self.request2, callback=callbacks.f)
batch.execute(http=http)
self.assertEqual(None, callbacks.responses['1'])
self.assertEqual(401, callbacks.exceptions['1'].resp.status)
self.assertEqual(
'Authorization Required', callbacks.exceptions['1'].resp.reason)
self.assertEqual({u'baz': u'qux'}, callbacks.responses['2'])
self.assertEqual(None, callbacks.exceptions['2'])
def getNewestVideos(self):
# Temporary fix to overcome oauth expiries, should only call once oauth
# is expired (to be fixed)
self.records = {}
# When subscription count is large it's important to batch all the
# HTTP requests together as 1 http request. This will break if
# Channel list is > 1000 (to be fixed)
batch = BatchHttpRequest(callback=self.getChannelNewestVideosCallback)
# Add each playlist to the batch request
for channel_id in self.channel_titles:
# We should be getting videos directly off the playlist items
# But YouTube API takes 15 - 60 mins to update this list
# So instead search.list is used at great quota cost
# Also since moving to batch we only get the last 50 results from
# a channel, TO DO: collate nextPageTokens if require more than 50
check_after = (datetime.utcnow() -
timedelta(days=self.set.days_uploaded_after))
check_after = check_after.isoformat("T") + "Z"
batch.add(
self.youtube.search().list(
part='snippet', maxResults=50, channelId=channel_id,
type='video', safeSearch='none', publishedAfter=check_after
)
)
for _ in range(500):
with ytLoginManager(self.login_timer) as request:
if request.relogin:
self.youtube = self.initilize_youtube(self.set)
batch.execute()
if request.success:
break
while not self.descq.empty():
try:
[YTid, cid, desc_contain, record] = self.descq.get()
ful_desc = self.getVideoDescription(YTid)
check_ful_desc = re.sub('[\W_]+', '', ful_desc).lower()
if desc_contain in check_ful_desc:
self.recq.put([YTid, cid, record])
except Exception:
continue
counter = 0
while not self.recq.empty():
try:
[YTid, cid, record] = self.recq.get()
self.records[YTid] = record
self.channel_videos[cid].append(YTid)
counter += 1
except Exception:
continue
return counter
def test_new_id(self):
batch = BatchHttpRequest()
id_ = batch._new_id()
self.assertEquals(id_, '1')
id_ = batch._new_id()
self.assertEquals(id_, '2')
batch.add(self.request1, request_id='3')
id_ = batch._new_id()
self.assertEquals(id_, '4')
def test_new_id(self):
batch = BatchHttpRequest()
id_ = batch._new_id()
self.assertEqual('1', id_)
id_ = batch._new_id()
self.assertEqual('2', id_)
batch.add(self.request1, request_id='3')
id_ = batch._new_id()
self.assertEqual('4', id_)
def new_batch_http_request(callback=None):
"""Create a BatchHttpRequest object based on the discovery document.
Args:
callback: callable, A callback to be called for each response, of the
form callback(id, response, exception). The first parameter is the
request id, and the second is the deserialized response object. The
third is an apiclient.errors.HttpError exception object if an HTTP
error occurred while processing the request, or None if no error
occurred.
Returns:
A BatchHttpRequest object based on the discovery document.
"""
return BatchHttpRequest(callback=callback, batch_uri=batch_uri)
def getGmailInbox(self):
#request = self.client.users().messages().list(userId='me')
request = self.client.users().messages().list(userId='me',
includeSpamTrash='true')
while request != None:
messages_doc = request.execute()
batch = BatchHttpRequest(callback=self.saveEmails)
for msg_id in messages_doc['messages']:
batch.add(self.client.users().messages().get(userId='me',
id=msg_id['id']))
batch.execute()
request = self.client.users().messages().list_next(
request, messages_doc)
def test_execute_global_callback(self):
callbacks = Callbacks()
batch = BatchHttpRequest(callback=callbacks.f)
batch.add(self.request1)
batch.add(self.request2)
http = HttpMockSequence([
({'status': '200',
'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'},
BATCH_RESPONSE),
])
batch.execute(http=http)
self.assertEqual({'foo': 42}, callbacks.responses['1'])
self.assertEqual({'baz': 'qux'}, callbacks.responses['2'])
def batchUpdateInventory(self, gproduct_qset):
GoogleProduct = get_model('gmerchant', 'GoogleProduct')
def product_updated(request_id, unused_response, exception):
if exception is not None:
# Do something with the exception.
print 'There was an error: ' + str(exception)
else:
gp = GoogleProduct.objects.get(google_shopping_id=offer_id)
gp.google_shopping_updated = datetime.now()
gp.save()
print 'Request ID: %s - Product was updated.' % (
str(request_id), )
merchant_id = self.merchant_id
batch = BatchHttpRequest(callback=product_updated)
for prod in gproduct_qset:
product = prod.product
new_status = {
#Update the price of the item
'price': {
'value': str(product.stockrecords.first().price_incl_tax),
'currency': 'GBP'
},
'description':
len(product.google_shopping_description) > 0 and bleach.clean(
smart_text(product.google_shopping_description),
strip=True) or bleach.clean(smart_text(
product.parent.google_shopping_description),
strip=True),
'link':
SITE_ROOT + product.get_absolute_url(),
'imageLink':
product.get_first_image_url(),
#Is it in stock?
'availability':
resolve_google_availability(product),
}
# Add product update to the batch.
batch.add(self.service.inventory().set(
merchantId=merchant_id,
productId=prod.google_shopping_id,
body=new_status))
try:
batch.execute()
except client.AccessTokenRefreshError:
warn_exp_token()
def main(argv):
# Authenticate and construct service.
service, flags = sample_tools.init(argv,
'content',
'v2',
__doc__,
__file__,
parents=[argparser])
merchant_id = flags.merchant_id
batch = BatchHttpRequest(callback=datafeed_inserted)
for _ in range(BATCH_SIZE):
name = 'feed%s' % shopping_common.get_unique_id()
datafeed = {
'name': name,
'contentType': 'products',
'attributeLanguage': 'en',
'contentLanguage': 'en',
'intendedDestinations': ['Shopping'],
# The file name must be unique per account. We only use unique names in
# these examples, so it's not an issue here.
'fileName': name,
'targetCountry': 'US',
# You can schedule monthly, weekly or daily.
#
# Monthly - set day of month ('dayOfMonth') and hour ('hour')
# Weekly - set day of week ('weekday') and hour ('hour')
# Daily - set just the hour ('hour')
'fetchSchedule': {
'weekday': 'monday',
'hour': 6,
'timeZone': 'America/Los_Angeles',
'fetchUrl': 'https://feeds.myshop.com/' + name
},
'format': {
'fileEncoding': 'utf-8',
'columnDelimiter': 'tab',
'quotingMode': 'value quoting'
}
}
# Add datafeed to the batch.
batch.add(service.datafeeds().insert(merchantId=merchant_id,
body=datafeed))
try:
batch.execute()
except client.AccessTokenRefreshError:
print(
'The credentials have been revoked or expired, please re-run the '
'application to re-authorize')
def infoUpdate(vSources, forceUpdate=False):
batch = BatchHttpRequest()
sourcesToUpdate = False
for vSource in vSources:
if forceUpdate or vSource.needsInfoUpdate():
request, callback = vSource.fetchInfoBatchRequest()
batch.add(request, callback=callback)
sourcesToUpdate = True
if sourcesToUpdate:
batch.execute()
def share(self, file_ids, emails, role='reader', callback=None):
"""
Share a list of files to a list of e-mails.
"""
if not isinstance(file_ids, (list, tuple)):
raise ValueError(
"We are expecting a list of file_ids, not %s" % file_ids
)
if not isinstance(emails, (list, tuple)):
raise ValueError(
"We are expecting a list of emails, not %s" % emails
)
self._pydrive.auth.Authorize()
perms = self._pydrive.auth.service.permissions()
http = self._pydrive.auth.http
batch_response = OrderedDict()
def batch_callback(request_id, response, exception):
file_id = request_id.split('__', 2)[1]
if exception:
logger.error("Error on drive batch operation for %s: %s",
request_id, exception)
batch_response[file_id].update({'exception': exception})
else:
batch_response[file_id].update(response)
batch_request = BatchHttpRequest(callback=batch_callback)
for file_id in list(set(file_ids)):
for email in list(set(emails)):
kwargs = {
'fileId': file_id,
'body': {
'value': email,
'type': 'user',
'role': role
}
}
batch_id = 'share__%s__%s' % (file_id, uuid4())
batch_request.add(perms.insert(**kwargs), request_id=batch_id)
logger.info(
"Batch share request added with ID %s and data %s",
batch_id, kwargs
)
batch_response[file_id] = {'insert_kwargs': kwargs}
batch_request.execute()
return batch_response
def execute(self):
"""Executes requests in the queue.
Removes items from the queue, and adds them to a BatchHttpRequest object.
Only removes up to set quota. and then calls the BatchHttPRequest object's
execute method.
"""
batch = BatchHttpRequest(callback=self.call_back)
for _ in range(self.quota):
if self.queue.qsize() == 0:
break
request, request_id = self.queue.get()
batch.add(request, request_id=request_id)
batch.execute(http=httplib2.Http())
def add_users(users, permissions):
"""Adds users to every view (profile) with the given permissions.
Args:
users: A list of user email addresses.
permissions: A list of user permissions.
Note: this code assumes you have MANAGE_USERS level permissions
to each profile and an authorized Google Analytics service object.
"""
# Get the a full set of account summaries.
account_summaries = analytics.management().accountSummaries().list(
).execute()
# Loop through each account.
for account in account_summaries.get('items', []):
account_id = account.get('id')
# Loop through each user.
for user in users:
# Create the BatchHttpRequest object.
batch = BatchHttpRequest(callback=call_back)
# Loop through each property.
for property_summary in account.get('webProperties', []):
property_id = property_summary.get('id')
# Loop through each view (profile).
for view in property_summary.get('profiles', []):
view_id = view.get('id')
# Construct the Profile User Link.
link = analytics.management().profileUserLinks().insert(
accountId=account_id,
webPropertyId=property_id,
profileId=view_id,
body={
'permissions': {
'local': permissions
},
'userRef': {
'email': user
}
})
batch.add(link)
# Execute the batch request for each user.
batch.execute()
