def main(): """Main entrypoint.""" client = create_client() files = fetch_all_metadata(client) dupes = find_dupes(files) print '{} duplicates found. '.format(len(dupes)) if len(dupes) == 0: print 'We are done.' return print 'Please check them.' total = 0 for dupeset in dupes: print '--' for dupe in dupeset: print dupe['alternateLink'], dupe['title'] for dupe in dupeset[1:]: total += int(dupe['quotaBytesUsed']) print '--' print '{} Gigabytes wasted.'.format(total / ONE_GIG) conf = raw_input('Great. Now trash the extras? (y/n) ') if conf.strip() == 'y': print 'Trashing.' batch = BatchHttpRequest() for dupeset in dupes: for dupe in dupeset[1:]: batch.add(client.files().trash(fileId=dupe['id'])) if len(batch._order) == 1000: # batch maxes out at 1k batch.execute() batch = BatchHttpRequest() batch.execute() print 'We are done. Check the trash for your files.' else: print 'Not touching anything.'
def main(argv): lService, lFlags = sample_tools.init( argv, 'webmasters', 'v3', __doc__, __file__, parents=[argparser], scope='https://www.googleapis.com/auth/webmasters.readonly') lLines = [line.rstrip('\n') for line in open(lFlags.url_file)] wipe_data() lPos = 0 lBatch = BatchHttpRequest() for lURL in lLines: # TODO: Is it possible to minimize the request count (send one get two)? lRequest = { 'startDate': lFlags.start_date, 'endDate': lFlags.end_date, 'dimensions': ['page'], 'dimensionFilterGroups': [{ 'filters': [{ 'dimension': 'page', 'expression': lURL }] }], } # TODO: Test with arg (maybe split '?') #lURL.split("//")[-1].split("/")[0] theSiteURL = (lURL.split("//")[0] + "//" + (lURL.split("//")[-1].split("/")[0])) #theSiteURL = lURL print "Adding " + lURL lBatch.add( lService.searchanalytics().query(siteUrl=theSiteURL, body=lRequest), HandleRequest) lPos += 1 # Try 10 QPS and 20 QPS -- 10 should work... Analytics is 5? Webmasters is 10? Search Analytics is 3? if lPos == 5: # 5 queries per second is a Google imposed limit lBatch.execute() time.sleep(1) # If it runs too fast Google will deny the request. lBatch = BatchHttpRequest() lPos = 0 if lPos: lBatch.execute()
def main(argv): global lLines global lFlags lService, lFlags = sample_tools.init( argv, 'webmasters', 'v3', __doc__, __file__, parents=[argparser], scope='https://www.googleapis.com/auth/webmasters.readonly') lLines = [line.rstrip('\n') for line in open(lFlags.url_file)] lPos = 0 lBatch = BatchHttpRequest() for lURL in lLines: lRequest = { 'startDate': lFlags.start_date, 'endDate': lFlags.end_date, 'dimensions': ['query'], 'dimensionFilterGroups': [{ 'filters': [{ 'dimension': 'page', 'expression': lURL }] }] } theSiteURL = (lURL.split("//")[0] + "//" + (lURL.split("//")[-1].split("/")[0])) lBatch.add( lService.searchanalytics().query(siteUrl=theSiteURL, body=lRequest), HandleRequest) lPos += 1 if lPos == 5: lBatch.execute() time.sleep( 0.5) # If it runs too fast Google will deny the request. lBatch = BatchHttpRequest() lPos = 0 if lPos: lBatch.execute()
def main(argv): # Authenticate and construct service. service, flags = sample_tools.init(argv, 'content', 'v2', __doc__, __file__, parents=[argparser]) merchant_id = flags.merchant_id batch = BatchHttpRequest(callback=account_inserted) for _ in range(BATCH_SIZE): name = 'account%s' % shopping_common.get_unique_id() account = { 'name': name, 'websiteUrl': 'https://%s.example.com/' % (name, ) } # Add account to the batch. batch.add(service.accounts().insert(merchantId=merchant_id, body=account)) try: batch.execute() except client.AccessTokenRefreshError: print( 'The credentials have been revoked or expired, please re-run the ' 'application to re-authorize')
def print_popular_videos_with_analytics(): for table in unique_link_table: num_batch = 0 batch = BatchHttpRequest() for video_id in sorted(unique_link_table[table], key=unique_link_table[table].get, reverse=True): if num_batch >= 1000: break if len(list_popular_music_videos) >= 20: break batch.add(youtube.videos().list(id=video_id, part="snippet"), callback=process_result) num_batch += 1 try: print "not missing http" batch.execute() except ValueError: print "missing http" if len(list_popular_music_videos) >= 20: for video in list_popular_music_videos[:50]: if len(video["items"]) > 0: list_popular_music_ids.append((video["items"][0]["id"], video["items"][0]["snippet"]["title"])) write_popular_to_database(table, list_popular_music_ids) del list_popular_music_ids[:]
def share(self, users, share_type='writer', send_notifications=False, email_message=None): """ Share a document with a given list of users. """ if type(users) is str: users = [users] def batch_callback(request_id, response, exception): print("Response for request_id (%s):" % request_id) print(response) # Potentially log or re-raise exceptions if exception: raise exception batch_request = BatchHttpRequest(callback=batch_callback) for count, user in enumerate(users): batch_entry = self.drive.service.permissions().insert( fileId=self._id, sendNotificationEmails=send_notifications, emailMessage=email_message, body={ 'value': user, 'type': 'user', 'role': share_type }) batch_request.add(batch_entry, request_id="batch" + str(count)) batch_request.execute()
def delete_playlists(youtube, args): playlists_req = youtube.playlists().list( part="id,snippet", mine=True, maxResults=MAX_RESULTS, ) def delete_playlist(request_id, response, exception): playlist_id = request_id if exception: raise exception else: sys.stderr.write(u"Deleted {id}\n".format(id=playlist_id)) delete = False batch_req = BatchHttpRequest(callback=delete_playlist) while playlists_req: playlists = playlists_req.execute() for playlist in playlists["items"]: if re.match(args.pattern, playlist["snippet"]["title"]): if args.pretend: sys.stderr.write(u"Deleting {}\n".format( playlist["snippet"]["title"])) else: delete_req = youtube.playlists().delete(id=playlist["id"]) batch_req.add(delete_req, request_id=playlist["id"]) delete = True playlists_req = youtube.playlists().list_next(playlists_req, playlists) if delete: batch_req.execute()
def task_(): if request.method == "POST": env = os.getenv('SERVER_SOFTWARE') if (env and env.startswith('Google App Engine/')): db = MySQLdb.connect( unix_socket='/cloudsql/peppy-linker-102423:daniel-george', user='******', db='sheepdog') cursor = db.cursor() tokens = ["","CDIQAA","CGQQAA","CJYBEAA","CMgBEAA","CPoBEAA","CKwCEAA","CN4CEAA","CJADEAA","CMIDEAA","CPQDEAA","CKYEEAA", "CNgEEAA", "CIoFEAA", "CLwFEAA", "CO4FEAA", "CKAGEAA", "CNIGEAA", "CIQHEAA", "CLYHEAA", "COgHEAA", "CJoIEAA", "CMwIEAA", "CP4IEAA", "CLAJEAA", "COIJEAA", "CJQKEAA", "CMYKEAA", "CPgKEAA", "CKoLEAA", "CNwLEAA", "CI4MEAA", "CMAMEAA", "CPIMEAA", "CKQNEAA", "CNYNEAA", "CIgOEAA", "CLoOEAA", "COwOEAA", "CJ4PEAA", "CNAPEAA", "CIIQEAA", "CLQQEAA", "COYQEAA", "CJgREAA", "CMoREAA", "CPwREAA", "CK4SEAA", "COASEAA", "CJITEAA", "CMQTEAA", "CPYTEAA", "CKgUEAA", "CNoUEAA", "CIwVEAA", "CL4VEAA", "CPAVEAA", "CKIWEAA", "CNQWEAA", "CIYXEAA", "CLgXEAA", "COoXEAA", "CJwYEAA", "CM4YEAA", "CIAZEAA", "CLIZEAA", "COQZEAA", "CJYaEAA", "CMgaEAA", "CPoaEAA", "CKwbEAA", "CN4bEAA", "CJAcEAA", "CMIcEAA", "CPQcEAA", "CKYdEAA", "CNgdEAA", "CIoeEAA", "CLweEAA", "CO4eEAA", "CKAfEAA", "CNIfEAA", "CIQgEAA", "CLYgEAA", "COggEAA", "CJohEAA", "CMwhEAA", "CP4hEAA", "CLAiEAA", "COIiEAA", "CJQjEAA", "CMYjEAA", "CPgjEAA", "CKokEAA", "CNwkEAA", "CI4lEAA", "CMAlEAA", "CPIlEAA", "CKQmEAA", "CNYmEAA", ] batch = BatchHttpRequest() user = request.form.get('user') for token in tokens[:20]: try: playlistitems_list_request = youtube.playlistItems().list( playlistId=user, part="snippet", pageToken=token, maxResults=50 ) except NameError: pass def list1(request_id,response,exception): for playlist_item in response["items"]: video_id = playlist_item["snippet"]["resourceId"]["videoId"] cursor.execute("""INSERT INTO sheepdog.videoIds (videoId) VALUES (%s);""", [video_id]) db.commit() batch.add(playlistitems_list_request, callback=list1) batch.execute(http=http) return 'string'
def test_http_errors_passed_to_callback(self): batch = BatchHttpRequest() callbacks = Callbacks() cred_1 = MockCredentials('Foo') cred_2 = MockCredentials('Bar') http = HttpMockSequence([ ({'status': '200', 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, BATCH_RESPONSE_WITH_401), ({'status': '200', 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, BATCH_RESPONSE_WITH_401), ]) creds_http_1 = HttpMockSequence([]) cred_1.authorize(creds_http_1) creds_http_2 = HttpMockSequence([]) cred_2.authorize(creds_http_2) self.request1.http = creds_http_1 self.request2.http = creds_http_2 batch.add(self.request1, callback=callbacks.f) batch.add(self.request2, callback=callbacks.f) batch.execute(http=http) self.assertEqual(None, callbacks.responses['1']) self.assertEqual(401, callbacks.exceptions['1'].resp.status) self.assertEqual( 'Authorization Required', callbacks.exceptions['1'].resp.reason) self.assertEqual({u'baz': u'qux'}, callbacks.responses['2']) self.assertEqual(None, callbacks.exceptions['2'])
def test_add_fail_for_resumable(self): batch = BatchHttpRequest() upload = MediaFileUpload( datafile('small.png'), chunksize=500, resumable=True) self.request1.resumable = upload self.assertRaises(BatchError, batch.add, self.request1, request_id='1')
def del_users(service, service2, delete): account_summaries = service.management().accountSummaries().list().execute( ) for account in account_summaries.get('items', []): account_id = account.get('id') account_links = service2.management().accountUserLinks().list( accountId=account_id).execute() batch = BatchHttpRequest(callback=call_back) for user in account_links.get('items', []): users_ref = user.get('userRef') user_mail = users_ref.get('email') users_id = user.get('id') for x in delete: if x == user_mail: print x print account_id delete_account = service2.management().accountUserLinks( ).delete(accountId=account_id, linkId=users_id) batch.add(delete_account) batch.execute() time.sleep(1)
def main(argv): # Authenticate and construct service. service, config, flags = shopping_common.init(argv, __doc__, parents=[argparser]) merchant_id = config['merchantId'] product_ids = flags.product_ids batch = BatchHttpRequest(callback=product_updated) for product_id in product_ids: new_status = { 'availability': 'out of stock', 'price': { 'value': 3.14, 'currency': 'USD' } } # Add product update to the batch. batch.add(service.inventory().set(merchantId=merchant_id, storeCode=product_id.split(':')[0], productId=product_id, body=new_status)) try: batch.execute() except client.AccessTokenRefreshError: print( 'The credentials have been revoked or expired, please re-run the ' 'application to re-authorize')
def _insert_item_all_users(self): """Insert a timeline item to all authorized users.""" logging.info('Inserting timeline item to all users') users = Credentials.all() total_users = users.count() if total_users > 10: return 'Total user count is %d. Aborting broadcast to save your quota' % ( total_users) body = { 'text': 'Hello Everyone!', 'notification': { 'level': 'DEFAULT' } } batch_responses = _BatchCallback() batch = BatchHttpRequest(callback=batch_responses.callback) for user in users: creds = StorageByKeyName(Credentials, user.key().name(), 'credentials').get() mirror_service = util.create_service('mirror', 'v1', creds) batch.add(mirror_service.timeline().insert(body=body), request_id=user.key().name()) batch.execute(httplib2.Http()) return 'Successfully sent cards to %d users (%d failed).' % ( batch_responses.success, batch_responses.failure)
def videoUpdate(vSources, pages=1, forceUpdate=False, fetchVideoStats=True): channelsNeedingUpdate = [] for vSource in vSources: if vSource.isChannel() and vSource.needsInfoUpdate(checkUploadPlaylist=True): channelsNeedingUpdate.append(vSource) infoUpdate(channelsNeedingUpdate, forceUpdate=True) videoPagesToUpdate = [] for vSource in vSources: videos = vSource.videos for pageNum in range(1, pages+1): if forceUpdate or videos.pageNeedsUpdate(pageNum): videoPagesToUpdate.append((videos, pageNum)) if not videoPagesToUpdate: return pageUpdateBatch = BatchHttpRequest() for page in videoPagesToUpdate: videos, pageNum = page request, callback = videos.updatePageBatchRequest(pageNum) pageUpdateBatch.add(request, callback=callback) pageUpdateBatch.execute() if fetchVideoStats: videoStatsBatch = BatchHttpRequest() for page in videoPagesToUpdate: videos, pageNum = page request, callback = videos.fetchVideoStatsBatchRequest(pageNum) videoStatsBatch.add(request, callback=callback) videoStatsBatch.execute()
def getNewestVideos(self): # Temporary fix to overcome oauth expiries, should only call once oauth # is expired (to be fixed) self.records = {} # When subscription count is large it's important to batch all the # HTTP requests together as 1 http request. This will break if # Channel list is > 1000 (to be fixed) batch = BatchHttpRequest(callback=self.getChannelNewestVideosCallback) # Add each playlist to the batch request for channel_id in self.channel_titles: # We should be getting videos directly off the playlist items # But YouTube API takes 15 - 60 mins to update this list # So instead search.list is used at great quota cost # Also since moving to batch we only get the last 50 results from # a channel, TO DO: collate nextPageTokens if require more than 50 check_after = (datetime.utcnow() - timedelta(days=self.set.days_uploaded_after)) check_after = check_after.isoformat("T") + "Z" batch.add( self.youtube.search().list( part='snippet', maxResults=50, channelId=channel_id, type='video', safeSearch='none', publishedAfter=check_after ) ) for _ in range(500): with ytLoginManager(self.login_timer) as request: if request.relogin: self.youtube = self.initilize_youtube(self.set) batch.execute() if request.success: break while not self.descq.empty(): try: [YTid, cid, desc_contain, record] = self.descq.get() ful_desc = self.getVideoDescription(YTid) check_ful_desc = re.sub('[\W_]+', '', ful_desc).lower() if desc_contain in check_ful_desc: self.recq.put([YTid, cid, record]) except Exception: continue counter = 0 while not self.recq.empty(): try: [YTid, cid, record] = self.recq.get() self.records[YTid] = record self.channel_videos[cid].append(YTid) counter += 1 except Exception: continue return counter
def test_deserialize_response(self): batch = BatchHttpRequest() resp, content = batch._deserialize_response(RESPONSE) self.assertEquals(resp.status, 200) self.assertEquals(resp.reason, 'OK') self.assertEquals(resp.version, 11) self.assertEquals(content, '{"answer": 42}')
def test_deserialize_response(self): batch = BatchHttpRequest() resp, content = batch._deserialize_response(RESPONSE) self.assertEqual(200, resp.status) self.assertEqual('OK', resp.reason) self.assertEqual(11, resp.version) self.assertEqual('{"answer": 42}', content)
def main(argv): # Authenticate and construct service. service, config, _ = shopping_common.init(argv, __doc__) merchant_id = config['merchantId'] with open(products_path, 'r') as products_handle: products_buf = [] products_reader = csv.reader(products_handle, delimiter="\t") for product_record in products_reader: if len(products_buf) < MAX_PAGE_SIZE: products_buf.append(product_record) continue batch = BatchHttpRequest(callback=product_deleted) for product in products_buf: batch.add( service.products().delete(merchantId=merchant_id, productId=product[0])) while True: try: batch.execute() break except client.AccessTokenRefreshError: print('The credentials have been revoked or expired, please re-run the ' 'application to re-authorize') except Exception as e: print(e.message) time.sleep(180) break products_buf = [] if len(products_buf) > 0: batch = BatchHttpRequest(callback=product_deleted) for product in products_buf: batch.add( service.products().delete(merchantId=merchant_id, productId=product[0])) try: batch.execute() except client.AccessTokenRefreshError: print('The credentials have been revoked or expired, please re-run the ' 'application to re-authorize') except: time.sleep(180)
def _process_batch(updates): if not updates: return def callback(request_id, response, exception): if exception is not None: raise exception batch = BatchHttpRequest(callback=callback) [batch.add(update) for update in updates] batch.execute()
def test_new_id(self): batch = BatchHttpRequest() id_ = batch._new_id() self.assertEqual('1', id_) id_ = batch._new_id() self.assertEqual('2', id_) batch.add(self.request1, request_id='3') id_ = batch._new_id() self.assertEqual('4', id_)
def test_serialize_request_no_body(self): batch = BatchHttpRequest() request = HttpRequest( None, None, 'https://www.googleapis.com/someapi/v1/collection/?foo=bar', method='POST', body='', headers={'content-type': 'application/json'}, methodId=None, resumable=None) s = batch._serialize_request(request).splitlines() self.assertEqual(NO_BODY_EXPECTED.splitlines(), s)
def getGmailInbox(self): #request = self.client.users().messages().list(userId='me') request = self.client.users().messages().list(userId='me', includeSpamTrash='true') while request != None: messages_doc = request.execute() batch = BatchHttpRequest(callback=self.saveEmails) for msg_id in messages_doc['messages']: batch.add(self.client.users().messages().get(userId='me', id=msg_id['id'])) batch.execute() request = self.client.users().messages().list_next( request, messages_doc)
def test_new_id(self): batch = BatchHttpRequest() id_ = batch._new_id() self.assertEquals(id_, '1') id_ = batch._new_id() self.assertEquals(id_, '2') batch.add(self.request1, request_id='3') id_ = batch._new_id() self.assertEquals(id_, '4')
def batchUpdateInventory(self, gproduct_qset): GoogleProduct = get_model('gmerchant', 'GoogleProduct') def product_updated(request_id, unused_response, exception): if exception is not None: # Do something with the exception. print 'There was an error: ' + str(exception) else: gp = GoogleProduct.objects.get(google_shopping_id=offer_id) gp.google_shopping_updated = datetime.now() gp.save() print 'Request ID: %s - Product was updated.' % ( str(request_id), ) merchant_id = self.merchant_id batch = BatchHttpRequest(callback=product_updated) for prod in gproduct_qset: product = prod.product new_status = { #Update the price of the item 'price': { 'value': str(product.stockrecords.first().price_incl_tax), 'currency': 'GBP' }, 'description': len(product.google_shopping_description) > 0 and bleach.clean( smart_text(product.google_shopping_description), strip=True) or bleach.clean(smart_text( product.parent.google_shopping_description), strip=True), 'link': SITE_ROOT + product.get_absolute_url(), 'imageLink': product.get_first_image_url(), #Is it in stock? 'availability': resolve_google_availability(product), } # Add product update to the batch. batch.add(self.service.inventory().set( merchantId=merchant_id, productId=prod.google_shopping_id, body=new_status)) try: batch.execute() except client.AccessTokenRefreshError: warn_exp_token()
def test_execute_global_callback(self): callbacks = Callbacks() batch = BatchHttpRequest(callback=callbacks.f) batch.add(self.request1) batch.add(self.request2) http = HttpMockSequence([ ({'status': '200', 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, BATCH_RESPONSE), ]) batch.execute(http=http) self.assertEqual({'foo': 42}, callbacks.responses['1']) self.assertEqual({'baz': 'qux'}, callbacks.responses['2'])
def infoUpdate(vSources, forceUpdate=False): batch = BatchHttpRequest() sourcesToUpdate = False for vSource in vSources: if forceUpdate or vSource.needsInfoUpdate(): request, callback = vSource.fetchInfoBatchRequest() batch.add(request, callback=callback) sourcesToUpdate = True if sourcesToUpdate: batch.execute()
def main(argv): # Authenticate and construct service. service, flags = sample_tools.init(argv, 'content', 'v2', __doc__, __file__, parents=[argparser]) merchant_id = flags.merchant_id batch = BatchHttpRequest(callback=datafeed_inserted) for _ in range(BATCH_SIZE): name = 'feed%s' % shopping_common.get_unique_id() datafeed = { 'name': name, 'contentType': 'products', 'attributeLanguage': 'en', 'contentLanguage': 'en', 'intendedDestinations': ['Shopping'], # The file name must be unique per account. We only use unique names in # these examples, so it's not an issue here. 'fileName': name, 'targetCountry': 'US', # You can schedule monthly, weekly or daily. # # Monthly - set day of month ('dayOfMonth') and hour ('hour') # Weekly - set day of week ('weekday') and hour ('hour') # Daily - set just the hour ('hour') 'fetchSchedule': { 'weekday': 'monday', 'hour': 6, 'timeZone': 'America/Los_Angeles', 'fetchUrl': 'https://feeds.myshop.com/' + name }, 'format': { 'fileEncoding': 'utf-8', 'columnDelimiter': 'tab', 'quotingMode': 'value quoting' } } # Add datafeed to the batch. batch.add(service.datafeeds().insert(merchantId=merchant_id, body=datafeed)) try: batch.execute() except client.AccessTokenRefreshError: print( 'The credentials have been revoked or expired, please re-run the ' 'application to re-authorize')
def share(self, file_ids, emails, role='reader', callback=None): """ Share a list of files to a list of e-mails. """ if not isinstance(file_ids, (list, tuple)): raise ValueError( "We are expecting a list of file_ids, not %s" % file_ids ) if not isinstance(emails, (list, tuple)): raise ValueError( "We are expecting a list of emails, not %s" % emails ) self._pydrive.auth.Authorize() perms = self._pydrive.auth.service.permissions() http = self._pydrive.auth.http batch_response = OrderedDict() def batch_callback(request_id, response, exception): file_id = request_id.split('__', 2)[1] if exception: logger.error("Error on drive batch operation for %s: %s", request_id, exception) batch_response[file_id].update({'exception': exception}) else: batch_response[file_id].update(response) batch_request = BatchHttpRequest(callback=batch_callback) for file_id in list(set(file_ids)): for email in list(set(emails)): kwargs = { 'fileId': file_id, 'body': { 'value': email, 'type': 'user', 'role': role } } batch_id = 'share__%s__%s' % (file_id, uuid4()) batch_request.add(perms.insert(**kwargs), request_id=batch_id) logger.info( "Batch share request added with ID %s and data %s", batch_id, kwargs ) batch_response[file_id] = {'insert_kwargs': kwargs} batch_request.execute() return batch_response
def execute(self): """Executes requests in the queue. Removes items from the queue, and adds them to a BatchHttpRequest object. Only removes up to set quota. and then calls the BatchHttPRequest object's execute method. """ batch = BatchHttpRequest(callback=self.call_back) for _ in range(self.quota): if self.queue.qsize() == 0: break request, request_id = self.queue.get() batch.add(request, request_id=request_id) batch.execute(http=httplib2.Http())
def new_batch_http_request(callback=None): """Create a BatchHttpRequest object based on the discovery document. Args: callback: callable, A callback to be called for each response, of the form callback(id, response, exception). The first parameter is the request id, and the second is the deserialized response object. The third is an apiclient.errors.HttpError exception object if an HTTP error occurred while processing the request, or None if no error occurred. Returns: A BatchHttpRequest object based on the discovery document. """ return BatchHttpRequest(callback=callback, batch_uri=batch_uri)