def queryTimeRange(self, minTime, maxTime, splitAttempts=0): if self.shouldStop: self.exitThread() response = photos_search(minTime, maxTime, self, config.photo_extras) if response == None: if splitAttempts < config.maxSplitAttempts: # Try to split the query into two parts midTime = (minTime + maxTime) / 2 self.queryTimeRange(minTime, midTime, splitAttempts + 1) self.queryTimeRange(midTime, maxTime, splitAttempts + 1) else: self.totalEmptyQueries += 1 return numImages = helper.numImagesInResponse(response) # If chunk is too large and we still have split-attempts remaining, then split # the chunk into smaller queries. if numImages > config.maxPhotosPerPage and splitAttempts < config.maxSplitAttempts: self.splitQuery(numImages, minTime, maxTime, splitAttempts) # If the chunk is a valid size or we need to force it to its max sie because we # have reached the maximum number of splitting attempts. elif numImages > 0: self.handleResponse(response, numImages)
def photos_search(minTime, maxTime, thread, extra_info): successfulQuery = False retries = -1 log = '' while not successfulQuery: retries += 1 if retries > config.numQueryRetries: log += 'ERROR: all retries failed' + '\n' return None elif retries > 0: log += 'Retrying query: retry ' + str(retries) + ' of ' + str(config.numQueryRetries) + '\n' try: time.sleep(3 * retries) # 0 sleep on first iteration if thread.shouldStop: thread.exitThread() flickr_api_key = config.flickrAPIKey flickr_per_page = str(config.maxPhotosPerPage) flickr_query_string = thread.queryString response = thread.flickrAPI.photos_search(api_key=flickr_api_key, ispublic='1', media='photos', per_page=flickr_per_page, sort='interestingness-desc', page='1', text=flickr_query_string, extras = extra_info, min_upload_date=str(minTime), max_upload_date=str(maxTime)) time.sleep(1) if thread.shouldStop: thread.exitThread() # Make sure that we can get the number of images from the response. numImages = helper.numImagesInResponse(response) # If no images were returned in the query, retry the query (it seems that # sometimes Flickr may just return an empty query if it is busy. if numImages == 0: log += 'WARNING: 0 images in response' + '\n' continue except KeyboardInterrupt: print(thread.processPrint + 'Rasing interrupt...') raise except Exception as error: log += 'ERROR: there was an exception while querying' + '\n' log += str(error) + '\n' print (log) continue successfulQuery = True return response