def check_moz_domain(m, cols, wait_time): """ Calls the Moz API for the url of the given URLMetric. Args: m (URLMetrics): The metrics object whose query_url to use in the call. cols (list): A list of fields the call should return. See the URLMetrics class for more details. wait_time (int): Number of seconds to wait before releasing the API lock after the call has returned. """ lock = MozAPILock() lock.acquire() params = AdminSetting.get_moz_params() params.append(('Cols', cols)) try: r = requests.get(AdminSetting.get_moz_api_url()+'url-metrics/'+m.query_url, params=params) rtext = r.text if r.status_code == 200: # Retrieve the JSON result rd = json.loads(rtext) # Store the fields in the URLMetrics object m.store_result(rd) # Update the status of the URLMetrics m.last_updated = timezone.now() m.save() r.close() print u'Done with %s, waiting (new)...' % m.query_url # Wait the specified time time.sleep(wait_time) except Exception as e: lock.release() raise lock.release()
def check_moz_update(): """ Calls the Moz API to determine when the Moz data was last updated. This information is recorded and used to determine whether URLMetrics are still up to date. """ if settings.DEBUG: logging.basicConfig() logging.getLogger().setLevel(logging.DEBUG) requests_log = logging.getLogger(u'requests.packages.urllib3') requests_log.setLevel(logging.DEBUG) requests_log.propagate = True params = AdminSetting.get_moz_params() r = requests.get(AdminSetting.get_moz_api_url() + 'metadata/last_update.json', params=params) status = r.status_code print status if status == 200: rtext = r.text print rtext rd = json.loads(rtext) mu = MozLastUpdate() mu.datetime = timezone.make_aware( datetime.datetime.fromtimestamp(int(rd['last_update'])), timezone.get_current_timezone()) mu.retrieved = timezone.now() mu.save() r.close()
def check_moz_domain(m, cols, wait_time): """ Calls the Moz API for the url of the given URLMetric. Args: m (URLMetrics): The metrics object whose query_url to use in the call. cols (list): A list of fields the call should return. See the URLMetrics class for more details. wait_time (int): Number of seconds to wait before releasing the API lock after the call has returned. """ lock = MozAPILock() lock.acquire() params = AdminSetting.get_moz_params() params.append(('Cols', cols)) try: r = requests.get(AdminSetting.get_moz_api_url() + 'url-metrics/' + m.query_url, params=params) rtext = r.text if r.status_code == 200: # Retrieve the JSON result rd = json.loads(rtext) # Store the fields in the URLMetrics object m.store_result(rd) # Update the status of the URLMetrics m.last_updated = timezone.now() m.save() r.close() print u'Done with %s, waiting (new)...' % m.query_url # Wait the specified time time.sleep(wait_time) except Exception as e: lock.release() raise lock.release()
def check_moz_update(): """ Calls the Moz API to determine when the Moz data was last updated. This information is recorded and used to determine whether URLMetrics are still up to date. """ if settings.DEBUG: logging.basicConfig() logging.getLogger().setLevel(logging.DEBUG) requests_log = logging.getLogger(u'requests.packages.urllib3') requests_log.setLevel(logging.DEBUG) requests_log.propagate = True params = AdminSetting.get_moz_params() r = requests.get(AdminSetting.get_moz_api_url()+'metadata/last_update.json', params=params) status = r.status_code print status if status == 200: rtext = r.text print rtext rd = json.loads(rtext) mu = MozLastUpdate() mu.datetime = timezone.make_aware(datetime.datetime.fromtimestamp(int(rd['last_update'])), timezone.get_current_timezone()) mu.retrieved = timezone.now() mu.save() r.close()