def associate_project_metrics(project): """ Checks to ensure that all the parsed domains of the given project are represented by a URLMetrics association. A new URLMetrics object is created if an appropriate one does not exist. Args: project (UserProject): The project to check. """ for pd in project.projectdomain_set.all(): metric_associated = False for um in project.urlmetrics.all(): if um.query_url == pd.domain: metric_associated = True break if not metric_associated: newum = None try: newum = URLMetrics.objects.get(query_url=pd.domain) except URLMetrics.DoesNotExist: newum = URLMetrics(query_url=query_url.pd_domain) newum.save() pm = ProjectMetrics(project=project, urlmetrics=newum, is_checked=False, is_extension=False) pm.save()
def handle(self, *args, **options): pm_total = 0 pm_checked = 0 dup_removed = 0 with transaction.atomic(): for p in UserProject.objects.all(): for pm in ProjectMetrics.objects.filter(project=p, is_extension=False): pm.delete() for pm in ProjectMetrics.objects.filter(project=p, is_extension=True): if pm.urlmetrics.is_uptodate(): pm_checked += 1 pm.is_checked = True else: pm.is_checked = False for pd in p.projectdomain_set.filter(state=u'available'): ums = URLMetrics.objects.filter( query_url=pd.domain).order_by(u'-last_updated') if len(ums) == 0: um = URLMetrics(query_url=pd.domain) um.save() pm = ProjectMetrics(project=p, urlmetrics=um, is_checked=False) pm.save() else: if len(ums) > 1: for dum in ums[1:]: dup_removed += 1 dum.delete() um = ums[0] pm = ProjectMetrics(project=p, urlmetrics=um) if um.is_uptodate(): pm.is_checked = True pm_checked += 1 else: pm.is_checked = False pm.save() pm_total += 1 self.stdout.write('Statistics:') self.stdout.write(' Project metric links: %d' % pm_total) self.stdout.write(' Checked/up-to-date: %d' % pm_checked) self.stdout.write(' Duplicate metric records removed: %d' % dup_removed) self.stdout.write(' Total metric records (post-removal): %d' % (len(URLMetrics.objects.all())))
def get_extensions(urlmetrics): """ Returns a list of URLMetrics that represent extensions (i.e. addition of 'www.') to the given URLMetrics. """ # ex_prefixes = ['www.'] ex_prefixes = [x.prefix+'.' for x in ExtensionPrefix.objects.all()] extensions = [] for ex in ex_prefixes: if urlmetrics.query_url.startswith(ex): # Skip if the given object already starts with the extension continue extension_url = ex+urlmetrics.query_url # Check if a URLMetrics object with the given extension already exists, otherwise create it try: exu = URLMetrics.objects.get(query_url=extension_url) except URLMetrics.DoesNotExist: exu = URLMetrics(query_url=extension_url) exu.extended_from = urlmetrics exu.save() extensions.append(exu) return extensions
def get_extensions(urlmetrics): """ Returns a list of URLMetrics that represent extensions (i.e. addition of 'www.') to the given URLMetrics. """ # ex_prefixes = ['www.'] ex_prefixes = [x.prefix + '.' for x in ExtensionPrefix.objects.all()] extensions = [] for ex in ex_prefixes: if urlmetrics.query_url.startswith(ex): # Skip if the given object already starts with the extension continue extension_url = ex + urlmetrics.query_url # Check if a URLMetrics object with the given extension already exists, otherwise create it try: exu = URLMetrics.objects.get(query_url=extension_url) except URLMetrics.DoesNotExist: exu = URLMetrics(query_url=extension_url) exu.extended_from = urlmetrics exu.save() extensions.append(exu) return extensions
def check_project_domains(project_id): """ Use the Namecheap API to update availability status for all the domains associated with the given project. Args: project_id (int): The ID of the project to check domains for. """ lock = NamecheapLock() project = UserProject.objects.get(id=project_id) # Enable debug output if settings.DEBUG: logging.basicConfig() logging.getLogger().setLevel(logging.DEBUG) requests_log = logging.getLogger(u'requests.packages.urllib3') requests_log.setLevel(logging.DEBUG) requests_log.propagate = True while True: lock.acquire() try: # Retrieve list of unchecked domains (limited by the set limit of domains per call) domain_list = project.projectdomain_set.filter(is_checked=False)[:AdminSetting.get_api_urls_per_request()] # If no domains unchecked, progress project to the next stage (usually metrics measuring) if domain_list.count() == 0: print u'No domains found.' project.update_state(save=False) project.save() lock.release() break # Fold the list into a dictionary for easy reference domains = dict([(d.domain, d) for d in domain_list]) domain_str = u','.join(domains.keys()) params = AdminSetting.get_api_params() params.append((u'Command', u'namecheap.domains.check')) params.append((u'DomainList', domain_str)) print u'Domains that will be checked: %s' % domain_str print params # Make the call to the Namecheap API (retry 3 times then fail) retries = 0 while True: try: r = requests.get(AdminSetting.get_api_url(), params=params) break except requests.exceptions.ConnectionError as ce: retries += 1 if retries >= 3: raise ce time.sleep(5) sc = r.status_code print u'Status code: %d' % sc if sc == 200: rxml = r.text.encode(u'utf-8') (domain_results, error_results) = parse_namecheap_result(rxml) if len(domain_results) == 0 and len(error_results) > 0: # Handle specific but rare Namecheap API errors gracefully for er in error_results: if int(er[u'number']) == 2030280: # TLD not found - assume same result for all for domain, d in domains.items(): d.state = u'error' d.error = u'API unable to parse TLD for this domain (possible encoding issue)' d.is_checked = True d.last_checked = timezone.now() d.save() break elif int(er[u'number']) == 3031510: # Denied authorization for this domain for domain, d in domains.items(): d.state = u'error' d.error = u'API denies authorisation to check this domain (reason not given)' d.is_checked = True d.last_checked = timezone.now() d.save() break else: # Assume catastrophic error error_str = u'the API backend returned the following unrecoverable error(s):\n\n' error_str += u'\n'.join([u' %d: [%s] %s' % (i+1, er[u'number'], er[u'description']) for i, er in enumerate(error_results)]) raise Exception(error_str) """ Match the call results to the domain list and store them. If appropriate, create and associate a metrics object for the project. """ for dr in domain_results: print u'Finding match for "%s"...' % (dr[u'domain']) for key in domains.keys(): # We use endswith to handle mailto: addresses, TODO: These should be handled at the parsing stage if key.endswith(dr[u'domain']): d = domains[key] if dr[u'errorno'] != 0: d.state = u'error' d.error = u'API error (%d): %s' % (dr[u'errorno'], dr[u'description']) print dr else: d.state = u'available' if dr[u'available'] else u'unavailable' d.description = None d.is_checked = True d.last_checked = timezone.now() d.save() if d.state == u'available': try: um = URLMetrics.objects.get(query_url=d.domain) except URLMetrics.DoesNotExist: um = URLMetrics(query_url=d.domain) um.save() pm = ProjectMetrics(project=project, urlmetrics=um, is_checked=False, is_extension=False) pm.save() break # Make a debug note if a requested domain does not appear in the results (likely an error occurred) for domain, d in domains.items(): if d.state == u'unchecked': print u'Domain result not found (will recheck later): %s' % domain else: print u'Warning: Unexpected response while calling API code: %d, will retry after delay' % sc r.close() time.sleep(AdminSetting.get_api_wait_time()) lock.release() except Exception as e: lock.release() # A fatal error has occurred, set the project state appropriately and send an email to the user. project.state = u'error' project.error = u'Error occurred while checking domains - %s' % str(e).encode('utf-8') project.updated = timezone.now() project.completed_datetime = timezone.now() project.save() reply_address = AdminSetting.get_value(u'noreply_address') server_address = AdminSetting.get_value(u'server_address') messagebody = (u'The project "%s" has encountered an error:\n\n' + \ u'%s\n\nYou can view the results at the following address:\n\n' + \ u'%s/project?id=%d\n\n' + \ u'Thank you for using Domain Checker.') % \ (project.name(), project.error, server_address, project.id) user = User.objects.get(id=project.user_id) send_mail(u'Domain Checker - Project "%s" Error' % (project.name(),), messagebody, reply_address, [user.email]) (exc_type, exc_value, exc_traceback) = sys.exc_info() admin_email = AdminSetting.get_value(u'admin_address') admin_messagebody = (u'The user "%s" has encountered an unrecoverable error for project id %d.\n\n%s') % \ (user.username, project.id, '\n'.join(traceback.format_exception(exc_type, exc_value, exc_traceback))) print admin_email print admin_messagebody send_mail(u'Domain Checker - User Unrecoverable Error', admin_messagebody, reply_address, [admin_email]) # Propagate error to Celery handler raise project.update_state() # If any domains require metrics retrieval, start the appropriate background task if project.state == u'measuring': update_project_metrics.delay(project.id)
def check_project_domains(project_id): """ Use the Namecheap API to update availability status for all the domains associated with the given project. Args: project_id (int): The ID of the project to check domains for. """ lock = NamecheapLock() project = UserProject.objects.get(id=project_id) # Enable debug output if settings.DEBUG: logging.basicConfig() logging.getLogger().setLevel(logging.DEBUG) requests_log = logging.getLogger(u'requests.packages.urllib3') requests_log.setLevel(logging.DEBUG) requests_log.propagate = True while True: lock.acquire() try: # Retrieve list of unchecked domains (limited by the set limit of domains per call) domain_list = project.projectdomain_set.filter( is_checked=False)[:AdminSetting.get_api_urls_per_request()] # If no domains unchecked, progress project to the next stage (usually metrics measuring) if domain_list.count() == 0: print u'No domains found.' project.update_state(save=False) project.save() lock.release() break # Fold the list into a dictionary for easy reference domains = dict([(d.domain, d) for d in domain_list]) domain_str = u','.join(domains.keys()) params = AdminSetting.get_api_params() params.append((u'Command', u'namecheap.domains.check')) params.append((u'DomainList', domain_str)) print u'Domains that will be checked: %s' % domain_str print params # Make the call to the Namecheap API (retry 3 times then fail) retries = 0 while True: try: r = requests.get(AdminSetting.get_api_url(), params=params) break except requests.exceptions.ConnectionError as ce: retries += 1 if retries >= 3: raise ce time.sleep(5) sc = r.status_code print u'Status code: %d' % sc if sc == 200: rxml = r.text.encode(u'utf-8') (domain_results, error_results) = parse_namecheap_result(rxml) if len(domain_results) == 0 and len(error_results) > 0: # Handle specific but rare Namecheap API errors gracefully for er in error_results: if int(er[u'number']) == 2030280: # TLD not found - assume same result for all for domain, d in domains.items(): d.state = u'error' d.error = u'API unable to parse TLD for this domain (possible encoding issue)' d.is_checked = True d.last_checked = timezone.now() d.save() break elif int(er[u'number']) == 3031510: # Denied authorization for this domain for domain, d in domains.items(): d.state = u'error' d.error = u'API denies authorisation to check this domain (reason not given)' d.is_checked = True d.last_checked = timezone.now() d.save() break else: # Assume catastrophic error error_str = u'the API backend returned the following unrecoverable error(s):\n\n' error_str += u'\n'.join([ u' %d: [%s] %s' % (i + 1, er[u'number'], er[u'description']) for i, er in enumerate(error_results) ]) raise Exception(error_str) """ Match the call results to the domain list and store them. If appropriate, create and associate a metrics object for the project. """ for dr in domain_results: print u'Finding match for "%s"...' % (dr[u'domain']) for key in domains.keys(): # We use endswith to handle mailto: addresses, TODO: These should be handled at the parsing stage if key.endswith(dr[u'domain']): d = domains[key] if dr[u'errorno'] != 0: d.state = u'error' d.error = u'API error (%d): %s' % ( dr[u'errorno'], dr[u'description']) print dr else: d.state = u'available' if dr[ u'available'] else u'unavailable' d.description = None d.is_checked = True d.last_checked = timezone.now() d.save() if d.state == u'available': try: um = URLMetrics.objects.get( query_url=d.domain) except URLMetrics.DoesNotExist: um = URLMetrics(query_url=d.domain) um.save() pm = ProjectMetrics(project=project, urlmetrics=um, is_checked=False, is_extension=False) pm.save() break # Make a debug note if a requested domain does not appear in the results (likely an error occurred) for domain, d in domains.items(): if d.state == u'unchecked': print u'Domain result not found (will recheck later): %s' % domain else: print u'Warning: Unexpected response while calling API code: %d, will retry after delay' % sc r.close() time.sleep(AdminSetting.get_api_wait_time()) lock.release() except Exception as e: lock.release() # A fatal error has occurred, set the project state appropriately and send an email to the user. project.state = u'error' project.error = u'Error occurred while checking domains - %s' % str( e).encode('utf-8') project.updated = timezone.now() project.completed_datetime = timezone.now() project.save() reply_address = AdminSetting.get_value(u'noreply_address') server_address = AdminSetting.get_value(u'server_address') messagebody = (u'The project "%s" has encountered an error:\n\n' + \ u'%s\n\nYou can view the results at the following address:\n\n' + \ u'%s/project?id=%d\n\n' + \ u'Thank you for using Domain Checker.') % \ (project.name(), project.error, server_address, project.id) user = User.objects.get(id=project.user_id) send_mail( u'Domain Checker - Project "%s" Error' % (project.name(), ), messagebody, reply_address, [user.email]) (exc_type, exc_value, exc_traceback) = sys.exc_info() admin_email = AdminSetting.get_value(u'admin_address') admin_messagebody = (u'The user "%s" has encountered an unrecoverable error for project id %d.\n\n%s') % \ (user.username, project.id, '\n'.join(traceback.format_exception(exc_type, exc_value, exc_traceback))) print admin_email print admin_messagebody send_mail(u'Domain Checker - User Unrecoverable Error', admin_messagebody, reply_address, [admin_email]) # Propagate error to Celery handler raise project.update_state() # If any domains require metrics retrieval, start the appropriate background task if project.state == u'measuring': update_project_metrics.delay(project.id)