def update_tlds(): """ Calls the Namecheap API to update the list of recognized and registerable top-level domains. This is currently initiated manually via the administration panel. """ params = AdminSetting.get_api_params() params.append((u'Command', u'namecheap.domains.gettldlist')) r = requests.get(AdminSetting.get_api_url(), params=params) rtext = r.text send_mail( u'Domain Checker - TLD Update', u'The following response was received from the TLD update (using %s):\n\n%s' % (AdminSetting.get_api_url(), rtext), AdminSetting.get_value(u'noreply_address'), [AdminSetting.get_value(u'admin_address')]) parser = etree.XMLParser(encoding=u'utf-8') header_len = len('<?xml version="1.0" encoding="utf-8"?>') rtext = rtext[header_len:] rtree = etree.fromstring(rtext, parser=parser) rels = rtree.findall( u'./{http://api.namecheap.com/xml.response}CommandResponse/{http://api.namecheap.com/xml.response}Tlds/{http://api.namecheap.com/xml.response}Tld' ) rels = dict([(r.attrib[u'Name'], r) for r in rels]) tlds = TLD.objects.all() with transaction.atomic(): for tld in tlds: if tld.domain in rels.keys(): rel = rels[tld.domain] tld.is_recognized = True tld.is_api_registerable = ( rel.attrib[u'IsApiRegisterable'] == u'true') tld.description = rel.text tld.type = rel.attrib[u'Type'] else: tld.is_recognized = False tld.is_api_registrable = False tld.type = u'unknown' tld.description = None tld.save() for ncd, rel in rels.items(): if len(TLD.objects.filter(domain=ncd)) == 0: new_tld = TLD(domain=ncd, is_recognized=True, is_api_registerable=( rel.attrib['IsApiRegisterable'] == True), description=rel.text, type=rel.attrib['Type']) new_tld.save() print u'New TLD added: %s' % ncd print u'Finished processing tlds.'
def update_tlds(): """ Calls the Namecheap API to update the list of recognized and registerable top-level domains. This is currently initiated manually via the administration panel. """ params = AdminSetting.get_api_params() params.append((u'Command', u'namecheap.domains.gettldlist')) r = requests.get(AdminSetting.get_api_url(), params=params) rtext = r.text send_mail(u'Domain Checker - TLD Update', u'The following response was received from the TLD update (using %s):\n\n%s' % (AdminSetting.get_api_url(), rtext), AdminSetting.get_value(u'noreply_address'), [AdminSetting.get_value(u'admin_address')]) parser = etree.XMLParser(encoding=u'utf-8') rtree = etree.fromstring(rtext, parser=parser) rels = rtree.findall(u'./{http://api.namecheap.com/xml.response}CommandResponse/{http://api.namecheap.com/xml.response}Tlds/{http://api.namecheap.com/xml.response}Tld') rels = dict([(r.attrib[u'Name'], r) for r in rels]) tlds = TLD.objects.all() with transaction.atomic(): for tld in tlds: if tld.domain in rels.keys(): rel = rels[tld.domain] tld.is_recognized = True tld.is_api_registerable = (rel.attrib[u'IsApiRegisterable'] == u'true') tld.description = rel.text tld.type = rel.attrib[u'Type'] else: tld.is_recognized = False tld.is_api_registrable = False tld.type = u'unknown' tld.description = None tld.save() for ncd, rel in rels.items(): if len(TLD.objects.filter(domain=ncd)) == 0: new_tld = TLD(domain=ncd, is_recognized=True, is_api_registerable=(rel.attrib['IsApiRegisterable'] == True), description=rel.text, type=rel.attrib['Type']) new_tld.save() print u'New TLD added: %s' % ncd print u'Finished processing tlds.'
def check_project_domains(project_id): """ Use the Namecheap API to update availability status for all the domains associated with the given project. Args: project_id (int): The ID of the project to check domains for. """ lock = NamecheapLock() project = UserProject.objects.get(id=project_id) # Enable debug output if settings.DEBUG: logging.basicConfig() logging.getLogger().setLevel(logging.DEBUG) requests_log = logging.getLogger(u'requests.packages.urllib3') requests_log.setLevel(logging.DEBUG) requests_log.propagate = True while True: lock.acquire() try: # Retrieve list of unchecked domains (limited by the set limit of domains per call) domain_list = project.projectdomain_set.filter(is_checked=False)[:AdminSetting.get_api_urls_per_request()] # If no domains unchecked, progress project to the next stage (usually metrics measuring) if domain_list.count() == 0: print u'No domains found.' project.update_state(save=False) project.save() lock.release() break # Fold the list into a dictionary for easy reference domains = dict([(d.domain, d) for d in domain_list]) domain_str = u','.join(domains.keys()) params = AdminSetting.get_api_params() params.append((u'Command', u'namecheap.domains.check')) params.append((u'DomainList', domain_str)) print u'Domains that will be checked: %s' % domain_str print params # Make the call to the Namecheap API (retry 3 times then fail) retries = 0 while True: try: r = requests.get(AdminSetting.get_api_url(), params=params) break except requests.exceptions.ConnectionError as ce: retries += 1 if retries >= 3: raise ce time.sleep(5) sc = r.status_code print u'Status code: %d' % sc if sc == 200: rxml = r.text.encode(u'utf-8') (domain_results, error_results) = parse_namecheap_result(rxml) if len(domain_results) == 0 and len(error_results) > 0: # Handle specific but rare Namecheap API errors gracefully for er in error_results: if int(er[u'number']) == 2030280: # TLD not found - assume same result for all for domain, d in domains.items(): d.state = u'error' d.error = u'API unable to parse TLD for this domain (possible encoding issue)' d.is_checked = True d.last_checked = timezone.now() d.save() break elif int(er[u'number']) == 3031510: # Denied authorization for this domain for domain, d in domains.items(): d.state = u'error' d.error = u'API denies authorisation to check this domain (reason not given)' d.is_checked = True d.last_checked = timezone.now() d.save() break else: # Assume catastrophic error error_str = u'the API backend returned the following unrecoverable error(s):\n\n' error_str += u'\n'.join([u' %d: [%s] %s' % (i+1, er[u'number'], er[u'description']) for i, er in enumerate(error_results)]) raise Exception(error_str) """ Match the call results to the domain list and store them. If appropriate, create and associate a metrics object for the project. """ for dr in domain_results: print u'Finding match for "%s"...' % (dr[u'domain']) for key in domains.keys(): # We use endswith to handle mailto: addresses, TODO: These should be handled at the parsing stage if key.endswith(dr[u'domain']): d = domains[key] if dr[u'errorno'] != 0: d.state = u'error' d.error = u'API error (%d): %s' % (dr[u'errorno'], dr[u'description']) print dr else: d.state = u'available' if dr[u'available'] else u'unavailable' d.description = None d.is_checked = True d.last_checked = timezone.now() d.save() if d.state == u'available': try: um = URLMetrics.objects.get(query_url=d.domain) except URLMetrics.DoesNotExist: um = URLMetrics(query_url=d.domain) um.save() pm = ProjectMetrics(project=project, urlmetrics=um, is_checked=False, is_extension=False) pm.save() break # Make a debug note if a requested domain does not appear in the results (likely an error occurred) for domain, d in domains.items(): if d.state == u'unchecked': print u'Domain result not found (will recheck later): %s' % domain else: print u'Warning: Unexpected response while calling API code: %d, will retry after delay' % sc r.close() time.sleep(AdminSetting.get_api_wait_time()) lock.release() except Exception as e: lock.release() # A fatal error has occurred, set the project state appropriately and send an email to the user. project.state = u'error' project.error = u'Error occurred while checking domains - %s' % str(e).encode('utf-8') project.updated = timezone.now() project.completed_datetime = timezone.now() project.save() reply_address = AdminSetting.get_value(u'noreply_address') server_address = AdminSetting.get_value(u'server_address') messagebody = (u'The project "%s" has encountered an error:\n\n' + \ u'%s\n\nYou can view the results at the following address:\n\n' + \ u'%s/project?id=%d\n\n' + \ u'Thank you for using Domain Checker.') % \ (project.name(), project.error, server_address, project.id) user = User.objects.get(id=project.user_id) send_mail(u'Domain Checker - Project "%s" Error' % (project.name(),), messagebody, reply_address, [user.email]) (exc_type, exc_value, exc_traceback) = sys.exc_info() admin_email = AdminSetting.get_value(u'admin_address') admin_messagebody = (u'The user "%s" has encountered an unrecoverable error for project id %d.\n\n%s') % \ (user.username, project.id, '\n'.join(traceback.format_exception(exc_type, exc_value, exc_traceback))) print admin_email print admin_messagebody send_mail(u'Domain Checker - User Unrecoverable Error', admin_messagebody, reply_address, [admin_email]) # Propagate error to Celery handler raise project.update_state() # If any domains require metrics retrieval, start the appropriate background task if project.state == u'measuring': update_project_metrics.delay(project.id)
def check_project_domains(project_id): """ Use the Namecheap API to update availability status for all the domains associated with the given project. Args: project_id (int): The ID of the project to check domains for. """ lock = NamecheapLock() project = UserProject.objects.get(id=project_id) # Enable debug output if settings.DEBUG: logging.basicConfig() logging.getLogger().setLevel(logging.DEBUG) requests_log = logging.getLogger(u'requests.packages.urllib3') requests_log.setLevel(logging.DEBUG) requests_log.propagate = True while True: lock.acquire() try: # Retrieve list of unchecked domains (limited by the set limit of domains per call) domain_list = project.projectdomain_set.filter( is_checked=False)[:AdminSetting.get_api_urls_per_request()] # If no domains unchecked, progress project to the next stage (usually metrics measuring) if domain_list.count() == 0: print u'No domains found.' project.update_state(save=False) project.save() lock.release() break # Fold the list into a dictionary for easy reference domains = dict([(d.domain, d) for d in domain_list]) domain_str = u','.join(domains.keys()) params = AdminSetting.get_api_params() params.append((u'Command', u'namecheap.domains.check')) params.append((u'DomainList', domain_str)) print u'Domains that will be checked: %s' % domain_str print params # Make the call to the Namecheap API (retry 3 times then fail) retries = 0 while True: try: r = requests.get(AdminSetting.get_api_url(), params=params) break except requests.exceptions.ConnectionError as ce: retries += 1 if retries >= 3: raise ce time.sleep(5) sc = r.status_code print u'Status code: %d' % sc if sc == 200: rxml = r.text.encode(u'utf-8') (domain_results, error_results) = parse_namecheap_result(rxml) if len(domain_results) == 0 and len(error_results) > 0: # Handle specific but rare Namecheap API errors gracefully for er in error_results: if int(er[u'number']) == 2030280: # TLD not found - assume same result for all for domain, d in domains.items(): d.state = u'error' d.error = u'API unable to parse TLD for this domain (possible encoding issue)' d.is_checked = True d.last_checked = timezone.now() d.save() break elif int(er[u'number']) == 3031510: # Denied authorization for this domain for domain, d in domains.items(): d.state = u'error' d.error = u'API denies authorisation to check this domain (reason not given)' d.is_checked = True d.last_checked = timezone.now() d.save() break else: # Assume catastrophic error error_str = u'the API backend returned the following unrecoverable error(s):\n\n' error_str += u'\n'.join([ u' %d: [%s] %s' % (i + 1, er[u'number'], er[u'description']) for i, er in enumerate(error_results) ]) raise Exception(error_str) """ Match the call results to the domain list and store them. If appropriate, create and associate a metrics object for the project. """ for dr in domain_results: print u'Finding match for "%s"...' % (dr[u'domain']) for key in domains.keys(): # We use endswith to handle mailto: addresses, TODO: These should be handled at the parsing stage if key.endswith(dr[u'domain']): d = domains[key] if dr[u'errorno'] != 0: d.state = u'error' d.error = u'API error (%d): %s' % ( dr[u'errorno'], dr[u'description']) print dr else: d.state = u'available' if dr[ u'available'] else u'unavailable' d.description = None d.is_checked = True d.last_checked = timezone.now() d.save() if d.state == u'available': try: um = URLMetrics.objects.get( query_url=d.domain) except URLMetrics.DoesNotExist: um = URLMetrics(query_url=d.domain) um.save() pm = ProjectMetrics(project=project, urlmetrics=um, is_checked=False, is_extension=False) pm.save() break # Make a debug note if a requested domain does not appear in the results (likely an error occurred) for domain, d in domains.items(): if d.state == u'unchecked': print u'Domain result not found (will recheck later): %s' % domain else: print u'Warning: Unexpected response while calling API code: %d, will retry after delay' % sc r.close() time.sleep(AdminSetting.get_api_wait_time()) lock.release() except Exception as e: lock.release() # A fatal error has occurred, set the project state appropriately and send an email to the user. project.state = u'error' project.error = u'Error occurred while checking domains - %s' % str( e).encode('utf-8') project.updated = timezone.now() project.completed_datetime = timezone.now() project.save() reply_address = AdminSetting.get_value(u'noreply_address') server_address = AdminSetting.get_value(u'server_address') messagebody = (u'The project "%s" has encountered an error:\n\n' + \ u'%s\n\nYou can view the results at the following address:\n\n' + \ u'%s/project?id=%d\n\n' + \ u'Thank you for using Domain Checker.') % \ (project.name(), project.error, server_address, project.id) user = User.objects.get(id=project.user_id) send_mail( u'Domain Checker - Project "%s" Error' % (project.name(), ), messagebody, reply_address, [user.email]) (exc_type, exc_value, exc_traceback) = sys.exc_info() admin_email = AdminSetting.get_value(u'admin_address') admin_messagebody = (u'The user "%s" has encountered an unrecoverable error for project id %d.\n\n%s') % \ (user.username, project.id, '\n'.join(traceback.format_exception(exc_type, exc_value, exc_traceback))) print admin_email print admin_messagebody send_mail(u'Domain Checker - User Unrecoverable Error', admin_messagebody, reply_address, [admin_email]) # Propagate error to Celery handler raise project.update_state() # If any domains require metrics retrieval, start the appropriate background task if project.state == u'measuring': update_project_metrics.delay(project.id)