def update_tlds():
"""
Calls the Namecheap API to update the list of recognized and registerable top-level domains. This is currently initiated manually via the administration panel.
"""
params = AdminSetting.get_api_params()
params.append((u'Command', u'namecheap.domains.gettldlist'))
r = requests.get(AdminSetting.get_api_url(), params=params)
rtext = r.text
send_mail(
u'Domain Checker - TLD Update',
u'The following response was received from the TLD update (using %s):\n\n%s'
% (AdminSetting.get_api_url(), rtext),
AdminSetting.get_value(u'noreply_address'),
[AdminSetting.get_value(u'admin_address')])
parser = etree.XMLParser(encoding=u'utf-8')
header_len = len('<?xml version="1.0" encoding="utf-8"?>')
rtext = rtext[header_len:]
rtree = etree.fromstring(rtext, parser=parser)
rels = rtree.findall(
u'./{http://api.namecheap.com/xml.response}CommandResponse/{http://api.namecheap.com/xml.response}Tlds/{http://api.namecheap.com/xml.response}Tld'
)
rels = dict([(r.attrib[u'Name'], r) for r in rels])
tlds = TLD.objects.all()
with transaction.atomic():
for tld in tlds:
if tld.domain in rels.keys():
rel = rels[tld.domain]
tld.is_recognized = True
tld.is_api_registerable = (
rel.attrib[u'IsApiRegisterable'] == u'true')
tld.description = rel.text
tld.type = rel.attrib[u'Type']
else:
tld.is_recognized = False
tld.is_api_registrable = False
tld.type = u'unknown'
tld.description = None
tld.save()
for ncd, rel in rels.items():
if len(TLD.objects.filter(domain=ncd)) == 0:
new_tld = TLD(domain=ncd,
is_recognized=True,
is_api_registerable=(
rel.attrib['IsApiRegisterable'] == True),
description=rel.text,
type=rel.attrib['Type'])
new_tld.save()
print u'New TLD added: %s' % ncd
print u'Finished processing tlds.'
def update_tlds():
"""
Calls the Namecheap API to update the list of recognized and registerable top-level domains. This is currently initiated manually via the administration panel.
"""
params = AdminSetting.get_api_params()
params.append((u'Command', u'namecheap.domains.gettldlist'))
r = requests.get(AdminSetting.get_api_url(), params=params)
rtext = r.text
send_mail(u'Domain Checker - TLD Update', u'The following response was received from the TLD update (using %s):\n\n%s' % (AdminSetting.get_api_url(), rtext), AdminSetting.get_value(u'noreply_address'), [AdminSetting.get_value(u'admin_address')])
parser = etree.XMLParser(encoding=u'utf-8')
rtree = etree.fromstring(rtext, parser=parser)
rels = rtree.findall(u'./{http://api.namecheap.com/xml.response}CommandResponse/{http://api.namecheap.com/xml.response}Tlds/{http://api.namecheap.com/xml.response}Tld')
rels = dict([(r.attrib[u'Name'], r) for r in rels])
tlds = TLD.objects.all()
with transaction.atomic():
for tld in tlds:
if tld.domain in rels.keys():
rel = rels[tld.domain]
tld.is_recognized = True
tld.is_api_registerable = (rel.attrib[u'IsApiRegisterable'] == u'true')
tld.description = rel.text
tld.type = rel.attrib[u'Type']
else:
tld.is_recognized = False
tld.is_api_registrable = False
tld.type = u'unknown'
tld.description = None
tld.save()
for ncd, rel in rels.items():
if len(TLD.objects.filter(domain=ncd)) == 0:
new_tld = TLD(domain=ncd, is_recognized=True, is_api_registerable=(rel.attrib['IsApiRegisterable'] == True), description=rel.text, type=rel.attrib['Type'])
new_tld.save()
print u'New TLD added: %s' % ncd
print u'Finished processing tlds.'
def check_project_domains(project_id):
"""
Use the Namecheap API to update availability status for all the domains associated with the given project.
Args:
project_id (int): The ID of the project to check domains for.
"""
lock = NamecheapLock()
project = UserProject.objects.get(id=project_id)
# Enable debug output
if settings.DEBUG:
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
requests_log = logging.getLogger(u'requests.packages.urllib3')
requests_log.setLevel(logging.DEBUG)
requests_log.propagate = True
while True:
lock.acquire()
try:
# Retrieve list of unchecked domains (limited by the set limit of domains per call)
domain_list = project.projectdomain_set.filter(is_checked=False)[:AdminSetting.get_api_urls_per_request()]
# If no domains unchecked, progress project to the next stage (usually metrics measuring)
if domain_list.count() == 0:
print u'No domains found.'
project.update_state(save=False)
project.save()
lock.release()
break
# Fold the list into a dictionary for easy reference
domains = dict([(d.domain, d) for d in domain_list])
domain_str = u','.join(domains.keys())
params = AdminSetting.get_api_params()
params.append((u'Command', u'namecheap.domains.check'))
params.append((u'DomainList', domain_str))
print u'Domains that will be checked: %s' % domain_str
print params
# Make the call to the Namecheap API (retry 3 times then fail)
retries = 0
while True:
try:
r = requests.get(AdminSetting.get_api_url(), params=params)
break
except requests.exceptions.ConnectionError as ce:
retries += 1
if retries >= 3:
raise ce
time.sleep(5)
sc = r.status_code
print u'Status code: %d' % sc
if sc == 200:
rxml = r.text.encode(u'utf-8')
(domain_results, error_results) = parse_namecheap_result(rxml)
if len(domain_results) == 0 and len(error_results) > 0:
# Handle specific but rare Namecheap API errors gracefully
for er in error_results:
if int(er[u'number']) == 2030280:
# TLD not found - assume same result for all
for domain, d in domains.items():
d.state = u'error'
d.error = u'API unable to parse TLD for this domain (possible encoding issue)'
d.is_checked = True
d.last_checked = timezone.now()
d.save()
break
elif int(er[u'number']) == 3031510:
# Denied authorization for this domain
for domain, d in domains.items():
d.state = u'error'
d.error = u'API denies authorisation to check this domain (reason not given)'
d.is_checked = True
d.last_checked = timezone.now()
d.save()
break
else:
# Assume catastrophic error
error_str = u'the API backend returned the following unrecoverable error(s):\n\n'
error_str += u'\n'.join([u' %d: [%s] %s' % (i+1, er[u'number'], er[u'description']) for i, er in enumerate(error_results)])
raise Exception(error_str)
"""
Match the call results to the domain list and store them. If appropriate, create and associate a metrics object for the project.
"""
for dr in domain_results:
print u'Finding match for "%s"...' % (dr[u'domain'])
for key in domains.keys():
# We use endswith to handle mailto: addresses, TODO: These should be handled at the parsing stage
if key.endswith(dr[u'domain']):
d = domains[key]
if dr[u'errorno'] != 0:
d.state = u'error'
d.error = u'API error (%d): %s' % (dr[u'errorno'], dr[u'description'])
print dr
else:
d.state = u'available' if dr[u'available'] else u'unavailable'
d.description = None
d.is_checked = True
d.last_checked = timezone.now()
d.save()
if d.state == u'available':
try:
um = URLMetrics.objects.get(query_url=d.domain)
except URLMetrics.DoesNotExist:
um = URLMetrics(query_url=d.domain)
um.save()
pm = ProjectMetrics(project=project, urlmetrics=um, is_checked=False, is_extension=False)
pm.save()
break
# Make a debug note if a requested domain does not appear in the results (likely an error occurred)
for domain, d in domains.items():
if d.state == u'unchecked':
print u'Domain result not found (will recheck later): %s' % domain
else:
print u'Warning: Unexpected response while calling API code: %d, will retry after delay' % sc
r.close()
time.sleep(AdminSetting.get_api_wait_time())
lock.release()
except Exception as e:
lock.release()
# A fatal error has occurred, set the project state appropriately and send an email to the user.
project.state = u'error'
project.error = u'Error occurred while checking domains - %s' % str(e).encode('utf-8')
project.updated = timezone.now()
project.completed_datetime = timezone.now()
project.save()
reply_address = AdminSetting.get_value(u'noreply_address')
server_address = AdminSetting.get_value(u'server_address')
messagebody = (u'The project "%s" has encountered an error:\n\n' + \
u'%s\n\nYou can view the results at the following address:\n\n' + \
u'%s/project?id=%d\n\n' + \
u'Thank you for using Domain Checker.') % \
(project.name(), project.error, server_address, project.id)
user = User.objects.get(id=project.user_id)
send_mail(u'Domain Checker - Project "%s" Error' % (project.name(),), messagebody, reply_address, [user.email])
(exc_type, exc_value, exc_traceback) = sys.exc_info()
admin_email = AdminSetting.get_value(u'admin_address')
admin_messagebody = (u'The user "%s" has encountered an unrecoverable error for project id %d.\n\n%s') % \
(user.username, project.id, '\n'.join(traceback.format_exception(exc_type, exc_value, exc_traceback)))
print admin_email
print admin_messagebody
send_mail(u'Domain Checker - User Unrecoverable Error', admin_messagebody, reply_address, [admin_email])
# Propagate error to Celery handler
raise
project.update_state()
# If any domains require metrics retrieval, start the appropriate background task
if project.state == u'measuring':
update_project_metrics.delay(project.id)
def check_project_domains(project_id):
"""
Use the Namecheap API to update availability status for all the domains associated with the given project.
Args:
project_id (int): The ID of the project to check domains for.
"""
lock = NamecheapLock()
project = UserProject.objects.get(id=project_id)
# Enable debug output
if settings.DEBUG:
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
requests_log = logging.getLogger(u'requests.packages.urllib3')
requests_log.setLevel(logging.DEBUG)
requests_log.propagate = True
while True:
lock.acquire()
try:
# Retrieve list of unchecked domains (limited by the set limit of domains per call)
domain_list = project.projectdomain_set.filter(
is_checked=False)[:AdminSetting.get_api_urls_per_request()]
# If no domains unchecked, progress project to the next stage (usually metrics measuring)
if domain_list.count() == 0:
print u'No domains found.'
project.update_state(save=False)
project.save()
lock.release()
break
# Fold the list into a dictionary for easy reference
domains = dict([(d.domain, d) for d in domain_list])
domain_str = u','.join(domains.keys())
params = AdminSetting.get_api_params()
params.append((u'Command', u'namecheap.domains.check'))
params.append((u'DomainList', domain_str))
print u'Domains that will be checked: %s' % domain_str
print params
# Make the call to the Namecheap API (retry 3 times then fail)
retries = 0
while True:
try:
r = requests.get(AdminSetting.get_api_url(), params=params)
break
except requests.exceptions.ConnectionError as ce:
retries += 1
if retries >= 3:
raise ce
time.sleep(5)
sc = r.status_code
print u'Status code: %d' % sc
if sc == 200:
rxml = r.text.encode(u'utf-8')
(domain_results, error_results) = parse_namecheap_result(rxml)
if len(domain_results) == 0 and len(error_results) > 0:
# Handle specific but rare Namecheap API errors gracefully
for er in error_results:
if int(er[u'number']) == 2030280:
# TLD not found - assume same result for all
for domain, d in domains.items():
d.state = u'error'
d.error = u'API unable to parse TLD for this domain (possible encoding issue)'
d.is_checked = True
d.last_checked = timezone.now()
d.save()
break
elif int(er[u'number']) == 3031510:
# Denied authorization for this domain
for domain, d in domains.items():
d.state = u'error'
d.error = u'API denies authorisation to check this domain (reason not given)'
d.is_checked = True
d.last_checked = timezone.now()
d.save()
break
else:
# Assume catastrophic error
error_str = u'the API backend returned the following unrecoverable error(s):\n\n'
error_str += u'\n'.join([
u' %d: [%s] %s' %
(i + 1, er[u'number'], er[u'description'])
for i, er in enumerate(error_results)
])
raise Exception(error_str)
"""
Match the call results to the domain list and store them. If appropriate, create and associate a metrics object for the project.
"""
for dr in domain_results:
print u'Finding match for "%s"...' % (dr[u'domain'])
for key in domains.keys():
# We use endswith to handle mailto: addresses, TODO: These should be handled at the parsing stage
if key.endswith(dr[u'domain']):
d = domains[key]
if dr[u'errorno'] != 0:
d.state = u'error'
d.error = u'API error (%d): %s' % (
dr[u'errorno'], dr[u'description'])
print dr
else:
d.state = u'available' if dr[
u'available'] else u'unavailable'
d.description = None
d.is_checked = True
d.last_checked = timezone.now()
d.save()
if d.state == u'available':
try:
um = URLMetrics.objects.get(
query_url=d.domain)
except URLMetrics.DoesNotExist:
um = URLMetrics(query_url=d.domain)
um.save()
pm = ProjectMetrics(project=project,
urlmetrics=um,
is_checked=False,
is_extension=False)
pm.save()
break
# Make a debug note if a requested domain does not appear in the results (likely an error occurred)
for domain, d in domains.items():
if d.state == u'unchecked':
print u'Domain result not found (will recheck later): %s' % domain
else:
print u'Warning: Unexpected response while calling API code: %d, will retry after delay' % sc
r.close()
time.sleep(AdminSetting.get_api_wait_time())
lock.release()
except Exception as e:
lock.release()
# A fatal error has occurred, set the project state appropriately and send an email to the user.
project.state = u'error'
project.error = u'Error occurred while checking domains - %s' % str(
e).encode('utf-8')
project.updated = timezone.now()
project.completed_datetime = timezone.now()
project.save()
reply_address = AdminSetting.get_value(u'noreply_address')
server_address = AdminSetting.get_value(u'server_address')
messagebody = (u'The project "%s" has encountered an error:\n\n' + \
u'%s\n\nYou can view the results at the following address:\n\n' + \
u'%s/project?id=%d\n\n' + \
u'Thank you for using Domain Checker.') % \
(project.name(), project.error, server_address, project.id)
user = User.objects.get(id=project.user_id)
send_mail(
u'Domain Checker - Project "%s" Error' % (project.name(), ),
messagebody, reply_address, [user.email])
(exc_type, exc_value, exc_traceback) = sys.exc_info()
admin_email = AdminSetting.get_value(u'admin_address')
admin_messagebody = (u'The user "%s" has encountered an unrecoverable error for project id %d.\n\n%s') % \
(user.username, project.id, '\n'.join(traceback.format_exception(exc_type, exc_value, exc_traceback)))
print admin_email
print admin_messagebody
send_mail(u'Domain Checker - User Unrecoverable Error',
admin_messagebody, reply_address, [admin_email])
# Propagate error to Celery handler
raise
project.update_state()
# If any domains require metrics retrieval, start the appropriate background task
if project.state == u'measuring':
update_project_metrics.delay(project.id)
