def get_zone_links():
global access_token
links_url = self.czds_base_url + "/czds/downloads/links"
links_response = do_get(links_url, access_token)
status_code = links_response.status_code
if status_code == 200:
zone_links = links_response.json()
print("{0}: The number of zone files to be downloaded is {1}".
format(datetime.datetime.now(), len(zone_links)))
return zone_links
elif status_code == 401:
print(
"The access_token has been expired. Re-authenticate user {0}"
.format(self.username))
access_token = authenticate(self.username, self.password,
self.authen_base_url)
get_zone_links(self.czds_base_url)
else:
sys.stderr.write(
"Failed to get zone links from {0} with error code {1}\n".
format(links_url, status_code))
return None
def download_one_zone(url, output_directory):
print("{0}: Downloading zone file from {1}".format(str(datetime.datetime.now()), url))
global access_token
download_zone_response = do_get(url, access_token)
status_code = download_zone_response.status_code
if status_code == 200:
# Try to get the filename from the header
_,option = cgi.parse_header(download_zone_response.headers['content-disposition'])
filename = option['filename']
# If could get a filename from the header, then makeup one like [tld].txt.gz
if not filename:
filename = url.rsplit('/', 1)[-1].rsplit('.')[-2] + '.txt.gz'
# This is where the zone file will be saved
path = '{0}/{1}'.format(output_directory, filename)
with open(path, 'wb') as f:
for chunk in download_zone_response.iter_content(1024):
f.write(chunk)
print("{0}: Completed downloading zone to file {1}".format(str(datetime.datetime.now()), path))
elif status_code == 401:
print("The access_token has been expired. Re-authenticate user {0}".format(username))
access_token = authenticate(username, password, authen_base_url)
download_one_zone(url, output_directory)
elif status_code == 404:
print("No zone file found for {0}".format(url))
else:
sys.stderr.write('Failed to download zone from {0} with code {1}\n'.format(url, status_code))
if aws_access_key_id != None:
aws_session.update({
'aws_access_key_id': aws_access_key_id,
'aws_secret_access_key': aws_secret_access_key
})
if aws_profile != None:
aws_session.update({'profile_name': aws_profile})
##############################################################################################################
# Second Step: authenticate the user to get an access_token.
# Note that the access_token is global for all the REST API calls afterwards
##############################################################################################################
print("Authenticate user {0}".format(username))
access_token = authenticate(username, password, authentication_base_url)
##############################################################################################################
# Third Step: Get the download zone file links
##############################################################################################################
# Function definition for listing the zone links
def get_zone_links(czds_base_url):
global access_token
links_url = czds_base_url + "/czds/downloads/links"
links_response = do_get(links_url, access_token)
status_code = links_response.status_code
def download_all_zone_files(self):
"""
Use the CZDS API to download all zone files. Each zone file should be
timestamped and written to the location specified in CZDS_API_CONFIG.
:return: True if completed with no exceptions. False otherwise.
"""
##############################################################################################################
# Second Step: authenticate the user to get an access_token.
# Note that the access_token is global for all the REST API calls afterwards
##############################################################################################################
global access_token
print("Authenticate user {0}".format(self.username))
access_token = authenticate(self.username, self.password,
self.authen_base_url)
##############################################################################################################
# Third Step: Get the download zone file links
##############################################################################################################
# Function definition for listing the zone links
def get_zone_links():
global access_token
links_url = self.czds_base_url + "/czds/downloads/links"
links_response = do_get(links_url, access_token)
status_code = links_response.status_code
if status_code == 200:
zone_links = links_response.json()
print("{0}: The number of zone files to be downloaded is {1}".
format(datetime.datetime.now(), len(zone_links)))
return zone_links
elif status_code == 401:
print(
"The access_token has been expired. Re-authenticate user {0}"
.format(self.username))
access_token = authenticate(self.username, self.password,
self.authen_base_url)
get_zone_links(self.czds_base_url)
else:
sys.stderr.write(
"Failed to get zone links from {0} with error code {1}\n".
format(links_url, status_code))
return None
# Get the zone links
zone_links = get_zone_links()
if not zone_links:
exit(1)
##############################################################################################################
# Fourth Step: download zone files
##############################################################################################################
# Function definition to download one zone file
def download_one_zone(url):
print("{0}: Downloading zone file from {1}".format(
str(datetime.datetime.now()), url))
global access_token
download_zone_response = do_get(url, access_token)
status_code = download_zone_response.status_code
if status_code == 200:
# Try to get the filename from the header
_, option = cgi.parse_header(
download_zone_response.headers['content-disposition'])
filename = option['filename']
# If could get a filename from the header, then makeup one like [tld].txt.gz
if not filename:
filename = url.rsplit('/',
1)[-1].rsplit('.')[-2] + '.txt.gz'
# This is where the zone file will be saved
path = '{0}/{1}'.format(self.output_directory, filename)
with open(path, 'wb') as f:
for chunk in download_zone_response.iter_content(1024):
f.write(chunk)
print("{0}: Completed downloading zone to file {1}".format(
str(datetime.datetime.now()), path))
elif status_code == 401:
print(
"The access_token has been expired. Re-authenticate user {0}"
.format(username))
access_token = authenticate(self.username, self.password,
self.authen_base_url)
download_one_zone(url, self.output_directory)
elif status_code == 404:
print("No zone file found for {0}".format(url))
else:
sys.stderr.write(
'Failed to download zone from {0} with code {1}\n'.format(
url, status_code))
# Function definition for downloading all the zone files
def download_zone_files(urls):
if not os.path.exists(self.working_directory + "/zonefiles/"):
os.makedirs(self.working_directory + "/zonefiles/")
if not os.path.exists(self.output_directory):
os.makedirs(self.output_directory)
# Download the zone files one by one
for link in urls:
download_one_zone(link)
# Finally, download all zone files
start_time = datetime.datetime.now()
download_zone_files(zone_links)
end_time = datetime.datetime.now()
print(
"{0}: DONE DONE. Completed downloading all zone files. Time spent: {1}"
.format(str(end_time), (end_time - start_time)))
return True