def acknowledge_downloaded_files():
"""Acknowledge the reception of the files
"""
requests_to_delete = jobtracker.query("SELECT * FROM requests " \
"WHERE status='finished'")
if len(requests_to_delete) > 0:
queries = []
for request_to_delete in requests_to_delete:
DownloaderSPAN512.delete_stagged_file(request_to_delete)
dlm_cout.outs("Report download (%s) succeeded." % request_to_delete['guid'])
queries.append("UPDATE requests " \
"SET status='cleaned_up', " \
"details='download complete', " \
"updated_at='%s' " \
"WHERE id=%d" % \
(jobtracker.nowstr(), request_to_delete['id']))
jobtracker.query(queries)
else: pass
def download(attempt):
"""Given a row from the job-tracker's download_attempts table,
actually attempt the download.
"""
file = jobtracker.query("SELECT * FROM files " \
"WHERE id=%d" % attempt['file_id'], \
fetchone=True)
request = jobtracker.query("SELECT * FROM requests " \
"WHERE id=%d" % file['request_id'], \
fetchone=True)
queries = []
# Download using bbftp
res = DownloaderSPAN512.exec_download(request, file)
# bbftp should report 'get filename OK' if the transfer is successfull
if res == 'OK':
queries.append("UPDATE files " \
"SET status='unverified', " \
"updated_at='%s', " \
"details='Download is complete - File is unverified' " \
"WHERE id=%d" % \
(jobtracker.nowstr(), file['id']))
queries.append("UPDATE download_attempts " \
"SET status='complete', " \
"details='Download is complete', " \
"updated_at='%s' " \
"WHERE id=%d" % \
(jobtracker.nowstr(), attempt['id']))
else:
queries.append("UPDATE files " \
"SET status='failed', " \
"updated_at='%s', " \
"details='Download failed - %s' " \
"WHERE id=%d" % \
(jobtracker.nowstr(), str(res), file['id']))
queries.append("UPDATE download_attempts " \
"SET status='download_failed', " \
"details='Download failed - %s', " \
"updated_at='%s' " \
"WHERE id=%d" % \
(str(res), jobtracker.nowstr(), attempt['id']))
jobtracker.query(queries)
def check_active_requests():
"""Check for any requests with status='waiting'. If there are
some, check if the files are ready for download.
"""
active_requests = jobtracker.query("SELECT * FROM requests " \
"WHERE status='waiting'")
for request in active_requests:
# Check requested status
if DownloaderSPAN512.check_request_done(request):
dlm_cout.outs("Restore (GUID: %s) has succeeded. Will create file entries.\n" % request['guid'])
create_file_entries(request)
else:
# dlm_cout.outs("Request (GUID: %s) has failed.\n" \
# "\tDatabase failed to report the data as restored." % request['guid'])
# jobtracker.query("UPDATE requests SET status='failed', " \
# "details='Request failed. Why ?', " \
# "updated_at='%s' " \
# "WHERE guid='%s'" % (jobtracker.nowstr(), request['guid']))
query = "SELECT (TO_SECONDS('%s')-TO_SECONDS(created_at)) " \
"AS deltaT_seconds " \
"FROM requests " \
"WHERE guid='%s'" % \
(jobtracker.nowstr(), request['guid'])
row = jobtracker.query(query, fetchone=True)
#if row['deltaT_seconds']/3600. > config.download.request_timeout:
if row/3600. > config.download.request_timeout:
dlm_cout.outs("Restore (GUID: %s) is over %d hr old " \
"and still not ready. Marking " \
"it as failed." % \
(request['guid'], config.download.request_timeout))
jobtracker.query("UPDATE requests " \
"SET status='failed', " \
"details='Request took too long (> %d hr)', " \
"updated_at='%s' " \
"WHERE guid='%s'" % \
(config.download.request_timeout, jobtracker.nowstr(), \
request['guid']))
def create_file_entries(request):
"""Given a row from the requests table in the job-tracker DB
check the data server for its files and create entries in
the files table.
Input:
request: A row from the requests table.
Outputs:
None
"""
files = DownloaderSPAN512.get_files_infos(request)
total_size = 0
num_files = 0
queries = []
for fn, size in files:
# Check if it's ok to add the file in the DB (check if it's already in the DB, may add further criteria...)
if not pipeline_utils.can_add_file_generic(fn):
dlm_cout.outs("Skipping %s" % fn)
continue
# Insert entry into DB's files table
queries.append("INSERT INTO files ( " \
"request_id, " \
"remote_filename, " \
"filename, " \
"status, " \
"created_at, " \
"updated_at, " \
"size) " \
"VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %d)" % \
(request['id'], fn, os.path.join(config.download.datadir, os.path.split(fn)[-1]), \
'new', jobtracker.nowstr(), jobtracker.nowstr(), size))
total_size += size
num_files += 1
if num_files:
dlm_cout.outs("Request (GUID: %s) has succeeded.\n" \
"\tNumber of files to be downloaded: %d" % \
(request['guid'], num_files))
queries.append("UPDATE requests " \
"SET size=%d, " \
"updated_at='%s', " \
"status='downloading', " \
"details='Request has been filled' " \
"WHERE id=%d" % \
(total_size, jobtracker.nowstr(), request['id']))
else:
dlm_cout.outs("Request (GUID: %s) has failed.\n" \
"\tThere are no files to be downloaded." % \
request['guid'])
# delete restore since there may be skipped files
DownloaderSPAN512.delete_stagged_file(request)
# redefine 'queries' because there are no files to update
queries = ["UPDATE requests " \
"SET updated_at='%s', " \
"status='failed', " \
"details='No files to download.' " \
"WHERE id=%d" % \
(jobtracker.nowstr(), request['id'])]
jobtracker.query(queries)
