def acknowledge_downloaded_files():
"""Acknowledge the reception of the files
"""
requests_to_delete = jobtracker.query("SELECT * FROM requests " \
"WHERE status='finished'")
if len(requests_to_delete) > 0:
queries = []
for request_to_delete in requests_to_delete:
DownloaderSPAN512.delete_stagged_file(request_to_delete)
dlm_cout.outs("Report download (%s) succeeded." % request_to_delete['guid'])
queries.append("UPDATE requests " \
"SET status='cleaned_up', " \
"details='download complete', " \
"updated_at='%s' " \
"WHERE id=%d" % \
(jobtracker.nowstr(), request_to_delete['id']))
jobtracker.query(queries)
else: pass
def create_file_entries(request):
"""Given a row from the requests table in the job-tracker DB
check the data server for its files and create entries in
the files table.
Input:
request: A row from the requests table.
Outputs:
None
"""
files = DownloaderSPAN512.get_files_infos(request)
total_size = 0
num_files = 0
queries = []
for fn, size in files:
# Check if it's ok to add the file in the DB (check if it's already in the DB, may add further criteria...)
if not pipeline_utils.can_add_file_generic(fn):
dlm_cout.outs("Skipping %s" % fn)
continue
# Insert entry into DB's files table
queries.append("INSERT INTO files ( " \
"request_id, " \
"remote_filename, " \
"filename, " \
"status, " \
"created_at, " \
"updated_at, " \
"size) " \
"VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %d)" % \
(request['id'], fn, os.path.join(config.download.datadir, os.path.split(fn)[-1]), \
'new', jobtracker.nowstr(), jobtracker.nowstr(), size))
total_size += size
num_files += 1
if num_files:
dlm_cout.outs("Request (GUID: %s) has succeeded.\n" \
"\tNumber of files to be downloaded: %d" % \
(request['guid'], num_files))
queries.append("UPDATE requests " \
"SET size=%d, " \
"updated_at='%s', " \
"status='downloading', " \
"details='Request has been filled' " \
"WHERE id=%d" % \
(total_size, jobtracker.nowstr(), request['id']))
else:
dlm_cout.outs("Request (GUID: %s) has failed.\n" \
"\tThere are no files to be downloaded." % \
request['guid'])
# delete restore since there may be skipped files
DownloaderSPAN512.delete_stagged_file(request)
# redefine 'queries' because there are no files to update
queries = ["UPDATE requests " \
"SET updated_at='%s', " \
"status='failed', " \
"details='No files to download.' " \
"WHERE id=%d" % \
(jobtracker.nowstr(), request['id'])]
jobtracker.query(queries)
