def acknowledge_downloaded_files(): """Acknowledge the reception of the files """ requests_to_delete = jobtracker.query("SELECT * FROM requests " \ "WHERE status='finished'") if len(requests_to_delete) > 0: queries = [] for request_to_delete in requests_to_delete: DownloaderSPAN512.delete_stagged_file(request_to_delete) dlm_cout.outs("Report download (%s) succeeded." % request_to_delete['guid']) queries.append("UPDATE requests " \ "SET status='cleaned_up', " \ "details='download complete', " \ "updated_at='%s' " \ "WHERE id=%d" % \ (jobtracker.nowstr(), request_to_delete['id'])) jobtracker.query(queries) else: pass
def download(attempt): """Given a row from the job-tracker's download_attempts table, actually attempt the download. """ file = jobtracker.query("SELECT * FROM files " \ "WHERE id=%d" % attempt['file_id'], \ fetchone=True) request = jobtracker.query("SELECT * FROM requests " \ "WHERE id=%d" % file['request_id'], \ fetchone=True) queries = [] # Download using bbftp res = DownloaderSPAN512.exec_download(request, file) # bbftp should report 'get filename OK' if the transfer is successfull if res == 'OK': queries.append("UPDATE files " \ "SET status='unverified', " \ "updated_at='%s', " \ "details='Download is complete - File is unverified' " \ "WHERE id=%d" % \ (jobtracker.nowstr(), file['id'])) queries.append("UPDATE download_attempts " \ "SET status='complete', " \ "details='Download is complete', " \ "updated_at='%s' " \ "WHERE id=%d" % \ (jobtracker.nowstr(), attempt['id'])) else: queries.append("UPDATE files " \ "SET status='failed', " \ "updated_at='%s', " \ "details='Download failed - %s' " \ "WHERE id=%d" % \ (jobtracker.nowstr(), str(res), file['id'])) queries.append("UPDATE download_attempts " \ "SET status='download_failed', " \ "details='Download failed - %s', " \ "updated_at='%s' " \ "WHERE id=%d" % \ (str(res), jobtracker.nowstr(), attempt['id'])) jobtracker.query(queries)
def check_active_requests(): """Check for any requests with status='waiting'. If there are some, check if the files are ready for download. """ active_requests = jobtracker.query("SELECT * FROM requests " \ "WHERE status='waiting'") for request in active_requests: # Check requested status if DownloaderSPAN512.check_request_done(request): dlm_cout.outs("Restore (GUID: %s) has succeeded. Will create file entries.\n" % request['guid']) create_file_entries(request) else: # dlm_cout.outs("Request (GUID: %s) has failed.\n" \ # "\tDatabase failed to report the data as restored." % request['guid']) # jobtracker.query("UPDATE requests SET status='failed', " \ # "details='Request failed. Why ?', " \ # "updated_at='%s' " \ # "WHERE guid='%s'" % (jobtracker.nowstr(), request['guid'])) query = "SELECT (TO_SECONDS('%s')-TO_SECONDS(created_at)) " \ "AS deltaT_seconds " \ "FROM requests " \ "WHERE guid='%s'" % \ (jobtracker.nowstr(), request['guid']) row = jobtracker.query(query, fetchone=True) #if row['deltaT_seconds']/3600. > config.download.request_timeout: if row/3600. > config.download.request_timeout: dlm_cout.outs("Restore (GUID: %s) is over %d hr old " \ "and still not ready. Marking " \ "it as failed." % \ (request['guid'], config.download.request_timeout)) jobtracker.query("UPDATE requests " \ "SET status='failed', " \ "details='Request took too long (> %d hr)', " \ "updated_at='%s' " \ "WHERE guid='%s'" % \ (config.download.request_timeout, jobtracker.nowstr(), \ request['guid']))
def create_file_entries(request): """Given a row from the requests table in the job-tracker DB check the data server for its files and create entries in the files table. Input: request: A row from the requests table. Outputs: None """ files = DownloaderSPAN512.get_files_infos(request) total_size = 0 num_files = 0 queries = [] for fn, size in files: # Check if it's ok to add the file in the DB (check if it's already in the DB, may add further criteria...) if not pipeline_utils.can_add_file_generic(fn): dlm_cout.outs("Skipping %s" % fn) continue # Insert entry into DB's files table queries.append("INSERT INTO files ( " \ "request_id, " \ "remote_filename, " \ "filename, " \ "status, " \ "created_at, " \ "updated_at, " \ "size) " \ "VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %d)" % \ (request['id'], fn, os.path.join(config.download.datadir, os.path.split(fn)[-1]), \ 'new', jobtracker.nowstr(), jobtracker.nowstr(), size)) total_size += size num_files += 1 if num_files: dlm_cout.outs("Request (GUID: %s) has succeeded.\n" \ "\tNumber of files to be downloaded: %d" % \ (request['guid'], num_files)) queries.append("UPDATE requests " \ "SET size=%d, " \ "updated_at='%s', " \ "status='downloading', " \ "details='Request has been filled' " \ "WHERE id=%d" % \ (total_size, jobtracker.nowstr(), request['id'])) else: dlm_cout.outs("Request (GUID: %s) has failed.\n" \ "\tThere are no files to be downloaded." % \ request['guid']) # delete restore since there may be skipped files DownloaderSPAN512.delete_stagged_file(request) # redefine 'queries' because there are no files to update queries = ["UPDATE requests " \ "SET updated_at='%s', " \ "status='failed', " \ "details='No files to download.' " \ "WHERE id=%d" % \ (jobtracker.nowstr(), request['id'])] jobtracker.query(queries)