def update(self):
newsize = pipeline_utils.get_file_size(self.name)
#newsize = 0
if newsize != self.currsize:
self.oldsize = self.currsize
self.oldtime = self.currtime
self.currsize = newsize
self.currtime = time.time()
def presubmission_check(fns):
"""Check to see if datafiles meet the critera for submission.
"""
# Check that files exist
missingfiles = [fn for fn in fns if pipeline_utils.get_file_size(fn) <=0 ]
if missingfiles:
errormsg = "The following files cannot be found:\n"
for missing in missingfiles:
errormsg += "\t%s\n" % missing
raise pipeline_utils.PipelineError(errormsg) # if files missing want to crash
def verify_files():
"""For all downloaded files with status 'unverify' verify the files.
Inputs:
None
Output:
numverified: The number of files successfully verified.
"""
toverify = jobtracker.query("SELECT * FROM files " \
"WHERE status='unverified'")
numverified = 0
for file in toverify:
actualsize = pipeline_utils.get_file_size(file['filename'])
expectedsize = file['size']
last_attempt_id = jobtracker.query("SELECT id " \
"FROM download_attempts " \
"WHERE file_id=%s " \
"ORDER BY id DESC " % file['id'], \
fetchone=True)
queries = []
if actualsize == expectedsize:
dlm_cout.outs("Download of %s is complete and verified." % \
os.path.split(file['filename'])[-1])
# Everything checks out!
queries.append("UPDATE files " \
"SET status='downloaded', " \
"details='Download is complete and verified', " \
"updated_at='%s'" \
"WHERE id=%d" % \
(jobtracker.nowstr(), file['id']))
queries.append("UPDATE download_attempts " \
"SET status='downloaded', " \
"details='Download is complete and verified', " \
"updated_at='%s'" \
"WHERE id=%d" % \
(jobtracker.nowstr(), last_attempt_id))
# Mark the beam as downloaded in the main database
#mark_beam_downloaded(os.path.split(file['filename'])[-1]))
numverified += 1
else:
dlm_cout.outs("Verification of %s failed. \n" \
"\tActual size (%d bytes) != Expected size (%d bytes)" % \
(os.path.split(file['filename'])[-1], actualsize, expectedsize))
# Boo... verification failed.
queries.append("UPDATE files " \
"SET status='failed', " \
"details='Downloaded file failed verification', " \
"updated_at='%s'" \
"WHERE id=%d" % \
(jobtracker.nowstr(), file['id']))
queries.append("UPDATE download_attempts " \
"SET status='verification_failed', " \
"details='Downloaded file failed verification', " \
"updated_at='%s'" \
"WHERE id=%d" % \
(jobtracker.nowstr(), last_attempt_id))
jobtracker.query(queries)
return numverified
