def update(self): newsize = pipeline_utils.get_file_size(self.name) #newsize = 0 if newsize != self.currsize: self.oldsize = self.currsize self.oldtime = self.currtime self.currsize = newsize self.currtime = time.time()
def presubmission_check(fns): """Check to see if datafiles meet the critera for submission. """ # Check that files exist missingfiles = [fn for fn in fns if pipeline_utils.get_file_size(fn) <=0 ] if missingfiles: errormsg = "The following files cannot be found:\n" for missing in missingfiles: errormsg += "\t%s\n" % missing raise pipeline_utils.PipelineError(errormsg) # if files missing want to crash
def verify_files(): """For all downloaded files with status 'unverify' verify the files. Inputs: None Output: numverified: The number of files successfully verified. """ toverify = jobtracker.query("SELECT * FROM files " \ "WHERE status='unverified'") numverified = 0 for file in toverify: actualsize = pipeline_utils.get_file_size(file['filename']) expectedsize = file['size'] last_attempt_id = jobtracker.query("SELECT id " \ "FROM download_attempts " \ "WHERE file_id=%s " \ "ORDER BY id DESC " % file['id'], \ fetchone=True) queries = [] if actualsize == expectedsize: dlm_cout.outs("Download of %s is complete and verified." % \ os.path.split(file['filename'])[-1]) # Everything checks out! queries.append("UPDATE files " \ "SET status='downloaded', " \ "details='Download is complete and verified', " \ "updated_at='%s'" \ "WHERE id=%d" % \ (jobtracker.nowstr(), file['id'])) queries.append("UPDATE download_attempts " \ "SET status='downloaded', " \ "details='Download is complete and verified', " \ "updated_at='%s'" \ "WHERE id=%d" % \ (jobtracker.nowstr(), last_attempt_id)) # Mark the beam as downloaded in the main database #mark_beam_downloaded(os.path.split(file['filename'])[-1])) numverified += 1 else: dlm_cout.outs("Verification of %s failed. \n" \ "\tActual size (%d bytes) != Expected size (%d bytes)" % \ (os.path.split(file['filename'])[-1], actualsize, expectedsize)) # Boo... verification failed. queries.append("UPDATE files " \ "SET status='failed', " \ "details='Downloaded file failed verification', " \ "updated_at='%s'" \ "WHERE id=%d" % \ (jobtracker.nowstr(), file['id'])) queries.append("UPDATE download_attempts " \ "SET status='verification_failed', " \ "details='Downloaded file failed verification', " \ "updated_at='%s'" \ "WHERE id=%d" % \ (jobtracker.nowstr(), last_attempt_id)) jobtracker.query(queries) return numverified