def checkFile(self, file_path, correct_size, correct_cksum): """ Check a file and return data to write to the receipt file """ if not os.path.exists(file_path): self.warn("%s: no such file" % file_path) return [ReceiptFile.NO_SUCH_FILE] try: actual_size = futils.getSize(file_path) if actual_size != correct_size: self.warn("%s actual size %s correct size %s" % (file_path, actual_size, correct_size)) return [ReceiptFile.BAD_SIZE, actual_size] else: actual_cksum = futils.calcChecksum(file_path) if actual_cksum != correct_cksum: self.warn("%s actual cksum %s correct cksum %s" % (file_path, actual_cksum, correct_cksum)) return [ReceiptFile.BAD_CKSUM, actual_size, actual_cksum] else: return [ReceiptFile.SUCCESS, actual_size, actual_cksum] except IOError: return [ReceiptFile.IO_ERROR]
def processTransfers(self): """ Runs forever until status changes. """ self.doSetup() if self.status == status.COMPLETE: return self.status self.status = status.RUNNING self.info("Processing Transfers...") tbc = TransferBaseController(self.dconfig) had_completion_file = False runLoop = True while runLoop == True: self.updateStatusAndConfig() tp = self.dconfig.get("outgoing.transfer_protocol") # If transfer protocol not set then exit if tp.strip().lower() in ("", "none"): self.status = status.STOPPED print "Exiting transfer controller for '%s' because 'outgoing.transfer_protocol' is set to '%s' which is invalid." % (self.dconfig.get("data_stream.name"), tp) else: # If transfer protocol is set then check valid target information is set for host and directory for check_item in ("outgoing.target_dir", "outgoing.target_host"): if (self.dconfig.checkSet(check_item) == False): self.status = status.STOPPED print "Exiting transfer controller for '%s' because '%s' not set in config file." % (self.dconfig.get("data_stream.name"), check_item) if self.status == status.STOPPED: return self.status # Get a list of items, either files or directories items = self.listDataDir() if items: self.info("Found a list of items to transfer of length '%d', starting with: %s" % (len(items), items[:3])) # completion condition is that we have seen the completion # file, and the data_stream directory is currently empty if (not items) and had_completion_file: self.status = status.COMPLETE return self.status # Start a counter to check status after every 5 items icount = 1 stat_diff = False for item in items: # Since we cannot guarantee that the file being placed in the incoming # directory is not still being written to we: # - check size and last mod time # - wait 2 seconds # - check size and last mod time again if os.path.isdir(item) == True: for dirname, dirnames, filenames in os.walk(item): for filename in filenames: full_path = os.path.join(self.dataset_dir, item) stat1 = (futils.getLastUpdatedTime(full_path), futils.getSize(full_path)) time.sleep(5) stat2 = (futils.getLastUpdatedTime(full_path), futils.getSize(full_path)) if stat1 != stat2: self.info("File is still writing so ignore: %s" % item) stat_diff = True continue else: full_path = os.path.join(self.dataset_dir, item) stat1 = (futils.getLastUpdatedTime(full_path), futils.getSize(full_path)) time.sleep(5) stat2 = (futils.getLastUpdatedTime(full_path), futils.getSize(full_path)) if stat1 != stat2 or stat_diff == True: self.info("File is still writing so ignore: %s" % item) continue # Every fifth item, test status has not been changed or stopped if (icount % 5) == 0: self.updateStatusAndConfig() if self.status == status.STOPPED: return self.status if self.doIgnore(item): continue # Copy the item to remote host. # don't actually do anything with the return code. as # currently coded, TUC will already have logged an error / # quarantined the files as necessary tresp = tbc.transfer(item) if tresp != None and str(tresp.code) == "Failure": if tresp.msg.find("Not all variables in") != -1: # this data_stream will not continue as not all variables have been set # exit cleanly self.status = status.STOPPED return self.status if item == self.completion_file: had_completion_file = True icount += 1 self.info("Sleeping for %d seconds..." % self.poll_interval) time.sleep(self.poll_interval) # if we get here - then all the files have been processed # if we are running oneoff then exit here if self.dconfig.get("global.oneoff") == True: runLoop = False # send a signal back to MiStaMoverController syaing that this # data_stream has completed os.kill(os.getppid(), signal.SIGUSR2)