def run(self): while self.alive: try: out = subprocess.check_output(shlex.split(DATE_FIND_CMD % (self.last_update-QUANTIZATION_CORRECTION)), stderr=subprocess.STDOUT).split() except subprocess.CalledProcessError: LOG.warning('Error while checking for new files') else: this_check = datetime.datetime.fromtimestamp(int(out[0])) remote_files = sorted([DataFile(*f.split(';')) for f in out[1:]], key=lambda rf: rf.mod_time) for rf in remote_files: if rf.name in self.monitored_files: mf = self.monitored_files[rf.name] if rf.size == mf.size and mf.needs_reaping: success = rf.reap(this_check) if success: nimsutil.update_reference_datetime(self.datetime_file, rf.mod_time) self.last_update = rf.mod_time elif mf.needs_reaping: LOG.info('Monitoring %s' % rf) elif rf.size == mf.size: rf.needs_reaping = False else: LOG.info('Discovered %s' % rf) self.monitored_files = dict(zip([rf.name for rf in remote_files], remote_files)) finally: time.sleep(SLEEP_TIME)
def run(self): while self.alive: try: reap_files = [ReapPFile(p, self) for p in glob.glob(self.data_glob)] if not reap_files: raise Warning('No matching files found (or error while checking for files)') except (OSError, Warning) as e: log.warning(e) else: reap_files = sorted(filter(lambda f: f.mod_time >= self.current_file_timestamp, reap_files), key=lambda f: f.mod_time) for rf in reap_files: if rf.path in self.monitored_files: mf = self.monitored_files[rf.path] if mf.needs_reaping and rf.size == mf.size: rf.reap() if not rf.needs_reaping: nimsutil.update_reference_datetime(self.datetime_file, rf.mod_time) self.current_file_timestamp = rf.mod_time elif mf.needs_reaping: log.info('Monitoring %s' % rf) elif rf.size == mf.size: rf.needs_reaping = False else: log.info('Discovered %s' % rf) self.monitored_files = dict(zip([rf.path for rf in reap_files], reap_files)) finally: time.sleep(self.sleep_time)
def run(self): while self.alive: reap_files = [ReapFile(f, self.id_, self.reap_stage, self.sort_stage, self.log) for f in glob.glob(data_glob)] reap_files = sorted(filter(lambda f: f.mod_time >= self.current_file_timestamp, reap_files), key=lambda f: f.mod_time) if not reap_files: self.log.warning('No matching files found (or error while checking for files)') time.sleep(self.sleep_time) continue for rf in reap_files: if rf.path in self.monitored_files: mf = self.monitored_files[rf.path] if rf.size == mf.size and mf.needs_reaping: success = rf.reap() if success: nimsutil.update_reference_datetime(self.datetime_file, rf.mod_time) self.current_file_timestamp = rf.mod_time elif mf.needs_reaping: self.log.info('Monitoring %s' % mf) elif rf.size == mf.size: rf.needs_reaping = False else: self.log.info('Discovered %s' % rf) self.monitored_files = dict(zip([rf.path for rf in reap_files], reap_files)) time.sleep(self.sleep_time)
def run(self): while self.alive: outstanding_exams = self.get_outstanding_exams() n_monitored_exams = len(self.monitored_exams) n_outstanding_exams = len(outstanding_exams) if n_monitored_exams > 0 and n_outstanding_exams > 0 and self.monitored_exams[0].id_ != outstanding_exams[0].id_: vanished_exam = self.monitored_exams.popleft() log.warning('Dropping %s (assumed deleted from scanner)' % vanished_exam) continue if n_monitored_exams > 1 and n_outstanding_exams > 1 and self.monitored_exams[1].id_ != outstanding_exams[1].id_: vanished_exam = self.monitored_exams.pop() log.warning('Dropping %s (assumed deleted from scanner)' % vanished_exam) continue next_exam = None if n_monitored_exams < 2 and n_monitored_exams < n_outstanding_exams: next_exam = outstanding_exams[n_monitored_exams] self.monitored_exams.append(next_exam) elif n_monitored_exams == 2 and n_monitored_exams < n_outstanding_exams: if not any([series.needs_reaping for series in self.monitored_exams[0].series_dict.itervalues()]): self.monitored_exams.popleft() next_exam = outstanding_exams[n_monitored_exams] self.monitored_exams.append(next_exam) self.current_exam_datetime = self.monitored_exams[0].datetime nimsutil.update_reference_datetime(self.datetime_file, self.current_exam_datetime) if next_exam: log.info('New %s' % self.monitored_exams[-1]) for exam in self.monitored_exams: if not self.alive: return exam.reap() time.sleep(self.sleep_time)
def run(self): while self.alive: outstanding_exams = self.get_outstanding_exams() n_monitored_exams = len(self.monitored_exams) n_outstanding_exams = len(outstanding_exams) if n_monitored_exams > 0 and n_outstanding_exams > 0 and self.monitored_exams[0].id_ != outstanding_exams[0].id_: vanished_exam = self.monitored_exams.popleft() self.log.warning('Dropping %s (assumed deleted from scanner)' % vanished_exam) continue if n_monitored_exams > 1 and n_outstanding_exams > 1 and self.monitored_exams[1].id_ != outstanding_exams[1].id_: vanished_exam = self.monitored_exams.pop() self.log.warning('Dropping %s (assumed deleted from scanner)' % vanished_exam) continue next_exam = None if n_monitored_exams < 2 and n_monitored_exams < n_outstanding_exams: next_exam = outstanding_exams[n_monitored_exams] self.monitored_exams.append(next_exam) elif n_monitored_exams == 2 and n_monitored_exams < n_outstanding_exams: if not any([series.needs_reaping for series in self.monitored_exams[0].series_dict.itervalues()]): self.monitored_exams.popleft() next_exam = outstanding_exams[n_monitored_exams] self.monitored_exams.append(next_exam) self.current_exam_datetime = self.monitored_exams[0].datetime nimsutil.update_reference_datetime(self.datetime_file, self.current_exam_datetime) if next_exam: self.log.info('New %s' % self.monitored_exams[-1]) for exam in self.monitored_exams: if not self.alive: return exam.reap() time.sleep(self.sleep_time)
def run(self): while self.alive: try: reap_files = [ ReapPFile(p, self) for p in glob.glob(self.data_glob) ] if not reap_files: raise Warning( 'No matching files found (or error while checking for files)' ) except (OSError, Warning) as e: log.warning(e) else: reap_files = sorted(filter( lambda f: f.mod_time >= self.current_file_timestamp, reap_files), key=lambda f: f.mod_time) for rf in reap_files: if rf.path in self.monitored_files: mf = self.monitored_files[rf.path] if mf.needs_reaping and rf.size == mf.size: rf.reap() if not rf.needs_reaping: nimsutil.update_reference_datetime( self.datetime_file, rf.mod_time) self.current_file_timestamp = rf.mod_time elif mf.needs_reaping: log.info('Monitoring %s' % rf) elif rf.size == mf.size: rf.needs_reaping = False else: log.info('Discovered %s' % rf) self.monitored_files = dict( zip([rf.path for rf in reap_files], reap_files)) finally: time.sleep(self.sleep_time)