def run(self):
while self.alive:
try:
out = subprocess.check_output(shlex.split(DATE_FIND_CMD % (self.last_update-QUANTIZATION_CORRECTION)), stderr=subprocess.STDOUT).split()
except subprocess.CalledProcessError:
LOG.warning('Error while checking for new files')
else:
this_check = datetime.datetime.fromtimestamp(int(out[0]))
remote_files = sorted([DataFile(*f.split(';')) for f in out[1:]], key=lambda rf: rf.mod_time)
for rf in remote_files:
if rf.name in self.monitored_files:
mf = self.monitored_files[rf.name]
if rf.size == mf.size and mf.needs_reaping:
success = rf.reap(this_check)
if success:
nimsutil.update_reference_datetime(self.datetime_file, rf.mod_time)
self.last_update = rf.mod_time
elif mf.needs_reaping:
LOG.info('Monitoring %s' % rf)
elif rf.size == mf.size:
rf.needs_reaping = False
else:
LOG.info('Discovered %s' % rf)
self.monitored_files = dict(zip([rf.name for rf in remote_files], remote_files))
finally:
time.sleep(SLEEP_TIME)
def run(self):
while self.alive:
try:
reap_files = [ReapPFile(p, self) for p in glob.glob(self.data_glob)]
if not reap_files:
raise Warning('No matching files found (or error while checking for files)')
except (OSError, Warning) as e:
log.warning(e)
else:
reap_files = sorted(filter(lambda f: f.mod_time >= self.current_file_timestamp, reap_files), key=lambda f: f.mod_time)
for rf in reap_files:
if rf.path in self.monitored_files:
mf = self.monitored_files[rf.path]
if mf.needs_reaping and rf.size == mf.size:
rf.reap()
if not rf.needs_reaping:
nimsutil.update_reference_datetime(self.datetime_file, rf.mod_time)
self.current_file_timestamp = rf.mod_time
elif mf.needs_reaping:
log.info('Monitoring %s' % rf)
elif rf.size == mf.size:
rf.needs_reaping = False
else:
log.info('Discovered %s' % rf)
self.monitored_files = dict(zip([rf.path for rf in reap_files], reap_files))
finally:
time.sleep(self.sleep_time)
def run(self):
while self.alive:
reap_files = [ReapFile(f, self.id_, self.reap_stage, self.sort_stage, self.log) for f in glob.glob(data_glob)]
reap_files = sorted(filter(lambda f: f.mod_time >= self.current_file_timestamp, reap_files), key=lambda f: f.mod_time)
if not reap_files:
self.log.warning('No matching files found (or error while checking for files)')
time.sleep(self.sleep_time)
continue
for rf in reap_files:
if rf.path in self.monitored_files:
mf = self.monitored_files[rf.path]
if rf.size == mf.size and mf.needs_reaping:
success = rf.reap()
if success:
nimsutil.update_reference_datetime(self.datetime_file, rf.mod_time)
self.current_file_timestamp = rf.mod_time
elif mf.needs_reaping:
self.log.info('Monitoring %s' % mf)
elif rf.size == mf.size:
rf.needs_reaping = False
else:
self.log.info('Discovered %s' % rf)
self.monitored_files = dict(zip([rf.path for rf in reap_files], reap_files))
time.sleep(self.sleep_time)
def run(self):
while self.alive:
outstanding_exams = self.get_outstanding_exams()
n_monitored_exams = len(self.monitored_exams)
n_outstanding_exams = len(outstanding_exams)
if n_monitored_exams > 0 and n_outstanding_exams > 0 and self.monitored_exams[0].id_ != outstanding_exams[0].id_:
vanished_exam = self.monitored_exams.popleft()
log.warning('Dropping %s (assumed deleted from scanner)' % vanished_exam)
continue
if n_monitored_exams > 1 and n_outstanding_exams > 1 and self.monitored_exams[1].id_ != outstanding_exams[1].id_:
vanished_exam = self.monitored_exams.pop()
log.warning('Dropping %s (assumed deleted from scanner)' % vanished_exam)
continue
next_exam = None
if n_monitored_exams < 2 and n_monitored_exams < n_outstanding_exams:
next_exam = outstanding_exams[n_monitored_exams]
self.monitored_exams.append(next_exam)
elif n_monitored_exams == 2 and n_monitored_exams < n_outstanding_exams:
if not any([series.needs_reaping for series in self.monitored_exams[0].series_dict.itervalues()]):
self.monitored_exams.popleft()
next_exam = outstanding_exams[n_monitored_exams]
self.monitored_exams.append(next_exam)
self.current_exam_datetime = self.monitored_exams[0].datetime
nimsutil.update_reference_datetime(self.datetime_file, self.current_exam_datetime)
if next_exam:
log.info('New %s' % self.monitored_exams[-1])
for exam in self.monitored_exams:
if not self.alive: return
exam.reap()
time.sleep(self.sleep_time)
def run(self):
while self.alive:
outstanding_exams = self.get_outstanding_exams()
n_monitored_exams = len(self.monitored_exams)
n_outstanding_exams = len(outstanding_exams)
if n_monitored_exams > 0 and n_outstanding_exams > 0 and self.monitored_exams[0].id_ != outstanding_exams[0].id_:
vanished_exam = self.monitored_exams.popleft()
self.log.warning('Dropping %s (assumed deleted from scanner)' % vanished_exam)
continue
if n_monitored_exams > 1 and n_outstanding_exams > 1 and self.monitored_exams[1].id_ != outstanding_exams[1].id_:
vanished_exam = self.monitored_exams.pop()
self.log.warning('Dropping %s (assumed deleted from scanner)' % vanished_exam)
continue
next_exam = None
if n_monitored_exams < 2 and n_monitored_exams < n_outstanding_exams:
next_exam = outstanding_exams[n_monitored_exams]
self.monitored_exams.append(next_exam)
elif n_monitored_exams == 2 and n_monitored_exams < n_outstanding_exams:
if not any([series.needs_reaping for series in self.monitored_exams[0].series_dict.itervalues()]):
self.monitored_exams.popleft()
next_exam = outstanding_exams[n_monitored_exams]
self.monitored_exams.append(next_exam)
self.current_exam_datetime = self.monitored_exams[0].datetime
nimsutil.update_reference_datetime(self.datetime_file, self.current_exam_datetime)
if next_exam:
self.log.info('New %s' % self.monitored_exams[-1])
for exam in self.monitored_exams:
if not self.alive: return
exam.reap()
time.sleep(self.sleep_time)
def run(self):
while self.alive:
try:
reap_files = [
ReapPFile(p, self) for p in glob.glob(self.data_glob)
]
if not reap_files:
raise Warning(
'No matching files found (or error while checking for files)'
)
except (OSError, Warning) as e:
log.warning(e)
else:
reap_files = sorted(filter(
lambda f: f.mod_time >= self.current_file_timestamp,
reap_files),
key=lambda f: f.mod_time)
for rf in reap_files:
if rf.path in self.monitored_files:
mf = self.monitored_files[rf.path]
if mf.needs_reaping and rf.size == mf.size:
rf.reap()
if not rf.needs_reaping:
nimsutil.update_reference_datetime(
self.datetime_file, rf.mod_time)
self.current_file_timestamp = rf.mod_time
elif mf.needs_reaping:
log.info('Monitoring %s' % rf)
elif rf.size == mf.size:
rf.needs_reaping = False
else:
log.info('Discovered %s' % rf)
self.monitored_files = dict(
zip([rf.path for rf in reap_files], reap_files))
finally:
time.sleep(self.sleep_time)
