def from_mrfile(cls, mrfile, nims_path, archived=True): series_uid = nimsutil.pack_dicom_uid(mrfile.series_uid) dataset = (cls.query.join(Epoch) .filter(Epoch.uid == series_uid) .filter(Epoch.acq == mrfile.acq_no) .filter(cls.filetype == mrfile.filetype) .first()) if not dataset: alt_dataset = (cls.query.join(Epoch) .filter(Epoch.uid == series_uid) .filter(Epoch.acq == mrfile.acq_no) .filter(cls.kind == u'primary') .first()) if not alt_dataset: kind = u'primary' elif alt_dataset.priority < mrfile.priority: kind = u'primary' alt_dataset.kind = u'secondary' else: kind = u'secondary' epoch = Epoch.from_mrfile(mrfile) dataset = cls( container=epoch, priority = mrfile.priority, filetype=mrfile.filetype, compressed=mrfile.compressed, kind=kind, label=cls.default_labels[mrfile.filetype], archived=archived, ) transaction.commit() DBSession.add(dataset) nimsutil.make_joined_path(nims_path, dataset.relpath) return dataset
def at_path(cls, nims_path, filename, datatype=None, archived=False): metadata = cls.get_metadata(filename) if metadata: dataset = cls.from_metadata(metadata) nimsutil.make_joined_path(nims_path, dataset.relpath) else: dataset = None return dataset
def __init__(self, db_uri, sort_path, preserve_path, nims_path, dir_mode, sleep_time): super(Sorter, self).__init__() self.sort_path = nimsutil.make_joined_path(sort_path) self.preserve_path = nimsutil.make_joined_path(preserve_path) if preserve_path else None self.nims_path = nimsutil.make_joined_path(nims_path) self.dir_mode = dir_mode self.sleep_time = sleep_time self.alive = True model.init_model(sqlalchemy.create_engine(db_uri))
def reap(self): try: log.info('Inspecting %s' % self) self.pfile = nimsdata.nimsraw.NIMSPFile(self.path) except nimsdata.nimsraw.NIMSPFileError as e: self.needs_reaping = False log.warning('Skipping %s (%s)' % (self, str(e))) return else: self.pat_id = self.pfile.patient_id stage_dir = '%s_%s' % (self.reaper.id_, datetime.datetime.now().strftime('%s.%f')) reap_path = nimsutil.make_joined_path(self.reaper.reap_stage, stage_dir) if self.pat_id.strip('/').lower() in reaper.discard_ids: self.needs_reaping = False log.info('Discarding %s' % self) return if self.reaper.pat_id and not re.match( self.reaper.pat_id.replace('*', '.*'), self.pat_id): self.needs_reaping = False log.info('Ignoring %s' % self) return try: log.info('Reaping %s' % self) shutil.copy2(self.path, reap_path) for fp in glob.glob(self.path + '_' + self.pfile.series_uid + '_*'): log.info('Reaping %s to %s' % (os.path.basename(fp), os.path.join( reap_path, '_' + self.basename + '_' + fp.rsplit('_', 1)[-1]))) shutil.copy2( fp, os.path.join( reap_path, '_' + self.basename + '_' + fp.rsplit('_', 1)[-1])) except KeyboardInterrupt: shutil.rmtree(reap_path) raise except (shutil.Error, IOError): log.warning('Error while reaping %s' % self) else: log.info('Compressing %s' % self) nimsutil.gzip_inplace(os.path.join(reap_path, self.basename), 0o644) shutil.move(reap_path, os.path.join(self.reaper.sort_stage, '.' + stage_dir)) os.rename(os.path.join(self.reaper.sort_stage, '.' + stage_dir), os.path.join(self.reaper.sort_stage, stage_dir)) self.needs_reaping = False log.info('Reaped %s' % self)
def sort_directory(self, dirpath, filenames): self.log.debug('Sorting %s in directory mode' % os.path.basename(dirpath)) dataset = self.dataset_at_path(self.nims_path, os.path.join(dirpath, filenames[0])) if dataset: for filepath in [os.path.join(dirpath, filename) for filename in filenames]: ext = dataset.filename_ext if os.path.splitext(filepath)[1] != dataset.filename_ext else '' shutil.move(filepath, os.path.join(self.nims_path, dataset.relpath, os.path.basename(filepath) + ext)) dataset.updatetime = datetime.datetime.now() dataset.untrash() transaction.commit() elif self.preserve_mode: unsort_path = nimsutil.make_joined_path(self.unsort_path, os.path.dirname(os.path.relpath(dirpath, self.stage_path))) shutil.move(dirpath, unsort_path)
def __init__(self, id_, pat_id, discard_ids, data_path, reap_path, sort_path, datetime_file, sleep_time): super(PFileReaper, self).__init__() self.id_ = id_ self.pat_id = pat_id self.discard_ids = discard_ids self.data_glob = os.path.join(data_path, 'P?????.7') self.reap_stage = nimsutil.make_joined_path(reap_path) self.sort_stage = nimsutil.make_joined_path(sort_path) self.datetime_file = datetime_file self.sleep_time = sleep_time self.current_file_timestamp = nimsutil.get_reference_datetime(self.datetime_file) self.monitored_files = {} self.alive = True # delete any files left behind from a previous run for item in os.listdir(self.reap_stage): if item.startswith(self.id_): shutil.rmtree(os.path.join(self.reap_stage, item)) for item in os.listdir(self.sort_stage): if item.startswith('.' + self.id_): shutil.rmtree(os.path.join(self.sort_stage, item))
def sort_file(self, filepath): self.log.debug('Sorting %s' % os.path.basename(filepath)) dataset = self.dataset_at_path(self.nims_path, filepath) if dataset: ext = dataset.filename_ext if os.path.splitext(filepath)[1] != dataset.filename_ext else '' shutil.move(filepath, os.path.join(self.nims_path, dataset.relpath, os.path.basename(filepath) + ext)) dataset.updatetime = datetime.datetime.now() dataset.untrash() transaction.commit() elif self.preserve_mode: unsort_path = nimsutil.make_joined_path(self.unsort_path, os.path.dirname(os.path.relpath(filepath, self.stage_path))) shutil.move(filepath, unsort_path) else: os.remove(filepath)
def __init__(self, id_, pat_id, discard_ids, data_path, reap_path, sort_path, datetime_file, sleep_time): super(PFileReaper, self).__init__() self.id_ = id_ self.pat_id = pat_id self.discard_ids = discard_ids self.data_glob = os.path.join(data_path, 'P?????.7') self.reap_stage = nimsutil.make_joined_path(reap_path) self.sort_stage = nimsutil.make_joined_path(sort_path) self.datetime_file = datetime_file self.sleep_time = sleep_time self.current_file_timestamp = nimsutil.get_reference_datetime( self.datetime_file) self.monitored_files = {} self.alive = True # delete any files left behind from a previous run for item in os.listdir(self.reap_stage): if item.startswith(self.id_): shutil.rmtree(os.path.join(self.reap_stage, item)) for item in os.listdir(self.sort_stage): if item.startswith('.' + self.id_): shutil.rmtree(os.path.join(self.sort_stage, item))
def reap(self, timestamp): stage_dir = '%s_%s_%s' % (REAPER_ID, os.path.basename(self.name), timestamp.strftime('%s')) reap_path = nimsutil.make_joined_path(REAP_STAGE, stage_dir) try: LOG.info('Reaping %s' % self) subprocess.check_output(shlex.split(REAP_CMD % (self.name, reap_path)), stderr=subprocess.STDOUT) except subprocess.CalledProcessError: shutil.rmtree(reap_path) success = False LOG.warning('Error while reaping %s' % self) else: os.rename(reap_path, os.path.join(SORT_STAGE, stage_dir)) self.needs_reaping = False success = True LOG.info('Reaped %s' % self) return success
def reap(self): reap_path = nimsutil.make_joined_path(self.reap_stage, '%s_%s' % (self.reaper_id, datetime.datetime.now().strftime('%s.%f'))) try: self.log.info('Reaping %s' % self) shutil.copy2(self.path, reap_path) except KeyboardInterrupt: shutil.rmtree(reap_path) raise except (shutil.Error, IOError): success = False self.log.warning('Error while reaping %s' % self) else: shutil.move(reap_path, sort_stage) self.needs_reaping = False success = True self.log.info('Reaped %s' % self) return success
def reap(self, new_image_count): if new_image_count > self.image_count: self.image_count = new_image_count self.needs_reaping = True self.reaper.log.info('Monitoring %s' % self) elif self.needs_reaping: # image count has stopped increasing self.reaper.log.info('Reaping %s' % self) now = datetime.datetime.now().strftime('%s') stage_dir = '%s_%s-%d_%s' % (self.reaper.id_, self.exam.id_, self.id_, now) reap_path = nimsutil.make_joined_path(self.reaper.reap_stage, stage_dir) reap_count = self.reaper.scu.move(scu.SeriesQuery(StudyID=self.exam.id_, SeriesNumber=self.id_), reap_path) if reap_count >= self.image_count: self.needs_reaping = False shutil.move(reap_path, reaper.sort_stage) self.reaper.log.info('Reaped %s' % self) else: shutil.rmtree(reap_path) self.reaper.log.warning('Incomplete %s, %d reaped' % (self, reap_count))
def sort_directory(self, dirpath, filenames, aux_paths): log.debug('Sorting %s in directory mode' % os.path.basename(dirpath)) try: mrfile = nimsdata.parse(os.path.join(dirpath, filenames[0])) except nimsdata.NIMSDataError: if self.preserve_path: preserve_path = nimsutil.make_joined_path(self.preserve_path, os.path.relpath(dirpath, self.sort_path)) for filename in os.listdir(dirpath): shutil.move(os.path.join(dirpath, filename), os.path.join(preserve_path, filename)) else: dataset = model.Dataset.from_mrfile(mrfile, self.nims_path) for filepath, aux_paths in [(os.path.join(dirpath, filename), aux_paths.get(filename, [])) for filename in filenames]: shutil.move(filepath, os.path.join(self.nims_path, dataset.relpath, os.path.basename(filepath))) for aux_path in aux_paths: shutil.move(aux_path, os.path.join(self.nims_path, dataset.relpath, os.path.basename(aux_path))) dataset.updatetime = datetime.datetime.now() dataset.untrash() transaction.commit() shutil.rmtree(dirpath)
def reap(self): try: log.info('Inspecting %s' % self) self.pfile = nimsdata.nimsraw.NIMSPFile(self.path) except nimsdata.nimsraw.NIMSPFileError as e: self.needs_reaping = False log.warning('Skipping %s (%s)' % (self, str(e))) return else: self.pat_id = self.pfile.patient_id self.exam = self.pfile.exam_no self.series = self.pfile.series_no self.acq = self.pfile.acq_no stage_dir = '%s_%s' % (self.reaper.id_, datetime.datetime.now().strftime('%s.%f')) reap_path = nimsutil.make_joined_path(self.reaper.reap_stage, stage_dir) aux_reap_files = [arf for arf in glob.glob(self.path + '_*') if self.is_aux_file(arf)] if self.pat_id.strip('/').lower() in reaper.discard_ids: self.needs_reaping = False log.info('Discarding %s' % self) return if self.reaper.pat_id and not re.match(self.reaper.pat_id.replace('*','.*'), self.pat_id): self.needs_reaping = False log.info('Ignoring %s' % self) return try: log.info('Reaping %s' % self) shutil.copy2(self.path, reap_path) for arf in aux_reap_files: shutil.copy2(arf, os.path.join(reap_path, '_' + os.path.basename(arf))) log.info('Reaping %s' % '_' + os.path.basename(arf)) except KeyboardInterrupt: shutil.rmtree(reap_path) raise except (shutil.Error, IOError): log.warning('Error while reaping %s' % self) else: log.info('Compressing %s' % self) nimsutil.gzip_inplace(os.path.join(reap_path, self.basename), 0o644) shutil.move(reap_path, os.path.join(self.reaper.sort_stage, '.' + stage_dir)) os.rename(os.path.join(self.reaper.sort_stage, '.' + stage_dir), os.path.join(self.reaper.sort_stage, stage_dir)) self.needs_reaping = False log.info('Reaped %s' % self)
def sort_files(self, dirpath, filenames, aux_paths): for filepath, filename in [(os.path.join(dirpath, fn), fn) for fn in filenames]: log.debug('Sorting %s' % filename) try: mrfile = nimsdata.parse(filepath) except nimsdata.NIMSDataError: if self.preserve_path: preserve_path = nimsutil.make_joined_path(self.preserve_path, os.path.dirname(os.path.relpath(filepath, self.sort_path))) shutil.move(filepath, os.path.join(preserve_path, filename)) else: dataset = model.Dataset.from_mrfile(mrfile, self.nims_path) new_filenames = [filename] shutil.move(filepath, os.path.join(self.nims_path, dataset.relpath, filename)) for aux_path in aux_paths.get(os.path.splitext(filename)[0] if dataset.compressed else filename, []): new_filenames.append(os.path.basename(aux_path)) shutil.move(aux_path, os.path.join(self.nims_path, dataset.relpath, os.path.basename(aux_path))) dataset.filenames = set(dataset.filenames + new_filenames) dataset.updatetime = datetime.datetime.now() dataset.untrash() transaction.commit() shutil.rmtree(dirpath)
class ArgumentParser(argparse.ArgumentParser): def __init__(self): super(ArgumentParser, self).__init__() self.add_argument('source_stage', help='path to source staging area') self.add_argument('data_host', help='username@hostname of data destination') self.add_argument('remote_stage', help='path to destination staging area') self.add_argument('-s', '--sleeptime', type=int, default=30, help='time to sleep before checking for new data') self.add_argument('-n', '--logname', default=os.path.splitext(os.path.basename(__file__))[0], help='process name for log') self.add_argument('-f', '--logfile', help='path to log file') self.add_argument('-l', '--loglevel', default='info', help='path to log file') if __name__ == "__main__": args = ArgumentParser().parse_args() log = nimsutil.get_logger(args.logname, args.logfile, args.loglevel) source_stage = nimsutil.make_joined_path(args.source_stage, 'sort') reap_stage = nimsutil.make_joined_path(args.remote_stage, 'reap') sort_stage = nimsutil.make_joined_path(args.remote_stage, 'sort') restager = Restager(source_stage, args.data_host, reap_stage, sort_stage, args.sleeptime, log) def term_handler(signum, stack): restager.halt() log.info('Received SIGTERM - shutting down...') signal.signal(signal.SIGTERM, term_handler) restager.run() log.warning('Process halted')
def at_path(cls, nims_path, filetype, label=None, archived=False): dataset = cls(filetype=filetype, label=(label if label else cls.default_labels[filetype]), archived=archived) transaction.commit() DBSession.add(dataset) nimsutil.make_joined_path(nims_path, dataset.relpath) return dataset
def __init__(self): super(ArgumentParser, self).__init__() self.add_argument('db_uri', help='database URI') self.add_argument('stage_path', help='path to staging area') self.add_argument('nims_path', help='data destination') self.add_argument('-d', '--dirmode', action='store_true', help='assume files are pre-sorted by directory') self.add_argument('-p', '--preserve', action='store_true', help='preserve unsortable files') self.add_argument('-s', '--sleeptime', type=int, default=10, help='time to sleep before checking for new files') self.add_argument('-n', '--logname', default=os.path.splitext(os.path.basename(__file__))[0], help='process name for log') self.add_argument('-f', '--logfile', help='path to log file') self.add_argument('-l', '--loglevel', default='info', help='path to log file') if __name__ == '__main__': args = ArgumentParser().parse_args() log = nimsutil.get_logger(args.logname, args.logfile, args.loglevel) stage_path = nimsutil.make_joined_path(args.stage_path, 'sort') unsort_path = nimsutil.make_joined_path(args.stage_path, 'unsortable') nims_path = nimsutil.make_joined_path(args.nims_path) sorter = Sorter(args.db_uri, stage_path, unsort_path, nims_path, args.dirmode, args.preserve, args.sleeptime, log) def term_handler(signum, stack): sorter.halt() log.info('Receieved SIGTERM - shutting down...') signal.signal(signal.SIGTERM, term_handler) sorter.run() log.warning('Process halted')
self.add_argument('-l', '--loglevel', default='info', help='log level (default: info)') self.add_argument('-q', '--quiet', action='store_true', default=False, help='disable console logging') if __name__ == "__main__": args = ArgumentParser().parse_args() nimsutil.configure_log(args.logfile, not args.quiet, args.loglevel) source_stage = nimsutil.make_joined_path(args.source_stage, 'sort') reap_stage = nimsutil.make_joined_path(args.remote_stage, 'reap') sort_stage = nimsutil.make_joined_path(args.remote_stage, 'sort') restager = Restager(source_stage, args.data_host, reap_stage, sort_stage, args.sleeptime) def term_handler(signum, stack): restager.halt() log.info('Received SIGTERM - shutting down...') signal.signal(signal.SIGTERM, term_handler) restager.run() log.warning('Process halted')
self.add_argument('data_host', help='username@hostname of data source') self.add_argument('data_path', help='path to data source') self.add_argument('-g', '--data_glob', default='*', help='glob format for files to move') self.add_argument('-s', '--sleeptime', type=int, default=60, help='time to sleep before checking for new data') self.add_argument('-n', '--logname', default=__file__, help='process name for log') self.add_argument('-f', '--logfile', help='path to log file') self.add_argument('-l', '--loglevel', default='info', help='path to log file') if __name__ == '__main__': args = ArgumentParser().parse_args() REAPER_ID = os.path.basename(args.data_path.rstrip('/')) LOG = nimsutil.get_logger(args.logname, args.logfile, args.loglevel) SLEEP_TIME = int(args.sleeptime) REAP_STAGE = nimsutil.make_joined_path(args.stage_path, 'reap') DATA_PATH = os.path.join(args.data_path, args.data_glob) SORT_STAGE = nimsutil.make_joined_path(args.stage_path, 'sort') DATE_CMD = 'date +%%s' FIND_CMD = 'find %s -maxdepth 1 -newermt "%%s" -exec stat -c "%%%%n;%%%%s;%%%%Y" {} +' % DATA_PATH DATE_FIND_CMD = 'ssh %s \'%s; %s\'' % (args.data_host, DATE_CMD, FIND_CMD) REAP_CMD = 'rsync -a %s:%%s %%s' % args.data_host datetime_file = os.path.join(os.path.dirname(__file__), '.%s.datetime' % REAPER_ID) reaper = DataReaper(datetime_file) def term_handler(signum, stack): reaper.halt() LOG.info('Received SIGTERM - shutting down...') signal.signal(signal.SIGTERM, term_handler)
super(ArgumentParser, self).__init__() self.add_argument('stage_path', help='path to staging area') self.add_argument('dicomserver', help='dicom server and port (hostname:port)') self.add_argument('aet', help='caller AE title') self.add_argument('aec', help='callee AE title') self.add_argument('-s', '--sleeptime', type=int, default=30, help='time to sleep before checking for new data') self.add_argument('-n', '--logname', default=os.path.splitext(os.path.basename(__file__))[0], help='process name for log') self.add_argument('-f', '--logfile', help='path to log file') self.add_argument('-l', '--loglevel', default='info', help='path to log file') if __name__ == '__main__': args = ArgumentParser().parse_args() host, port = args.dicomserver.split(':') log = nimsutil.get_logger(args.logname, args.logfile, args.loglevel) scu_ = scu.SCU(host, port, args.aet, args.aec, log=log) reap_stage = nimsutil.make_joined_path(args.stage_path, 'reap') sort_stage = nimsutil.make_joined_path(args.stage_path, 'sort') datetime_file = os.path.join(os.path.dirname(__file__), '.%s.datetime' % host) reaper = DicomReaper(host, scu_, reap_stage, sort_stage, datetime_file, args.sleeptime, log) def term_handler(signum, stack): reaper.halt() log.warning('Received SIGTERM - shutting down...') signal.signal(signal.SIGTERM, term_handler) reaper.run() log.warning('Process halted')
def at_path(cls, nims_path, filename=None, datatype=None, archived=False): dataset = cls(datatype=datatype, archived=archived) transaction.commit() DBSession.add(dataset) nimsutil.make_joined_path(nims_path, dataset.relpath) return dataset