def from_mrfile(cls, mrfile, nims_path, archived=True):
series_uid = nimsutil.pack_dicom_uid(mrfile.series_uid)
dataset = (cls.query.join(Epoch)
.filter(Epoch.uid == series_uid)
.filter(Epoch.acq == mrfile.acq_no)
.filter(cls.filetype == mrfile.filetype)
.first())
if not dataset:
alt_dataset = (cls.query.join(Epoch)
.filter(Epoch.uid == series_uid)
.filter(Epoch.acq == mrfile.acq_no)
.filter(cls.kind == u'primary')
.first())
if not alt_dataset:
kind = u'primary'
elif alt_dataset.priority < mrfile.priority:
kind = u'primary'
alt_dataset.kind = u'secondary'
else:
kind = u'secondary'
epoch = Epoch.from_mrfile(mrfile)
dataset = cls(
container=epoch,
priority = mrfile.priority,
filetype=mrfile.filetype,
compressed=mrfile.compressed,
kind=kind,
label=cls.default_labels[mrfile.filetype],
archived=archived,
)
transaction.commit()
DBSession.add(dataset)
nimsutil.make_joined_path(nims_path, dataset.relpath)
return dataset
def at_path(cls, nims_path, filename, datatype=None, archived=False):
metadata = cls.get_metadata(filename)
if metadata:
dataset = cls.from_metadata(metadata)
nimsutil.make_joined_path(nims_path, dataset.relpath)
else:
dataset = None
return dataset
def __init__(self, db_uri, sort_path, preserve_path, nims_path, dir_mode, sleep_time):
super(Sorter, self).__init__()
self.sort_path = nimsutil.make_joined_path(sort_path)
self.preserve_path = nimsutil.make_joined_path(preserve_path) if preserve_path else None
self.nims_path = nimsutil.make_joined_path(nims_path)
self.dir_mode = dir_mode
self.sleep_time = sleep_time
self.alive = True
model.init_model(sqlalchemy.create_engine(db_uri))
def reap(self):
try:
log.info('Inspecting %s' % self)
self.pfile = nimsdata.nimsraw.NIMSPFile(self.path)
except nimsdata.nimsraw.NIMSPFileError as e:
self.needs_reaping = False
log.warning('Skipping %s (%s)' % (self, str(e)))
return
else:
self.pat_id = self.pfile.patient_id
stage_dir = '%s_%s' % (self.reaper.id_,
datetime.datetime.now().strftime('%s.%f'))
reap_path = nimsutil.make_joined_path(self.reaper.reap_stage,
stage_dir)
if self.pat_id.strip('/').lower() in reaper.discard_ids:
self.needs_reaping = False
log.info('Discarding %s' % self)
return
if self.reaper.pat_id and not re.match(
self.reaper.pat_id.replace('*', '.*'), self.pat_id):
self.needs_reaping = False
log.info('Ignoring %s' % self)
return
try:
log.info('Reaping %s' % self)
shutil.copy2(self.path, reap_path)
for fp in glob.glob(self.path + '_' + self.pfile.series_uid +
'_*'):
log.info('Reaping %s to %s' %
(os.path.basename(fp),
os.path.join(
reap_path, '_' + self.basename + '_' +
fp.rsplit('_', 1)[-1])))
shutil.copy2(
fp,
os.path.join(
reap_path,
'_' + self.basename + '_' + fp.rsplit('_', 1)[-1]))
except KeyboardInterrupt:
shutil.rmtree(reap_path)
raise
except (shutil.Error, IOError):
log.warning('Error while reaping %s' % self)
else:
log.info('Compressing %s' % self)
nimsutil.gzip_inplace(os.path.join(reap_path, self.basename),
0o644)
shutil.move(reap_path,
os.path.join(self.reaper.sort_stage, '.' + stage_dir))
os.rename(os.path.join(self.reaper.sort_stage, '.' + stage_dir),
os.path.join(self.reaper.sort_stage, stage_dir))
self.needs_reaping = False
log.info('Reaped %s' % self)
def sort_directory(self, dirpath, filenames):
self.log.debug('Sorting %s in directory mode' % os.path.basename(dirpath))
dataset = self.dataset_at_path(self.nims_path, os.path.join(dirpath, filenames[0]))
if dataset:
for filepath in [os.path.join(dirpath, filename) for filename in filenames]:
ext = dataset.filename_ext if os.path.splitext(filepath)[1] != dataset.filename_ext else ''
shutil.move(filepath, os.path.join(self.nims_path, dataset.relpath, os.path.basename(filepath) + ext))
dataset.updatetime = datetime.datetime.now()
dataset.untrash()
transaction.commit()
elif self.preserve_mode:
unsort_path = nimsutil.make_joined_path(self.unsort_path, os.path.dirname(os.path.relpath(dirpath, self.stage_path)))
shutil.move(dirpath, unsort_path)
def __init__(self, id_, pat_id, discard_ids, data_path, reap_path, sort_path, datetime_file, sleep_time):
super(PFileReaper, self).__init__()
self.id_ = id_
self.pat_id = pat_id
self.discard_ids = discard_ids
self.data_glob = os.path.join(data_path, 'P?????.7')
self.reap_stage = nimsutil.make_joined_path(reap_path)
self.sort_stage = nimsutil.make_joined_path(sort_path)
self.datetime_file = datetime_file
self.sleep_time = sleep_time
self.current_file_timestamp = nimsutil.get_reference_datetime(self.datetime_file)
self.monitored_files = {}
self.alive = True
# delete any files left behind from a previous run
for item in os.listdir(self.reap_stage):
if item.startswith(self.id_):
shutil.rmtree(os.path.join(self.reap_stage, item))
for item in os.listdir(self.sort_stage):
if item.startswith('.' + self.id_):
shutil.rmtree(os.path.join(self.sort_stage, item))
def sort_file(self, filepath):
self.log.debug('Sorting %s' % os.path.basename(filepath))
dataset = self.dataset_at_path(self.nims_path, filepath)
if dataset:
ext = dataset.filename_ext if os.path.splitext(filepath)[1] != dataset.filename_ext else ''
shutil.move(filepath, os.path.join(self.nims_path, dataset.relpath, os.path.basename(filepath) + ext))
dataset.updatetime = datetime.datetime.now()
dataset.untrash()
transaction.commit()
elif self.preserve_mode:
unsort_path = nimsutil.make_joined_path(self.unsort_path, os.path.dirname(os.path.relpath(filepath, self.stage_path)))
shutil.move(filepath, unsort_path)
else:
os.remove(filepath)
def __init__(self, id_, pat_id, discard_ids, data_path, reap_path,
sort_path, datetime_file, sleep_time):
super(PFileReaper, self).__init__()
self.id_ = id_
self.pat_id = pat_id
self.discard_ids = discard_ids
self.data_glob = os.path.join(data_path, 'P?????.7')
self.reap_stage = nimsutil.make_joined_path(reap_path)
self.sort_stage = nimsutil.make_joined_path(sort_path)
self.datetime_file = datetime_file
self.sleep_time = sleep_time
self.current_file_timestamp = nimsutil.get_reference_datetime(
self.datetime_file)
self.monitored_files = {}
self.alive = True
# delete any files left behind from a previous run
for item in os.listdir(self.reap_stage):
if item.startswith(self.id_):
shutil.rmtree(os.path.join(self.reap_stage, item))
for item in os.listdir(self.sort_stage):
if item.startswith('.' + self.id_):
shutil.rmtree(os.path.join(self.sort_stage, item))
def reap(self, timestamp):
stage_dir = '%s_%s_%s' % (REAPER_ID, os.path.basename(self.name), timestamp.strftime('%s'))
reap_path = nimsutil.make_joined_path(REAP_STAGE, stage_dir)
try:
LOG.info('Reaping %s' % self)
subprocess.check_output(shlex.split(REAP_CMD % (self.name, reap_path)), stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
shutil.rmtree(reap_path)
success = False
LOG.warning('Error while reaping %s' % self)
else:
os.rename(reap_path, os.path.join(SORT_STAGE, stage_dir))
self.needs_reaping = False
success = True
LOG.info('Reaped %s' % self)
return success
def reap(self):
reap_path = nimsutil.make_joined_path(self.reap_stage, '%s_%s' % (self.reaper_id, datetime.datetime.now().strftime('%s.%f')))
try:
self.log.info('Reaping %s' % self)
shutil.copy2(self.path, reap_path)
except KeyboardInterrupt:
shutil.rmtree(reap_path)
raise
except (shutil.Error, IOError):
success = False
self.log.warning('Error while reaping %s' % self)
else:
shutil.move(reap_path, sort_stage)
self.needs_reaping = False
success = True
self.log.info('Reaped %s' % self)
return success
def reap(self, new_image_count):
if new_image_count > self.image_count:
self.image_count = new_image_count
self.needs_reaping = True
self.reaper.log.info('Monitoring %s' % self)
elif self.needs_reaping: # image count has stopped increasing
self.reaper.log.info('Reaping %s' % self)
now = datetime.datetime.now().strftime('%s')
stage_dir = '%s_%s-%d_%s' % (self.reaper.id_, self.exam.id_, self.id_, now)
reap_path = nimsutil.make_joined_path(self.reaper.reap_stage, stage_dir)
reap_count = self.reaper.scu.move(scu.SeriesQuery(StudyID=self.exam.id_, SeriesNumber=self.id_), reap_path)
if reap_count >= self.image_count:
self.needs_reaping = False
shutil.move(reap_path, reaper.sort_stage)
self.reaper.log.info('Reaped %s' % self)
else:
shutil.rmtree(reap_path)
self.reaper.log.warning('Incomplete %s, %d reaped' % (self, reap_count))
def sort_directory(self, dirpath, filenames, aux_paths):
log.debug('Sorting %s in directory mode' % os.path.basename(dirpath))
try:
mrfile = nimsdata.parse(os.path.join(dirpath, filenames[0]))
except nimsdata.NIMSDataError:
if self.preserve_path:
preserve_path = nimsutil.make_joined_path(self.preserve_path, os.path.relpath(dirpath, self.sort_path))
for filename in os.listdir(dirpath):
shutil.move(os.path.join(dirpath, filename), os.path.join(preserve_path, filename))
else:
dataset = model.Dataset.from_mrfile(mrfile, self.nims_path)
for filepath, aux_paths in [(os.path.join(dirpath, filename), aux_paths.get(filename, [])) for filename in filenames]:
shutil.move(filepath, os.path.join(self.nims_path, dataset.relpath, os.path.basename(filepath)))
for aux_path in aux_paths:
shutil.move(aux_path, os.path.join(self.nims_path, dataset.relpath, os.path.basename(aux_path)))
dataset.updatetime = datetime.datetime.now()
dataset.untrash()
transaction.commit()
shutil.rmtree(dirpath)
def reap(self):
try:
log.info('Inspecting %s' % self)
self.pfile = nimsdata.nimsraw.NIMSPFile(self.path)
except nimsdata.nimsraw.NIMSPFileError as e:
self.needs_reaping = False
log.warning('Skipping %s (%s)' % (self, str(e)))
return
else:
self.pat_id = self.pfile.patient_id
self.exam = self.pfile.exam_no
self.series = self.pfile.series_no
self.acq = self.pfile.acq_no
stage_dir = '%s_%s' % (self.reaper.id_, datetime.datetime.now().strftime('%s.%f'))
reap_path = nimsutil.make_joined_path(self.reaper.reap_stage, stage_dir)
aux_reap_files = [arf for arf in glob.glob(self.path + '_*') if self.is_aux_file(arf)]
if self.pat_id.strip('/').lower() in reaper.discard_ids:
self.needs_reaping = False
log.info('Discarding %s' % self)
return
if self.reaper.pat_id and not re.match(self.reaper.pat_id.replace('*','.*'), self.pat_id):
self.needs_reaping = False
log.info('Ignoring %s' % self)
return
try:
log.info('Reaping %s' % self)
shutil.copy2(self.path, reap_path)
for arf in aux_reap_files:
shutil.copy2(arf, os.path.join(reap_path, '_' + os.path.basename(arf)))
log.info('Reaping %s' % '_' + os.path.basename(arf))
except KeyboardInterrupt:
shutil.rmtree(reap_path)
raise
except (shutil.Error, IOError):
log.warning('Error while reaping %s' % self)
else:
log.info('Compressing %s' % self)
nimsutil.gzip_inplace(os.path.join(reap_path, self.basename), 0o644)
shutil.move(reap_path, os.path.join(self.reaper.sort_stage, '.' + stage_dir))
os.rename(os.path.join(self.reaper.sort_stage, '.' + stage_dir), os.path.join(self.reaper.sort_stage, stage_dir))
self.needs_reaping = False
log.info('Reaped %s' % self)
def sort_files(self, dirpath, filenames, aux_paths):
for filepath, filename in [(os.path.join(dirpath, fn), fn) for fn in filenames]:
log.debug('Sorting %s' % filename)
try:
mrfile = nimsdata.parse(filepath)
except nimsdata.NIMSDataError:
if self.preserve_path:
preserve_path = nimsutil.make_joined_path(self.preserve_path, os.path.dirname(os.path.relpath(filepath, self.sort_path)))
shutil.move(filepath, os.path.join(preserve_path, filename))
else:
dataset = model.Dataset.from_mrfile(mrfile, self.nims_path)
new_filenames = [filename]
shutil.move(filepath, os.path.join(self.nims_path, dataset.relpath, filename))
for aux_path in aux_paths.get(os.path.splitext(filename)[0] if dataset.compressed else filename, []):
new_filenames.append(os.path.basename(aux_path))
shutil.move(aux_path, os.path.join(self.nims_path, dataset.relpath, os.path.basename(aux_path)))
dataset.filenames = set(dataset.filenames + new_filenames)
dataset.updatetime = datetime.datetime.now()
dataset.untrash()
transaction.commit()
shutil.rmtree(dirpath)
class ArgumentParser(argparse.ArgumentParser):
def __init__(self):
super(ArgumentParser, self).__init__()
self.add_argument('source_stage', help='path to source staging area')
self.add_argument('data_host', help='username@hostname of data destination')
self.add_argument('remote_stage', help='path to destination staging area')
self.add_argument('-s', '--sleeptime', type=int, default=30, help='time to sleep before checking for new data')
self.add_argument('-n', '--logname', default=os.path.splitext(os.path.basename(__file__))[0], help='process name for log')
self.add_argument('-f', '--logfile', help='path to log file')
self.add_argument('-l', '--loglevel', default='info', help='path to log file')
if __name__ == "__main__":
args = ArgumentParser().parse_args()
log = nimsutil.get_logger(args.logname, args.logfile, args.loglevel)
source_stage = nimsutil.make_joined_path(args.source_stage, 'sort')
reap_stage = nimsutil.make_joined_path(args.remote_stage, 'reap')
sort_stage = nimsutil.make_joined_path(args.remote_stage, 'sort')
restager = Restager(source_stage, args.data_host, reap_stage, sort_stage, args.sleeptime, log)
def term_handler(signum, stack):
restager.halt()
log.info('Received SIGTERM - shutting down...')
signal.signal(signal.SIGTERM, term_handler)
restager.run()
log.warning('Process halted')
def at_path(cls, nims_path, filetype, label=None, archived=False):
dataset = cls(filetype=filetype, label=(label if label else cls.default_labels[filetype]), archived=archived)
transaction.commit()
DBSession.add(dataset)
nimsutil.make_joined_path(nims_path, dataset.relpath)
return dataset
def __init__(self):
super(ArgumentParser, self).__init__()
self.add_argument('db_uri', help='database URI')
self.add_argument('stage_path', help='path to staging area')
self.add_argument('nims_path', help='data destination')
self.add_argument('-d', '--dirmode', action='store_true', help='assume files are pre-sorted by directory')
self.add_argument('-p', '--preserve', action='store_true', help='preserve unsortable files')
self.add_argument('-s', '--sleeptime', type=int, default=10, help='time to sleep before checking for new files')
self.add_argument('-n', '--logname', default=os.path.splitext(os.path.basename(__file__))[0], help='process name for log')
self.add_argument('-f', '--logfile', help='path to log file')
self.add_argument('-l', '--loglevel', default='info', help='path to log file')
if __name__ == '__main__':
args = ArgumentParser().parse_args()
log = nimsutil.get_logger(args.logname, args.logfile, args.loglevel)
stage_path = nimsutil.make_joined_path(args.stage_path, 'sort')
unsort_path = nimsutil.make_joined_path(args.stage_path, 'unsortable')
nims_path = nimsutil.make_joined_path(args.nims_path)
sorter = Sorter(args.db_uri, stage_path, unsort_path, nims_path, args.dirmode, args.preserve, args.sleeptime, log)
def term_handler(signum, stack):
sorter.halt()
log.info('Receieved SIGTERM - shutting down...')
signal.signal(signal.SIGTERM, term_handler)
sorter.run()
log.warning('Process halted')
self.add_argument('-l',
'--loglevel',
default='info',
help='log level (default: info)')
self.add_argument('-q',
'--quiet',
action='store_true',
default=False,
help='disable console logging')
if __name__ == "__main__":
args = ArgumentParser().parse_args()
nimsutil.configure_log(args.logfile, not args.quiet, args.loglevel)
source_stage = nimsutil.make_joined_path(args.source_stage, 'sort')
reap_stage = nimsutil.make_joined_path(args.remote_stage, 'reap')
sort_stage = nimsutil.make_joined_path(args.remote_stage, 'sort')
restager = Restager(source_stage, args.data_host, reap_stage, sort_stage,
args.sleeptime)
def term_handler(signum, stack):
restager.halt()
log.info('Received SIGTERM - shutting down...')
signal.signal(signal.SIGTERM, term_handler)
restager.run()
log.warning('Process halted')
self.add_argument('data_host', help='username@hostname of data source')
self.add_argument('data_path', help='path to data source')
self.add_argument('-g', '--data_glob', default='*', help='glob format for files to move')
self.add_argument('-s', '--sleeptime', type=int, default=60, help='time to sleep before checking for new data')
self.add_argument('-n', '--logname', default=__file__, help='process name for log')
self.add_argument('-f', '--logfile', help='path to log file')
self.add_argument('-l', '--loglevel', default='info', help='path to log file')
if __name__ == '__main__':
args = ArgumentParser().parse_args()
REAPER_ID = os.path.basename(args.data_path.rstrip('/'))
LOG = nimsutil.get_logger(args.logname, args.logfile, args.loglevel)
SLEEP_TIME = int(args.sleeptime)
REAP_STAGE = nimsutil.make_joined_path(args.stage_path, 'reap')
DATA_PATH = os.path.join(args.data_path, args.data_glob)
SORT_STAGE = nimsutil.make_joined_path(args.stage_path, 'sort')
DATE_CMD = 'date +%%s'
FIND_CMD = 'find %s -maxdepth 1 -newermt "%%s" -exec stat -c "%%%%n;%%%%s;%%%%Y" {} +' % DATA_PATH
DATE_FIND_CMD = 'ssh %s \'%s; %s\'' % (args.data_host, DATE_CMD, FIND_CMD)
REAP_CMD = 'rsync -a %s:%%s %%s' % args.data_host
datetime_file = os.path.join(os.path.dirname(__file__), '.%s.datetime' % REAPER_ID)
reaper = DataReaper(datetime_file)
def term_handler(signum, stack):
reaper.halt()
LOG.info('Received SIGTERM - shutting down...')
signal.signal(signal.SIGTERM, term_handler)
super(ArgumentParser, self).__init__()
self.add_argument('stage_path', help='path to staging area')
self.add_argument('dicomserver', help='dicom server and port (hostname:port)')
self.add_argument('aet', help='caller AE title')
self.add_argument('aec', help='callee AE title')
self.add_argument('-s', '--sleeptime', type=int, default=30, help='time to sleep before checking for new data')
self.add_argument('-n', '--logname', default=os.path.splitext(os.path.basename(__file__))[0], help='process name for log')
self.add_argument('-f', '--logfile', help='path to log file')
self.add_argument('-l', '--loglevel', default='info', help='path to log file')
if __name__ == '__main__':
args = ArgumentParser().parse_args()
host, port = args.dicomserver.split(':')
log = nimsutil.get_logger(args.logname, args.logfile, args.loglevel)
scu_ = scu.SCU(host, port, args.aet, args.aec, log=log)
reap_stage = nimsutil.make_joined_path(args.stage_path, 'reap')
sort_stage = nimsutil.make_joined_path(args.stage_path, 'sort')
datetime_file = os.path.join(os.path.dirname(__file__), '.%s.datetime' % host)
reaper = DicomReaper(host, scu_, reap_stage, sort_stage, datetime_file, args.sleeptime, log)
def term_handler(signum, stack):
reaper.halt()
log.warning('Received SIGTERM - shutting down...')
signal.signal(signal.SIGTERM, term_handler)
reaper.run()
log.warning('Process halted')
def at_path(cls, nims_path, filename=None, datatype=None, archived=False):
dataset = cls(datatype=datatype, archived=archived)
transaction.commit()
DBSession.add(dataset)
nimsutil.make_joined_path(nims_path, dataset.relpath)
return dataset
