Esempio n. 1
0
 def reap(self, _id, item, tempdir):
     reap_start = datetime.datetime.utcnow()
     log.info('reaping      %s' % self.state_str(_id, item['state']))
     reap_cnt = 0
     reap_path = tempdir + '/' + 'reap'
     os.mkdir(reap_path)
     metadata_path = tempdir + '/' + 'METADATA.json'
     for fn in os.listdir(item['path']):
         fp = item['path'] + '/' + fn
         if fn == 'metadata.json' or fn == 'METADATA.json':
             shutil.move(fp, metadata_path)
         else:
             try:
                 dcm = dicom.read_file(fp, stop_before_pixels=True) # ensure file is dicom
             except:
                 pass
             else:
                 reap_cnt += 1
                 shutil.copyfile(fp, reap_path + '/' + fn)
     log.info('reaped       %s (%d images) in %.1fs' % (_id, reap_cnt, (datetime.datetime.utcnow() - reap_start).total_seconds()))
     metadata = {}
     if os.path.exists(metadata_path):
         with open(metadata_path, 'r') as metadata_file:
             metadata = json.load(metadata_file, object_hook=reaper.datetime_decoder)
         os.remove(metadata_path)
     metadata['filetype'] = 'dicom'
     log.info('compressing  %s' % _id)
     reaper.create_archive(reap_path+'.tgz', reap_path, os.path.basename(reap_path), metadata, compresslevel=6)
     shutil.rmtree(reap_path)
     return True
Esempio n. 2
0
    def reap(self, _id, item, tempdir):
        reap_start = datetime.datetime.utcnow()
        log.info('reaping      %s' % self.state_str(_id, item['state']))
        reap_cnt = 0
        reap_path = os.path.join(tempdir, 'reap')
        os.mkdir(reap_path)
        for fn in os.listdir(item['path']):
            fp = os.path.join(item['path'], fn)
            try:
                # catch warnings here because some recordings have a
                # shielding mode that will throw a warning on read
                import mne
                with warnings.catch_warnings(record=True):
                    mne.io.read_raw_fif(fp, allow_maxshield=True)
            except Exception:
                pass
            else:
                reap_cnt += 1
                shutil.copyfile(fp, reap_path + '/' + fn)
        t = (datetime.datetime.utcnow() - reap_start).total_seconds()
        log.info('reaped       %s (%d images) in %.1fs' % (_id, reap_cnt, t))

        metadata = dict(filetype=u'meeg', timezone=self.timezone,
                        header={})
        log.info('compressing  %s' % _id)
        reaper.create_archive(reap_path + '.zip', reap_path,
                              os.path.basename(reap_path), metadata)
        shutil.rmtree(reap_path)
        return True
Esempio n. 3
0
 def reap(self, _id, item, tempdir):
     try:
         pfile = scipfile.PFile(item['path'], timezone=self.timezone)
     except (IOError, scipfile.PFileError):
         success = None
         log.warning('skipping     %s (disappeared or unparsable)' % _id)
     else:
         if pfile.patient_id.strip('/').lower() in self.blacklist:
             success = None
             log.info('discarding   %s' % _id)
         elif not re.match(self.whitelist, pfile.patient_id):
             success = None
             log.info('ignoring     %s (non-matching patient ID)' % _id)
         else:
             name_prefix = pfile.series_uid + '_' + str(pfile.acq_no)
             reap_path = tempdir + '/' + name_prefix + '_' + scipfile.PFile.filetype
             os.mkdir(reap_path)
             auxfiles = [(ap, _id + '_' + ap.rsplit('_', 1)[-1]) for ap in glob.glob(item['path'] + '_' + pfile.series_uid + '_*')]
             log.debug('staging      %s%s' % (_id, ', ' + ', '.join([af[1] for af in auxfiles]) if auxfiles else ''))
             os.symlink(item['path'], os.path.join(reap_path, _id))
             for af in auxfiles:
                 os.symlink(af[0], os.path.join(reap_path, af[1]))
             pfile_size = reaper.hrsize(item['state']['size'])
             metadata = {
                 'filetype': scipfile.PFile.filetype,
                 'timezone': self.timezone,
                 'header': {
                     'group': pfile.nims_group_id,
                     'project': pfile.nims_project,
                     'session': pfile.nims_session_id,
                     'session_no': pfile.series_no,
                     'session_desc': pfile.series_desc,
                     'acquisition': pfile.nims_acquisition_id,
                     'acquisition_no': pfile.acq_no,
                     'timestamp': pfile.nims_timestamp,
                 },
             }
             reap_start = datetime.datetime.utcnow()
             auxfile_str = ' + %d aux files' % len(auxfiles) if auxfiles else ''
             log.info('reaping.zip  %s [%s%s]' % (_id, pfile_size, auxfile_str))
             try:
                 reaper.create_archive(reap_path+'.zip', reap_path, os.path.basename(reap_path), metadata)
                 shutil.rmtree(reap_path)
             except (IOError):
                 success = False
                 log.warning('reap error   %s%s' % (_id, ' or aux files' if auxfiles else ''))
             else:
                 success = True
                 reap_time = (datetime.datetime.utcnow() - reap_start).total_seconds()
                 log.info('reaped.zip   %s [%s%s] in %.1fs' % (_id, pfile_size, auxfile_str, reap_time))
                 self.reap_peripheral_data(tempdir, pfile, name_prefix, _id)
     return success
Esempio n. 4
0
def reap(name, data_path, reap_path, reap_data, reap_name, log, log_info, tempdir):
    if not reap_data.psd_name or not re.match(r'^[a-zA-Z0-9_+-]+$', reap_data.psd_name):
        log.warning('periph data  %s %s invalid PSD name' % (log_info, name))
        return
    lower_time_bound = reap_data.timestamp + datetime.timedelta(seconds=reap_data.prescribed_duration or 0) - datetime.timedelta(seconds=15)
    upper_time_bound = lower_time_bound + datetime.timedelta(seconds=180)
    sleep_time = (upper_time_bound - datetime.datetime.now()).total_seconds()
    if sleep_time > 0:
        log.info('periph data  %s waiting for %s for %ds' % (log_info, name, sleep_time))
        time.sleep(sleep_time)
    for i in range(15):
        try:
            physio_files = os.listdir(data_path)
        except OSError:
            physio_files = []
        if physio_files:
            break
        else:
            log.warning('periph data  %s %s temporarily unavailable' % (log_info, name))
            time.sleep(60)
    else:
        log.error('periph data  %s %s permanently unavailable - giving up' % (log_info, name))
        return
    physio_tuples = filter(lambda pt: pt[0], [(re.match('.+_%s_([0-9_]{18,20})$' % reap_data.psd_name, pfn), pfn) for pfn in physio_files])
    physio_tuples = [(datetime.datetime.strptime(pts.group(1), '%m%d%Y%H_%M_%S_%f'), pfn) for pts, pfn in physio_tuples]
    physio_tuples = filter(lambda pt: lower_time_bound <= pt[0] <= upper_time_bound, physio_tuples)
    if physio_tuples:
        log.info('periph data  %s %s found' % (log_info, name))
        with tempfile.TemporaryDirectory(dir=tempdir) as tempdir_path:
            metadata = {
                    'filetype': scitran.data.medimg.gephysio.GEPhysio.filetype,
                    'timezone': reap_data.nims_timezone,
                    'header': {
                        'group': reap_data.nims_group_id,
                        'project': reap_data.nims_project,
                        'session': reap_data.nims_session_id,
                        'acquisition': reap_data.nims_acquisition_id,
                        'timestamp': reap_data.nims_timestamp,
                        },
                    }
            physio_reap_path = os.path.join(tempdir_path, reap_name)
            os.mkdir(physio_reap_path)
            for pts, pfn in physio_tuples:
                shutil.copy2(os.path.join(data_path, pfn), physio_reap_path)
            reaper.create_archive(os.path.join(reap_path, reap_name+'.tgz'), physio_reap_path, reap_name, metadata, compresslevel=6)
    else:
        log.info('periph data  %s %s not found' % (log_info, name))
Esempio n. 5
0
 def split_into_acquisitions(self, _id, item, path, filepaths):
     if self.anonymize:
         log.info('anonymizing  %s' % _id)
     dcm_dict = {}
     for filepath in filepaths:
         dcm = self.DicomFile(filepath, self.anonymize)
         if os.path.basename(filepath).startswith('(none)'):
             new_filepath = filepath.replace('(none)', 'NA')
             os.rename(filepath, new_filepath)
             filepath = new_filepath
         os.utime(filepath, (int(dcm.timestamp.strftime('%s')), int(dcm.timestamp.strftime('%s'))))  # correct timestamps
         dcm_dict.setdefault(dcm.acq_no, []).append(filepath)
     log.info('compressing  %s' % _id)
     acq_info = []
     for acq_no, acq_paths in dcm_dict.iteritems():
         name_prefix = _id + ('_' + acq_no if acq_no is not None else '')
         dir_name = name_prefix + '_' + scidcm.Dicom.filetype
         arcdir_path = os.path.join(path, dir_name)
         os.mkdir(arcdir_path)
         for filepath in acq_paths:
             os.rename(filepath, '%s.dcm' % os.path.join(arcdir_path, os.path.basename(filepath)))
         metadata = {
                 'filetype': scidcm.Dicom.filetype,
                 'timezone': self.timezone,
                 'overwrite': {
                     'firstname_hash': dcm.firstname_hash,
                     'lastname_hash': dcm.lastname_hash,
                     }
                 }
         reaper.create_archive(arcdir_path+'.zip', arcdir_path, dir_name, metadata)
         shutil.rmtree(arcdir_path)
         acq_info.append({
                 'path': arcdir_path+'.zip',
                 'prefix': name_prefix,
                 'log_info': '%s%s' % (_id, '.' + acq_no if acq_no is not None else ''),
                 })
     return acq_info