def __init__(self, nas_path=None, rig_path=None, archive_path=None, backup_path=None): self._nas_path = None self._rig_path = None self._archive_path = None self._backup_path = None if nas_path is not None: self._nas_path = nas_path self.site_dh = getDirHandle(nas_path) elif rig_path is not None: self._rig_path = rig_path self.site_dh = getDirHandle(rig_path) elif archive_path is not None: self._archive_path = archive_path self.site_dh = getDirHandle(archive_path) elif backup_path is not None: self._backup_path = backup_path self.site_dh = getDirHandle(backup_path) self.site_info = self.site_dh.info() self._slice_info = None self._expt_info = None self._specimen_info = None
def create_db_entries(cls, job_id, session): slices = all_slices() path = slices[job_id] dh = getDirHandle(path) info = dh.info() parent_info = dh.parent().info() # pull some metadata from LIMS sid = info['specimen_ID'].strip() limsdata = lims.specimen_info(sid) quality = info.get('slice quality', None) try: quality = int(quality) except Exception: quality = None # Interpret slice time slice_time = parent_info.get('time_of_dissection', None) if slice_time == '': slice_time = None if slice_time is not None: m = re.match(r'((20\d\d)-(\d{1,2})-(\d{1,2}) )?(\d+):(\d+)', slice_time.strip()) if m is not None: _, year, mon, day, hh, mm = m.groups() if year is None: date = datetime.fromtimestamp(dh.parent().info()['__timestamp__']) slice_time = datetime(date.year, date.month, date.day, int(hh), int(mm)) else: slice_time = datetime(int(year), int(mon), int(day), int(hh), int(mm)) # construct full genotype string genotype = limsdata['genotype'] or '' for info in (parent_info, info): inj = info.get('injections') if inj in (None, ''): continue if inj not in constants.INJECTIONS: raise KeyError("Injection %r is unknown in constants.INJECTIONS" % inj) genotype = ';'.join(genotype.split(';') + [constants.INJECTIONS[inj]]) fields = { 'acq_timestamp': info['__timestamp__'], 'species': limsdata['organism'], 'date_of_birth': limsdata['date_of_birth'], 'age': limsdata['age'], 'sex': limsdata['sex'], 'genotype': genotype, 'orientation': limsdata['plane_of_section'], 'surface': limsdata['exposed_surface'], 'hemisphere': limsdata['hemisphere'], 'quality': quality, 'slice_time': slice_time, 'slice_conditions': {}, 'lims_specimen_name': sid, 'storage_path': dh.name(relativeTo=dh.parent().parent()), } sl = db.Slice(**fields) session.add(sl)
def _get_raw_paths(self): expt_subpath = self.expt_subpath # find the local primary/archive paths that contain this experiment found_paths = False rig_name = self.rig_name if rig_name is None: return rig_data_paths = config.rig_data_paths.get(rig_name, []) for path_set in rig_data_paths: for root in path_set.values(): test_path = os.path.join(root, expt_subpath) if not os.path.isdir(test_path): continue dh = getDirHandle(test_path) if self.site_info is None: raise Exception ('%s %s missing index file' % (self, self.path)) if dh.info()['__timestamp__'] == self.site_info['__timestamp__']: found_paths = True # set self._primary_path, self._archive_path, etc. for k,v in path_set.items(): setattr(self, '_'+k+'_path', os.path.join(v, expt_subpath)) break if found_paths: break
def find_submittable_expts(): """Search synphys data storage for experiments that are ready to be submitted to LIMS. """ # Start by finding all site paths that have an nwb file and a file_manifest.yml all_nwbs = glob.glob( os.path.join(config.synphys_data, '*', 'slice_*', 'site_*', '*.nwb')) all_sites = OrderedDict() for nwb in all_nwbs: path = os.path.dirname(nwb) if 'file_manifest.yml' in os.listdir(path): all_sites[path] = 1 # filter out anything that has been submitted already ready_sites = [] for path in all_sites.keys(): site_dh = getDirHandle(path) acq_ts = site_dh.info()['__timestamp__'] spec_name = site_dh.parent().info()['specimen_ID'].strip() spec_id = lims.specimen_id_from_name(spec_name) subs = lims.expt_submissions(spec_id, acq_ts) if len(subs) == 0: ready_sites.append(site_dh) return ready_sites
def sync_experiment(site_dir): dh = getDirHandle(site_dir) sub = RawDataSubmission(dh) err, warn = sub.check() if len(err) > 0: return [], err, warn sub.submit() return sub.changes, err, warn
def __init__(self, path=None): Experiment.__init__(self, verify=False) self._site_path = path self.site_dh = getDirHandle(path) self._rig_name = None self._primary_path = None self._archive_path = None self._backup_path = None # reassign path based on order of most likely to be updated for path_typ in ['primary', 'archive', 'nas', 'backup']: path = getattr(self, path_typ + '_path') if path is not None and os.path.exists(path): self._site_path = path self.site_dh = getDirHandle(path) # reset values loaded while determining path self._expt_info = None self._slice_info = None self._site_info = None break
def all_slices(): """Return a dict mapping {slice_timestamp: path} for all known slices. This is only generated once per running process; set _all_slices = None to force the list to be regenerated. """ global _all_slices if _all_slices is not None: return _all_slices # Speed things up by caching this list with a 4 hour timeout cachefile = os.path.join(config.cache_path, 'all_slices.pkl') if os.path.exists(cachefile): age = time.time() - os.stat(cachefile).st_mtime if age < 4 * 3600: print("Loaded slice timestamps from cache (%0.1f hours old)" % (age/3600.)) return pickle.load(open(cachefile, 'r')) #slice_dirs = sorted(glob.glob(os.path.join(config.synphys_data, '*', 'slice_*'))) expt_csv = config.experiment_csv csv_entries = [] with open(expt_csv, 'r') as csv_file: reader = csv.DictReader(csv_file) for row in reader: csv_entries.append(row) slice_dirs = sorted([os.path.split(os.path.join(config.synphys_data, exp['site_path']))[0] for exp in csv_entries]) _all_slices = OrderedDict() for path in slice_dirs: dh = getDirHandle(path) ts = dh.info().get('__timestamp__') if ts is None: #print("MISSING TIMESTAMP: %s" % path) _all_slices.update([('%.3f'%0.0, 'place_holder')]) continue ts = '%0.3f'%ts ## convert timestamp to string here, make sure it has 3 decimal places _all_slices[ts] = path try: tmpfile = cachefile+'.tmp' pickle.dump(_all_slices, open(tmpfile, 'w')) os.rename(tmpfile, cachefile) except: if os.path.exists(tmpfile): os.remove(tmpfile) return _all_slices
def loadClicked(self): try: startDir = self.manager.getCurrentDir() except Exception: startDir = self.manager.getBaseDir() dirname = Qt.QFileDialog.getExistingDirectory(self.win, "Open Record", startDir.name()) if dirname == '': return self.recordDir = getDirHandle(dirname) self.recordWritable = False self.updateFileLabel() self.clearChannels() for dev in self.recordDir.ls(): w = self.addChannel(dev, mode=None, recordDir=self.recordDir)
def sync_experiment(site_dir): """Synchronize all files for an experiment to the server. Argument must be the path of an experiment _site_ folder. This will also cause synchronization for the parent (slice) and grandparent (day) folders to ensure that all slice images and metadata are copied. Sibling site and slice folders will _not_ be copied. Return a list of changes made. """ site_dh = getDirHandle(site_dir) changes = [] slice_dh = site_dh.parent() expt_dh = slice_dh.parent() now = time.strftime('%Y-%m-%d_%H:%M:%S') log("========== %s : Sync %s to server" % (now, site_dh.name())) skipped = 0 try: # Decide how the top-level directory will be named on the remote server # (it may already be there from a previous slice/site, or the current # name may already be taken by another rig.) server_expt_path = os.path.join(config.synphys_data, get_server_path(expt_dh)) log(" using server path: %s" % server_expt_path) skipped += _sync_paths(expt_dh.name(), server_expt_path, changes) # Copy slice files if needed server_slice_path = os.path.join(server_expt_path, slice_dh.shortName()) skipped += _sync_paths(slice_dh.name(), server_slice_path, changes) # Copy site files if needed server_site_path = os.path.join(server_slice_path, site_dh.shortName()) skipped += _sync_paths(site_dh.name(), server_site_path, changes) log(" Done; skipped %d files." % skipped) except Exception: err = traceback.format_exc() changes.append(('error', site_dh.name(), err)) log(err) return changes
def poll(self): # Find all available site paths across all data sources count = 0 path = self.search_path self.update.emit(path, "Updating...") root_dh = getDirHandle(path) # iterate over all expt sites in this path for day_name in sorted(os.listdir(root_dh.name()), reverse=True): for expt_path in glob.iglob( os.path.join(root_dh.name(), day_name, 'slice_*', 'site_*')): if self._stop or not self.enable_polling: return try: expt = ExperimentMetadata(path=expt_path) ts = expt.timestamp except: print('Error loading %s, ignoring and moving on...' % expt_path) sys.excepthook(*sys.exc_info()) continue # Couldn't get timestamp; show an error message if ts is None: print("Error getting timestamp for %s" % expt) continue with self.known_expts_lock: now = time.time() if ts in self.known_expts: expt, last_update = self.known_expts[ts] if now - last_update < self.interval: # We've already seen this expt recently; skip continue self.known_expts[ts] = (expt, now) # Add this expt to the queue to be checked self.expt_queue.put((-ts, expt)) count += 1 if self.limit > 0 and count >= self.limit: return self.update.emit(path, "Finished")
def poll(self): self.session = database.Session() expts = {} print("loading site paths..") #site_paths = glob.glob(os.path.join(config.synphys_data, '*', 'slice_*', 'site_*')) root_dh = getDirHandle(config.synphys_data) print(root_dh.name()) for site_dh in self.list_expts(root_dh): expt = ExperimentMetadata(nas_path=site_dh.name()) if expt.timestamp in expts: continue expts[expt.timestamp] = expt self.check(expt) if self.limit > 0 and len(expts) > self.limit: break
def generate_expt_path_cache(): global _expt_path_cache _expt_path_cache = {} root = getDirHandle(config.synphys_data) for f in root.ls(): if 'recycle' in f.lower(): continue dh = root[f] if not dh.isDir(): continue try: acq_timestamp = dh.info()['__timestamp__'] except KeyError: print("NO TIMESTAMP:", dh.name()) sys.exit(-1) if acq_timestamp in _expt_path_cache: raise Exception("timestamp %s appears twice in synphys data!!" % acq_timestamp) _expt_path_cache[acq_timestamp] = dh.name(relativeTo=root) write_expt_path_cache()
def poll(self): # Find all available site paths across all data sources count = 0 path = self.search_path self.update.emit(path, "Updating...") root_dh = getDirHandle(path) # iterate over all expt sites in this path for day_name in sorted(os.listdir(root_dh.name()), reverse=True): for expt_path in glob.iglob(os.path.join(root_dh.name(), day_name, 'slice_*', 'site_*')): if self._stop or not self.enable_polling: return try: expt = ExperimentMetadata(path=expt_path) ts = expt.timestamp except: print ('Error loading %s, ignoring and moving on...' % expt_path) continue # Couldn't get timestamp; show an error message if ts is None: print("Error getting timestamp for %s" % expt) continue with self.known_expts_lock: now = time.time() if ts in self.known_expts: expt, last_update = self.known_expts[ts] if now - last_update < self.interval: # We've already seen this expt recently; skip continue self.known_expts[ts] = (expt, now) # Add this expt to the queue to be checked self.expt_queue.put((-ts, expt)) count += 1 if self.limit > 0 and count >= self.limit: return self.update.emit(path, "Finished")
def all_slices(): """Return a dict mapping {slice_timestamp: path} for all known slices. This is only generated once per running process; set _all_slices = None to force the list to be regenerated. """ global _all_slices if _all_slices is not None: return _all_slices # Speed things up by caching this list with a 4 hour timeout cachefile = os.path.join(config.cache_path, 'all_slices.pkl') if os.path.exists(cachefile): age = time.time() - os.stat(cachefile).st_mtime if age < 4 * 3600: print("Loaded slice timestamps from cache (%0.1f hours old)" % (age / 3600.)) return pickle.load(open(cachefile, 'rb')) slice_dirs = sorted( glob.glob(os.path.join(config.synphys_data, '*', 'slice_*'))) _all_slices = OrderedDict() for path in slice_dirs: dh = getDirHandle(path) ts = dh.info().get('__timestamp__') if ts is None: print("MISSING TIMESTAMP: %s" % path) continue _all_slices["%0.3f" % ts] = path try: tmpfile = cachefile + '.tmp' pickle.dump(_all_slices, open(tmpfile, 'wb')) os.rename(tmpfile, cachefile) except: if os.path.exists(tmpfile): os.remove(tmpfile) return _all_slices
def poll(self): print("loading site paths..") #site_paths = glob.glob(os.path.join(config.synphys_data, '*', 'slice_*', 'site_*')) root_dh = getDirHandle(config.synphys_data) print(root_dh.name()) for expt in root_dh.ls(): expt_dh = root_dh[expt] print(expt_dh.name()) if not expt_dh.isDir(): continue for slice_name in expt_dh.ls(): slice_dh = expt_dh[slice_name] if not slice_dh.isDir(): continue print(slice_dh.name()) for site_name in slice_dh.ls(): site_dh = slice_dh[site_name] if not site_dh.isDir(): continue self.check(site_dh)
def all_slices(): """Return a dict mapping {slice_timestamp: path} for all known slices. This is only generated once per running process; set _all_slices = None to force the list to be regenerated. """ global _all_slices if _all_slices is not None: return _all_slices # Speed things up by caching this list with a 4 hour timeout cachefile = os.path.join(config.cache_path, 'all_slices.pkl') if os.path.exists(cachefile): age = time.time() - os.stat(cachefile).st_mtime if age < 4 * 3600: print("Loaded slice timestamps from cache (%0.1f hours old)" % (age/3600.)) return pickle.load(open(cachefile, 'r')) slice_dirs = sorted(glob.glob(os.path.join(config.synphys_data, '*', 'slice_*'))) _all_slices = OrderedDict() for path in slice_dirs: dh = getDirHandle(path) ts = dh.info().get('__timestamp__') if ts is None: print("MISSING TIMESTAMP: %s" % path) continue _all_slices[ts] = path try: tmpfile = cachefile+'.tmp' pickle.dump(_all_slices, open(tmpfile, 'w')) os.rename(tmpfile, cachefile) except: if os.path.exists(tmpfile): os.remove(tmpfile) return _all_slices
def find_submittable_expts(): """Search synphys data storage for experiments that are ready to be submitted to LIMS. """ # Start by finding all site paths that have an nwb file and a file_manifest.yml all_nwbs = glob.glob(os.path.join(config.synphys_data, '*', 'slice_*', 'site_*', '*.nwb')) all_sites = OrderedDict() for nwb in all_nwbs: path = os.path.dirname(nwb) if 'file_manifest.yml' in os.listdir(path): all_sites[path] = 1 # filter out anything that has been submitted already ready_sites = [] for path in all_sites.keys(): site_dh = getDirHandle(path) acq_ts = site_dh.info()['__timestamp__'] spec_name = site_dh.parent().info()['specimen_ID'].strip() spec_id = lims.specimen_id_from_name(spec_name) subs = lims.expt_submissions(spec_id, acq_ts) if len(subs) == 0: ready_sites.append(site_dh) return ready_sites
def get_experiment_server_path(dh): server_path = config.synphys_data acq_timestamp = dh.info()['__timestamp__'] # First check the cache cache = experiment_path_cache() if acq_timestamp in cache: return os.path.join(server_path, cache[acq_timestamp]) # We have not already submitted a site from this experiment folder; # look for a suitable new directory name on the server expt_base_name = dh.shortName().split('_')[0] expt_dirs = set(os.listdir(server_path)) i = 0 while True: expt_name = expt_base_name + '_%03d' % i if expt_name not in expt_dirs: break i += 1 server_expt_path = os.path.join(server_path, expt_name) assert not os.path.exists(server_expt_path) os.mkdir(server_expt_path) try: dh = getDirHandle(server_expt_path) # temporarily mark with timestamp; should be overwritten later. dh.setInfo(__timestamp__=acq_timestamp) except Exception: if os.path.exists(server_expt_path): shutil.rmtree(server_expt_path) raise cache[acq_timestamp] = expt_name write_expt_path_cache() return server_expt_path
return None root = sys.argv[1] parser = argparse.ArgumentParser() parser.add_argument('--set-data', action='store_true', default=False, dest='set-data') args = vars(parser.parse_args(sys.argv[2:])) set_data = args['set-data'] # find all subject folders that contain at least one site folder sites = glob.glob(os.path.join(root, '*', 'slice_*', 'site_*')) checked_days = set() checked_slices = set() for path in sites: site_dh = getDirHandle(path) slice_dh = site_dh.parent() day_dh = slice_dh.parent() sub_id = day_dh.info().get('animal_ID', None) expt_date = datetime.datetime.fromtimestamp(day_dh.info()['__timestamp__']).date() species = day_dh.info().get('species', None) if species != 'human': genotype = species if sub_id is not None and species is None: try: species = day_dh.info().get('LIMS_specimen_info')['organism'] genotype = day_dh.info().get('LIMS_specimen_info')['genotype'] except TypeError: try: species = day_dh.info().get('LIMS_donor_info')['organism'] except TypeError:
root = sys.argv[1] parser = argparse.ArgumentParser() parser.add_argument('--set-data', action='store_true', default=False, dest='set-data') args = vars(parser.parse_args(sys.argv[2:])) set_data = args['set-data'] # find all subject folders that contain at least one site folder sites = glob.glob(os.path.join(root, '*', 'slice_*', 'site_*')) checked_days = set() checked_slices = set() for path in sites: site_dh = getDirHandle(path) slice_dh = site_dh.parent() day_dh = slice_dh.parent() sub_id = day_dh.info().get('animal_ID', None) expt_date = datetime.datetime.fromtimestamp( day_dh.info()['__timestamp__']).date() species = day_dh.info().get('species', None) if species != 'human': genotype = species if sub_id is not None and species is None: try: species = day_dh.info().get('LIMS_specimen_info')['organism'] genotype = day_dh.info().get('LIMS_specimen_info')['genotype'] except TypeError: try: species = day_dh.info().get('LIMS_donor_info')['organism']
def ready_jobs(self): """Return an ordered dict of all jobs that are ready to be processed (all dependencies are present) and the dates that dependencies were created. """ slice_module = self.pipeline.get_module('opto_slice') finished_slices = slice_module.finished_jobs() # cache = synphys_cache.get_cache() # all_expts = cache.list_experiments() db = self.database session = db.session() slices = session.query(db.Slice.storage_path).all() slice_paths = [s[0] for s in slices] #ymls = [] #for rec in slices: # path = rec[0] # ymls.extend(glob.glob(os.path.join(config.synphys_data, path, 'site_*', 'pipettes.yml'))) expts = read_expt_csvs() n_errors = {} n_no_slice = [] ready = OrderedDict() print("checking for ready expts....") for i, expt in enumerate(expts['expt_list']): #print("Checking experiment %i/%i"%(i, len(expts['expt_list']))) site_path = os.path.join(config.synphys_data, expt['site_path']) slice_path = getDirHandle(os.path.split(site_path)[0]).name( relativeTo=getDirHandle(config.synphys_data)) #print slice_paths #if not slice_path in slice_paths: # #print("Did not find slice path for %s"%slice_path) # n_no_slice += 1 # continue try: if expt['site_path'] == '': cnx_json = os.path.join(config.connections_dir, expt['experiment']) ex = AI_Experiment( loader=OptoExperimentLoader(load_file=cnx_json), meta_info=expt) else: ex = AI_Experiment(loader=OptoExperimentLoader( site_path=site_path)) raw_data_mtime = ex.last_modification_time slice_ts = ex.info.get('slice_info', {}).get('__timestamp__') if slice_ts is None: slice_ts = 0.0 slice_mtime, slice_success = finished_slices.get( '%.3f' % slice_ts, (None, None)) #print('found expt for path:', site_path) except Exception as exc: raise n_errors[expt['experiment']] = exc continue if slice_mtime is None or slice_success is False: # slice_mtime = 0 n_no_slice.append(expt['experiment']) continue ready[ex.ext_id] = { 'dep_time': max(raw_data_mtime, slice_mtime), 'meta': { 'source': site_path } } print( "Found %d experiments; %d are able to be processed, %d were skipped due to errors, %d were skipped due to missing or failed slice entries." % (len(expts['expt_list']), len(ready), len(n_errors), len(n_no_slice))) if len(n_errors) > 0 or len(n_no_slice) > 0: print("-------- skipped experiments: ----------") for e, exc in n_errors.items(): print(' %s: Error - %s' % (e.split('_conn')[0], exc)) for e in n_no_slice: print(' %s: skipped due to problem with slice' % e.split('_conn')[0]) return ready
def create_db_entries(cls, job, session): job_id = job['job_id'] db = job['database'] slices = all_slices() path = slices[job_id] dh = getDirHandle(path) info = dh.info() parent_info = dh.parent().info() # pull some metadata from LIMS sid = info['specimen_ID'].strip() limsdata = lims.specimen_info(sid) quality = info.get('slice quality', None) try: quality = int(quality) except Exception: quality = None # Interpret slice time slice_time = parent_info.get('time_of_dissection', None) if slice_time == '': slice_time = None if slice_time is not None: m = re.match(r'((20\d\d)-(\d{1,2})-(\d{1,2}) )?(\d+):(\d+)', slice_time.strip()) if m is not None: _, year, mon, day, hh, mm = m.groups() if year is None: date = datetime.fromtimestamp(dh.parent().info()['__timestamp__']) slice_time = datetime(date.year, date.month, date.day, int(hh), int(mm)) else: slice_time = datetime(int(year), int(mon), int(day), int(hh), int(mm)) # construct full genotype string genotype = limsdata['genotype'] or '' for info in (parent_info, info): inj = info.get('injections') if inj in (None, ''): continue if inj not in constants.INJECTIONS: raise KeyError("Injection %r is unknown in constants.INJECTIONS" % inj) genotype = ';'.join(genotype.split(';') + [constants.INJECTIONS[inj]]) fields = { 'ext_id': job_id, 'acq_timestamp': info['__timestamp__'], 'species': limsdata['organism'], 'date_of_birth': limsdata['date_of_birth'], 'age': limsdata['age'], 'sex': limsdata['sex'], 'genotype': genotype, 'orientation': limsdata['plane_of_section'], 'surface': limsdata['exposed_surface'], 'hemisphere': limsdata['hemisphere'], 'quality': quality, 'slice_time': slice_time, 'slice_conditions': {}, 'lims_specimen_name': sid, 'storage_path': dh.name(relativeTo=dh.parent().parent()), } sl = db.Slice(**fields) session.add(sl)
def create_db_entries(cls, job, session): job_id = job['job_id'] db = job['database'] slices = all_slices() path = slices[job_id] if path == 'place_holder': sl = db.Slice(storage_path='place_holder', acq_timestamp=0.0) session.add(sl) return dh = getDirHandle(path) info = dh.info() parent_info = dh.parent().info() # pull some metadata from LIMS #sid = self.find_specimen_name(dh) sids = data_model.find_lims_specimen_ids(dh) #print('sids:', sids) if len(sids) == 0: limsdata = {} elif len(sids) == 1: limsdata = lims.specimen_info(specimen_id=sids[0]) elif len(sids) > 1: data = [] for i in sids: data.append(lims.specimen_info(specimen_id=i)) limsdata = {} for key in ['organism', 'date_of_birth', 'age', 'sex', 'plane_of_section', 'exposed_surface', 'hemisphere', 'specimen_name', 'genotype']: vals = list(set([d[key] for d in data])) if len(vals) == 1: limsdata[key] = vals[0] quality = info.get('slice quality', None) try: quality = int(quality) except Exception: quality = None # Interpret slice time slice_time = parent_info.get('time_of_dissection', None) if slice_time is not None: m = re.match(r'((20\d\d)-(\d{1,2})-(\d{1,2}) )?(\d+):(\d+)', slice_time.strip()) if m is not None: _, year, mon, day, hh, mm = m.groups() if year is None: date = datetime.fromtimestamp(dh.parent().info()['__timestamp__']) slice_time = datetime(date.year, date.month, date.day, int(hh), int(mm)) else: slice_time = datetime(int(year), int(mon), int(day), int(hh), int(mm)) # construct full genotype string genotype = limsdata.get('genotype', '') for info in (parent_info, info): inj = info.get('injections') if inj in (None, ''): continue if inj not in constants.INJECTIONS: raise KeyError("Injection %r is unknown in constants.INJECTIONS" % inj) genotype = genotype + ';' + constants.INJECTIONS[inj] fields = { 'ext_id':'%.3f'%info['__timestamp__'], 'acq_timestamp': info['__timestamp__'], 'species': limsdata.get('organism'), 'date_of_birth': limsdata.get('date_of_birth'), 'age': limsdata.get('age'), 'sex': limsdata.get('sex'), 'genotype': genotype, 'orientation': limsdata.get('plane_of_section'), 'surface': limsdata.get('exposed_surface'), 'hemisphere': limsdata.get('hemisphere'), 'quality': quality, 'slice_time': slice_time, 'slice_conditions': {}, 'lims_specimen_name': limsdata.get('specimen_name'), 'storage_path': dh.name(relativeTo=getDirHandle(config.synphys_data)), } sl = db.Slice(**fields) session.add(sl) session.commit()
def __init__(self, slice_dir): self.slice_dir = slice_dir self.dh = getDirHandle(self.slice_dir) self._fields = None
import os, datetime, re import config import database from acq4.util.DataManager import getDirHandle for path in config.raw_data_paths: for fname in os.listdir(path): fname = os.path.join(path, fname) if not os.path.isdir(fname): continue dh = getDirHandle(fname) if not dh.isManaged(): continue for sl in dh.ls(): if not sl.startswith('slice_'): continue slice_dh = dh[sl] if not slice_dh.isDir(): continue for site in slice_dh.ls(): if not site.startswith('site_'): continue site_dh = slice_dh[site] if not site_dh.isDir(): continue ts = site_dh.info()['__timestamp__'] date = datetime.datetime.fromtimestamp(ts)