def _make_labnumber(self, li, root, options, prog): #make dir for labnumber ident = li.identifier ln_root = os.path.join(root, ident) r_mkdir(ln_root) prog.change_message('Making {} for {}'.format(self._tag, ident)) #filter invalid analyses ans = filter(lambda x: not x.tag == 'invalid', li.analyses) #group by stepheat vs fusion pred = lambda x: bool(x.step) ans = sorted(ans, key=pred) stepheat, fusion = map(list, partition(ans, pred)) apred = lambda x: x.aliquot stepheat = sorted(stepheat, key=apred) project = 'Minna Bluff' li = li.identifier if stepheat: key = lambda x: x.aliquot stepheat = sorted(stepheat, key=key) for aliquot, ais in groupby(stepheat, key=key): name = make_runid(li, aliquot, '') self._make_editor(ais, 'step_heat', options, prog, False, (ln_root, name, name, project, (li, ))) if fusion: self._make_editor(fusion, 'fusion', options, prog, False, (ln_root, li, li, project, (li, )))
def _do_syn_extract(self, spec, script, post_script): self.debug('Executing SynExtraction mode="{}"'.format(spec.mode)) #modify the persister. the original persister for the automated run is saved at self.persister #arun.persister reset to original when syn extraction stops identifier = spec.identifier last_aq = self.arun.persister.get_last_aliquot(identifier) if last_aq is None: self.warning( 'invalid identifier "{}". Does not exist in database'.format( identifier)) else: runid = make_runid(identifier, last_aq + 1) self.arun.info('Starting SynExtraction run {}'.format(runid)) self.arun.persister.trait_set(use_massspec_database=False, runid=runid, uuid=str(uuid.uuid4())) if self.arun.do_measurement(script=script, use_post_on_fail=False): if post_script: self.debug('starting post measurement') if not self.arun.do_post_measurement(script=post_script): return self.debug('delay between syn extractions {}'.format( spec.delay_between)) self.arun.wait(spec.delay_between) return True
def _sync_meas_analysis_attributes(self, meas_analysis): # copy meas_analysis attrs nocast = lambda x: x attrs = [ ('labnumber', 'labnumber', lambda x: x.identifier), ('aliquot', 'aliquot', int), ('step', 'step', str), # ('status', 'status', int), ('comment', 'comment', str), ('uuid', 'uuid', str), ('rundate', 'analysis_timestamp', nocast), ('timestamp', 'analysis_timestamp', lambda x: time.mktime(x.timetuple())), ] for key, attr, cast in attrs: v = getattr(meas_analysis, attr) setattr(self, key, cast(v)) self.record_id = make_runid(self.labnumber, self.aliquot, self.step) tag = meas_analysis.tag if tag: tag = meas_analysis.tag_item self.set_tag(tag)
def _sync_meas_analysis_attributes(self, meas_analysis): # copy meas_analysis attrs nocast = lambda x: x attrs = [ ('labnumber', 'labnumber', lambda x: x.identifier), ('aliquot', 'aliquot', int), ('step', 'step', str), # ('status', 'status', int), ('comment', 'comment', str), ('uuid', 'uuid', str), ('rundate', 'analysis_timestamp', nocast), ('timestamp', 'analysis_timestamp', lambda x: time.mktime(x.timetuple()) ), ] for key, attr, cast in attrs: v = getattr(meas_analysis, attr) setattr(self, key, cast(v)) self.record_id = make_runid(self.labnumber, self.aliquot, self.step) tag = meas_analysis.tag if tag: tag = meas_analysis.tag_item self.set_tag(tag)
def _do_syn_extract(self, spec, script, post_script): self.debug('Executing SynExtraction mode="{}"'.format(spec.mode)) #modify the persister. the original persister for the automated run is saved at self.persister #arun.persister reset to original when syn extraction stops identifier=spec.identifier last_aq=self.arun.persister.get_last_aliquot(identifier) if last_aq is None: self.warning('invalid identifier "{}". Does not exist in database'.format(identifier)) else: runid=make_runid(identifier, last_aq+1) self.arun.info('Starting SynExtraction run {}'.format(runid)) self.arun.persister.trait_set(use_secondary_database=False, runid=runid, uuid=str(uuid.uuid4())) if self.arun.do_measurement(script=script, use_post_on_fail=False): if post_script: self.debug('starting post measurement') if not self.arun.do_post_measurement(script=post_script): return self.debug('delay between syn extractions {}'.format(spec.delay_between)) self.arun.wait(spec.delay_between) return True
def _make(self, ep): project = 'Minna Bluff' db = self.db with db.session_ctx(): prj = db.get_project(project) Ar40, Ar39, Ar38, Ar37, Ar36 = [], [], [], [], [] for dev in (('Eurotherm', 'Furnace'), ('CO2')): for si in prj.samples: for li in si.labnumbers: self.debug('blanks for {},{}'.format( si.name, li.identifier)) for ai in li.analyses: if ai.extraction.extraction_device.name in dev: bs = self._extract_blanks(ai) if bs is not None: r = make_runid(li.identifier, ai.aliquot, ai.step) # self.debug('blanks for {} {}'.format(r,bs)) Ar40.append(bs[0]) Ar39.append(bs[1]) Ar38.append(bs[2]) Ar37.append(bs[3]) Ar36.append(bs[4]) reg = WeightedMeanRegressor() print 'blanks for {}'.format(dev) for iso in (Ar40, Ar39, Ar38, Ar37, Ar36): ys, es = zip(*iso) reg.trait_set(ys=ys, yserr=es) print reg.predict()
def __init__(self, dbrecord): self.uuid = dbrecord.uuid ln = dbrecord.labnumber labnumber = str(ln.identifier) aliquot = dbrecord.aliquot step = dbrecord.step self.record_id = make_runid(labnumber, aliquot, step) self.rundate = dbrecord.analysis_timestamp self.timestamp = time.mktime(self.rundate.timetuple()) self.tag = dbrecord.tag or '' self.is_plateau_step = False meas = dbrecord.measurement if meas is not None: if meas.analysis_type: self.analysis_type = meas.analysis_type.name if meas.mass_spectrometer: self.mass_spectrometer = meas.mass_spectrometer.name sam = ln.sample if sam: self.sample = sam.name if sam.project: self.project = sam.project.name.lower()
def _make_labnumber(self, li, root, options, prog): #make dir for labnumber ident = li.identifier ln_root = os.path.join(root, ident) r_mkdir(ln_root) prog.change_message('Making {} for {}'.format(self._tag, ident)) #filter invalid analyses ans=filter(lambda x: not x.tag=='invalid', li.analyses) #group by stepheat vs fusion pred = lambda x: bool(x.step) ans = sorted(ans, key=pred) stepheat, fusion = map(list, partition(ans, pred)) apred = lambda x: x.aliquot stepheat = sorted(stepheat, key=apred) project='Minna Bluff' li = li.identifier if stepheat: key=lambda x: x.aliquot stepheat=sorted(stepheat, key=key) for aliquot, ais in groupby(stepheat, key=key): name=make_runid(li, aliquot, '') self._make_editor(ais, 'step_heat', options, prog, False, (ln_root, name, name, project, (li,))) if fusion: self._make_editor(fusion, 'fusion', options, prog, False, (ln_root, li, li, project, (li,)))
def _make(self, ep): project = "Minna Bluff" db = self.db with db.session_ctx(): prj = db.get_project(project) Ar40, Ar39, Ar38, Ar37, Ar36 = [], [], [], [], [] for dev in (("Eurotherm", "Furnace"), ("CO2")): for si in prj.samples: for li in si.labnumbers: self.debug("blanks for {},{}".format(si.name, li.identifier)) for ai in li.analyses: if ai.extraction.extraction_device.name in dev: bs = self._extract_blanks(ai) if bs is not None: r = make_runid(li.identifier, ai.aliquot, ai.step) # self.debug('blanks for {} {}'.format(r,bs)) Ar40.append(bs[0]) Ar39.append(bs[1]) Ar38.append(bs[2]) Ar37.append(bs[3]) Ar36.append(bs[4]) reg = WeightedMeanRegressor() print "blanks for {}".format(dev) for iso in (Ar40, Ar39, Ar38, Ar37, Ar36): ys, es = zip(*iso) reg.trait_set(ys=ys, yserr=es) print reg.predict()
def _modify_secondary(self, ans, new_ans): self.info('modifying analyses in secondary db. dry={}'.format(self.dry)) db = self.secondary_db if not db.connect(): self.debug('Not connected to secondary db') return with db.session_ctx(commit=not self.dry): for ai, ni in zip(ans, new_ans): rid = ai.record_id dban = db.get_analysis_rid(rid) if dban: ident = ni.identifier dbirradpos = db.get_irradiation_position(ident) if dbirradpos: dban.RID = make_runid(ident, ni.aliquot, ni.step) self.debug('setting {} to {}'.format(ident, dban.RID)) dban.Aliquot = '{:02d}'.format(int(ni.aliquot)) dban.Increment = ni.step dban.Aliquot_pychron = ni.aliquot dban.IrradPosition = ident dban.RedundantSampleID = dbirradpos.SampleID else: self.unique_warning('Labnumber {} does not exist in Secondary DB'.format(ident)) else: self.warning('Analysis {} does not exist in Secondary DB'.format(rid))
def create(self, dbrecord, fast_load=False): # print 'asdfsadfsdaf', dbrecord, dbrecord.labnumber, dbrecord.uuid try: if dbrecord is None or not dbrecord.labnumber: return ln = dbrecord.labnumber self.labnumber = str(ln.identifier) self.identifier = self.labnumber self.aliquot = dbrecord.aliquot self.step = dbrecord.step self.record_id = make_runid(self.labnumber, self.aliquot, self.step) self.uuid = dbrecord.uuid self.tag = dbrecord.tag or '' self.rundate = dbrecord.analysis_timestamp sam = ln.sample if sam: self.sample = sam.name if sam.project: if isinstance(sam.project, (str, unicode)): self.project = sam.project.lower() else: self.project = sam.project.name.lower() irp = ln.irradiation_position if irp is not None: irl = irp.level ir = irl.irradiation self.irradiation_info = '{}{} {}'.format(ir.name, irl.name, irp.position) try: self.mass_spectrometer = dbrecord.mass_spectrometer except AttributeError: pass meas = dbrecord.measurement if meas: self.mass_spectrometer = meas.mass_spectrometer.name.lower() try: self.analysis_type = meas.analysis_type.name except AttributeError, e: pass # print 'IsotopeRecord create meas 1 {}'.format(e) ext = dbrecord.extraction if ext: self.extract_value = ext.extract_value self.cleanup = ext.cleanup_duration self.duration = ext.extract_duration try: if ext.extraction_device: self.extract_device = ext.extraction_device.name except AttributeError, e: pass
def create(self, dbrecord): # print 'asdfsadfsdaf', dbrecord, dbrecord.labnumber, dbrecord.uuid try: if dbrecord is None or not dbrecord.labnumber: return ln = dbrecord.labnumber self.labnumber = str(ln.identifier) self.identifier = self.labnumber self.aliquot = dbrecord.aliquot self.step = dbrecord.step self.uuid = dbrecord.uuid self.tag = dbrecord.tag or '' self.rundate = dbrecord.analysis_timestamp self.timestamp = time.mktime(self.rundate.timetuple()) self.record_id = make_runid(self.labnumber, self.aliquot, self.step) # print self.record_id, self.uuid if ln.sample: self.sample = ln.sample.name if dbrecord.labnumber.sample: self.sample = dbrecord.labnumber.sample.name irp = ln.irradiation_position if irp is not None: irl = irp.level ir = irl.irradiation self.irradiation_info = '{}{} {}'.format(ir.name, irl.name, irp.position) else: self.irradiation_info = '' meas = dbrecord.measurement if meas is not None: self.mass_spectrometer = meas.mass_spectrometer.name.lower() if meas.analysis_type: self.analysis_type = meas.analysis_type.name ext = dbrecord.extraction if ext: if ext.extraction_device: self.extract_device = ext.extraction_device.name self.flux_fit_status = self._get_flux_fit_status(dbrecord) self.blank_fit_status = self._get_selected_history_item(dbrecord, 'selected_blanks_id') self.ic_fit_status = self._get_selected_history_item(dbrecord, 'selected_det_intercal_id') self.iso_fit_status = self._get_selected_history_item(dbrecord, 'selected_fits_id') return True except Exception, e: import traceback traceback.print_exc() print e
def init(self): if self.increment >= 0: self.step = ALPHAS[self.increment] else: self.step = '' rid = make_runid(self.identifier, self.aliquot, self.step) if self.use_repository_suffix: rid = '{}-{}'.format(rid, self.repository_identifier) self.record_id = rid
def _construct_analysis(self, rec, prog, calculate_age=True, unpack=False, load_changes=False): atype = None if isinstance(rec, meas_AnalysisTable): rid = make_runid(rec.labnumber.identifier, rec.aliquot, rec.step) atype = rec.measurement.analysis_type.name elif hasattr(rec, 'record_id'): rid = rec.record_id else: rid = id(rec) graph_id = 0 group_id = 0 if hasattr(rec, 'group_id'): group_id = rec.group_id if hasattr(rec, 'graph_id'): graph_id = rec.graph_id if atype is None: atype = rec.analysis_type if prog: show_age = calculate_age and atype in ('unknown', 'cocktail') m = 'calculating age' if show_age else '' msg = 'loading {}. {}'.format(rid, m) prog.change_message(msg) meas_analysis = self.db.get_analysis_uuid(rec.uuid) klass = DBAnalysis if not self.use_vcs else VCSAnalysis ai = klass(group_id=group_id, graph_id=graph_id) # if not self.use_vcs: synced = False if atype in ('unknown', 'cocktail'): if calculate_age: ai.sync(meas_analysis, load_changes=load_changes) ai.calculate_age(force=not self.use_vcs) synced = True if not synced: ai.sync(meas_analysis, unpack=unpack, load_changes=load_changes) return ai
def make_summary(self): s = 'Pr' f = self.fit if f: if not f in INTERPOLATE_TYPES: f = f[:1].upper() s = '{}{}'.format(self.isotope, f) if self.preceding_id: p = self.preceding_analysis rid = make_runid(p.labnumber.identifier, p.aliquot, p.step) s = '{} ({})'.format(s, rid) return s
def test_modifier_secondarydb(self): self.modifier.use_main = False self.modifier.use_secondary = True an = self.analyses[0] can = copy.copy(an) can.identifier = self.new_identifier self.modifier.modify_analyses([an], [can]) db = self.modifier.secondary_db with db.session_ctx(): old_an = db.get_analysis(an.labnumber, an.aliquot, an.step) self.assertIsNone(old_an) new_an = db.get_analysis(self.new_identifier, '{:02d}'.format(an.aliquot), an.step) self.assertEqual(new_an.RID, make_runid(self.new_identifier, an.aliquot, an.step)) self.assertEqual(new_an.sample.Sample, 'Bar')
def generate_omassspecdb(): src = os.path.join(get_data_dir(), 'omassspecdata.db') db = massspec_db_factory(src) with db.session_ctx(): s = db.add_sample('Foo') db.flush() db.add_irradiation_position('1000', 'NM-100A', 1, sample=s.SampleID) s = db.add_sample('Bar') db.flush() db.add_irradiation_position('2000', 'NM-100A', 2, sample=s.SampleID) db.flush() for i in range(3): rid = make_runid('1000', i + 1) db.add_analysis(rid, i + 1, '', '1000', 1)
def _construct_analysis(self, rec, prog, calculate_age=True, unpack=False, load_changes=False): atype = None if isinstance(rec, meas_AnalysisTable): rid = make_runid(rec.labnumber.identifier, rec.aliquot, rec.step) atype = rec.measurement.analysis_type.name elif hasattr(rec, 'record_id'): rid = rec.record_id else: rid = id(rec) graph_id = 0 group_id = 0 if hasattr(rec, 'group_id'): group_id = rec.group_id if hasattr(rec, 'graph_id'): graph_id = rec.graph_id if atype is None: atype = rec.analysis_type if prog: show_age = calculate_age and atype in ('unknown', 'cocktail') m = 'calculating age' if show_age else '' msg = 'loading {}. {}'.format(rid, m) prog.change_message(msg) meas_analysis = self.db.get_analysis_uuid(rec.uuid) klass=DBAnalysis if not self.use_vcs else VCSAnalysis ai = klass(group_id=group_id, graph_id=graph_id) # if not self.use_vcs: synced=False if atype in ('unknown', 'cocktail'): if calculate_age: ai.sync(meas_analysis, load_changes=load_changes) ai.calculate_age(force=not self.use_vcs) synced=True if not synced: ai.sync(meas_analysis, unpack=unpack, load_changes=load_changes) return ai
def add_analysis(self, spec, commit=True): db = self.db for i in range(3): with self.db.session_ctx(): irradpos = spec.irradpos rid = spec.runid trid = rid.lower() identifier = spec.labnumber if trid.startswith('b'): runtype = 'Blank' irradpos = -1 elif trid.startswith('a'): runtype = 'Air' irradpos = -2 elif trid.startswith('c'): runtype = 'Unknown' identifier = irradpos = self.get_identifier(spec) else: runtype = 'Unknown' rid = make_runid(identifier, spec.aliquot, spec.step) self._analysis = None db.reraise = True try: ret = self._add_analysis(db.session, spec, irradpos, rid, runtype) db.commit() return ret except Exception, e: import traceback tb = traceback.format_exc() self.debug('Mass Spec save exception. {}\n {}'.format( e, tb)) if i == 2: self.message('Could not save spec.runid={} rid={} ' 'to Mass Spec database.\n {}'.format( spec.runid, rid, tb)) else: self.debug('retry mass spec save') # if commit: db.rollback() finally: self.db.reraise = True
def add_analysis(self, spec, commit=True): db = self.db # for i in range(3): with db.session_ctx(use_parent_session=False) as session: irradpos = spec.irradpos rid = spec.runid trid = rid.lower() identifier = spec.labnumber if trid.startswith('b'): runtype = 'Blank' irradpos = -1 elif trid.startswith('a'): runtype = 'Air' irradpos = -2 elif trid.startswith('c'): runtype = 'Unknown' identifier = irradpos = self.get_identifier(spec) else: runtype = 'Unknown' rid = make_runid(identifier, spec.aliquot, spec.step) self._analysis = None db.reraise = True try: ret = self._add_analysis(session, spec, irradpos, rid, runtype) db.commit() return ret except Exception as e: import traceback self.debug('Mass Spec save exception. {}'.format(e)) tb = traceback.format_exc() self.debug(tb) self.message('Could not save spec. runid={} rid={} to MassSpec DB.\n{}'.format(spec.runid, rid, tb)) # if i == 2: # self.message('Could not save spec.runid={} rid={} ' # 'to Mass Spec database.\n {}'.format(spec.runid, rid, tb)) # else: # self.debug('retry mass spec save') # # if commit: # db.rollback() finally: self.db.reraise = True
def _bulk_runid(self, ai, aliquot, step): if not aliquot: aliquot = ai.aliquot if not step: step = ai.step self.dvc.db.modify_aliquot_step(ai.uuid, aliquot, alpha_to_int(step)) def modify_meta(p): jd = dvc_load(p) jd['aliquot'] = aliquot jd['increment'] = alpha_to_int(step) dvc_dump(jd, p) ps = [] repo_id = ai.repository_identifier sp = analysis_path(('', ai.record_id), repo_id) if sp: modify_meta(sp) ps.append(sp) # using runid path name new_runid = make_runid(ai.identifier, aliquot, step) for m in NPATH_MODIFIERS: sp = analysis_path(('', ai.record_id), repo_id, modifier=m) dp = analysis_path(('', new_runid), repo_id, modifier=m, mode='w') if sp and os.path.isfile(sp): if os.path.isfile(dp) and m != 'extraction': continue ps.append(sp) ps.append(dp) shutil.move(sp, dp) else: # using uuid path name # only need to modify metadata file sp = analysis_path(ai, repo_id) modify_meta(sp, aliquot, step) ps.append(sp) return repo_id, ps
def test_modifier_secondarydb(self): self.modifier.use_main = False self.modifier.use_secondary = True an = self.analyses[0] can = copy.copy(an) can.identifier = self.new_identifier self.modifier.modify_analyses([an], [can]) db = self.modifier.secondary_db with db.session_ctx(): old_an = db.get_analysis(an.labnumber, an.aliquot, an.step) self.assertIsNone(old_an) new_an = db.get_analysis(self.new_identifier, '{:02d}'.format(an.aliquot), an.step) self.assertEqual( new_an.RID, make_runid(self.new_identifier, an.aliquot, an.step)) self.assertEqual(new_an.sample.Sample, 'Bar')
def _set_device_sensitivity(self, s, e, mi, ed, pr, v, err, prog): """ get all analyses between s, e where mass_spectrometer=mi, extract_device=ed, and project=pr set sensitivity to v +/-1s e """ db = self.db with db.session_ctx(): ans = db.get_analyses_date_range(s, e, mass_spectrometer=mi, extract_device=ed, project=pr) ans = filter(lambda x: not x.uuid in self._uuids, ans) self._uuids.extend([ai.uuid for ai in ans]) self.debug('setting {} analyses'.format(len(ans))) prog.increase_max(len(ans)) for ai in ans: r = make_runid(ai.labnumber.identifier, ai.aliquot, ai.step) prog.change_message('Saving sensitivity for {}'.format(r)) db.set_analysis_sensitivity(ai, v, err)
def _set_device_sensitivity(self, s, e, mi, ed, pr, v, err,prog): """ get all analyses between s, e where mass_spectrometer=mi, extract_device=ed, and project=pr set sensitivity to v +/-1s e """ db=self.db with db.session_ctx(): ans = db.get_analyses_date_range(s, e, mass_spectrometer=mi, extract_device=ed, project=pr) ans=filter(lambda x: not x.uuid in self._uuids, ans) self._uuids.extend([ai.uuid for ai in ans]) self.debug('setting {} analyses'.format(len(ans))) prog.increase_max(len(ans)) for ai in ans: r = make_runid(ai.labnumber.identifier, ai.aliquot, ai.step) prog.change_message('Saving sensitivity for {}'.format(r)) db.set_analysis_sensitivity(ai, v, err)
def _sync(self, path, *args, **kw): self.path = path yd = self._load_yaml(path) attrs = ANALYSIS_ATTRS + () for attr in attrs: if isinstance(attr, tuple): attr, ydattr = attr else: attr, ydattr = attr, attr setattr(self, attr, yd[ydattr]) self.analysis_type = 'unknown' self.rundate = datetime.fromtimestamp(self.timestamp) self.set_j(yd['j'], yd['j_err']) self.record_id = make_runid(self.labnumber, self.aliquot, self.step) self.interference_corrections = yd['interference_corrections'] self.production_ratios = yd['production_ratios'] self._sync_isotopes(yd)
def add_analysis(self, spec, commit=True): with self.db.session_ctx(commit=False) as sess: irradpos = spec.irradpos rid = spec.runid trid = rid.lower() identifier = spec.labnumber if trid.startswith("b"): runtype = "Blank" irradpos = -1 elif trid.startswith("a"): runtype = "Air" irradpos = -2 elif trid.startswith("c"): runtype = "Unknown" identifier = irradpos = self.get_identifier(spec) else: runtype = "Unknown" # paliquot = self.db.get_latest_analysis_aliquot(identifier) # if paliquot is None: # paliquot=0 # # rid = '{}-{:02n}'.format(identifier, spec.aliquot, spec.step) # self.info('Saving analysis {} to database as {}'.format(spec.rid, rid)) rid = make_runid(identifier, spec.aliquot, spec.step) self._analysis = None try: return self._add_analysis(sess, spec, irradpos, rid, runtype) except Exception, e: import traceback tb = traceback.format_exc() self.message( "Could not save spec.runid={} rid={} to Mass Spec database.\n {}".format(spec.runid, rid, tb) ) if commit: sess.rollback()
def add_analysis(self, spec, commit=True): with self.db.session_ctx(commit=False) as sess: irradpos = spec.irradpos rid = spec.runid trid = rid.lower() identifier = spec.labnumber if trid.startswith('b'): runtype = 'Blank' irradpos = -1 elif trid.startswith('a'): runtype = 'Air' irradpos = -2 elif trid.startswith('c'): runtype = 'Unknown' identifier = irradpos = self.get_identifier(spec) else: runtype = 'Unknown' # paliquot = self.db.get_latest_analysis_aliquot(identifier) # if paliquot is None: # paliquot=0 # #rid = '{}-{:02n}'.format(identifier, spec.aliquot, spec.step) # self.info('Saving analysis {} to database as {}'.format(spec.rid, rid)) rid = make_runid(identifier, spec.aliquot, spec.step) self._analysis = None try: return self._add_analysis(sess, spec, irradpos, rid, runtype) except Exception, e: import traceback tb = traceback.format_exc() self.message( 'Could not save spec.runid={} rid={} to Mass Spec database.\n {}' .format(spec.runid, rid, tb)) if commit: sess.rollback()
def get_isotope_data(self, idx, k): v = self.get_isotope(idx, k) i = self.get_identifier(idx) a = self.get_aliquot(idx) s = self.get_step(idx) rid = make_runid(i, a, s) if isinstance(v, float): return [], [] else: parser = self.parser sh = parser.sheet def g(): parser.set_sheet(v, header_idx=0) cx, cy = parser.get_index('{}_xs'.format(k)), parser.get_index('{}_ys'.format(k)) for row in parser.iterblock(0, rid): yield row[cx].value, row[cy].value data = list(g()) self.parser.set_sheet(sh) return map(list, zip(*data))
def get_isotope_data(self, idx, k): v = self.get_isotope(idx, k) i = self.get_identifier(idx) a = self.get_aliquot(idx) s = self.get_step(idx) rid = make_runid(i, a, s) if isinstance(v, float): return [], [] else: parser = self.parser sh = parser.sheet def g(): parser.set_sheet(v, header_idx=0) cx, cy = parser.get_index('{}_xs'.format(k)), parser.get_index( '{}_ys'.format(k)) for row in parser.iterblock(0, rid): yield row[cx].value, row[cy].value data = list(g()) self.parser.set_sheet(sh) return map(list, zip(*data))
def _get_runid(self): return make_runid(self.labnumber, self.aliquot, self.step)
def record_id(self): return make_runid(self.labnumber.identifier, self.aliquot, self.increment)
def _get_record_id(self): record_id = self._record_id if not record_id: record_id = make_runid(self.labnumber, self.aliquot, self.step) return record_id
def _transfer_analysis(self, rec, exp, overwrite=True, monitor_mapping=None): dest = self.dvc.db proc = self.processor src = proc.db # args = rec.split('-') # idn = '-'.join(args[:-1]) # t = args[-1] # try: # aliquot = int(t) # step = None # except ValueError: # aliquot = int(t[:-1]) # step = t[-1] m = IDENTIFIER_REGEX.match(rec) if not m: m = SPECIAL_IDENTIFIER_REGEX.match(rec) if not m: self.warning('invalid runid {}'.format(rec)) return else: idn = m.group('identifier') aliquot = m.group('aliquot') try: step = m.group('step') or None except IndexError: step = None if idn == '4359': idn = 'c-01-j' elif idn == '4358': idn = 'c-01-o' # check if analysis already exists. skip if it does if dest.get_analysis_runid(idn, aliquot, step): self.warning('{} already exists'.format( make_runid(idn, aliquot, step))) return dban = src.get_analysis_runid(idn, aliquot, step) iv = IsotopeRecordView() iv.uuid = dban.uuid self.debug('make analysis idn:{}, aliquot:{} step:{}'.format( idn, aliquot, step)) # try: an = proc.make_analysis(iv, unpack=True, use_cache=False, use_progress=False) # except BaseException as e: # self.warning('Failed to make {}'.format(make_runid(idn, aliquot, step))) # self.warning('exception: {}'.format(e)) # return self._transfer_meta(dest, dban, monitor_mapping) # return dblab = dban.labnumber if dblab.irradiation_position: irrad = dblab.irradiation_position.level.irradiation.name level = dblab.irradiation_position.level.name irradpos = dblab.irradiation_position.position else: irrad = 'NoIrradiation' level = 'A' irradpos = self._get_irradpos(dest, irrad, level, dblab.identifier) # irrad, level, irradpos = '', '', 0 extraction = dban.extraction ms = dban.measurement.mass_spectrometer.name if not dest.get_mass_spectrometer(ms): self.debug('adding mass spectrometer {}'.format(ms)) dest.add_mass_spectrometer(ms) dest.commit() ed = extraction.extraction_device.name if extraction.extraction_device else None if not ed: ed = 'No Extract Device' if not dest.get_extraction_device(ed): self.debug('adding extract device {}'.format(ed)) dest.add_extraction_device(ed) dest.commit() if step is None: inc = -1 else: inc = alpha_to_int(step) username = '' if dban.user: username = dban.user.name if not dest.get_user(username): self.debug('adding user. username:{}'.format(username)) dest.add_user(username) dest.commit() if monitor_mapping: sample_name, material_name, project_name = monitor_mapping else: dbsam = dblab.sample sample_name = dbsam.name material_name = dbsam.material.name project_name = format_repository_identifier(dbsam.project.name) rs = AutomatedRunSpec(labnumber=idn, username=username, material=material_name, project=project_name, sample=sample_name, irradiation=irrad, irradiation_level=level, irradiation_position=irradpos, repository_identifier=exp, mass_spectrometer=ms, uuid=dban.uuid, _step=inc, comment=dban.comment.decode('utf-8') or '', aliquot=int(aliquot), extract_device=ed, duration=extraction.extract_duration, cleanup=extraction.cleanup_duration, beam_diameter=extraction.beam_diameter, extract_units=extraction.extract_units or '', extract_value=extraction.extract_value, pattern=extraction.pattern or '', weight=extraction.weight, ramp_duration=extraction.ramp_duration or 0, ramp_rate=extraction.ramp_rate or 0, collection_version='0.1:0.1', queue_conditionals_name='', tray='') meas = dban.measurement # get spectrometer parameters # gains gains = {} gain_history = dban.gain_history if gain_history: gains = {d.detector.name: d.value for d in gain_history.gains} # deflections deflections = {d.detector.name: d.deflection for d in meas.deflections} # source src = { k: getattr(meas.spectrometer_parameters, k) for k in QTEGRA_SOURCE_KEYS } ps = PersistenceSpec( run_spec=rs, tag=an.tag.name, isotope_group=an, timestamp=dban.analysis_timestamp, defl_dict=deflections, gains=gains, spec_dict=src, use_repository_association=True, positions=[p.position for p in extraction.positions]) self.debug('transfer analysis with persister') self.persister.per_spec_save(ps, commit=False, commit_tag='Database Transfer') return True
def record_id(self): return make_runid(self.uuid, self.aliquot, self.step)
def record_id(self): return make_runid(self.irradiation_position.identifier, self.aliquot, self.increment)
def create(self, dbrecord): # print 'asdfsadfsdaf', dbrecord, dbrecord.labnumber, dbrecord.uuid try: if dbrecord is None or not dbrecord.labnumber: return ln = dbrecord.labnumber self.labnumber = str(ln.identifier) self.identifier = self.labnumber self.aliquot = dbrecord.aliquot self.step = dbrecord.step self.uuid = dbrecord.uuid self.tag = dbrecord.tag or '' self.rundate = dbrecord.analysis_timestamp self.timestamp = time.mktime(self.rundate.timetuple()) self.record_id = make_runid(self.labnumber, self.aliquot, self.step) # print self.record_id, self.uuid if ln.sample: self.sample = ln.sample.name if dbrecord.labnumber.sample: self.sample = dbrecord.labnumber.sample.name irp = ln.irradiation_position if irp is not None: irl = irp.level ir = irl.irradiation self.irradiation_info = '{}{} {}'.format( ir.name, irl.name, irp.position) else: self.irradiation_info = '' meas = dbrecord.measurement if meas is not None: self.mass_spectrometer = meas.mass_spectrometer.name.lower() if meas.analysis_type: self.analysis_type = meas.analysis_type.name ext = dbrecord.extraction if ext: if ext.extraction_device: self.extract_device = ext.extraction_device.name self.flux_fit_status = self._get_flux_fit_status(dbrecord) self.blank_fit_status = self._get_selected_history_item( dbrecord, 'selected_blanks_id') self.ic_fit_status = self._get_selected_history_item( dbrecord, 'selected_det_intercal_id') self.iso_fit_status = self._get_selected_history_item( dbrecord, 'selected_fits_id') return True except Exception, e: import traceback traceback.print_exc() print e
def record_id(self): return make_runid(self.identifier, self.aliquot, self.step)
def record_id(self): v = self._record_id if v is None: v = make_runid(self.labnumber, self.aliquot, self.step) return v
def _construct_analysis(self, rec, group, prog, calculate_age=True, calculate_F=False, unpack=False, load_aux=False): atype = None if isinstance(rec, meas_AnalysisTable): rid = make_runid(rec.labnumber.identifier, rec.aliquot, rec.step) atype = rec.measurement.analysis_type.name elif hasattr(rec, 'record_id'): rid = rec.record_id else: rid = id(rec) graph_id = 0 group_id = 0 if hasattr(rec, 'group_id'): group_id = rec.group_id if hasattr(rec, 'graph_id'): graph_id = rec.graph_id if atype is None: atype = rec.analysis_type if prog: m = '' if calculate_age: show_age = atype in ('unknown', 'cocktail') m = 'calculating age' if show_age else '' elif calculate_F: m = 'calculating F' msg = 'loading {}. {}'.format(rid, m) prog.change_message(msg) if isinstance(rec, DBAnalysis): ai = rec if load_aux: ai.sync_aux(group) else: ai.sync(group, unpack=unpack, load_aux=load_aux) else: ai = DBAnalysis() # if not self.use_vcs else VCSAnalysis # print ai ai.sync(group, unpack=unpack, load_aux=load_aux) # print ai, group # ai = klass(group_id=group_id, # graph_id=graph_id) # ai.trait_set(group_id=group_id, # graph_id=graph_id) # if not self.use_vcs: # # timethis(ai.sync, args=(group,), # kwargs=dict(unpack=unpack, load_aux=load_aux)) if atype in ('unknown', 'cocktail'): if calculate_age: # timethis(ai.sync, args=(meas_analysis, ), # kwargs=dict(unpack=unpack, load_aux=load_aux)) # timethis(ai.calculate_age, kwargs=dict(force=not self.use_vcs)) ai.calculate_age() # timethis(ai.sync, args=(meas_analysis,), # kwargs=dict(unpack=unpack, load_aux=load_aux)) # timethis(ai.calculate_age) # synced = True if calculate_F: ai.calculate_f() # if not synced: # ai.sync(group, unpack=unpack, load_aux=load_aux) return ai
def runid(self): if self._runid: return self._runid else: return make_runid(self.labnumber, self.aliquot, self.step)
def _find_run(self, aid): return next((a for a in self.automated_runs if make_runid(a.labnumber, a.aliquot, a.step) == aid), None)
def _construct_analysis(self, rec, group, prog, calculate_age=True, calculate_F=False, unpack=False, load_aux=False): atype = None if isinstance(rec, meas_AnalysisTable): rid = make_runid(rec.labnumber.identifier, rec.aliquot, rec.step) atype = rec.measurement.analysis_type.name elif hasattr(rec, 'record_id'): rid = rec.record_id else: rid = id(rec) graph_id = 0 group_id = 0 if hasattr(rec, 'group_id'): group_id = rec.group_id if hasattr(rec, 'graph_id'): graph_id = rec.graph_id if atype is None: atype = rec.analysis_type if prog: m='' if calculate_age: show_age = atype in ('unknown', 'cocktail') m = 'calculating age' if show_age else '' elif calculate_F: m = 'calculating F' msg = 'loading {}. {}'.format(rid, m) prog.change_message(msg) if isinstance(rec, DBAnalysis): ai = rec if load_aux: ai.sync_aux(group) else: ai.sync(group, unpack=unpack, load_aux=load_aux) else: ai = DBAnalysis() # if not self.use_vcs else VCSAnalysis # print ai ai.sync(group, unpack=unpack, load_aux=load_aux) # print ai, group # ai = klass(group_id=group_id, # graph_id=graph_id) ai.trait_set(group_id=group_id, graph_id=graph_id) # if not self.use_vcs: # # timethis(ai.sync, args=(group,), # kwargs=dict(unpack=unpack, load_aux=load_aux)) if atype in ('unknown', 'cocktail'): if calculate_age: # timethis(ai.sync, args=(meas_analysis, ), # kwargs=dict(unpack=unpack, load_aux=load_aux)) # timethis(ai.calculate_age, kwargs=dict(force=not self.use_vcs)) ai.calculate_age() #force=not self.use_vcs) # timethis(ai.sync, args=(meas_analysis,), # kwargs=dict(unpack=unpack, load_aux=load_aux)) # timethis(ai.calculate_age) # synced = True if calculate_F: ai.calculate_F() # if not synced: # ai.sync(group, unpack=unpack, load_aux=load_aux) return ai
def _transfer_analysis(self, rec, exp, overwrite=True, monitor_mapping=None): dest = self.dvc.db proc = self.processor src = proc.db # args = rec.split('-') # idn = '-'.join(args[:-1]) # t = args[-1] # try: # aliquot = int(t) # step = None # except ValueError: # aliquot = int(t[:-1]) # step = t[-1] m = IDENTIFIER_REGEX.match(rec) if not m: m = SPECIAL_IDENTIFIER_REGEX.match(rec) if not m: self.warning('invalid runid {}'.format(rec)) return else: idn = m.group('identifier') aliquot = m.group('aliquot') try: step = m.group('step') or None except IndexError: step = None if idn == '4359': idn = 'c-01-j' elif idn == '4358': idn = 'c-01-o' # check if analysis already exists. skip if it does if dest.get_analysis_runid(idn, aliquot, step): self.warning('{} already exists'.format(make_runid(idn, aliquot, step))) return dban = src.get_analysis_runid(idn, aliquot, step) iv = IsotopeRecordView() iv.uuid = dban.uuid self.debug('make analysis idn:{}, aliquot:{} step:{}'.format(idn, aliquot, step)) try: an = proc.make_analysis(iv, unpack=True, use_cache=False, use_progress=False) except: self.warning('Failed to make {}'.format(make_runid(idn, aliquot, step))) return self._transfer_meta(dest, dban, monitor_mapping) # return dblab = dban.labnumber if dblab.irradiation_position: irrad = dblab.irradiation_position.level.irradiation.name level = dblab.irradiation_position.level.name irradpos = dblab.irradiation_position.position else: irrad = 'NoIrradiation' level = 'A' irradpos = self._get_irradpos(dest, irrad, level, dblab.identifier) # irrad, level, irradpos = '', '', 0 extraction = dban.extraction ms = dban.measurement.mass_spectrometer.name if not dest.get_mass_spectrometer(ms): self.debug('adding mass spectrometer {}'.format(ms)) dest.add_mass_spectrometer(ms) dest.commit() ed = extraction.extraction_device.name if extraction.extraction_device else None if not ed: ed = 'No Extract Device' if not dest.get_extraction_device(ed): self.debug('adding extract device {}'.format(ed)) dest.add_extraction_device(ed) dest.commit() if step is None: inc = -1 else: inc = ALPHAS.index(step) username = '' if dban.user: username = dban.user.name if not dest.get_user(username): self.debug('adding user. username:{}'.format(username)) dest.add_user(username) dest.commit() if monitor_mapping: sample_name, material_name, project_name = monitor_mapping else: dbsam = dblab.sample sample_name = dbsam.name material_name = dbsam.material.name project_name = format_repository_identifier(dbsam.project.name) rs = AutomatedRunSpec(labnumber=idn, username=username, material=material_name, project=project_name, sample=sample_name, irradiation=irrad, irradiation_level=level, irradiation_position=irradpos, repository_identifier=exp, mass_spectrometer=ms, uuid=dban.uuid, _step=inc, comment=dban.comment or '', aliquot=int(aliquot), extract_device=ed, duration=extraction.extract_duration, cleanup=extraction.cleanup_duration, beam_diameter=extraction.beam_diameter, extract_units=extraction.extract_units or '', extract_value=extraction.extract_value, pattern=extraction.pattern or '', weight=extraction.weight, ramp_duration=extraction.ramp_duration or 0, ramp_rate=extraction.ramp_rate or 0, collection_version='0.1:0.1', queue_conditionals_name='', tray='') meas = dban.measurement # get spectrometer parameters # gains gains = {} gain_history = dban.gain_history if gain_history: gains = {d.detector.name: d.value for d in gain_history.gains} # deflections deflections = {d.detector.name: d.deflection for d in meas.deflections} # source src = {k: getattr(meas.spectrometer_parameters, k) for k in QTEGRA_SOURCE_KEYS} ps = PersistenceSpec(run_spec=rs, tag=an.tag.name, arar_age=an, timestamp=dban.analysis_timestamp, defl_dict=deflections, gains=gains, spec_dict=src, use_repository_association=True, positions=[p.position for p in extraction.positions]) self.debug('transfer analysis with persister') self.persister.per_spec_save(ps, commit=False, msg_prefix='Database Transfer') return True