def test_save_persister(): paths.build('_dev') dvc = DVC(bind=False) dvc.db.connect() per = DVCPersister(dvc=dvc) run_spec = AutomatedRunSpec() run_spec.labnumber = '10001' run_spec.project = 'Test' arar = ArArAge() arar.isotopes['Ar40'] = Isotope(xs=[1, 2, 3], ys=[1, 2, 3], name='Ar40', detector='H1') sd = {} dd = {'H1': 100} gd = {'H1': 1.021} per_spec = PersistenceSpec(run_spec=run_spec, arar_age=arar, spec_dict=sd, defl_dict=dd, gains=gd, positions=[1, ], experiment_queue_name='testexp.txt', measurement_name='jan_unknown.py', extraction_name='foo.py') per.per_spec_save(per_spec)
def setUpClass(cls): aspec = AutomatedRunSpec() aspec.mass_spectrometer = 'jan' aspec.labnumber = '17005' aspec.aliquot = 95 a = AutomatedRun() a.script_info.measurement_script_name = 'unknown_peak_hop' s = ArgusSpectrometerManager() ion = IonOpticsManager(spectrometer=s.spectrometer) s.load(db_mol_weights=False) a.spectrometer_manager = s a.ion_optics_manager = ion a.isotope_group = ArArAge() a.isotope_group.j = ufloat(0.001, 1e-6) a._alive = True a.uuid = '12345-ABCDE' a.spec = aspec a._measured = True a._save_enabled = True cls.arun = a
def setUpClass(cls): aspec = AutomatedRunSpec() aspec.mass_spectrometer = 'jan' aspec.labnumber = '17005' aspec.aliquot = 95 a = AutomatedRun() a.script_info.measurement_script_name = 'unknown_peak_hop' s = ArgusSpectrometerManager() ion = IonOpticsManager(spectrometer=s.spectrometer) s.load(db_mol_weights=False) a.spectrometer_manager = s a.ion_optics_manager = ion a.isotope_group = ArArAge() a.isotope_group.j=ufloat(0.001, 1e-6) a._alive = True a.uuid = '12345-ABCDE' a.spec = aspec a._measured = True a._save_enabled = True cls.arun = a
def run(): hp = hpy() globalv.experiment_debug = True # t = ExperimentEditorTask() # p = os.path.join(paths.experiment_dir, 'demo.txt') # t._open_experiment(p) q = ExperimentQueue(delay_before_analyses=0, delay_between_analyses=0) q.automated_runs = [ AutomatedRunSpec(labnumber='61312', duration=2.0, extraction_script='pause', measurement_script='air', mass_spectrometer='jan'), AutomatedRunSpec(labnumber='61312', duration=2.0, extraction_script='pause', measurement_script='air', mass_spectrometer='jan'), AutomatedRunSpec(labnumber='61312', duration=2.0, extraction_script='pause', measurement_script='air', mass_spectrometer='jan'), AutomatedRunSpec(labnumber='61312', duration=2.0, extraction_script='pause', measurement_script='air', mass_spectrometer='jan'), AutomatedRunSpec(labnumber='61312', duration=2.0, extraction_script='pause', measurement_script='air', mass_spectrometer='jan') ] dbman = isotope_manager_factory() specman = SpectrometerManager() specman.load() ex = ExperimentExecutor( db=dbman.db, spectrometer_manager=specman, extraction_line_manager='', executable=True, ) ex.monitor = None ex.experiment_queues = [ q, ] ex.experiment_queue = q hp.setrelheap() t = ex.execute() t.join() # count_instances(group='sqlalchemy') return hp
def setUpClass(cls): aspec = AutomatedRunSpec() aspec.mass_spectrometer = 'jan' aspec.labnumber = '17005' aspec.aliquot = 82 aspec.syn_extraction='test' a = AutomatedRun() a.script_info.measurement_script_name = 'unknown' a.script_info.extraction_script_name = 'pause' s = ArgusSpectrometerManager() ion = IonOpticsManager(spectrometer=s.spectrometer) s.load() a.spectrometer_manager = s a.ion_optics_manager = ion a.isotope_group = ArArAge() a._alive = True a.uuid = '12345-ABCDE' a.spec = aspec a._measured = True a.persister.save_enabled = True a.persister.datahub=dh=Datahub(bind_mainstore=False) dh.mainstore.db.kind='mysql' dh.mainstore.db.username='******' dh.mainstore.db.name='pychrondata_dev' dh.mainstore.db.password='******' dh.mainstore.db.connect() cls.arun = a
def test3(self): d = DummyExecutor() spec = AutomatedRunSpec(identifier='bu-FD-j', aliquot=1) d.retroactive_repository_identifiers(spec) spec = AutomatedRunSpec(identifier='bu-FD-j', aliquot=2) d.retroactive_repository_identifiers(spec) spec = AutomatedRunSpec(identifier='10000', repository_identifier='foo') d.retroactive_repository_identifiers(spec) spec = AutomatedRunSpec(identifier='20000', repository_identifier='bar') d.retroactive_repository_identifiers(spec) spec = AutomatedRunSpec(identifier='20000', aliquot=1, repository_identifier='bar') d.retroactive_repository_identifiers(spec) spec = AutomatedRunSpec(identifier='bu-FD-j', aliquot=3) d.retroactive_repository_identifiers(spec) self.assertListEqual(d.associations, [ ('foo','10000-00'), ('foo', 'bu-FD-j-01'), ('foo','bu-FD-j-02'), ('bar', '20000-00'), ('bar','20000-01'), ('bar','bu-FD-j-03'), ])
def setUpClass(cls): aspec = AutomatedRunSpec() aspec.mass_spectrometer = 'jan' aspec.labnumber = '17005' aspec.aliquot = 82 a = AutomatedRun() a.script_info.measurement_script_name = 'unknown' s = SpectrometerManager() ion = IonOpticsManager(spectrometer=s.spectrometer) s.load(db_mol_weights=False) a.spectrometer_manager = s a.ion_optics_manager = ion a.arar_age = ArArAge() a._alive = True a.uuid = '12345-ABCDE' a.spec = aspec a._measured = True a._save_enabled = True cls.arun = a
def main(): from pychron.paths import paths paths.build('_dev') from pychron.core.helpers.logger_setup import logging_setup from pychron.experiment.automated_run.spec import AutomatedRunSpec logging_setup('dvcdb') from pychron.dvc.dvc_database import DVCDatabase from itertools import groupby db = DVCDatabase(kind='mysql', host='localhost', username='******', name='pychronmeta', password='******') db.connect() identifiers = ['63290', '63291'] runs = [ AutomatedRunSpec(identifier='63290', repository_identifier='Cather_McIntoshd') ] cr = ConflictResolver() experiments = {} with db.session_ctx(): cr.available_ids = db.get_repository_identifiers() eas = db.get_associated_repositories(identifiers) for idn, exps in groupby(eas, key=lambda x: x[1]): experiments[idn] = [e[0] for e in exps] conflicts = [] for ai in runs: identifier = ai.identifier es = experiments[identifier] if ai.repository_identifier not in es: conflicts.append((ai, es)) if conflicts: cr.add_conflicts('Foo', conflicts) if cr.conflicts: info = cr.edit_traits(kind='livemodal') if info.result: cr.apply()
def gen_mock_runspec(): for p, l, _ in runs: rs = AutomatedRunSpec() rs.project = p rs.labnumber = l yield rs
def get_analysis_import_spec(self, p, delimiter=None): pspec = PersistenceSpec() rspec = AutomatedRunSpec() pspec.run_spec = rspec # spec = ImportSpec # analysis = Analysis() # spec.analysis = analysis # position = Position() # analysis.position = position if delimiter is None: delim = self._delimiter def gen(): with open(p, 'r') as rfile: for line in rfile: yield line.strip().split(delim) f = gen() row = f.next() # analysis.runid = row[0] rspec.runid = row[0] # irrad = Irradiation() rspec.irradiation = row[1] row = f.next() rspec.irradiation_level = row[1] row = f.next() rspec.sample = row[1] row = f.next() rspec.material = row[1] row = f.next() rspec.project = row[1] row = f.next() j = float(row[1]) row = f.next() j_err = float(row[1]) pspec.j, pspec.j_err = j, j_err row = f.next() d = row[1] row = f.next() t = row[1] rspec.analysis_timestamp = datetime.strptime('{} {}'.format(d, t), '%m/%d/%Y %H:%M:%S') row = f.next() abundance_sens = float(row[0]) row = f.next() abundance_sens_err = float(row[0]) row = f.next() air = float(row[0]) disc = 295.5 / air pspec.discrimination = disc row = f.next() # MD errpr row = f.next() # peakhop cycles n40 = int(f.next()[0]) n39 = int(f.next()[0]) n38 = int(f.next()[0]) n37 = int(f.next()[0]) n36 = int(f.next()[0]) n35 = int(f.next()[0]) n355 = int(f.next()[0]) f.next() row = f.next() b40 = map(float, row) b39 = map(float, row) b38 = map(float, row) b37 = map(float, row) b36 = map(float, row) b35 = map(float, row) for i, row in enumerate(f): if i > 36: break isotopes = {} isotopes['Ar40'] = self._get_isotope(f, 'Ar40', n40, b40) isotopes['Ar39'] = self._get_isotope(f, 'Ar39', n39, b39) isotopes['Ar38'] = self._get_isotope(f, 'Ar38', n38, b38) isotopes['Ar37'] = self._get_isotope(f, 'Ar37', n37, b37) isotopes['Ar36'] = self._get_isotope(f, 'Ar36', n36, b36) isotopes['Ar35'] = self._get_isotope(f, 'Ar35', n35, b35) isotopes['Ar35.5'] = self._get_isotope(f, 'Ar35.5', n355, b355) try: f.next() self.warning('Extra data in file') except StopIteration: pass pspec.isotope_group = IsotopeGroup(isotopes=isotopes) return pspec
def _make_per_spec(self, lt): run_spec = AutomatedRunSpec() per_spec = PersistenceSpec() arar_age = ArArAge() # populate per_spec per_spec.run_spec = run_spec per_spec.arar_age = arar_age # popluate run_spec run_spec.identifier = lt.labnumber run_spec.aliquot = lt.aliquot run_spec.step = lt.step run_spec.username = '******' run_spec.uuid = lt.uuid cp = lt.collection_path man = H5DataManager() man.open_file(cp) # add signal/isotopes group = man.get_group('signal') for grp in man.get_groups(group): isok = grp._v_name iso = Isotope(name=isok, fit='linear') # only handle one detector per isotope tbl = man.get_tables(grp)[0] iso.detector = tbl._v_name xs = array([x['time'] for x in tbl.iterrows()]) ys = array([x['value'] for x in tbl.iterrows()]) iso.xs = xs iso.ys = ys arar_age.isotopes[isok] = iso # add sniffs group = man.get_group('sniff') for k, iso in arar_age.isotopes.iteritems(): grp = man.get_group(k, group) tbl = man.get_tables(grp)[0] iso.sniff.detector = tbl._v_name xs = array([x['time'] for x in tbl.iterrows()]) ys = array([x['value'] for x in tbl.iterrows()]) iso.sniff.xs = xs iso.sniff.ys = ys # add baselines group = man.get_group('baseline') for dettbl in man.get_tables(group): detname = dettbl._v_name xs = array([x['time'] for x in dettbl.iterrows()]) ys = array([x['value'] for x in dettbl.iterrows()]) for iso in arar_age.isotopes.itervalues(): if iso.detector == detname: iso.baseline.xs = xs iso.baseline.ys = ys iso.baseline.fit = 'average' return per_spec
def _make_per_spec(self, lt): run_spec = AutomatedRunSpec() per_spec = PersistenceSpec() arar_age = ArArAge() # populate per_spec per_spec.run_spec = run_spec per_spec.arar_age = arar_age # popluate run_spec run_spec.identifier = lt.labnumber run_spec.aliquot = lt.aliquot run_spec.step = lt.step run_spec.username = '******' run_spec.uuid = lt.uuid cp = lt.collection_path man = H5DataManager() man.open_file(cp) # add signal/isotopes group = man.get_group('signal') for grp in man.get_groups(group): isok = grp._v_name iso = Isotope(name=isok, fit='linear') # only handle one detector per isotope tbl = man.get_tables(grp)[0] iso.detector = tbl._v_name xs = array([x['time'] for x in tbl.iterrows()]) ys = array([x['value'] for x in tbl.iterrows()]) iso.xs = xs iso.ys = ys arar_age.isotopes[isok] = iso # add sniffs group = man.get_group('sniff') for k, iso in arar_age.isotopes.iteritems(): grp = man.get_group(k, group) tbl = man.get_tables(grp)[0] iso.sniff.detector = tbl._v_name xs = array([x['time'] for x in tbl.iterrows()]) ys = array([x['value'] for x in tbl.iterrows()]) iso.sniff.xs = xs iso.sniff.ys = ys # add baselines group = man.get_group('baseline') for dettbl in man.get_tables(group): detname = dettbl._v_name xs = array([x['time'] for x in dettbl.iterrows()]) ys = array([x['value'] for x in dettbl.iterrows()]) for iso in arar_age.isotopes.itervalues(): if iso.detector == detname: iso.baseline.xs = xs iso.baseline.ys = ys iso.baseline.fit = 'average' return per_spec
def _transfer_analysis(self, rec, exp, overwrite=True, monitor_mapping=None): dest = self.dvc.db proc = self.processor src = proc.db # args = rec.split('-') # idn = '-'.join(args[:-1]) # t = args[-1] # try: # aliquot = int(t) # step = None # except ValueError: # aliquot = int(t[:-1]) # step = t[-1] m = IDENTIFIER_REGEX.match(rec) if not m: m = SPECIAL_IDENTIFIER_REGEX.match(rec) if not m: self.warning('invalid runid {}'.format(rec)) return else: idn = m.group('identifier') aliquot = m.group('aliquot') try: step = m.group('step') or None except IndexError: step = None if idn == '4359': idn = 'c-01-j' elif idn == '4358': idn = 'c-01-o' # check if analysis already exists. skip if it does if dest.get_analysis_runid(idn, aliquot, step): self.warning('{} already exists'.format( make_runid(idn, aliquot, step))) return dban = src.get_analysis_runid(idn, aliquot, step) iv = IsotopeRecordView() iv.uuid = dban.uuid self.debug('make analysis idn:{}, aliquot:{} step:{}'.format( idn, aliquot, step)) # try: an = proc.make_analysis(iv, unpack=True, use_cache=False, use_progress=False) # except BaseException as e: # self.warning('Failed to make {}'.format(make_runid(idn, aliquot, step))) # self.warning('exception: {}'.format(e)) # return self._transfer_meta(dest, dban, monitor_mapping) # return dblab = dban.labnumber if dblab.irradiation_position: irrad = dblab.irradiation_position.level.irradiation.name level = dblab.irradiation_position.level.name irradpos = dblab.irradiation_position.position else: irrad = 'NoIrradiation' level = 'A' irradpos = self._get_irradpos(dest, irrad, level, dblab.identifier) # irrad, level, irradpos = '', '', 0 extraction = dban.extraction ms = dban.measurement.mass_spectrometer.name if not dest.get_mass_spectrometer(ms): self.debug('adding mass spectrometer {}'.format(ms)) dest.add_mass_spectrometer(ms) dest.commit() ed = extraction.extraction_device.name if extraction.extraction_device else None if not ed: ed = 'No Extract Device' if not dest.get_extraction_device(ed): self.debug('adding extract device {}'.format(ed)) dest.add_extraction_device(ed) dest.commit() if step is None: inc = -1 else: inc = alpha_to_int(step) username = '' if dban.user: username = dban.user.name if not dest.get_user(username): self.debug('adding user. username:{}'.format(username)) dest.add_user(username) dest.commit() if monitor_mapping: sample_name, material_name, project_name = monitor_mapping else: dbsam = dblab.sample sample_name = dbsam.name material_name = dbsam.material.name project_name = format_repository_identifier(dbsam.project.name) rs = AutomatedRunSpec(labnumber=idn, username=username, material=material_name, project=project_name, sample=sample_name, irradiation=irrad, irradiation_level=level, irradiation_position=irradpos, repository_identifier=exp, mass_spectrometer=ms, uuid=dban.uuid, _step=inc, comment=dban.comment.decode('utf-8') or '', aliquot=int(aliquot), extract_device=ed, duration=extraction.extract_duration, cleanup=extraction.cleanup_duration, beam_diameter=extraction.beam_diameter, extract_units=extraction.extract_units or '', extract_value=extraction.extract_value, pattern=extraction.pattern or '', weight=extraction.weight, ramp_duration=extraction.ramp_duration or 0, ramp_rate=extraction.ramp_rate or 0, collection_version='0.1:0.1', queue_conditionals_name='', tray='') meas = dban.measurement # get spectrometer parameters # gains gains = {} gain_history = dban.gain_history if gain_history: gains = {d.detector.name: d.value for d in gain_history.gains} # deflections deflections = {d.detector.name: d.deflection for d in meas.deflections} # source src = { k: getattr(meas.spectrometer_parameters, k) for k in QTEGRA_SOURCE_KEYS } ps = PersistenceSpec( run_spec=rs, tag=an.tag.name, isotope_group=an, timestamp=dban.analysis_timestamp, defl_dict=deflections, gains=gains, spec_dict=src, use_repository_association=True, positions=[p.position for p in extraction.positions]) self.debug('transfer analysis with persister') self.persister.per_spec_save(ps, commit=False, commit_tag='Database Transfer') return True
def new_persistence_spec(self): pspec = PersistenceSpec() rspec = AutomatedRunSpec() pspec.run_spec = rspec return pspec