def test_save_persister():
paths.build('_dev')
dvc = DVC(bind=False)
dvc.db.connect()
per = DVCPersister(dvc=dvc)
run_spec = AutomatedRunSpec()
run_spec.labnumber = '10001'
run_spec.project = 'Test'
arar = ArArAge()
arar.isotopes['Ar40'] = Isotope(xs=[1, 2, 3], ys=[1, 2, 3],
name='Ar40', detector='H1')
sd = {}
dd = {'H1': 100}
gd = {'H1': 1.021}
per_spec = PersistenceSpec(run_spec=run_spec,
arar_age=arar,
spec_dict=sd,
defl_dict=dd,
gains=gd,
positions=[1, ],
experiment_queue_name='testexp.txt',
measurement_name='jan_unknown.py',
extraction_name='foo.py')
per.per_spec_save(per_spec)
def setUpClass(cls):
aspec = AutomatedRunSpec()
aspec.mass_spectrometer = 'jan'
aspec.labnumber = '17005'
aspec.aliquot = 95
a = AutomatedRun()
a.script_info.measurement_script_name = 'unknown_peak_hop'
s = ArgusSpectrometerManager()
ion = IonOpticsManager(spectrometer=s.spectrometer)
s.load(db_mol_weights=False)
a.spectrometer_manager = s
a.ion_optics_manager = ion
a.isotope_group = ArArAge()
a.isotope_group.j = ufloat(0.001, 1e-6)
a._alive = True
a.uuid = '12345-ABCDE'
a.spec = aspec
a._measured = True
a._save_enabled = True
cls.arun = a
def setUpClass(cls):
aspec = AutomatedRunSpec()
aspec.mass_spectrometer = 'jan'
aspec.labnumber = '17005'
aspec.aliquot = 95
a = AutomatedRun()
a.script_info.measurement_script_name = 'unknown_peak_hop'
s = ArgusSpectrometerManager()
ion = IonOpticsManager(spectrometer=s.spectrometer)
s.load(db_mol_weights=False)
a.spectrometer_manager = s
a.ion_optics_manager = ion
a.isotope_group = ArArAge()
a.isotope_group.j=ufloat(0.001, 1e-6)
a._alive = True
a.uuid = '12345-ABCDE'
a.spec = aspec
a._measured = True
a._save_enabled = True
cls.arun = a
def run():
hp = hpy()
globalv.experiment_debug = True
# t = ExperimentEditorTask()
# p = os.path.join(paths.experiment_dir, 'demo.txt')
# t._open_experiment(p)
q = ExperimentQueue(delay_before_analyses=0, delay_between_analyses=0)
q.automated_runs = [
AutomatedRunSpec(labnumber='61312',
duration=2.0,
extraction_script='pause',
measurement_script='air',
mass_spectrometer='jan'),
AutomatedRunSpec(labnumber='61312',
duration=2.0,
extraction_script='pause',
measurement_script='air',
mass_spectrometer='jan'),
AutomatedRunSpec(labnumber='61312',
duration=2.0,
extraction_script='pause',
measurement_script='air',
mass_spectrometer='jan'),
AutomatedRunSpec(labnumber='61312',
duration=2.0,
extraction_script='pause',
measurement_script='air',
mass_spectrometer='jan'),
AutomatedRunSpec(labnumber='61312',
duration=2.0,
extraction_script='pause',
measurement_script='air',
mass_spectrometer='jan')
]
dbman = isotope_manager_factory()
specman = SpectrometerManager()
specman.load()
ex = ExperimentExecutor(
db=dbman.db,
spectrometer_manager=specman,
extraction_line_manager='',
executable=True,
)
ex.monitor = None
ex.experiment_queues = [
q,
]
ex.experiment_queue = q
hp.setrelheap()
t = ex.execute()
t.join()
# count_instances(group='sqlalchemy')
return hp
def setUpClass(cls):
aspec = AutomatedRunSpec()
aspec.mass_spectrometer = 'jan'
aspec.labnumber = '17005'
aspec.aliquot = 82
aspec.syn_extraction='test'
a = AutomatedRun()
a.script_info.measurement_script_name = 'unknown'
a.script_info.extraction_script_name = 'pause'
s = ArgusSpectrometerManager()
ion = IonOpticsManager(spectrometer=s.spectrometer)
s.load()
a.spectrometer_manager = s
a.ion_optics_manager = ion
a.isotope_group = ArArAge()
a._alive = True
a.uuid = '12345-ABCDE'
a.spec = aspec
a._measured = True
a.persister.save_enabled = True
a.persister.datahub=dh=Datahub(bind_mainstore=False)
dh.mainstore.db.kind='mysql'
dh.mainstore.db.username='******'
dh.mainstore.db.name='pychrondata_dev'
dh.mainstore.db.password='******'
dh.mainstore.db.connect()
cls.arun = a
def test3(self):
d = DummyExecutor()
spec = AutomatedRunSpec(identifier='bu-FD-j', aliquot=1)
d.retroactive_repository_identifiers(spec)
spec = AutomatedRunSpec(identifier='bu-FD-j', aliquot=2)
d.retroactive_repository_identifiers(spec)
spec = AutomatedRunSpec(identifier='10000', repository_identifier='foo')
d.retroactive_repository_identifiers(spec)
spec = AutomatedRunSpec(identifier='20000', repository_identifier='bar')
d.retroactive_repository_identifiers(spec)
spec = AutomatedRunSpec(identifier='20000', aliquot=1, repository_identifier='bar')
d.retroactive_repository_identifiers(spec)
spec = AutomatedRunSpec(identifier='bu-FD-j', aliquot=3)
d.retroactive_repository_identifiers(spec)
self.assertListEqual(d.associations, [
('foo','10000-00'),
('foo', 'bu-FD-j-01'),
('foo','bu-FD-j-02'),
('bar', '20000-00'),
('bar','20000-01'),
('bar','bu-FD-j-03'),
])
def setUpClass(cls):
aspec = AutomatedRunSpec()
aspec.mass_spectrometer = 'jan'
aspec.labnumber = '17005'
aspec.aliquot = 82
a = AutomatedRun()
a.script_info.measurement_script_name = 'unknown'
s = SpectrometerManager()
ion = IonOpticsManager(spectrometer=s.spectrometer)
s.load(db_mol_weights=False)
a.spectrometer_manager = s
a.ion_optics_manager = ion
a.arar_age = ArArAge()
a._alive = True
a.uuid = '12345-ABCDE'
a.spec = aspec
a._measured = True
a._save_enabled = True
cls.arun = a
def main():
from pychron.paths import paths
paths.build('_dev')
from pychron.core.helpers.logger_setup import logging_setup
from pychron.experiment.automated_run.spec import AutomatedRunSpec
logging_setup('dvcdb')
from pychron.dvc.dvc_database import DVCDatabase
from itertools import groupby
db = DVCDatabase(kind='mysql',
host='localhost',
username='******',
name='pychronmeta',
password='******')
db.connect()
identifiers = ['63290', '63291']
runs = [
AutomatedRunSpec(identifier='63290',
repository_identifier='Cather_McIntoshd')
]
cr = ConflictResolver()
experiments = {}
with db.session_ctx():
cr.available_ids = db.get_repository_identifiers()
eas = db.get_associated_repositories(identifiers)
for idn, exps in groupby(eas, key=lambda x: x[1]):
experiments[idn] = [e[0] for e in exps]
conflicts = []
for ai in runs:
identifier = ai.identifier
es = experiments[identifier]
if ai.repository_identifier not in es:
conflicts.append((ai, es))
if conflicts:
cr.add_conflicts('Foo', conflicts)
if cr.conflicts:
info = cr.edit_traits(kind='livemodal')
if info.result:
cr.apply()
def gen_mock_runspec():
for p, l, _ in runs:
rs = AutomatedRunSpec()
rs.project = p
rs.labnumber = l
yield rs
def get_analysis_import_spec(self, p, delimiter=None):
pspec = PersistenceSpec()
rspec = AutomatedRunSpec()
pspec.run_spec = rspec
# spec = ImportSpec
# analysis = Analysis()
# spec.analysis = analysis
# position = Position()
# analysis.position = position
if delimiter is None:
delim = self._delimiter
def gen():
with open(p, 'r') as rfile:
for line in rfile:
yield line.strip().split(delim)
f = gen()
row = f.next()
# analysis.runid = row[0]
rspec.runid = row[0]
# irrad = Irradiation()
rspec.irradiation = row[1]
row = f.next()
rspec.irradiation_level = row[1]
row = f.next()
rspec.sample = row[1]
row = f.next()
rspec.material = row[1]
row = f.next()
rspec.project = row[1]
row = f.next()
j = float(row[1])
row = f.next()
j_err = float(row[1])
pspec.j, pspec.j_err = j, j_err
row = f.next()
d = row[1]
row = f.next()
t = row[1]
rspec.analysis_timestamp = datetime.strptime('{} {}'.format(d, t),
'%m/%d/%Y %H:%M:%S')
row = f.next()
abundance_sens = float(row[0])
row = f.next()
abundance_sens_err = float(row[0])
row = f.next()
air = float(row[0])
disc = 295.5 / air
pspec.discrimination = disc
row = f.next() # MD errpr
row = f.next() # peakhop cycles
n40 = int(f.next()[0])
n39 = int(f.next()[0])
n38 = int(f.next()[0])
n37 = int(f.next()[0])
n36 = int(f.next()[0])
n35 = int(f.next()[0])
n355 = int(f.next()[0])
f.next()
row = f.next()
b40 = map(float, row)
b39 = map(float, row)
b38 = map(float, row)
b37 = map(float, row)
b36 = map(float, row)
b35 = map(float, row)
for i, row in enumerate(f):
if i > 36:
break
isotopes = {}
isotopes['Ar40'] = self._get_isotope(f, 'Ar40', n40, b40)
isotopes['Ar39'] = self._get_isotope(f, 'Ar39', n39, b39)
isotopes['Ar38'] = self._get_isotope(f, 'Ar38', n38, b38)
isotopes['Ar37'] = self._get_isotope(f, 'Ar37', n37, b37)
isotopes['Ar36'] = self._get_isotope(f, 'Ar36', n36, b36)
isotopes['Ar35'] = self._get_isotope(f, 'Ar35', n35, b35)
isotopes['Ar35.5'] = self._get_isotope(f, 'Ar35.5', n355, b355)
try:
f.next()
self.warning('Extra data in file')
except StopIteration:
pass
pspec.isotope_group = IsotopeGroup(isotopes=isotopes)
return pspec
def _make_per_spec(self, lt):
run_spec = AutomatedRunSpec()
per_spec = PersistenceSpec()
arar_age = ArArAge()
# populate per_spec
per_spec.run_spec = run_spec
per_spec.arar_age = arar_age
# popluate run_spec
run_spec.identifier = lt.labnumber
run_spec.aliquot = lt.aliquot
run_spec.step = lt.step
run_spec.username = '******'
run_spec.uuid = lt.uuid
cp = lt.collection_path
man = H5DataManager()
man.open_file(cp)
# add signal/isotopes
group = man.get_group('signal')
for grp in man.get_groups(group):
isok = grp._v_name
iso = Isotope(name=isok,
fit='linear')
# only handle one detector per isotope
tbl = man.get_tables(grp)[0]
iso.detector = tbl._v_name
xs = array([x['time'] for x in tbl.iterrows()])
ys = array([x['value'] for x in tbl.iterrows()])
iso.xs = xs
iso.ys = ys
arar_age.isotopes[isok] = iso
# add sniffs
group = man.get_group('sniff')
for k, iso in arar_age.isotopes.iteritems():
grp = man.get_group(k, group)
tbl = man.get_tables(grp)[0]
iso.sniff.detector = tbl._v_name
xs = array([x['time'] for x in tbl.iterrows()])
ys = array([x['value'] for x in tbl.iterrows()])
iso.sniff.xs = xs
iso.sniff.ys = ys
# add baselines
group = man.get_group('baseline')
for dettbl in man.get_tables(group):
detname = dettbl._v_name
xs = array([x['time'] for x in dettbl.iterrows()])
ys = array([x['value'] for x in dettbl.iterrows()])
for iso in arar_age.isotopes.itervalues():
if iso.detector == detname:
iso.baseline.xs = xs
iso.baseline.ys = ys
iso.baseline.fit = 'average'
return per_spec
def _make_per_spec(self, lt):
run_spec = AutomatedRunSpec()
per_spec = PersistenceSpec()
arar_age = ArArAge()
# populate per_spec
per_spec.run_spec = run_spec
per_spec.arar_age = arar_age
# popluate run_spec
run_spec.identifier = lt.labnumber
run_spec.aliquot = lt.aliquot
run_spec.step = lt.step
run_spec.username = '******'
run_spec.uuid = lt.uuid
cp = lt.collection_path
man = H5DataManager()
man.open_file(cp)
# add signal/isotopes
group = man.get_group('signal')
for grp in man.get_groups(group):
isok = grp._v_name
iso = Isotope(name=isok, fit='linear')
# only handle one detector per isotope
tbl = man.get_tables(grp)[0]
iso.detector = tbl._v_name
xs = array([x['time'] for x in tbl.iterrows()])
ys = array([x['value'] for x in tbl.iterrows()])
iso.xs = xs
iso.ys = ys
arar_age.isotopes[isok] = iso
# add sniffs
group = man.get_group('sniff')
for k, iso in arar_age.isotopes.iteritems():
grp = man.get_group(k, group)
tbl = man.get_tables(grp)[0]
iso.sniff.detector = tbl._v_name
xs = array([x['time'] for x in tbl.iterrows()])
ys = array([x['value'] for x in tbl.iterrows()])
iso.sniff.xs = xs
iso.sniff.ys = ys
# add baselines
group = man.get_group('baseline')
for dettbl in man.get_tables(group):
detname = dettbl._v_name
xs = array([x['time'] for x in dettbl.iterrows()])
ys = array([x['value'] for x in dettbl.iterrows()])
for iso in arar_age.isotopes.itervalues():
if iso.detector == detname:
iso.baseline.xs = xs
iso.baseline.ys = ys
iso.baseline.fit = 'average'
return per_spec
def _transfer_analysis(self,
rec,
exp,
overwrite=True,
monitor_mapping=None):
dest = self.dvc.db
proc = self.processor
src = proc.db
# args = rec.split('-')
# idn = '-'.join(args[:-1])
# t = args[-1]
# try:
# aliquot = int(t)
# step = None
# except ValueError:
# aliquot = int(t[:-1])
# step = t[-1]
m = IDENTIFIER_REGEX.match(rec)
if not m:
m = SPECIAL_IDENTIFIER_REGEX.match(rec)
if not m:
self.warning('invalid runid {}'.format(rec))
return
else:
idn = m.group('identifier')
aliquot = m.group('aliquot')
try:
step = m.group('step') or None
except IndexError:
step = None
if idn == '4359':
idn = 'c-01-j'
elif idn == '4358':
idn = 'c-01-o'
# check if analysis already exists. skip if it does
if dest.get_analysis_runid(idn, aliquot, step):
self.warning('{} already exists'.format(
make_runid(idn, aliquot, step)))
return
dban = src.get_analysis_runid(idn, aliquot, step)
iv = IsotopeRecordView()
iv.uuid = dban.uuid
self.debug('make analysis idn:{}, aliquot:{} step:{}'.format(
idn, aliquot, step))
# try:
an = proc.make_analysis(iv,
unpack=True,
use_cache=False,
use_progress=False)
# except BaseException as e:
# self.warning('Failed to make {}'.format(make_runid(idn, aliquot, step)))
# self.warning('exception: {}'.format(e))
# return
self._transfer_meta(dest, dban, monitor_mapping)
# return
dblab = dban.labnumber
if dblab.irradiation_position:
irrad = dblab.irradiation_position.level.irradiation.name
level = dblab.irradiation_position.level.name
irradpos = dblab.irradiation_position.position
else:
irrad = 'NoIrradiation'
level = 'A'
irradpos = self._get_irradpos(dest, irrad, level, dblab.identifier)
# irrad, level, irradpos = '', '', 0
extraction = dban.extraction
ms = dban.measurement.mass_spectrometer.name
if not dest.get_mass_spectrometer(ms):
self.debug('adding mass spectrometer {}'.format(ms))
dest.add_mass_spectrometer(ms)
dest.commit()
ed = extraction.extraction_device.name if extraction.extraction_device else None
if not ed:
ed = 'No Extract Device'
if not dest.get_extraction_device(ed):
self.debug('adding extract device {}'.format(ed))
dest.add_extraction_device(ed)
dest.commit()
if step is None:
inc = -1
else:
inc = alpha_to_int(step)
username = ''
if dban.user:
username = dban.user.name
if not dest.get_user(username):
self.debug('adding user. username:{}'.format(username))
dest.add_user(username)
dest.commit()
if monitor_mapping:
sample_name, material_name, project_name = monitor_mapping
else:
dbsam = dblab.sample
sample_name = dbsam.name
material_name = dbsam.material.name
project_name = format_repository_identifier(dbsam.project.name)
rs = AutomatedRunSpec(labnumber=idn,
username=username,
material=material_name,
project=project_name,
sample=sample_name,
irradiation=irrad,
irradiation_level=level,
irradiation_position=irradpos,
repository_identifier=exp,
mass_spectrometer=ms,
uuid=dban.uuid,
_step=inc,
comment=dban.comment.decode('utf-8') or '',
aliquot=int(aliquot),
extract_device=ed,
duration=extraction.extract_duration,
cleanup=extraction.cleanup_duration,
beam_diameter=extraction.beam_diameter,
extract_units=extraction.extract_units or '',
extract_value=extraction.extract_value,
pattern=extraction.pattern or '',
weight=extraction.weight,
ramp_duration=extraction.ramp_duration or 0,
ramp_rate=extraction.ramp_rate or 0,
collection_version='0.1:0.1',
queue_conditionals_name='',
tray='')
meas = dban.measurement
# get spectrometer parameters
# gains
gains = {}
gain_history = dban.gain_history
if gain_history:
gains = {d.detector.name: d.value for d in gain_history.gains}
# deflections
deflections = {d.detector.name: d.deflection for d in meas.deflections}
# source
src = {
k: getattr(meas.spectrometer_parameters, k)
for k in QTEGRA_SOURCE_KEYS
}
ps = PersistenceSpec(
run_spec=rs,
tag=an.tag.name,
isotope_group=an,
timestamp=dban.analysis_timestamp,
defl_dict=deflections,
gains=gains,
spec_dict=src,
use_repository_association=True,
positions=[p.position for p in extraction.positions])
self.debug('transfer analysis with persister')
self.persister.per_spec_save(ps,
commit=False,
commit_tag='Database Transfer')
return True
def new_persistence_spec(self):
pspec = PersistenceSpec()
rspec = AutomatedRunSpec()
pspec.run_spec = rspec
return pspec
