def _make_labnumber(self, li, root, options, prog):
#make dir for labnumber
ident = li.identifier
ln_root = os.path.join(root, ident)
r_mkdir(ln_root)
prog.change_message('Making {} for {}'.format(self._tag, ident))
#filter invalid analyses
ans = filter(lambda x: not x.tag == 'invalid', li.analyses)
#group by stepheat vs fusion
pred = lambda x: bool(x.step)
ans = sorted(ans, key=pred)
stepheat, fusion = map(list, partition(ans, pred))
apred = lambda x: x.aliquot
stepheat = sorted(stepheat, key=apred)
project = 'Minna Bluff'
li = li.identifier
if stepheat:
key = lambda x: x.aliquot
stepheat = sorted(stepheat, key=key)
for aliquot, ais in groupby(stepheat, key=key):
name = make_runid(li, aliquot, '')
self._make_editor(ais, 'step_heat', options, prog, False,
(ln_root, name, name, project, (li, )))
if fusion:
self._make_editor(fusion, 'fusion', options, prog, False,
(ln_root, li, li, project, (li, )))
def _do_syn_extract(self, spec, script, post_script):
self.debug('Executing SynExtraction mode="{}"'.format(spec.mode))
#modify the persister. the original persister for the automated run is saved at self.persister
#arun.persister reset to original when syn extraction stops
identifier = spec.identifier
last_aq = self.arun.persister.get_last_aliquot(identifier)
if last_aq is None:
self.warning(
'invalid identifier "{}". Does not exist in database'.format(
identifier))
else:
runid = make_runid(identifier, last_aq + 1)
self.arun.info('Starting SynExtraction run {}'.format(runid))
self.arun.persister.trait_set(use_massspec_database=False,
runid=runid,
uuid=str(uuid.uuid4()))
if self.arun.do_measurement(script=script, use_post_on_fail=False):
if post_script:
self.debug('starting post measurement')
if not self.arun.do_post_measurement(script=post_script):
return
self.debug('delay between syn extractions {}'.format(
spec.delay_between))
self.arun.wait(spec.delay_between)
return True
def _sync_meas_analysis_attributes(self, meas_analysis):
# copy meas_analysis attrs
nocast = lambda x: x
attrs = [
('labnumber', 'labnumber', lambda x: x.identifier),
('aliquot', 'aliquot', int),
('step', 'step', str),
# ('status', 'status', int),
('comment', 'comment', str),
('uuid', 'uuid', str),
('rundate', 'analysis_timestamp', nocast),
('timestamp', 'analysis_timestamp',
lambda x: time.mktime(x.timetuple())),
]
for key, attr, cast in attrs:
v = getattr(meas_analysis, attr)
setattr(self, key, cast(v))
self.record_id = make_runid(self.labnumber, self.aliquot, self.step)
tag = meas_analysis.tag
if tag:
tag = meas_analysis.tag_item
self.set_tag(tag)
def _sync_meas_analysis_attributes(self, meas_analysis):
# copy meas_analysis attrs
nocast = lambda x: x
attrs = [
('labnumber', 'labnumber', lambda x: x.identifier),
('aliquot', 'aliquot', int),
('step', 'step', str),
# ('status', 'status', int),
('comment', 'comment', str),
('uuid', 'uuid', str),
('rundate', 'analysis_timestamp', nocast),
('timestamp', 'analysis_timestamp',
lambda x: time.mktime(x.timetuple())
),
]
for key, attr, cast in attrs:
v = getattr(meas_analysis, attr)
setattr(self, key, cast(v))
self.record_id = make_runid(self.labnumber, self.aliquot, self.step)
tag = meas_analysis.tag
if tag:
tag = meas_analysis.tag_item
self.set_tag(tag)
def _do_syn_extract(self, spec, script, post_script):
self.debug('Executing SynExtraction mode="{}"'.format(spec.mode))
#modify the persister. the original persister for the automated run is saved at self.persister
#arun.persister reset to original when syn extraction stops
identifier=spec.identifier
last_aq=self.arun.persister.get_last_aliquot(identifier)
if last_aq is None:
self.warning('invalid identifier "{}". Does not exist in database'.format(identifier))
else:
runid=make_runid(identifier, last_aq+1)
self.arun.info('Starting SynExtraction run {}'.format(runid))
self.arun.persister.trait_set(use_secondary_database=False,
runid=runid,
uuid=str(uuid.uuid4()))
if self.arun.do_measurement(script=script, use_post_on_fail=False):
if post_script:
self.debug('starting post measurement')
if not self.arun.do_post_measurement(script=post_script):
return
self.debug('delay between syn extractions {}'.format(spec.delay_between))
self.arun.wait(spec.delay_between)
return True
def _make(self, ep):
project = 'Minna Bluff'
db = self.db
with db.session_ctx():
prj = db.get_project(project)
Ar40, Ar39, Ar38, Ar37, Ar36 = [], [], [], [], []
for dev in (('Eurotherm', 'Furnace'), ('CO2')):
for si in prj.samples:
for li in si.labnumbers:
self.debug('blanks for {},{}'.format(
si.name, li.identifier))
for ai in li.analyses:
if ai.extraction.extraction_device.name in dev:
bs = self._extract_blanks(ai)
if bs is not None:
r = make_runid(li.identifier, ai.aliquot,
ai.step)
# self.debug('blanks for {} {}'.format(r,bs))
Ar40.append(bs[0])
Ar39.append(bs[1])
Ar38.append(bs[2])
Ar37.append(bs[3])
Ar36.append(bs[4])
reg = WeightedMeanRegressor()
print 'blanks for {}'.format(dev)
for iso in (Ar40, Ar39, Ar38, Ar37, Ar36):
ys, es = zip(*iso)
reg.trait_set(ys=ys, yserr=es)
print reg.predict()
def __init__(self, dbrecord):
self.uuid = dbrecord.uuid
ln = dbrecord.labnumber
labnumber = str(ln.identifier)
aliquot = dbrecord.aliquot
step = dbrecord.step
self.record_id = make_runid(labnumber, aliquot, step)
self.rundate = dbrecord.analysis_timestamp
self.timestamp = time.mktime(self.rundate.timetuple())
self.tag = dbrecord.tag or ''
self.is_plateau_step = False
meas = dbrecord.measurement
if meas is not None:
if meas.analysis_type:
self.analysis_type = meas.analysis_type.name
if meas.mass_spectrometer:
self.mass_spectrometer = meas.mass_spectrometer.name
sam = ln.sample
if sam:
self.sample = sam.name
if sam.project:
self.project = sam.project.name.lower()
def __init__(self, dbrecord):
self.uuid = dbrecord.uuid
ln = dbrecord.labnumber
labnumber = str(ln.identifier)
aliquot = dbrecord.aliquot
step = dbrecord.step
self.record_id = make_runid(labnumber, aliquot, step)
self.rundate = dbrecord.analysis_timestamp
self.timestamp = time.mktime(self.rundate.timetuple())
self.tag = dbrecord.tag or ''
self.is_plateau_step = False
meas = dbrecord.measurement
if meas is not None:
if meas.analysis_type:
self.analysis_type = meas.analysis_type.name
if meas.mass_spectrometer:
self.mass_spectrometer = meas.mass_spectrometer.name
sam = ln.sample
if sam:
self.sample = sam.name
if sam.project:
self.project = sam.project.name.lower()
def _make_labnumber(self, li, root, options, prog):
#make dir for labnumber
ident = li.identifier
ln_root = os.path.join(root, ident)
r_mkdir(ln_root)
prog.change_message('Making {} for {}'.format(self._tag, ident))
#filter invalid analyses
ans=filter(lambda x: not x.tag=='invalid', li.analyses)
#group by stepheat vs fusion
pred = lambda x: bool(x.step)
ans = sorted(ans, key=pred)
stepheat, fusion = map(list, partition(ans, pred))
apred = lambda x: x.aliquot
stepheat = sorted(stepheat, key=apred)
project='Minna Bluff'
li = li.identifier
if stepheat:
key=lambda x: x.aliquot
stepheat=sorted(stepheat, key=key)
for aliquot, ais in groupby(stepheat, key=key):
name=make_runid(li, aliquot, '')
self._make_editor(ais, 'step_heat', options, prog, False,
(ln_root, name, name, project, (li,)))
if fusion:
self._make_editor(fusion, 'fusion', options, prog, False,
(ln_root, li, li, project, (li,)))
def _make(self, ep):
project = "Minna Bluff"
db = self.db
with db.session_ctx():
prj = db.get_project(project)
Ar40, Ar39, Ar38, Ar37, Ar36 = [], [], [], [], []
for dev in (("Eurotherm", "Furnace"), ("CO2")):
for si in prj.samples:
for li in si.labnumbers:
self.debug("blanks for {},{}".format(si.name, li.identifier))
for ai in li.analyses:
if ai.extraction.extraction_device.name in dev:
bs = self._extract_blanks(ai)
if bs is not None:
r = make_runid(li.identifier, ai.aliquot, ai.step)
# self.debug('blanks for {} {}'.format(r,bs))
Ar40.append(bs[0])
Ar39.append(bs[1])
Ar38.append(bs[2])
Ar37.append(bs[3])
Ar36.append(bs[4])
reg = WeightedMeanRegressor()
print "blanks for {}".format(dev)
for iso in (Ar40, Ar39, Ar38, Ar37, Ar36):
ys, es = zip(*iso)
reg.trait_set(ys=ys, yserr=es)
print reg.predict()
def _modify_secondary(self, ans, new_ans):
self.info('modifying analyses in secondary db. dry={}'.format(self.dry))
db = self.secondary_db
if not db.connect():
self.debug('Not connected to secondary db')
return
with db.session_ctx(commit=not self.dry):
for ai, ni in zip(ans, new_ans):
rid = ai.record_id
dban = db.get_analysis_rid(rid)
if dban:
ident = ni.identifier
dbirradpos = db.get_irradiation_position(ident)
if dbirradpos:
dban.RID = make_runid(ident, ni.aliquot, ni.step)
self.debug('setting {} to {}'.format(ident, dban.RID))
dban.Aliquot = '{:02d}'.format(int(ni.aliquot))
dban.Increment = ni.step
dban.Aliquot_pychron = ni.aliquot
dban.IrradPosition = ident
dban.RedundantSampleID = dbirradpos.SampleID
else:
self.unique_warning('Labnumber {} does not exist in Secondary DB'.format(ident))
else:
self.warning('Analysis {} does not exist in Secondary DB'.format(rid))
def create(self, dbrecord, fast_load=False):
# print 'asdfsadfsdaf', dbrecord, dbrecord.labnumber, dbrecord.uuid
try:
if dbrecord is None or not dbrecord.labnumber:
return
ln = dbrecord.labnumber
self.labnumber = str(ln.identifier)
self.identifier = self.labnumber
self.aliquot = dbrecord.aliquot
self.step = dbrecord.step
self.record_id = make_runid(self.labnumber, self.aliquot, self.step)
self.uuid = dbrecord.uuid
self.tag = dbrecord.tag or ''
self.rundate = dbrecord.analysis_timestamp
sam = ln.sample
if sam:
self.sample = sam.name
if sam.project:
if isinstance(sam.project, (str, unicode)):
self.project = sam.project.lower()
else:
self.project = sam.project.name.lower()
irp = ln.irradiation_position
if irp is not None:
irl = irp.level
ir = irl.irradiation
self.irradiation_info = '{}{} {}'.format(ir.name, irl.name, irp.position)
try:
self.mass_spectrometer = dbrecord.mass_spectrometer
except AttributeError:
pass
meas = dbrecord.measurement
if meas:
self.mass_spectrometer = meas.mass_spectrometer.name.lower()
try:
self.analysis_type = meas.analysis_type.name
except AttributeError, e:
pass
# print 'IsotopeRecord create meas 1 {}'.format(e)
ext = dbrecord.extraction
if ext:
self.extract_value = ext.extract_value
self.cleanup = ext.cleanup_duration
self.duration = ext.extract_duration
try:
if ext.extraction_device:
self.extract_device = ext.extraction_device.name
except AttributeError, e:
pass
def create(self, dbrecord):
# print 'asdfsadfsdaf', dbrecord, dbrecord.labnumber, dbrecord.uuid
try:
if dbrecord is None or not dbrecord.labnumber:
return
ln = dbrecord.labnumber
self.labnumber = str(ln.identifier)
self.identifier = self.labnumber
self.aliquot = dbrecord.aliquot
self.step = dbrecord.step
self.uuid = dbrecord.uuid
self.tag = dbrecord.tag or ''
self.rundate = dbrecord.analysis_timestamp
self.timestamp = time.mktime(self.rundate.timetuple())
self.record_id = make_runid(self.labnumber, self.aliquot, self.step)
# print self.record_id, self.uuid
if ln.sample:
self.sample = ln.sample.name
if dbrecord.labnumber.sample:
self.sample = dbrecord.labnumber.sample.name
irp = ln.irradiation_position
if irp is not None:
irl = irp.level
ir = irl.irradiation
self.irradiation_info = '{}{} {}'.format(ir.name, irl.name, irp.position)
else:
self.irradiation_info = ''
meas = dbrecord.measurement
if meas is not None:
self.mass_spectrometer = meas.mass_spectrometer.name.lower()
if meas.analysis_type:
self.analysis_type = meas.analysis_type.name
ext = dbrecord.extraction
if ext:
if ext.extraction_device:
self.extract_device = ext.extraction_device.name
self.flux_fit_status = self._get_flux_fit_status(dbrecord)
self.blank_fit_status = self._get_selected_history_item(dbrecord, 'selected_blanks_id')
self.ic_fit_status = self._get_selected_history_item(dbrecord, 'selected_det_intercal_id')
self.iso_fit_status = self._get_selected_history_item(dbrecord, 'selected_fits_id')
return True
except Exception, e:
import traceback
traceback.print_exc()
print e
def init(self):
if self.increment >= 0:
self.step = ALPHAS[self.increment]
else:
self.step = ''
rid = make_runid(self.identifier, self.aliquot, self.step)
if self.use_repository_suffix:
rid = '{}-{}'.format(rid, self.repository_identifier)
self.record_id = rid
def _construct_analysis(self,
rec,
prog,
calculate_age=True,
unpack=False,
load_changes=False):
atype = None
if isinstance(rec, meas_AnalysisTable):
rid = make_runid(rec.labnumber.identifier, rec.aliquot, rec.step)
atype = rec.measurement.analysis_type.name
elif hasattr(rec, 'record_id'):
rid = rec.record_id
else:
rid = id(rec)
graph_id = 0
group_id = 0
if hasattr(rec, 'group_id'):
group_id = rec.group_id
if hasattr(rec, 'graph_id'):
graph_id = rec.graph_id
if atype is None:
atype = rec.analysis_type
if prog:
show_age = calculate_age and atype in ('unknown', 'cocktail')
m = 'calculating age' if show_age else ''
msg = 'loading {}. {}'.format(rid, m)
prog.change_message(msg)
meas_analysis = self.db.get_analysis_uuid(rec.uuid)
klass = DBAnalysis if not self.use_vcs else VCSAnalysis
ai = klass(group_id=group_id, graph_id=graph_id)
# if not self.use_vcs:
synced = False
if atype in ('unknown', 'cocktail'):
if calculate_age:
ai.sync(meas_analysis, load_changes=load_changes)
ai.calculate_age(force=not self.use_vcs)
synced = True
if not synced:
ai.sync(meas_analysis, unpack=unpack, load_changes=load_changes)
return ai
def make_summary(self):
s = 'Pr'
f = self.fit
if f:
if not f in INTERPOLATE_TYPES:
f = f[:1].upper()
s = '{}{}'.format(self.isotope, f)
if self.preceding_id:
p = self.preceding_analysis
rid = make_runid(p.labnumber.identifier, p.aliquot, p.step)
s = '{} ({})'.format(s, rid)
return s
def test_modifier_secondarydb(self):
self.modifier.use_main = False
self.modifier.use_secondary = True
an = self.analyses[0]
can = copy.copy(an)
can.identifier = self.new_identifier
self.modifier.modify_analyses([an], [can])
db = self.modifier.secondary_db
with db.session_ctx():
old_an = db.get_analysis(an.labnumber, an.aliquot, an.step)
self.assertIsNone(old_an)
new_an = db.get_analysis(self.new_identifier, '{:02d}'.format(an.aliquot), an.step)
self.assertEqual(new_an.RID, make_runid(self.new_identifier, an.aliquot, an.step))
self.assertEqual(new_an.sample.Sample, 'Bar')
def generate_omassspecdb():
src = os.path.join(get_data_dir(), 'omassspecdata.db')
db = massspec_db_factory(src)
with db.session_ctx():
s = db.add_sample('Foo')
db.flush()
db.add_irradiation_position('1000', 'NM-100A', 1, sample=s.SampleID)
s = db.add_sample('Bar')
db.flush()
db.add_irradiation_position('2000', 'NM-100A', 2, sample=s.SampleID)
db.flush()
for i in range(3):
rid = make_runid('1000', i + 1)
db.add_analysis(rid, i + 1, '', '1000', 1)
def generate_omassspecdb():
src = os.path.join(get_data_dir(), 'omassspecdata.db')
db = massspec_db_factory(src)
with db.session_ctx():
s = db.add_sample('Foo')
db.flush()
db.add_irradiation_position('1000', 'NM-100A', 1, sample=s.SampleID)
s = db.add_sample('Bar')
db.flush()
db.add_irradiation_position('2000', 'NM-100A', 2, sample=s.SampleID)
db.flush()
for i in range(3):
rid = make_runid('1000', i + 1)
db.add_analysis(rid, i + 1, '', '1000', 1)
def _construct_analysis(self, rec, prog, calculate_age=True, unpack=False, load_changes=False):
atype = None
if isinstance(rec, meas_AnalysisTable):
rid = make_runid(rec.labnumber.identifier, rec.aliquot, rec.step)
atype = rec.measurement.analysis_type.name
elif hasattr(rec, 'record_id'):
rid = rec.record_id
else:
rid = id(rec)
graph_id = 0
group_id = 0
if hasattr(rec, 'group_id'):
group_id = rec.group_id
if hasattr(rec, 'graph_id'):
graph_id = rec.graph_id
if atype is None:
atype = rec.analysis_type
if prog:
show_age = calculate_age and atype in ('unknown', 'cocktail')
m = 'calculating age' if show_age else ''
msg = 'loading {}. {}'.format(rid, m)
prog.change_message(msg)
meas_analysis = self.db.get_analysis_uuid(rec.uuid)
klass=DBAnalysis if not self.use_vcs else VCSAnalysis
ai = klass(group_id=group_id,
graph_id=graph_id)
# if not self.use_vcs:
synced=False
if atype in ('unknown', 'cocktail'):
if calculate_age:
ai.sync(meas_analysis, load_changes=load_changes)
ai.calculate_age(force=not self.use_vcs)
synced=True
if not synced:
ai.sync(meas_analysis, unpack=unpack, load_changes=load_changes)
return ai
def add_analysis(self, spec, commit=True):
db = self.db
for i in range(3):
with self.db.session_ctx():
irradpos = spec.irradpos
rid = spec.runid
trid = rid.lower()
identifier = spec.labnumber
if trid.startswith('b'):
runtype = 'Blank'
irradpos = -1
elif trid.startswith('a'):
runtype = 'Air'
irradpos = -2
elif trid.startswith('c'):
runtype = 'Unknown'
identifier = irradpos = self.get_identifier(spec)
else:
runtype = 'Unknown'
rid = make_runid(identifier, spec.aliquot, spec.step)
self._analysis = None
db.reraise = True
try:
ret = self._add_analysis(db.session, spec, irradpos, rid,
runtype)
db.commit()
return ret
except Exception, e:
import traceback
tb = traceback.format_exc()
self.debug('Mass Spec save exception. {}\n {}'.format(
e, tb))
if i == 2:
self.message('Could not save spec.runid={} rid={} '
'to Mass Spec database.\n {}'.format(
spec.runid, rid, tb))
else:
self.debug('retry mass spec save')
# if commit:
db.rollback()
finally:
self.db.reraise = True
def add_analysis(self, spec, commit=True):
db = self.db
# for i in range(3):
with db.session_ctx(use_parent_session=False) as session:
irradpos = spec.irradpos
rid = spec.runid
trid = rid.lower()
identifier = spec.labnumber
if trid.startswith('b'):
runtype = 'Blank'
irradpos = -1
elif trid.startswith('a'):
runtype = 'Air'
irradpos = -2
elif trid.startswith('c'):
runtype = 'Unknown'
identifier = irradpos = self.get_identifier(spec)
else:
runtype = 'Unknown'
rid = make_runid(identifier, spec.aliquot, spec.step)
self._analysis = None
db.reraise = True
try:
ret = self._add_analysis(session, spec, irradpos, rid, runtype)
db.commit()
return ret
except Exception as e:
import traceback
self.debug('Mass Spec save exception. {}'.format(e))
tb = traceback.format_exc()
self.debug(tb)
self.message('Could not save spec. runid={} rid={} to MassSpec DB.\n{}'.format(spec.runid, rid, tb))
# if i == 2:
# self.message('Could not save spec.runid={} rid={} '
# 'to Mass Spec database.\n {}'.format(spec.runid, rid, tb))
# else:
# self.debug('retry mass spec save')
# # if commit:
# db.rollback()
finally:
self.db.reraise = True
def _bulk_runid(self, ai, aliquot, step):
if not aliquot:
aliquot = ai.aliquot
if not step:
step = ai.step
self.dvc.db.modify_aliquot_step(ai.uuid, aliquot, alpha_to_int(step))
def modify_meta(p):
jd = dvc_load(p)
jd['aliquot'] = aliquot
jd['increment'] = alpha_to_int(step)
dvc_dump(jd, p)
ps = []
repo_id = ai.repository_identifier
sp = analysis_path(('', ai.record_id), repo_id)
if sp:
modify_meta(sp)
ps.append(sp)
# using runid path name
new_runid = make_runid(ai.identifier, aliquot, step)
for m in NPATH_MODIFIERS:
sp = analysis_path(('', ai.record_id), repo_id, modifier=m)
dp = analysis_path(('', new_runid),
repo_id,
modifier=m,
mode='w')
if sp and os.path.isfile(sp):
if os.path.isfile(dp) and m != 'extraction':
continue
ps.append(sp)
ps.append(dp)
shutil.move(sp, dp)
else:
# using uuid path name
# only need to modify metadata file
sp = analysis_path(ai, repo_id)
modify_meta(sp, aliquot, step)
ps.append(sp)
return repo_id, ps
def test_modifier_secondarydb(self):
self.modifier.use_main = False
self.modifier.use_secondary = True
an = self.analyses[0]
can = copy.copy(an)
can.identifier = self.new_identifier
self.modifier.modify_analyses([an], [can])
db = self.modifier.secondary_db
with db.session_ctx():
old_an = db.get_analysis(an.labnumber, an.aliquot, an.step)
self.assertIsNone(old_an)
new_an = db.get_analysis(self.new_identifier,
'{:02d}'.format(an.aliquot), an.step)
self.assertEqual(
new_an.RID, make_runid(self.new_identifier, an.aliquot,
an.step))
self.assertEqual(new_an.sample.Sample, 'Bar')
def _set_device_sensitivity(self, s, e, mi, ed, pr, v, err, prog):
"""
get all analyses between s, e
where mass_spectrometer=mi, extract_device=ed, and project=pr
set sensitivity to v +/-1s e
"""
db = self.db
with db.session_ctx():
ans = db.get_analyses_date_range(s,
e,
mass_spectrometer=mi,
extract_device=ed,
project=pr)
ans = filter(lambda x: not x.uuid in self._uuids, ans)
self._uuids.extend([ai.uuid for ai in ans])
self.debug('setting {} analyses'.format(len(ans)))
prog.increase_max(len(ans))
for ai in ans:
r = make_runid(ai.labnumber.identifier, ai.aliquot, ai.step)
prog.change_message('Saving sensitivity for {}'.format(r))
db.set_analysis_sensitivity(ai, v, err)
def _set_device_sensitivity(self, s, e, mi, ed, pr, v, err,prog):
"""
get all analyses between s, e
where mass_spectrometer=mi, extract_device=ed, and project=pr
set sensitivity to v +/-1s e
"""
db=self.db
with db.session_ctx():
ans = db.get_analyses_date_range(s, e,
mass_spectrometer=mi,
extract_device=ed,
project=pr)
ans=filter(lambda x: not x.uuid in self._uuids, ans)
self._uuids.extend([ai.uuid for ai in ans])
self.debug('setting {} analyses'.format(len(ans)))
prog.increase_max(len(ans))
for ai in ans:
r = make_runid(ai.labnumber.identifier,
ai.aliquot, ai.step)
prog.change_message('Saving sensitivity for {}'.format(r))
db.set_analysis_sensitivity(ai, v, err)
def _sync(self, path, *args, **kw):
self.path = path
yd = self._load_yaml(path)
attrs = ANALYSIS_ATTRS + ()
for attr in attrs:
if isinstance(attr, tuple):
attr, ydattr = attr
else:
attr, ydattr = attr, attr
setattr(self, attr, yd[ydattr])
self.analysis_type = 'unknown'
self.rundate = datetime.fromtimestamp(self.timestamp)
self.set_j(yd['j'], yd['j_err'])
self.record_id = make_runid(self.labnumber, self.aliquot, self.step)
self.interference_corrections = yd['interference_corrections']
self.production_ratios = yd['production_ratios']
self._sync_isotopes(yd)
def _sync(self, path, *args, **kw):
self.path = path
yd = self._load_yaml(path)
attrs = ANALYSIS_ATTRS + ()
for attr in attrs:
if isinstance(attr, tuple):
attr, ydattr = attr
else:
attr, ydattr = attr, attr
setattr(self, attr, yd[ydattr])
self.analysis_type = 'unknown'
self.rundate = datetime.fromtimestamp(self.timestamp)
self.set_j(yd['j'], yd['j_err'])
self.record_id = make_runid(self.labnumber, self.aliquot, self.step)
self.interference_corrections = yd['interference_corrections']
self.production_ratios = yd['production_ratios']
self._sync_isotopes(yd)
def add_analysis(self, spec, commit=True):
with self.db.session_ctx(commit=False) as sess:
irradpos = spec.irradpos
rid = spec.runid
trid = rid.lower()
identifier = spec.labnumber
if trid.startswith("b"):
runtype = "Blank"
irradpos = -1
elif trid.startswith("a"):
runtype = "Air"
irradpos = -2
elif trid.startswith("c"):
runtype = "Unknown"
identifier = irradpos = self.get_identifier(spec)
else:
runtype = "Unknown"
# paliquot = self.db.get_latest_analysis_aliquot(identifier)
# if paliquot is None:
# paliquot=0
#
# rid = '{}-{:02n}'.format(identifier, spec.aliquot, spec.step)
# self.info('Saving analysis {} to database as {}'.format(spec.rid, rid))
rid = make_runid(identifier, spec.aliquot, spec.step)
self._analysis = None
try:
return self._add_analysis(sess, spec, irradpos, rid, runtype)
except Exception, e:
import traceback
tb = traceback.format_exc()
self.message(
"Could not save spec.runid={} rid={} to Mass Spec database.\n {}".format(spec.runid, rid, tb)
)
if commit:
sess.rollback()
def add_analysis(self, spec, commit=True):
with self.db.session_ctx(commit=False) as sess:
irradpos = spec.irradpos
rid = spec.runid
trid = rid.lower()
identifier = spec.labnumber
if trid.startswith('b'):
runtype = 'Blank'
irradpos = -1
elif trid.startswith('a'):
runtype = 'Air'
irradpos = -2
elif trid.startswith('c'):
runtype = 'Unknown'
identifier = irradpos = self.get_identifier(spec)
else:
runtype = 'Unknown'
# paliquot = self.db.get_latest_analysis_aliquot(identifier)
# if paliquot is None:
# paliquot=0
#
#rid = '{}-{:02n}'.format(identifier, spec.aliquot, spec.step)
# self.info('Saving analysis {} to database as {}'.format(spec.rid, rid))
rid = make_runid(identifier, spec.aliquot, spec.step)
self._analysis = None
try:
return self._add_analysis(sess, spec, irradpos, rid, runtype)
except Exception, e:
import traceback
tb = traceback.format_exc()
self.message(
'Could not save spec.runid={} rid={} to Mass Spec database.\n {}'
.format(spec.runid, rid, tb))
if commit:
sess.rollback()
def get_isotope_data(self, idx, k):
v = self.get_isotope(idx, k)
i = self.get_identifier(idx)
a = self.get_aliquot(idx)
s = self.get_step(idx)
rid = make_runid(i, a, s)
if isinstance(v, float):
return [], []
else:
parser = self.parser
sh = parser.sheet
def g():
parser.set_sheet(v, header_idx=0)
cx, cy = parser.get_index('{}_xs'.format(k)), parser.get_index('{}_ys'.format(k))
for row in parser.iterblock(0, rid):
yield row[cx].value, row[cy].value
data = list(g())
self.parser.set_sheet(sh)
return map(list, zip(*data))
def get_isotope_data(self, idx, k):
v = self.get_isotope(idx, k)
i = self.get_identifier(idx)
a = self.get_aliquot(idx)
s = self.get_step(idx)
rid = make_runid(i, a, s)
if isinstance(v, float):
return [], []
else:
parser = self.parser
sh = parser.sheet
def g():
parser.set_sheet(v, header_idx=0)
cx, cy = parser.get_index('{}_xs'.format(k)), parser.get_index(
'{}_ys'.format(k))
for row in parser.iterblock(0, rid):
yield row[cx].value, row[cy].value
data = list(g())
self.parser.set_sheet(sh)
return map(list, zip(*data))
def _get_runid(self):
return make_runid(self.labnumber, self.aliquot, self.step)
def record_id(self):
return make_runid(self.labnumber.identifier, self.aliquot, self.increment)
def _get_runid(self):
return make_runid(self.labnumber, self.aliquot, self.step)
def _get_record_id(self):
record_id = self._record_id
if not record_id:
record_id = make_runid(self.labnumber, self.aliquot, self.step)
return record_id
def _transfer_analysis(self,
rec,
exp,
overwrite=True,
monitor_mapping=None):
dest = self.dvc.db
proc = self.processor
src = proc.db
# args = rec.split('-')
# idn = '-'.join(args[:-1])
# t = args[-1]
# try:
# aliquot = int(t)
# step = None
# except ValueError:
# aliquot = int(t[:-1])
# step = t[-1]
m = IDENTIFIER_REGEX.match(rec)
if not m:
m = SPECIAL_IDENTIFIER_REGEX.match(rec)
if not m:
self.warning('invalid runid {}'.format(rec))
return
else:
idn = m.group('identifier')
aliquot = m.group('aliquot')
try:
step = m.group('step') or None
except IndexError:
step = None
if idn == '4359':
idn = 'c-01-j'
elif idn == '4358':
idn = 'c-01-o'
# check if analysis already exists. skip if it does
if dest.get_analysis_runid(idn, aliquot, step):
self.warning('{} already exists'.format(
make_runid(idn, aliquot, step)))
return
dban = src.get_analysis_runid(idn, aliquot, step)
iv = IsotopeRecordView()
iv.uuid = dban.uuid
self.debug('make analysis idn:{}, aliquot:{} step:{}'.format(
idn, aliquot, step))
# try:
an = proc.make_analysis(iv,
unpack=True,
use_cache=False,
use_progress=False)
# except BaseException as e:
# self.warning('Failed to make {}'.format(make_runid(idn, aliquot, step)))
# self.warning('exception: {}'.format(e))
# return
self._transfer_meta(dest, dban, monitor_mapping)
# return
dblab = dban.labnumber
if dblab.irradiation_position:
irrad = dblab.irradiation_position.level.irradiation.name
level = dblab.irradiation_position.level.name
irradpos = dblab.irradiation_position.position
else:
irrad = 'NoIrradiation'
level = 'A'
irradpos = self._get_irradpos(dest, irrad, level, dblab.identifier)
# irrad, level, irradpos = '', '', 0
extraction = dban.extraction
ms = dban.measurement.mass_spectrometer.name
if not dest.get_mass_spectrometer(ms):
self.debug('adding mass spectrometer {}'.format(ms))
dest.add_mass_spectrometer(ms)
dest.commit()
ed = extraction.extraction_device.name if extraction.extraction_device else None
if not ed:
ed = 'No Extract Device'
if not dest.get_extraction_device(ed):
self.debug('adding extract device {}'.format(ed))
dest.add_extraction_device(ed)
dest.commit()
if step is None:
inc = -1
else:
inc = alpha_to_int(step)
username = ''
if dban.user:
username = dban.user.name
if not dest.get_user(username):
self.debug('adding user. username:{}'.format(username))
dest.add_user(username)
dest.commit()
if monitor_mapping:
sample_name, material_name, project_name = monitor_mapping
else:
dbsam = dblab.sample
sample_name = dbsam.name
material_name = dbsam.material.name
project_name = format_repository_identifier(dbsam.project.name)
rs = AutomatedRunSpec(labnumber=idn,
username=username,
material=material_name,
project=project_name,
sample=sample_name,
irradiation=irrad,
irradiation_level=level,
irradiation_position=irradpos,
repository_identifier=exp,
mass_spectrometer=ms,
uuid=dban.uuid,
_step=inc,
comment=dban.comment.decode('utf-8') or '',
aliquot=int(aliquot),
extract_device=ed,
duration=extraction.extract_duration,
cleanup=extraction.cleanup_duration,
beam_diameter=extraction.beam_diameter,
extract_units=extraction.extract_units or '',
extract_value=extraction.extract_value,
pattern=extraction.pattern or '',
weight=extraction.weight,
ramp_duration=extraction.ramp_duration or 0,
ramp_rate=extraction.ramp_rate or 0,
collection_version='0.1:0.1',
queue_conditionals_name='',
tray='')
meas = dban.measurement
# get spectrometer parameters
# gains
gains = {}
gain_history = dban.gain_history
if gain_history:
gains = {d.detector.name: d.value for d in gain_history.gains}
# deflections
deflections = {d.detector.name: d.deflection for d in meas.deflections}
# source
src = {
k: getattr(meas.spectrometer_parameters, k)
for k in QTEGRA_SOURCE_KEYS
}
ps = PersistenceSpec(
run_spec=rs,
tag=an.tag.name,
isotope_group=an,
timestamp=dban.analysis_timestamp,
defl_dict=deflections,
gains=gains,
spec_dict=src,
use_repository_association=True,
positions=[p.position for p in extraction.positions])
self.debug('transfer analysis with persister')
self.persister.per_spec_save(ps,
commit=False,
commit_tag='Database Transfer')
return True
def record_id(self):
return make_runid(self.uuid, self.aliquot, self.step)
def record_id(self):
return make_runid(self.irradiation_position.identifier, self.aliquot, self.increment)
def create(self, dbrecord):
# print 'asdfsadfsdaf', dbrecord, dbrecord.labnumber, dbrecord.uuid
try:
if dbrecord is None or not dbrecord.labnumber:
return
ln = dbrecord.labnumber
self.labnumber = str(ln.identifier)
self.identifier = self.labnumber
self.aliquot = dbrecord.aliquot
self.step = dbrecord.step
self.uuid = dbrecord.uuid
self.tag = dbrecord.tag or ''
self.rundate = dbrecord.analysis_timestamp
self.timestamp = time.mktime(self.rundate.timetuple())
self.record_id = make_runid(self.labnumber, self.aliquot,
self.step)
# print self.record_id, self.uuid
if ln.sample:
self.sample = ln.sample.name
if dbrecord.labnumber.sample:
self.sample = dbrecord.labnumber.sample.name
irp = ln.irradiation_position
if irp is not None:
irl = irp.level
ir = irl.irradiation
self.irradiation_info = '{}{} {}'.format(
ir.name, irl.name, irp.position)
else:
self.irradiation_info = ''
meas = dbrecord.measurement
if meas is not None:
self.mass_spectrometer = meas.mass_spectrometer.name.lower()
if meas.analysis_type:
self.analysis_type = meas.analysis_type.name
ext = dbrecord.extraction
if ext:
if ext.extraction_device:
self.extract_device = ext.extraction_device.name
self.flux_fit_status = self._get_flux_fit_status(dbrecord)
self.blank_fit_status = self._get_selected_history_item(
dbrecord, 'selected_blanks_id')
self.ic_fit_status = self._get_selected_history_item(
dbrecord, 'selected_det_intercal_id')
self.iso_fit_status = self._get_selected_history_item(
dbrecord, 'selected_fits_id')
return True
except Exception, e:
import traceback
traceback.print_exc()
print e
def record_id(self):
return make_runid(self.identifier, self.aliquot, self.step)
def record_id(self):
v = self._record_id
if v is None:
v = make_runid(self.labnumber, self.aliquot, self.step)
return v
def _construct_analysis(self,
rec,
group,
prog,
calculate_age=True,
calculate_F=False,
unpack=False,
load_aux=False):
atype = None
if isinstance(rec, meas_AnalysisTable):
rid = make_runid(rec.labnumber.identifier, rec.aliquot, rec.step)
atype = rec.measurement.analysis_type.name
elif hasattr(rec, 'record_id'):
rid = rec.record_id
else:
rid = id(rec)
graph_id = 0
group_id = 0
if hasattr(rec, 'group_id'):
group_id = rec.group_id
if hasattr(rec, 'graph_id'):
graph_id = rec.graph_id
if atype is None:
atype = rec.analysis_type
if prog:
m = ''
if calculate_age:
show_age = atype in ('unknown', 'cocktail')
m = 'calculating age' if show_age else ''
elif calculate_F:
m = 'calculating F'
msg = 'loading {}. {}'.format(rid, m)
prog.change_message(msg)
if isinstance(rec, DBAnalysis):
ai = rec
if load_aux:
ai.sync_aux(group)
else:
ai.sync(group, unpack=unpack, load_aux=load_aux)
else:
ai = DBAnalysis() # if not self.use_vcs else VCSAnalysis
# print ai
ai.sync(group, unpack=unpack, load_aux=load_aux)
# print ai, group
# ai = klass(group_id=group_id,
# graph_id=graph_id)
# ai.trait_set(group_id=group_id,
# graph_id=graph_id)
# if not self.use_vcs:
#
# timethis(ai.sync, args=(group,),
# kwargs=dict(unpack=unpack, load_aux=load_aux))
if atype in ('unknown', 'cocktail'):
if calculate_age:
# timethis(ai.sync, args=(meas_analysis, ),
# kwargs=dict(unpack=unpack, load_aux=load_aux))
# timethis(ai.calculate_age, kwargs=dict(force=not self.use_vcs))
ai.calculate_age()
# timethis(ai.sync, args=(meas_analysis,),
# kwargs=dict(unpack=unpack, load_aux=load_aux))
# timethis(ai.calculate_age)
# synced = True
if calculate_F:
ai.calculate_f()
# if not synced:
# ai.sync(group, unpack=unpack, load_aux=load_aux)
return ai
def record_id(self):
v = self._record_id
if v is None:
v = make_runid(self.labnumber, self.aliquot, self.step)
return v
def runid(self):
if self._runid:
return self._runid
else:
return make_runid(self.labnumber, self.aliquot, self.step)
def _find_run(self, aid):
return next((a for a in self.automated_runs
if make_runid(a.labnumber, a.aliquot, a.step) == aid), None)
def record_id(self):
return make_runid(self.uuid, self.aliquot, self.step)
def _find_run(self, aid):
return next((a for a in self.automated_runs
if make_runid(a.labnumber, a.aliquot, a.step) == aid), None)
def record_id(self):
return make_runid(self.labnumber.identifier, self.aliquot, self.increment)
def record_id(self):
return make_runid(self.identifier, self.aliquot, self.step)
def _get_record_id(self):
record_id = self._record_id
if not record_id:
record_id = make_runid(self.labnumber, self.aliquot, self.step)
return record_id
def runid(self):
if self._runid:
return self._runid
else:
return make_runid(self.labnumber, self.aliquot, self.step)
def _construct_analysis(self, rec, group, prog, calculate_age=True, calculate_F=False,
unpack=False, load_aux=False):
atype = None
if isinstance(rec, meas_AnalysisTable):
rid = make_runid(rec.labnumber.identifier, rec.aliquot, rec.step)
atype = rec.measurement.analysis_type.name
elif hasattr(rec, 'record_id'):
rid = rec.record_id
else:
rid = id(rec)
graph_id = 0
group_id = 0
if hasattr(rec, 'group_id'):
group_id = rec.group_id
if hasattr(rec, 'graph_id'):
graph_id = rec.graph_id
if atype is None:
atype = rec.analysis_type
if prog:
m=''
if calculate_age:
show_age = atype in ('unknown', 'cocktail')
m = 'calculating age' if show_age else ''
elif calculate_F:
m = 'calculating F'
msg = 'loading {}. {}'.format(rid, m)
prog.change_message(msg)
if isinstance(rec, DBAnalysis):
ai = rec
if load_aux:
ai.sync_aux(group)
else:
ai.sync(group, unpack=unpack, load_aux=load_aux)
else:
ai = DBAnalysis() # if not self.use_vcs else VCSAnalysis
# print ai
ai.sync(group, unpack=unpack, load_aux=load_aux)
# print ai, group
# ai = klass(group_id=group_id,
# graph_id=graph_id)
ai.trait_set(group_id=group_id,
graph_id=graph_id)
# if not self.use_vcs:
#
# timethis(ai.sync, args=(group,),
# kwargs=dict(unpack=unpack, load_aux=load_aux))
if atype in ('unknown', 'cocktail'):
if calculate_age:
# timethis(ai.sync, args=(meas_analysis, ),
# kwargs=dict(unpack=unpack, load_aux=load_aux))
# timethis(ai.calculate_age, kwargs=dict(force=not self.use_vcs))
ai.calculate_age() #force=not self.use_vcs)
# timethis(ai.sync, args=(meas_analysis,),
# kwargs=dict(unpack=unpack, load_aux=load_aux))
# timethis(ai.calculate_age)
# synced = True
if calculate_F:
ai.calculate_F()
# if not synced:
# ai.sync(group, unpack=unpack, load_aux=load_aux)
return ai
def record_id(self):
return make_runid(self.irradiation_position.identifier, self.aliquot, self.increment)
def _transfer_analysis(self, rec, exp, overwrite=True, monitor_mapping=None):
dest = self.dvc.db
proc = self.processor
src = proc.db
# args = rec.split('-')
# idn = '-'.join(args[:-1])
# t = args[-1]
# try:
# aliquot = int(t)
# step = None
# except ValueError:
# aliquot = int(t[:-1])
# step = t[-1]
m = IDENTIFIER_REGEX.match(rec)
if not m:
m = SPECIAL_IDENTIFIER_REGEX.match(rec)
if not m:
self.warning('invalid runid {}'.format(rec))
return
else:
idn = m.group('identifier')
aliquot = m.group('aliquot')
try:
step = m.group('step') or None
except IndexError:
step = None
if idn == '4359':
idn = 'c-01-j'
elif idn == '4358':
idn = 'c-01-o'
# check if analysis already exists. skip if it does
if dest.get_analysis_runid(idn, aliquot, step):
self.warning('{} already exists'.format(make_runid(idn, aliquot, step)))
return
dban = src.get_analysis_runid(idn, aliquot, step)
iv = IsotopeRecordView()
iv.uuid = dban.uuid
self.debug('make analysis idn:{}, aliquot:{} step:{}'.format(idn, aliquot, step))
try:
an = proc.make_analysis(iv, unpack=True, use_cache=False, use_progress=False)
except:
self.warning('Failed to make {}'.format(make_runid(idn, aliquot, step)))
return
self._transfer_meta(dest, dban, monitor_mapping)
# return
dblab = dban.labnumber
if dblab.irradiation_position:
irrad = dblab.irradiation_position.level.irradiation.name
level = dblab.irradiation_position.level.name
irradpos = dblab.irradiation_position.position
else:
irrad = 'NoIrradiation'
level = 'A'
irradpos = self._get_irradpos(dest, irrad, level, dblab.identifier)
# irrad, level, irradpos = '', '', 0
extraction = dban.extraction
ms = dban.measurement.mass_spectrometer.name
if not dest.get_mass_spectrometer(ms):
self.debug('adding mass spectrometer {}'.format(ms))
dest.add_mass_spectrometer(ms)
dest.commit()
ed = extraction.extraction_device.name if extraction.extraction_device else None
if not ed:
ed = 'No Extract Device'
if not dest.get_extraction_device(ed):
self.debug('adding extract device {}'.format(ed))
dest.add_extraction_device(ed)
dest.commit()
if step is None:
inc = -1
else:
inc = ALPHAS.index(step)
username = ''
if dban.user:
username = dban.user.name
if not dest.get_user(username):
self.debug('adding user. username:{}'.format(username))
dest.add_user(username)
dest.commit()
if monitor_mapping:
sample_name, material_name, project_name = monitor_mapping
else:
dbsam = dblab.sample
sample_name = dbsam.name
material_name = dbsam.material.name
project_name = format_repository_identifier(dbsam.project.name)
rs = AutomatedRunSpec(labnumber=idn,
username=username,
material=material_name,
project=project_name,
sample=sample_name,
irradiation=irrad,
irradiation_level=level,
irradiation_position=irradpos,
repository_identifier=exp,
mass_spectrometer=ms,
uuid=dban.uuid,
_step=inc,
comment=dban.comment or '',
aliquot=int(aliquot),
extract_device=ed,
duration=extraction.extract_duration,
cleanup=extraction.cleanup_duration,
beam_diameter=extraction.beam_diameter,
extract_units=extraction.extract_units or '',
extract_value=extraction.extract_value,
pattern=extraction.pattern or '',
weight=extraction.weight,
ramp_duration=extraction.ramp_duration or 0,
ramp_rate=extraction.ramp_rate or 0,
collection_version='0.1:0.1',
queue_conditionals_name='',
tray='')
meas = dban.measurement
# get spectrometer parameters
# gains
gains = {}
gain_history = dban.gain_history
if gain_history:
gains = {d.detector.name: d.value for d in gain_history.gains}
# deflections
deflections = {d.detector.name: d.deflection for d in meas.deflections}
# source
src = {k: getattr(meas.spectrometer_parameters, k) for k in QTEGRA_SOURCE_KEYS}
ps = PersistenceSpec(run_spec=rs,
tag=an.tag.name,
arar_age=an,
timestamp=dban.analysis_timestamp,
defl_dict=deflections,
gains=gains,
spec_dict=src,
use_repository_association=True,
positions=[p.position for p in extraction.positions])
self.debug('transfer analysis with persister')
self.persister.per_spec_save(ps, commit=False, msg_prefix='Database Transfer')
return True
