def _make_level(self, doc, irrad, level,ids, ans):
root = doc['root']
options = doc['options']
lroot = os.path.join(root, irrad, level)
r_mkdir(lroot)
n=len(ans)
prog = self.open_progress(n, close_at_end=False)
ans = self.make_analyses(ans, progress=prog, use_cache=False)
#group by stepheat vs fusion
pred = lambda x: bool(x.step)
ans = sorted(ans, key=pred)
stepheat, fusion = map(list, partition(ans, pred))
# apred = lambda x: x.aliquot
# stepheat = sorted(stepheat, key=apred)
# if stepheat:
# self._make_editor(stepheat, 'step_heat', options, prog, ln_root, li)
project='J'
# lns=[li.identifier for li in level.labnumbers]
if fusion:
save_args=(lroot, level, '{} {}'.format(irrad, level),
project, ids)
self._make_editor(fusion, ('fusion','fusion_grouped'),
options, prog, 'aliquot',
save_args)
prog.close()
def group_level(self, level, irradiation=None, monitor_filter=None):
if monitor_filter is None:
def monitor_filter(pos):
if pos.sample == 'FC-2':
return True
db=self.db
with db.session_ctx():
if isinstance(level, str):
level = db.get_level(level, irradiation)
refs = []
unks = []
if level:
positions = level.positions
if positions:
def pos_factory(px):
ip = IrradiationPositionRecord()
ip.create(px)
return ip
positions = [pos_factory(pp) for pp in positions]
refs, unks = partition(positions, monitor_filter)
return refs, unks
def _make_labnumber(self, li, root, options, prog):
#make dir for labnumber
ident = li.identifier
ln_root = os.path.join(root, ident)
r_mkdir(ln_root)
prog.change_message('Making {} for {}'.format(self._tag, ident))
#filter invalid analyses
ans = filter(lambda x: not x.tag == 'invalid', li.analyses)
#group by stepheat vs fusion
pred = lambda x: bool(x.step)
ans = sorted(ans, key=pred)
stepheat, fusion = map(list, partition(ans, pred))
apred = lambda x: x.aliquot
stepheat = sorted(stepheat, key=apred)
project = 'Minna Bluff'
li = li.identifier
if stepheat:
key = lambda x: x.aliquot
stepheat = sorted(stepheat, key=key)
for aliquot, ais in groupby(stepheat, key=key):
name = make_runid(li, aliquot, '')
self._make_editor(ais, 'step_heat', options, prog, False,
(ln_root, name, name, project, (li, )))
if fusion:
self._make_editor(fusion, 'fusion', options, prog, False,
(ln_root, li, li, project, (li, )))
def _load_associated_labnumbers(self, names):
"""
names: list of project names
"""
db = self.db
sams = []
with db.session_ctx():
self._recent_mass_spectrometers = []
warned = False
rnames, onames = partition(names, lambda x: x.startswith('RECENT'))
for name in rnames:
# load associated samples
if not self.search_criteria.recent_hours:
if not warned:
self.warning_dialog('Set "Recent Hours" in Preferences.\n'
'"Recent Hours" is located in the "Processing" category')
warned = True
else:
sams.extend(self._retrieve_recent_labnumbers(name))
if list(onames):
sams.extend(self._retrieve_labnumbers())
self.samples = sams
self.osamples = sams
def _make_paths(self, root, lpost=None, hpost=None):
xs = [xi for xi in os.listdir(root) if not xi.startswith('.')]
dirs, files = partition(xs, lambda x: not os.path.isfile(os.path.join(root, x)))
if self.filter_hierarchy_str:
files = (ci for ci in files if ci.startswith(self.filter_hierarchy_str))
if self.date_filter == 'Modified':
func = modified_datetime
else:
func = created_datetime
if lpost:
try:
lpost = lpost.date()
except AttributeError:
pass
files = (ci for ci in files
if func(os.path.join(root, ci), strformat=None).date() >= lpost)
if hpost:
try:
hpost = hpost.date()
except AttributeError:
pass
files = (ci for ci in files
if func(os.path.join(root, ci), strformat=None).date() <= hpost)
files = [self._child_factory(ci) for ci in files]
dirs = [self._directory_factory(di) for di in dirs]
return dirs, files
def _make_level(self, doc, irrad, level, ids, ans):
root = doc['root']
options = doc['options']
lroot = os.path.join(root, irrad, level)
r_mkdir(lroot)
n = len(ans)
prog = self.open_progress(n, close_at_end=False)
ans = self.make_analyses(ans, progress=prog, use_cache=False)
#group by stepheat vs fusion
pred = lambda x: bool(x.step)
ans = sorted(ans, key=pred)
stepheat, fusion = map(list, partition(ans, pred))
# apred = lambda x: x.aliquot
# stepheat = sorted(stepheat, key=apred)
# if stepheat:
# self._make_editor(stepheat, 'step_heat', options, prog, ln_root, li)
project = 'J'
# lns=[li.identifier for li in level.labnumbers]
if fusion:
save_args = (lroot, level, '{} {}'.format(irrad,
level), project, ids)
self._make_editor(fusion, ('fusion', 'fusion_grouped'), options,
prog, 'aliquot', save_args)
prog.close()
def _make_paths(self, root, lpost=None, hpost=None):
xs = [xi for xi in os.listdir(root) if not xi.startswith('.')]
dirs, files = partition(xs, lambda x: not os.path.isfile(os.path.join(root, x)))
if self.filter_hierarchy_str:
files = (ci for ci in files if ci.startswith(self.filter_hierarchy_str))
if self.date_filter == 'Modified':
func = modified_datetime
else:
func = created_datetime
if lpost:
try:
lpost = lpost.date()
except AttributeError:
pass
files = (ci for ci in files
if func(os.path.join(root, ci), strformat=None).date() >= lpost)
if hpost:
try:
hpost = hpost.date()
except AttributeError:
pass
files = (ci for ci in files
if func(os.path.join(root, ci), strformat=None).date() <= hpost)
files = [self._child_factory(ci) for ci in files]
dirs = [self._directory_factory(di) for di in dirs]
return dirs, files
def group_level(self, level, irradiation=None, monitor_filter=None):
if monitor_filter is None:
def monitor_filter(pos):
if pos.sample == 'FC-2':
return True
db = self.db
with db.session_ctx():
if isinstance(level, str):
level = db.get_level(level, irradiation)
refs = []
unks = []
if level:
positions = level.positions
if positions:
def pos_factory(px):
ip = IrradiationPositionRecord()
ip.create(px)
return ip
positions = [pos_factory(pp) for pp in positions]
refs, unks = partition(positions, monitor_filter)
return refs, unks
def _load_associated_labnumbers(self, names):
"""
names: list of project names
"""
db = self.db
sams = []
with db.session_ctx():
self._recent_mass_spectrometers = []
warned = False
rnames, onames = partition(names, lambda x: x.startswith('RECENT'))
for name in rnames:
# load associated samples
if not self.search_criteria.recent_hours:
if not warned:
self.warning_dialog(
'Set "Recent Hours" in Preferences.\n'
'"Recent Hours" is located in the "Processing" category'
)
warned = True
else:
sams.extend(self._retrieve_recent_labnumbers(name))
if list(onames):
sams.extend(self._retrieve_labnumbers())
self.samples = sams
self.osamples = sams
def _make_labnumber(self, li, root, options, prog):
#make dir for labnumber
ident = li.identifier
ln_root = os.path.join(root, ident)
r_mkdir(ln_root)
prog.change_message('Making {} for {}'.format(self._tag, ident))
#filter invalid analyses
ans=filter(lambda x: not x.tag=='invalid', li.analyses)
#group by stepheat vs fusion
pred = lambda x: bool(x.step)
ans = sorted(ans, key=pred)
stepheat, fusion = map(list, partition(ans, pred))
apred = lambda x: x.aliquot
stepheat = sorted(stepheat, key=apred)
project='Minna Bluff'
li = li.identifier
if stepheat:
key=lambda x: x.aliquot
stepheat=sorted(stepheat, key=key)
for aliquot, ais in groupby(stepheat, key=key):
name=make_runid(li, aliquot, '')
self._make_editor(ais, 'step_heat', options, prog, False,
(ln_root, name, name, project, (li,)))
if fusion:
self._make_editor(fusion, 'fusion', options, prog, False,
(ln_root, li, li, project, (li,)))
def make_sample_points(geo):
db = geo.db
pr = 'Minna Bluff'
missing = []
groups = []
with db.session_ctx():
v = db.get_project(pr)
samples = v.samples
yr1, yr2 = partition(samples, lambda x: x.name.startswith('MB06'))
for name, sams in (('MB06_all_samples', yr1), ('MB07_all_samples',
yr2)):
pts = []
for s in sams:
if not s.lat:
missing.append((s.name, s.alt_name))
else:
print 'adding {} lat: {:0.5f} long: {:0.5f}'.format(
s.name, s.lat, s.long)
x, y = proj_pt(s.lat, s.long)
pt = SamplePoint(
x, y, s.name, s.material.name if s.material else '',
','.join([li.identifier for li in s.labnumbers]))
pts.append(pt)
groups.append((name, pts))
return groups
def save(self, name=None):
if name is None:
name = self.name
bfs, sfs = partition(self.fits, lambda x: x.is_baseline)
yd = {'signal': self._dump(sfs), 'baseline': self._dump(bfs)}
p = os.path.join(paths.fits_dir, '{}.yaml'.format(name))
with open(p, 'w') as wfile:
yaml.dump(yd, wfile, default_flow_style=False)
def save(self, name=None):
if name is None:
name = self.name
bfs, sfs = partition(self.fits, lambda x: x.is_baseline)
yd = {'signal': self._dump(sfs),
'baseline': self._dump(bfs)}
p = os.path.join(paths.fits_dir, '{}.yaml'.format(name))
with open(p, 'w') as wfile:
yaml.dump(yd, wfile, default_flow_style=False)
def _get_analyses(self, li):
#filter invalid analyses
ans = filter(lambda x: not x.tag == 'invalid', li.analyses)
#group by stepheat vs fusion
pred = lambda x: bool(x.step)
ans = sorted(ans, key=pred)
stepheat, fusion = map(list, partition(ans, pred))
return stepheat
def _get_analyses(self, li):
#filter invalid analyses
ans = filter(lambda x: not x.tag == 'invalid', li.analyses)
#group by stepheat vs fusion
pred = lambda x: bool(x.step)
ans = sorted(ans, key=pred)
stepheat, fusion = map(list, partition(ans, pred))
return stepheat
def _get_positions(self):
hkey = lambda x: x.hole_id
if self.tool.group_positions:
mons, unks = map(list, partition(self.monitor_positions.itervalues(),
lambda x: x.sample == self.tool.monitor.sample))
return [ai for ps in (mons, unks)
for ai in sorted(ps, key=hkey)]
else:
return sorted(self.monitor_positions.itervalues(), key=hkey)
def _get_positions(self):
hkey = lambda x: x.hole_id
if self.tool.group_positions:
mons, unks = map(
list,
partition(self.monitor_positions.itervalues(),
lambda x: x.sample == self.tool.monitor.sample))
return [ai for ps in (mons, unks) for ai in sorted(ps, key=hkey)]
else:
return sorted(self.monitor_positions.itervalues(), key=hkey)
def run(self, state):
dvc = self.dvc
rs = []
for ri in self.repositories:
ans = dvc.get_repoository_analyses(ri)
rs.extend(ans)
unks, refs = partition(
rs, predicate=lambda x: x.analysis_type == 'unknown')
state.unknowns = unks
state.references = refs
self.unknowns = unks
self.references = refs
def _dclicked_analysis_group_changed(self):
if self.active_editor:
if self.selected_analysis_groups:
g = self.selected_analysis_groups[0]
db = self.manager.db
with db.session_ctx():
dbg = db.get_analysis_group(g.id, key='id')
unks, b = partition(dbg.analyses, lambda x: x.analysis_type.name == 'unknown')
self.active_editor.auto_find = False
self.active_editor.set_items([ai.analysis for ai in unks])
self._dclicked_analysis_group_hook(unks, b)
def renumber_aliquots(aruns):
akey = attrgetter('user_defined_aliquot')
for ln, ans in groupby_key(aruns, 'labnumber'):
if is_special(ln):
continue
b, a = partition(ans, akey)
b = list(b)
if b:
minaliquot = min([bi.user_defined_aliquot for bi in b])
for i, (al, ans) in enumerate(groupby(b, key=akey)):
for ai in ans:
ai.user_defined_aliquot = minaliquot + i
def randomize_unknowns(self):
"""
1. get indices of non unknowns
2. partition into two lists unks, non-unks
3. randomize unks
4. insert non-unks back in using original indices
:return:
"""
aruns = self.automated_runs[:]
def predicate(x):
return not x.skip
skip_idx = [i for i, a in enumerate(aruns) if not predicate(a)]
aruns, skipped = partition(aruns, predicate=predicate)
def predicate(x):
return x.analysis_type == 'unknown'
idx = [i for i, a in enumerate(aruns) if not predicate(a)]
unks, refs = partition(aruns, predicate=predicate)
unks = list(unks)
refs = list(refs)
for i, r in list(zip(idx, refs)):
unks.insert(i, r)
for i, r in list(zip(skip_idx, skipped)):
unks.insert(i, r)
self.automated_runs = unks
self.refresh_table_needed = True
def _dclicked_analysis_group_changed(self):
if self.active_editor:
if self.browser_model.selected_analysis_groups:
g = self.browser_model.selected_analysis_groups[0]
db = self.manager.db
with db.session_ctx():
dbg = db.get_analysis_group(g.id, key='id')
unks, b = partition(
dbg.analyses,
lambda x: x.analysis_type.name == 'unknown')
self.active_editor.auto_find = False
self.active_editor.set_items([ai.analysis for ai in unks])
self._dclicked_analysis_group_hook(unks, b)
def _retrieve_labnumbers(self):
es = []
ps = []
ms = []
if self.mass_spectrometers_enabled:
if self.mass_spectrometer_includes:
ms = self.mass_spectrometer_includes
principal_investigator = None
if self.principal_investigator_enabled:
principal_investigator = self.principal_investigator
if self.repository_enabled:
if self.selected_repositories:
es = [e.name for e in self.selected_repositories]
if self.project_enabled:
if self.selected_projects:
rs, ps = partition([p.name for p in self.selected_projects],
lambda x: x.startswith('RECENT'))
ps, rs = list(ps), list(rs)
if rs:
hpost = datetime.now()
lpost = hpost - timedelta(
hours=self.search_criteria.recent_hours)
self._low_post = lpost
self.use_high_post = False
self.use_low_post = True
self.trait_property_changed('low_post', self._low_post)
for ri in rs:
mi = extract_mass_spectrometer_name(ri)
if mi not in ms:
ms.append(mi)
self._recent_mass_spectrometers.append(mi)
ls = self.db.get_labnumbers(
principal_investigator=principal_investigator,
projects=ps,
repositories=es,
mass_spectrometers=ms,
irradiation=self.irradiation if self.irradiation_enabled else None,
level=self.level if self.irradiation_enabled else None,
analysis_types=self.analysis_include_types
if self.use_analysis_type_filtering else None,
high_post=self.high_post if self.use_high_post else None,
low_post=self.low_post if self.use_low_post else None)
return ls
def dump_fits(self, keys, reviewed=False):
sisos = self.isotopes
isoks, dks = map(tuple, partition(keys, lambda x: x in sisos))
def update(d, i):
fd = i.filter_outliers_dict
d.update(fit=i.fit,
value=float(i.value),
error=float(i.error),
n=i.n,
fn=i.fn,
include_baseline_error=i.include_baseline_error,
filter_outliers=fd.get('filter_outliers', False),
iterations=fd.get('iterations', 0),
std_devs=fd.get('std_devs', 0))
# save intercepts
if isoks:
isos, path = self._get_json('intercepts')
for k in isoks:
try:
iso = isos[k]
siso = sisos[k]
update(iso, siso)
except KeyError:
pass
isos['reviewed'] = reviewed
self._dump(isos, path)
# save baselines
if dks:
baselines, path = self._get_json('baselines')
for di in dks:
try:
det = baselines[di]
except KeyError:
det = {}
baselines[di] = det
bs = next(
(iso.baseline
for iso in sisos.itervalues() if iso.detector == di),
None)
update(det, bs)
self._dump(baselines, path)
def renumber_aliquots(aruns):
key = lambda x: x.labnumber
akey = lambda x: x.user_defined_aliquot
aruns = sorted(aruns, key=key)
for ln, ans in groupby(aruns, key=key):
if is_special(ln):
continue
b, a = partition(ans, akey)
b = list(b)
if b:
minaliquot = min([bi.user_defined_aliquot for bi in b])
for i, (al, ans) in enumerate(groupby(b, key=akey)):
for ai in ans:
ai.user_defined_aliquot = minaliquot + i
def save_analysis_data_table(self, p):
# ans=[]
db=self.processor.db
with db.session_ctx():
ias=self.interpreted_ages[:10]
ans = [si.analysis for ia in ias
for si in db.get_interpreted_age_history(ia.id).interpreted_age.sets
if si.analysis.tag!='invalid']
prog = self.processor.open_progress(len(ans), close_at_end=False)
# hid = db.get_interpreted_age_history(ia.id)
# dbia = hid.interpreted_age
# ans.extend([si.analysis for si in db.get_interpreted_age_history(ia.id).interpreted_age.sets
# if not si.analysis.tag == 'invalid'])
# groups=[]
def gfactory(klass, ia):
hid = db.get_interpreted_age_history(ia.id)
ans = (si.analysis for si in hid.interpreted_age.sets \
if not si.analysis.tag == 'invalid')
ans = self.processor.make_analyses(ans,
calculate_age=True, use_cache=False,
progress=prog)
return klass(sample=ans[0].sample, analyses=ans)
#partition fusion vs stepheat
fusion, step_heat=partition(ias, lambda x: x.age_kind=='Weighted Mean')
# for ia in step_heat:
# groups.append(klass(sample=ans[0].sample,
# analyses=ans))
shgroups=[gfactory(StepHeatAnalysisGroup, ia) for ia in step_heat]
fgroups=[gfactory(AnalysisGroup, ia) for ia in fusion]
prog.close()
head, ext=os.path.splitext(p)
if shgroups:
w = StepHeatPDFTableWriter()
p='{}.step_heat_data{}'.format(head, ext)
w.build(p, shgroups, title=self.get_title())
if fgroups:
w=FusionPDFTableWriter()
p='{}.fusion_data{}'.format(head, ext)
w.build(p,fgroups, title=self.get_title())
def dump_fits(self, keys, reviewed=False):
sisos = self.isotopes
isoks, dks = list(map(tuple, partition(keys, lambda x: x in sisos)))
def update(d, i):
d.update(fit=i.fit, value=float(i.value), error=float(i.error),
n=i.n, fn=i.fn,
reviewed=reviewed,
include_baseline_error=i.include_baseline_error,
filter_outliers_dict=i.filter_outliers_dict,
user_excluded=i.user_excluded,
outlier_excluded=i.outlier_excluded)
# save intercepts
if isoks:
isos, path = self._get_json('intercepts')
for k in isoks:
try:
iso = isos[k]
except KeyError:
iso = {}
isos[k] = iso
siso = sisos[k]
if siso:
update(iso, siso)
self._dump(isos, path)
# save baselines
if dks:
baselines, path = self._get_json('baselines')
for di in dks:
try:
det = baselines[di]
except KeyError:
det = {}
baselines[di] = det
# bs = next((iso.baseline for iso in six.itervalues(sisos) if iso.detector == di), None)
bs = self.get_isotope(detector=di, kind='baseline')
if bs:
update(det, bs)
self._dump(baselines, path)
def save_analysis_data_table(self, p):
# ans=[]
db = self.processor.db
with db.session_ctx():
ias = self.interpreted_ages[:10]
ans = [
si.analysis
for ia in ias
for si in db.get_interpreted_age_history(ia.id).interpreted_age.sets
if si.analysis.tag != "invalid"
]
prog = self.processor.open_progress(len(ans), close_at_end=False)
# hid = db.get_interpreted_age_history(ia.id)
# dbia = hid.interpreted_age
# ans.extend([si.analysis for si in db.get_interpreted_age_history(ia.id).interpreted_age.sets
# if not si.analysis.tag == 'invalid'])
# groups=[]
def gfactory(klass, ia):
hid = db.get_interpreted_age_history(ia.id)
ans = (si.analysis for si in hid.interpreted_age.sets if not si.analysis.tag == "invalid")
ans = self.processor.make_analyses(ans, calculate_age=True, use_cache=False, progress=prog)
return klass(sample=ans[0].sample, analyses=ans)
# partition fusion vs stepheat
fusion, step_heat = partition(ias, lambda x: x.age_kind == "Weighted Mean")
# for ia in step_heat:
# groups.append(klass(sample=ans[0].sample,
# analyses=ans))
shgroups = [gfactory(StepHeatAnalysisGroup, ia) for ia in step_heat]
fgroups = [gfactory(AnalysisGroup, ia) for ia in fusion]
prog.close()
head, ext = os.path.splitext(p)
if shgroups:
w = StepHeatPDFTableWriter()
p = "{}.step_heat_data{}".format(head, ext)
w.build(p, shgroups, title=self.get_title())
if fgroups:
w = FusionPDFTableWriter()
p = "{}.fusion_data{}".format(head, ext)
w.build(p, fgroups, title=self.get_title())
def _retrieve_labnumbers(self):
es = []
ps = []
ms = []
if self.mass_spectrometers_enabled:
if self.mass_spectrometer_includes:
ms = self.mass_spectrometer_includes
principal_investigator = None
if self.principal_investigator_enabled:
principal_investigator = self.principal_investigator
if self.repository_enabled:
if self.selected_repositories:
es = [e.name for e in self.selected_repositories]
if self.project_enabled:
if self.selected_projects:
rs, ps = partition([p.name for p in self.selected_projects], lambda x: x.startswith('RECENT'))
ps, rs = list(ps), list(rs)
if rs:
hpost = datetime.now()
lpost = hpost - timedelta(hours=self.search_criteria.recent_hours)
self._low_post = lpost
self.use_high_post = False
self.use_low_post = True
self.trait_property_changed('low_post', self._low_post)
for ri in rs:
mi = extract_mass_spectrometer_name(ri)
if mi not in ms:
ms.append(mi)
self._recent_mass_spectrometers.append(mi)
ls = self.db.get_labnumbers(principal_investigator=principal_investigator,
projects=ps, repositories=es,
mass_spectrometers=ms,
irradiation=self.irradiation if self.irradiation_enabled else None,
level=self.level if self.irradiation_enabled else None,
analysis_types=self.analysis_include_types if self.use_analysis_type_filtering else None,
high_post=self.high_post if self.use_high_post else None,
low_post=self.low_post if self.use_low_post else None)
return ls
def dump_fits(self, keys, reviewed=False):
sisos = self.isotopes
isoks, dks = list(map(tuple, partition(keys, lambda x: x in sisos)))
def update(d, i):
fd = i.filter_outliers_dict
d.update(fit=i.fit, value=float(i.value), error=float(i.error),
n=i.n, fn=i.fn,
reviewed=reviewed,
include_baseline_error=i.include_baseline_error,
filter_outliers_dict=fd)
# save intercepts
if isoks:
isos, path = self._get_json('intercepts')
for k in isoks:
try:
iso = isos[k]
siso = sisos[k]
if siso:
update(iso, siso)
except KeyError:
pass
self._dump(isos, path)
# save baselines
if dks:
baselines, path = self._get_json('baselines')
for di in dks:
try:
det = baselines[di]
except KeyError:
det = {}
baselines[di] = det
# bs = next((iso.baseline for iso in six.itervalues(sisos) if iso.detector == di), None)
bs = self.get_isotope(detector=di, kind='baseline')
if bs:
update(det, bs)
self._dump(baselines, path)
def dump_fits(self, keys, reviewed=False):
sisos = self.isotopes
isoks, dks = map(tuple, partition(keys, lambda x: x in sisos))
def update(d, i):
fd = i.filter_outliers_dict
d.update(fit=i.fit, value=float(i.value), error=float(i.error),
n=i.n, fn=i.fn,
include_baseline_error=i.include_baseline_error,
filter_outliers=fd.get('filter_outliers', False),
iterations=fd.get('iterations', 0),
std_devs=fd.get('std_devs', 0))
# save intercepts
if isoks:
isos, path = self._get_json('intercepts')
for k in isoks:
try:
iso = isos[k]
siso = sisos[k]
update(iso, siso)
except KeyError:
pass
isos['reviewed'] = reviewed
self._dump(isos, path)
# save baselines
if dks:
baselines, path = self._get_json('baselines')
for di in dks:
try:
det = baselines[di]
except KeyError:
det = {}
baselines[di] = det
bs = next((iso.baseline for iso in sisos.itervalues() if iso.detector == di), None)
update(det, bs)
self._dump(baselines, path)
def _assemble_groups(self, ias):
db = self.processor.db
with db.session_ctx():
# ias = [ia for ia in ias if ia.age_kind == 'Weighted Mean'][:1]
ans = [si.analysis for ia in ias
for si in db.get_interpreted_age_history(ia.id).interpreted_age.sets]
prog = self.processor.open_progress(len(ans), close_at_end=False)
def gfactory(klass, dbia):
hid = db.get_interpreted_age_history(dbia.id)
ia_ans = (si.analysis for si in hid.interpreted_age.sets)
all_ans = self.processor.make_analyses(ia_ans,
calculate_age=True,
use_cache=False,
progress=prog)
#overwrite the tags for the analyses
for ai, sai in zip(all_ans, ia_ans):
ai.set_tag(sai.tag)
ais = [ai for ai in all_ans if not 'omit' in ai.tag]
return klass(sample=ais[0].sample,
all_analyses=all_ans,
analyses=ais)
#partition fusion vs stepheat
fusion, step_heat = partition(ias, lambda x: x.age_kind == 'Weighted Mean')
shgroups = [(ia, gfactory(StepHeatAnalysisGroup, ia)) for ia in step_heat]
# shgroups = [(ia, gfactory(StepHeatAnalysisGroup, ia)) for ia in list(step_heat)[:3]]
# shgroups =[]
# fgroups = [(ia, gfactory(AnalysisGroup, ia)) for ia in fusion]
# fgroups = [(ia, gfactory(AnalysisGroup, ia)) for ia in list(fusion)[:3]]
fgroups = []
prog.close()
return shgroups, fgroups
def save_analysis_data_tables(self, root, pdf=True, xls=True,
xls_summary=False,
auto_view=False):
ias = self.interpreted_ages
db = self.processor.db
with db.session_ctx():
#partition into map/argus
def pred(ia):
hid = db.get_interpreted_age_history(ia.id)
ref = hid.interpreted_age.sets[0].analysis
return ref.measurement.mass_spectrometer.name.lower() == 'map'
part = partition(ias, pred)
map_spec, argus = map(list, part)
if pdf:
self.debug('saving pdf tables')
step_heat_title = 'Table E.1 MAP Step Heat <sup>40</sup>Ar/<sup>39</sup>Ar Data'
fusion_title = 'Table D.1 MAP Fusion <sup>40</sup>Ar/<sup>39</sup>Ar Data'
# self._save_pdf_data_table(root, map_spec, step_heat_title, fusion_title, 'map',
# auto_view=auto_view)
step_heat_title = 'Table G.1 Argus Step Heat <sup>40</sup>Ar/<sup>39</sup>Ar Data'
fusion_title = 'Table F.1 Argus Fusion <sup>40</sup>Ar/<sup>39</sup>Ar Data'
self._save_pdf_data_table(root, argus, step_heat_title, fusion_title, 'argus',
auto_view=auto_view)
if xls:
self.debug('saving xls tables')
step_heat_title = 'Table 1. MAP Step heat 40Ar/39Ar Data'
fusion_title = 'Table 2. MAP Fusion 40Ar/39Ar Data'
self._save_xls_data_table(root, map_spec, step_heat_title, fusion_title, 'map',
summary_sheet=xls_summary,
auto_view=auto_view)
step_heat_title = 'Table 3. Argus Step heat 40Ar/39Ar Data'
fusion_title = 'Table 4. Argus Fusion 40Ar/39Ar Data'
self._save_xls_data_table(root, argus, step_heat_title, fusion_title, 'argus',
summary_sheet=xls_summary,
auto_view=auto_view)
def _make_multi_panel_labnumbers(self, ans, cnt):
root = self._config_options['root']
options = self._config_options['options']
# ans = [ai for li in lns for ai in li.analyses]
# ans = filter(lambda x: not x.tag == 'invalid', ans)
# prog = self.open_progress(len(ans), close_at_end=False)
# ans = self.make_analyses(ans,
# progress=prog,
# use_cache=False)
# print lns
lns = list({ai.labnumber.identifier for ai in ans})
print len(lns)
prog = None
pred = lambda x: bool(x.step)
ident = ','.join([li for li in lns])
li = ident
ident = '{:03d}-{}'.format(cnt, ident)
ln_root = os.path.join(root, ident)
r_mkdir(ln_root)
ans = sorted(ans, key=pred)
stepheat, fusion = map(list, partition(ans, pred))
project = 'Minna Bluff'
if stepheat and options.has_key('step_heat'):
# key = lambda x: x.aliquot
# stepheat = sorted(stepheat, key=key)
# for aliquot, ais in groupby(stepheat, key=key):
# name = make_runid(li, aliquot, '')
self._make_editor(
ans, 'step_heat', options, prog, False,
lambda x: '{}-{}'.format(x.identifier, x.aliquot),
(ln_root, 'spec', li, project, (li, )))
if fusion and options.has_key('fusion'):
self._make_editor(fusion, 'fusion', options, prog, False,
lambda x: x.identifier,
(ln_root, 'fig', li, project, (li, )))
def make_sample_points(geo):
db = geo.db
pr = 'Minna Bluff'
missing = []
groups = []
with db.session_ctx():
v = db.get_project(pr)
samples = v.samples
yr1, yr2 = partition(samples, lambda x: x.name.startswith('MB06'))
for name, sams in (('MB06_all_samples', yr1),
('MB07_all_samples', yr2)):
pts = []
for s in sams:
if not s.lat:
missing.append((s.name, s.alt_name))
else:
print 'adding {} lat: {:0.5f} long: {:0.5f}'.format(s.name, s.lat, s.long)
x, y = proj_pt(s.lat, s.long)
pt = SamplePoint(x, y, s.name, s.material.name if s.material else '',
','.join([li.identifier for li in s.labnumbers]))
pts.append(pt)
groups.append((name, pts))
return groups
def _calculate_cached_result_points(self, comp):
choles, fholes = partition(self.results, lambda r: r[1])
cpts = comp.map_screen([(x, y) for (x, y), _ in choles])
fpts = comp.map_screen([(x, y) for (x, y), _ in fholes])
self._cached_result_pts = (cpts, (0, 1, 1), 2), (fpts, (1, 0, 0), 3)
def group_labnumbers(self, ls):
def monitor_filter(pos):
return pos.sample.name == 'FC-2'
return partition(ls, monitor_filter)
def _calculate_cached_result_points(self, comp):
choles, fholes = partition(self.results, lambda r: r[1])
cpts = comp.map_screen([(x, y) for (x, y), _ in choles])
fpts = comp.map_screen([(x, y) for (x, y), _ in fholes])
self._cached_result_pts = (cpts, (0, 1, 1), 2), (fpts, (1, 0, 0), 3)
def make_analyses(self, ans,
progress=None,
exclude=None,
use_cache=True,
**kw):
"""
loading the analysis' signals appears to be the most expensive operation.
the majority of the load time is in _construct_analysis
"""
if exclude:
ans = self.filter_analysis_tag(ans, exclude)
if not ans:
return []
db=self.db
with db.session_ctx():
if ans:
#partition into DBAnalysis vs IsotopeRecordView
db_ans, no_db_ans = map(list, partition(ans, lambda x: isinstance(x, DBAnalysis)))
if no_db_ans:
#partition into cached and non cached analyses
cached_ans, no_db_ans = partition(no_db_ans,
lambda x: x.uuid in ANALYSIS_CACHE)
cached_ans = list(cached_ans)
#add analyses from cache to db_ans
db_ans.extend([ANALYSIS_CACHE[ci.uuid] for ci in cached_ans])
#increment value in cache_count
for ci in cached_ans:
self._add_to_cache(ci)
#load remaining analyses
no_db_ans = list(no_db_ans)
n = len(no_db_ans)
if n:
if self.use_vcs:
#clone the necessary project repositories
def f(x):
try:
return x.labnumber.sample.project.name
except AttributeError:
pass
prs=filter(lambda x: not x is None, (f(ai) for ai in no_db_ans))
self.vcs.clone_project_repos(prs)
if n > 1:
if progress is not None:
if progress.max < (n + progress.get_value()):
progress.increase_max(n+2)
else:
progress = self._open_progress(n+2)
new_ans=[]
for i, ai in enumerate(no_db_ans):
if progress:
if progress.canceled:
self.debug('canceling make analyses')
db_ans=[]
new_ans=[]
break
elif progress.accepted:
self.debug('accepting {}/{} analyses'.format(i, n))
break
a = self._construct_analysis(ai, progress, **kw)
if a:
if use_cache:
self._add_to_cache(a)
new_ans.append(a)
# if progress:
# progress.on_trait_change(self._progress_closed,
# 'closed', remove=True)
db_ans.extend(new_ans)
# self.debug('use vcs {}'.format(self.use_vcs))
# if self.use_vcs:
# if progress:
# progress.increase_max(len(new_ans)+1)
# progress.change_message('Adding analyses to vcs')
#
# self.vcs.add_analyses(new_ans, progress=progress)
self.debug('use offline database {}'.format(self.use_offline_database))
if self.use_offline_database:
if progress:
progress.increase_max(len(new_ans)+1)
progress.change_message('Transfering analyses for offline usage')
self.offline_bridge.add_analyses(db, new_ans, progress=progress)
if progress:
progress.soft_close()
return db_ans
def _easy_func(self, ep, manager):
db = self.manager.db
doc = ep.doc('blanks')
fits = doc['blank_fit_isotopes']
projects = doc['projects']
unks = [ai for proj in projects
for si in db.get_samples(project=proj)
for ln in si.labnumbers
for ai in ln.analyses
if ai.measurement.mass_spectrometer.name == 'MAP'
and ai.extraction.extraction_device.name in ('Furnace', 'Eurotherm')]
# for proj in projects:
# for si in db.get_samples(project=proj):
# for ln in si.labnumbers:
# for ai in ln.analyses:
# print ai.measurement.mass_spectrometer.name,ai.extraction.extraction_device.name
# print ai.measurement.mass_spectrometer.name == 'nmgrl map' and ai.extraction.extraction_device.name in ('Furnace','Eurotherm')
print len(unks)
prog = manager.progress
# prog = self.manager.open_progress(len(ans) + 1)
#bin analyses
prog.increase_max(len(unks))
preceding_fits, non_preceding_fits = map(list, partition(fits, lambda x: x['fit'] == 'preceding'))
if preceding_fits:
for ai in unks:
if prog.canceled:
return
elif prog.accepted:
break
l, a, s = ai.labnumber.identifier, ai.aliquot, ai.step
prog.change_message('Save preceding blank for {}-{:02n}{}'.format(l, a, s))
hist = db.add_history(ai, 'blanks')
ai.selected_histories.selected_blanks = hist
for fi in preceding_fits:
self._preceding_correct(db, fi, ai, hist)
#make figure root dir
if doc['save_figures']:
root = doc['figure_root']
r_mkdir(root)
with no_auto_ctx(self.active_editor):
if non_preceding_fits:
for fi in self.active_editor.tool.fits:
fi.fit = 'average'
fi.error_type = 'SEM'
fi.filter_outliers = True
fi.filter_iterations = 1
fi.filter_std_devs = 2
for ais in bin_analyses(unks):
if prog.canceled:
return
elif prog.accepted:
break
self.active_editor.set_items(ais, progress=prog)
self.active_editor.find_references(progress=prog)
#refresh graph
# invoke_in_main_thread(self.active_editor.rebuild_graph)
#
# if not manager.wait_for_user():
# return
#save a figure
if doc['save_figures']:
title = self.active_editor.make_title()
p = os.path.join(root, add_extension(title, '.pdf'))
self.active_editor.save_file(p)
self.active_editor.save(progress=prog)
self.active_editor.dump_tool()
return True
def make_analyses(self, ans,
progress=None,
use_progress=True,
exclude=None,
use_cache=True,
unpack=False,
calculate_age=False,
calculate_F=False,
load_aux=False,
**kw):
"""
loading the analysis' signals appears to be the most expensive operation.
the majority of the load time is in _construct_analysis
"""
if exclude:
ans = self.filter_analysis_tag(ans, exclude)
if not ans:
self.debug('no analyses to load')
return []
db = self.db
with db.session_ctx():
# partition into DBAnalysis vs IsotopeRecordView
db_ans, no_db_ans = map(list, partition(ans, lambda x: isinstance(x, DBAnalysis)))
self._calculate_cached_ages(db_ans, calculate_age, calculate_F)
if unpack:
for di in db_ans:
if not di.has_raw_data:
no_db_ans.append(di)
db_ans.remove(di)
if load_aux:
for di in db_ans:
if not di.has_changes:
if di not in no_db_ans:
no_db_ans.append(di)
db_ans.remove(di)
if no_db_ans:
if use_cache:
# partition into cached and non cached analyses
cached_ans, no_db_ans = partition(no_db_ans,
lambda x: x.uuid in ANALYSIS_CACHE)
cached_ans = list(cached_ans)
no_db_ans = list(no_db_ans)
cns = [ANALYSIS_CACHE[ci.uuid] for ci in cached_ans]
# if unpack is true make sure cached analyses have raw data
if unpack or load_aux:
if unpack:
a, b = self._unpack_cached_analyses(cns, calculate_age, calculate_F)
db_ans.extend(a)
no_db_ans.extend(b)
if load_aux:
a, b = self._load_aux_cached_analyses(cns)
db_ans.extend(a)
no_db_ans.extend(b)
else:
self._calculate_cached_ages(cns, calculate_age, calculate_F)
# add analyses from cache to db_ans
db_ans.extend(cns)
# increment value in cache_count
self._increment_cache(cached_ans, use_cache)
# load remaining analyses
n = len(no_db_ans)
if n:
# self._clone_vcs_repos(no_db_ans)
progress = self._setup_progress(n, progress, use_progress)
db_ans, new_ans = self._construct_analyses(no_db_ans, db_ans, progress,
calculate_age, calculate_F,
unpack, use_cache,
load_aux=load_aux, **kw)
db_ans.extend(new_ans)
# self.debug('use vcs {}'.format(self.use_vcs))
# if self.use_vcs:
# if progress:
# progress.increase_max(len(new_ans)+1)
# progress.change_message('Adding analyses to vcs')
#
# self.vcs.add_analyses(new_ans, progress=progress)
# self.debug('use offline database {}'.format(self.use_offline_database))
# if self.use_offline_database:
# if progress:
# progress.increase_max(len(new_ans) + 1)
# progress.change_message('Transfering analyses for offline usage')
# self.offline_bridge.add_analyses(db, new_ans, progress=progress)
if progress:
progress.soft_close()
return db_ans
def group_labnumbers(self, ls):
def monitor_filter(pos):
return pos.sample.name == 'FC-2'
return partition(ls, monitor_filter)
def make_analyses(self,
ans,
progress=None,
exclude=None,
use_cache=True,
**kw):
"""
loading the analysis' signals appears to be the most expensive operation.
the majority of the load time is in _construct_analysis
"""
if exclude:
ans = self.filter_analysis_tag(ans, exclude)
if not ans:
return []
db = self.db
with db.session_ctx():
if ans:
#partition into DBAnalysis vs IsotopeRecordView
db_ans, no_db_ans = map(
list, partition(ans, lambda x: isinstance(x, DBAnalysis)))
if no_db_ans:
#partition into cached and non cached analyses
cached_ans, no_db_ans = partition(
no_db_ans, lambda x: x.uuid in ANALYSIS_CACHE)
cached_ans = list(cached_ans)
#add analyses from cache to db_ans
db_ans.extend(
[ANALYSIS_CACHE[ci.uuid] for ci in cached_ans])
#increment value in cache_count
for ci in cached_ans:
self._add_to_cache(ci)
#load remaining analyses
no_db_ans = list(no_db_ans)
n = len(no_db_ans)
if n:
if self.use_vcs:
#clone the necessary project repositories
def f(x):
try:
return x.labnumber.sample.project.name
except AttributeError:
pass
prs = filter(lambda x: not x is None,
(f(ai) for ai in no_db_ans))
self.vcs.clone_project_repos(prs)
if n > 1:
if progress is not None:
if progress.max < (n + progress.get_value()):
progress.increase_max(n + 2)
else:
progress = self._open_progress(n + 2)
new_ans = []
for i, ai in enumerate(no_db_ans):
if progress:
if progress.canceled:
self.debug('canceling make analyses')
db_ans = []
new_ans = []
break
elif progress.accepted:
self.debug(
'accepting {}/{} analyses'.format(
i, n))
break
a = self._construct_analysis(ai, progress, **kw)
if a:
if use_cache:
self._add_to_cache(a)
new_ans.append(a)
# if progress:
# progress.on_trait_change(self._progress_closed,
# 'closed', remove=True)
db_ans.extend(new_ans)
# self.debug('use vcs {}'.format(self.use_vcs))
# if self.use_vcs:
# if progress:
# progress.increase_max(len(new_ans)+1)
# progress.change_message('Adding analyses to vcs')
#
# self.vcs.add_analyses(new_ans, progress=progress)
self.debug('use offline database {}'.format(
self.use_offline_database))
if self.use_offline_database:
if progress:
progress.increase_max(len(new_ans) + 1)
progress.change_message(
'Transfering analyses for offline usage')
self.offline_bridge.add_analyses(db,
new_ans,
progress=progress)
if progress:
progress.soft_close()
return db_ans
def _easy_func(self, ep, manager):
db=self.manager.db
doc = ep.doc('blanks')
fits = doc['blank_fit_isotopes']
projects = doc['projects']
unks = [ai for proj in projects
for si in db.get_samples(project=proj)
for ln in si.labnumbers
for ai in ln.analyses]
prog=manager.progress
# prog = self.manager.open_progress(len(ans) + 1)
#bin analyses
prog.increase_max(len(unks))
preceding_fits, non_preceding_fits=map(list,partition(fits, lambda x: x['fit']=='preceding'))
if preceding_fits:
for ai in unks:
if prog.canceled:
return
elif prog.accepted:
break
l, a, s = ai.labnumber.identifier, ai.aliquot, ai.step
prog.change_message('Save preceding blank for {}-{:02n}{}'.format(l, a, s))
hist = db.add_history(ai, 'blanks')
ai.selected_histories.selected_blanks = hist
for fi in preceding_fits:
self._preceding_correct(db, fi, ai, hist)
#make figure root dir
if doc['save_figures']:
root = doc['figure_root']
r_mkdir(root)
with no_auto_ctx(self.active_editor):
if non_preceding_fits:
for ais in bin_analyses(unks):
if prog.canceled:
return
elif prog.accepted:
break
self.active_editor.set_items(ais, progress=prog)
self.active_editor.find_references(progress=prog)
#refresh graph
invoke_in_main_thread(self.active_editor.rebuild_graph)
if not manager.wait_for_user():
return
#save a figure
if doc['save_figures']:
title=self.active_editor.make_title()
p=os.path.join(root, add_extension(title,'.pdf'))
self.active_editor.save_file(p)
self.active_editor.save(progress=prog)
self.active_editor.dump_tool()
return True
def make_analyses(self,
ans,
progress=None,
use_progress=True,
exclude=None,
use_cache=True,
unpack=False,
calculate_age=False,
calculate_F=False,
load_aux=False,
**kw):
"""
loading the analysis' signals appears to be the most expensive operation.
the majority of the load time is in _construct_analysis
"""
if exclude:
ans = self.filter_analysis_tag(ans, exclude)
if not ans:
self.debug('no analyses to load')
return []
db = self.db
with db.session_ctx():
# partition into DBAnalysis vs IsotopeRecordView
db_ans, no_db_ans = list(
map(list, partition(ans, lambda x: isinstance(x, DBAnalysis))))
self._calculate_cached_ages(db_ans, calculate_age, calculate_F)
if unpack:
for di in db_ans:
if not di.has_raw_data:
no_db_ans.append(di)
db_ans.remove(di)
if load_aux:
for di in db_ans:
if not di.has_changes:
if di not in no_db_ans:
no_db_ans.append(di)
db_ans.remove(di)
if no_db_ans:
if use_cache:
# partition into cached and non cached analyses
cached_ans, no_db_ans = partition(
no_db_ans, lambda x: x.uuid in ANALYSIS_CACHE)
cached_ans = list(cached_ans)
no_db_ans = list(no_db_ans)
cns = [ANALYSIS_CACHE[ci.uuid] for ci in cached_ans]
# if unpack is true make sure cached analyses have raw data
if unpack or load_aux:
if unpack:
a, b = self._unpack_cached_analyses(
cns, calculate_age, calculate_F)
db_ans.extend(a)
no_db_ans.extend(b)
if load_aux:
a, b = self._load_aux_cached_analyses(cns)
db_ans.extend(a)
no_db_ans.extend(b)
else:
self._calculate_cached_ages(cns, calculate_age,
calculate_F)
# add analyses from cache to db_ans
db_ans.extend(cns)
# increment value in cache_count
self._increment_cache(cached_ans, use_cache)
# load remaining analyses
n = len(no_db_ans)
if n:
# self._clone_vcs_repos(no_db_ans)
progress = self._setup_progress(n, progress, use_progress)
db_ans, new_ans = self._construct_analyses(
no_db_ans,
db_ans,
progress,
calculate_age,
calculate_F,
unpack,
use_cache,
use_progress,
load_aux=load_aux,
**kw)
db_ans.extend(new_ans)
# self.debug('use vcs {}'.format(self.use_vcs))
# if self.use_vcs:
# if progress:
# progress.increase_max(len(new_ans)+1)
# progress.change_message('Adding analyses to vcs')
#
# self.vcs.add_analyses(new_ans, progress=progress)
# self.debug('use offline database {}'.format(self.use_offline_database))
# if self.use_offline_database:
# if progress:
# progress.increase_max(len(new_ans) + 1)
# progress.change_message('Transferring analyses for offline usage')
# self.offline_bridge.add_analyses(db, new_ans, progress=progress)
if progress:
progress.soft_close()
return db_ans
