def _get_selected_hexsha(self, commits, lc, rc, view_klass=None, auto_select=True,
tags=None, **kw):
if view_klass is None:
view_klass = CommitView
lha = lc.hexsha[:7] if lc else ''
rha = rc.hexsha[:7] if rc else ''
ld = get_datetime(float(lc.committed_date)).strftime('%m-%d-%Y')
rd = get_datetime(float(rc.committed_date)).strftime('%m-%d-%Y') if rc else ''
n = len(commits)
h = UpdateGitHistory(n=n, branchname=self.branch,
local_commit='{} ({})'.format(ld, lha),
head_hexsha=lc.hexsha,
latest_remote_commit='{} ({})'.format(rd, rha),
**kw)
repo = self._repo
commits = [repo.commit(i) for i in commits]
h.set_items(commits, auto_select=auto_select)
if tags:
h.set_tags(tags)
cv = view_klass(model=h)
info = cv.edit_traits()
if info.result:
if h.selected:
return h.selected.hexsha
def find_project_overlaps(self, projects):
tol_hrs = 6
src = self.processor.db
pdict = {}
for p in projects:
for ms in ('jan', 'obama'):
ts, idxs = self._get_project_timestamps(p, ms, tol_hrs=tol_hrs)
for i, ais in enumerate(array_split(ts, idxs + 1)):
if not ais.shape[0]:
self.debug('skipping {}'.format(i))
continue
low = get_datetime(ais[0]) - timedelta(hours=tol_hrs / 2.)
high = get_datetime(ais[-1]) + timedelta(hours=tol_hrs / 2.)
print '========{}, {}, {}'.format(ms, low, high)
with src.session_ctx():
runs = src.get_analyses_date_range(low, high,
projects=('REFERENCES',),
mass_spectrometers=(ms,))
pdict[p] = [ai.record_id for ai in runs]
for p in projects:
for o in projects:
if p == o:
continue
pruns = pdict[p]
oruns = pdict[o]
for ai in pruns:
if ai in oruns:
print p, o, ai
def bulk_import_project(self, project, principal_investigator, source_name=None, dry=True):
src = self.processor.db
tol_hrs = 6
self.debug('bulk import project={}, pi={}'.format(project, principal_investigator))
oruns = []
repository_identifier = project
# def filterfunc(x):
# a = x.labnumber.irradiation_position is None
# b = False
# if not a:
# b = x.labnumber.irradiation_position.level.irradiation.name == irradname
#
# d = False
# if x.extraction:
# ed = x.extraction.extraction_device
# if not ed:
# d = True
# else:
# d = ed.name == 'Fusions CO2'
#
# return (a or b) and d
if source_name is None:
source_name = project
for ms in ('jan', 'obama', 'felix'):
ts, idxs = self._get_project_timestamps(source_name, ms, tol_hrs=tol_hrs)
for i, ais in enumerate(array_split(ts, idxs + 1)):
if not ais.shape[0]:
self.debug('skipping {}'.format(i))
continue
low = get_datetime(ais[0]) - timedelta(hours=tol_hrs / 2.)
high = get_datetime(ais[-1]) + timedelta(hours=tol_hrs / 2.)
print('========{}, {}, {}'.format(ms, low, high))
with src.session_ctx():
runs = src.get_analyses_date_range(low, high,
# labnumber=(63630, 63632, 63634, 63636, 63638, 63646, 63648),
projects=('REFERENCES', ),
# projects=('REFERENCES', source_name),
mass_spectrometers=(ms,))
if dry:
for ai in runs:
oruns.append(ai.record_id)
print(ai.measurement.mass_spectrometer.name, ai.record_id, ai.labnumber.sample.name, \
ai.analysis_timestamp)
else:
self.debug('================= Do Export i: {} low: {} high: {}'.format(i, low, high))
self.debug('N runs: {}'.format(len(runs)))
self.do_export([ai.record_id for ai in runs], repository_identifier, principal_investigator)
return oruns
def bulk_import_project(self, project, principal_investigator, dry=True):
src = self.processor.db
tol_hrs = 6
self.debug('bulk import project={}, pi={}'.format(project, principal_investigator))
oruns = []
repository_identifier = project
# def filterfunc(x):
# a = x.labnumber.irradiation_position is None
# b = False
# if not a:
# b = x.labnumber.irradiation_position.level.irradiation.name == irradname
#
# d = False
# if x.extraction:
# ed = x.extraction.extraction_device
# if not ed:
# d = True
# else:
# d = ed.name == 'Fusions CO2'
#
# return (a or b) and d
#
for ms in ('jan', 'obama'):
ts, idxs = self._get_project_timestamps(project, ms, tol_hrs=tol_hrs)
for i, ais in enumerate(array_split(ts, idxs + 1)):
if not ais.shape[0]:
self.debug('skipping {}'.format(i))
continue
low = get_datetime(ais[0]) - timedelta(hours=tol_hrs / 2.)
high = get_datetime(ais[-1]) + timedelta(hours=tol_hrs / 2.)
print '========{}, {}, {}'.format(ms, low, high)
with src.session_ctx():
runs = src.get_analyses_date_range(low, high,
projects=('REFERENCES', project),
mass_spectrometers=(ms,))
if dry:
for ai in runs:
oruns.append(ai.record_id)
print ai.measurement.mass_spectrometer.name, ai.record_id, ai.labnumber.sample.name, \
ai.analysis_timestamp
else:
self.debug('================= Do Export i: {} low: {} high: {}'.format(i, low, high))
self.debug('N runs: {}'.format(len(runs)))
self.do_export([ai.record_id for ai in runs], repository_identifier, principal_investigator)
return oruns
def _get_find_parameters(self):
f = FindAssociatedParametersDialog()
ais = self.active_editor.analyses
if ais:
unks = ais
elif self.analysis_table.selected:
ans = self.analysis_table.selected
unks = ans[:]
elif self.selected_samples:
ans = self.analysis_table.analyses
unks = ans[:]
elif self.selected_projects:
with self.manager.db.session_ctx():
ans = self._get_projects_analyzed_analyses(self.selected_projects)
unks = [IsotopeRecordView(ai) for ai in ans]
else:
self.information_dialog('Select a list of projects, samples or analyses')
return
ts = [get_datetime(ai.timestamp) for ai in unks]
lpost, hpost = min(ts), max(ts)
f.model.nominal_lpost_date = lpost.date()
f.model.nominal_hpost_date = hpost.date()
f.model.nominal_lpost_time = lpost.time()
f.model.nominal_hpost_time = hpost.time()
ms = list(set([ai.mass_spectrometer for ai in unks]))
f.model.available_mass_spectrometers = ms
f.model.mass_spectrometers = ms
info = f.edit_traits(kind='livemodal')
if info.result:
return f.model
def find_associated_analyses(self, found=None, use_cache=True, progress=None):
if self.active_editor:
unks = self.active_editor.analyses
key = lambda x: x.labnumber
unks = sorted(unks, key=key)
db = self.manager.db
with db.session_ctx():
tans = []
if found is None:
uuids = []
else:
uuids = found
ngroups = len(list(groupby(unks, key=key)))
if progress is None:
progress = self.manager.open_progress(ngroups + 1)
else:
progress.increase_max(ngroups + 1)
for ln, ais in groupby(unks, key=key):
msg = 'find associated analyses for labnumber {}'.format(ln)
self.debug(msg)
progress.change_message(msg)
ais = list(ais)
ts = [get_datetime(ai.timestamp) for ai in ais]
ref = ais[0]
ms = ref.mass_spectrometer
ed = ref.extract_device
self.debug("{} {}".format(ms, ed))
for atype in ('blank_unknown', 'blank_air', 'blank_cocktail',
'air', 'cocktail'):
for i in range(10):
td = timedelta(hours=6 * (i + 1))
lpost, hpost = min(ts) - td, max(ts) + td
ans = db.get_date_range_analyses(lpost, hpost,
atype=atype,
spectrometer=ms)
if ans:
self.debug('{} {} to {}. nanalyses={}'.format(atype, lpost, hpost, len(ans)))
ans = [ai for ai in ans if ai.uuid not in uuids]
self.debug('new ans {}'.format(len(ans)))
if ans:
tans.extend(ans)
uuids.extend([ai.uuid for ai in ans])
break
progress.soft_close()
self.active_editor.set_items(tans, is_append=True,
use_cache=use_cache, progress=progress)
return uuids
def pre_extraction_save(self):
"""
set runtime and rundate
"""
d = get_datetime()
self.runtime = d.time()
self.rundate = d.date()
self.info('Analysis started at {}'.format(self.runtime))
self._pre_extraction_save_hook()
def pre_extraction_save(self):
"""
set runtime and rundate
"""
d = get_datetime()
self.runtime = d.time()
self.rundate = d.date()
self.info('Analysis started at {}'.format(self.runtime))
self._pre_extraction_save_hook()
def factory(l):
l = l.strip()
if l:
l, t, p = l.split(',')
n = os.path.basename(p)
n, _ = os.path.splitext(n)
l = get_datetime(float(l)).strftime('%a %H:%M %m-%d-%Y')
return LaunchItem(name=n, path=p, last_run_time=l, total_launches=int(t))
def factory(l):
l = l.strip()
if l:
l, t, p = l.split(',')
n = os.path.basename(p)
n, _ = os.path.splitext(n)
l = get_datetime(float(l)).strftime('%a %H:%M %m-%d-%Y')
return LaunchItem(name=n, path=p, last_run_time=l, total_launches=int(t))
def get_project_bins(project):
# src = self.processor.db
src = IsotopeDatabaseManager(
dict(
host='localhost',
username=os.environ.get('LOCALHOST_DB_USER'),
password=os.environ.get('LOCALHOST_DB_PWD'),
kind='mysql',
# echo=True,
name='pychrondata'))
tol_hrs = 6
# self.debug('bulk import project={}, pi={}'.format(project, principal_investigator))
ts, idxs = get_project_timestamps(src.db, project, tol_hrs=tol_hrs)
# repository_identifier = project
# def filterfunc(x):
# a = x.labnumber.irradiation_position is None
# b = False
# if not a:
# b = x.labnumber.irradiation_position.level.irradiation.name == irradname
#
# d = False
# if x.extraction:
# ed = x.extraction.extraction_device
# if not ed:
# d = True
# else:
# d = ed.name == 'Fusions CO2'
#
# return (a or b) and d
#
for ms in ('jan', 'obama'):
for i, ais in enumerate(array_split(ts, idxs + 1)):
if not ais.shape[0]:
continue
low = get_datetime(ais[0]) - timedelta(hours=tol_hrs / 2.)
high = get_datetime(ais[-1]) + timedelta(hours=tol_hrs / 2.)
print ms, low, high
def seasonal_subseries(x, y, **kw):
ybins = [[], [], [], [], [], [], [], [], [], [], [], [],
[], [], [], [], [], [], [], [], [], [], [], []]
xbins = [[], [], [], [], [], [], [], [], [], [], [], [],
[], [], [], [], [], [], [], [], [], [], [], []]
m = 3600 * 24. / len(x)
for xi, yi in zip(x, y):
i = get_datetime(xi).hour
ybins[i - 1].append(yi)
xbins[i - 1].append((i - 1) * 3600 + (len(xbins[i - 1])) * m)
ms = [np.mean(x) for x in ybins]
return xbins, ybins, ms
def seasonal_subseries(x, y, **kw):
ybins = [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [],
[], [], [], [], [], [], [], []]
xbins = [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [],
[], [], [], [], [], [], [], []]
m = 3600 * 24. / len(x)
for xi, yi in zip(x, y):
i = get_datetime(xi).hour
ybins[i - 1].append(yi)
xbins[i - 1].append((i - 1) * 3600 + (len(xbins[i - 1])) * m)
ms = [np.mean(x) for x in ybins]
return xbins, ybins, ms
def get_project_bins(project):
# src = self.processor.db
src = IsotopeDatabaseManager(dict(host='localhost',
username=os.environ.get('LOCALHOST_DB_USER'),
password=os.environ.get('LOCALHOST_DB_PWD'),
kind='mysql',
# echo=True,
name='pychrondata'))
tol_hrs = 6
# self.debug('bulk import project={}, pi={}'.format(project, principal_investigator))
ts, idxs = get_project_timestamps(src.db, project, tol_hrs=tol_hrs)
# repository_identifier = project
# def filterfunc(x):
# a = x.labnumber.irradiation_position is None
# b = False
# if not a:
# b = x.labnumber.irradiation_position.level.irradiation.name == irradname
#
# d = False
# if x.extraction:
# ed = x.extraction.extraction_device
# if not ed:
# d = True
# else:
# d = ed.name == 'Fusions CO2'
#
# return (a or b) and d
#
for ms in ('jan', 'obama'):
for i, ais in enumerate(array_split(ts, idxs + 1)):
if not ais.shape[0]:
continue
low = get_datetime(ais[0]) - timedelta(hours=tol_hrs / 2.)
high = get_datetime(ais[-1]) + timedelta(hours=tol_hrs / 2.)
print(ms, low, high)
def seasonal_subseries(x, y, **kw):
from pychron.core.helpers.datetime_tools import get_datetime
ybins = [[], [], [], [], [], [], [], [], [], [], [], [],
[], [], [], [], [], [], [], [], [], [], [], []]
xbins = [[], [], [], [], [], [], [], [], [], [], [], [],
[], [], [], [], [], [], [], [], [], [], [], []]
m = 3600 * 24. / len(x)
for xi, yi in zip(x, y):
i = get_datetime(xi).hour
ybins[i - 1].append(yi)
xbins[i - 1].append((i - 1) * 3600 + (len(xbins[i - 1])) * m)
ms = [mean(x) for x in ybins]
return xbins, ybins, ms
def seasonal_subseries(x, y, **kw):
from pychron.core.helpers.datetime_tools import get_datetime
ybins = [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [],
[], [], [], [], [], [], [], []]
xbins = [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [],
[], [], [], [], [], [], [], []]
m = 3600 * 24. / len(x)
for xi, yi in zip(x, y):
i = get_datetime(xi).hour
ybins[i - 1].append(yi)
xbins[i - 1].append((i - 1) * 3600 + (len(xbins[i - 1])) * m)
ms = [mean(x) for x in ybins]
return xbins, ybins, ms
def _factory(self, ii):
li = LogItem()
li.name = 'foo'
json_level = str(ii.get('log_level'))
if 'debug' in json_level:
li.level = 'DEBUG'
elif 'info' in json_level:
li.level = 'WARNING'
else:
li.level = 'INFO'
li.timestamp = get_datetime(ii.get('log_time'))
fmt = ii.get('log_format')
li.message = str(fmt.format(**{k: tostr(v) for k, v in ii.items()}))
return li
def _factory(self, ii):
li = LogItem()
li.name = 'foo'
json_level = str(ii.get('log_level'))
if 'debug' in json_level:
li.level = 'DEBUG'
elif 'info' in json_level:
li.level = 'WARNING'
else:
li.level = 'INFO'
li.timestamp = get_datetime(ii.get('log_time'))
fmt = ii.get('log_format')
li.message = str(fmt.format(**{k: tostr(v) for k, v in ii.items()}))
return li
def _get_find_parameters(self):
f = FindAssociatedParametersDialog()
ais = self.active_editor.analyses
if ais:
unks = ais
elif self.analysis_table.selected:
ans = self.analysis_table.selected
unks = ans[:]
elif self.selected_samples:
ans = self.analysis_table.analyses
unks = ans[:]
elif self.selected_projects:
with self.manager.db.session_ctx():
ans = self._get_projects_analyzed_analyses(
self.selected_projects)
unks = [IsotopeRecordView(ai) for ai in ans]
else:
self.information_dialog(
'Select a list of projects, samples or analyses')
return
ts = [get_datetime(ai.timestamp) for ai in unks]
lpost, hpost = min(ts), max(ts)
f.model.nominal_lpost_date = lpost.date()
f.model.nominal_hpost_date = hpost.date()
f.model.nominal_lpost_time = lpost.time()
f.model.nominal_hpost_time = hpost.time()
ms = list(set([ai.mass_spectrometer for ai in unks]))
f.model.available_mass_spectrometers = ms
f.model.mass_spectrometers = ms
info = f.edit_traits(kind='livemodal')
if info.result:
return f.model
def post_measurement_save(self, save_local=True):
"""
check for runid conflicts. automatically update runid if conflict
#. save to primary database (aka mainstore)
#. save detector_ic to csv if applicable
#. save to secondary database
"""
if DEBUG:
self.debug('Not measurement saving to database')
return
self.info('post measurement save')
if not self.save_enabled:
self.info('Database saving disabled')
return
# check for conflicts immediately before saving
# automatically update if there is an issue
run_spec = self.per_spec.run_spec
conflict = self.datahub.is_conflict(run_spec)
if conflict:
self.debug('post measurement datastore conflict found. Automatically updating the aliquot and step')
self.datahub.update_spec(run_spec)
cp = self._current_data_frame
ln = run_spec.labnumber
aliquot = run_spec.aliquot
if save_local:
# save to local sqlite database for backup and reference
self._local_db_save()
# save to a database
db = self.datahub.mainstore.db
if not db or not db.connected:
self.warning('No database instanc. Not saving post measurement to primary database')
else:
with db.session_ctx() as sess:
pt = time.time()
lab = db.get_labnumber(ln)
endtime = get_datetime().time()
self.info('analysis finished at {}'.format(endtime))
un = run_spec.username
dbuser = db.get_user(un)
if dbuser is None:
self.debug('user= {} does not existing. adding to database now'.format(un))
dbuser = db.add_user(un)
self.debug('adding analysis identifier={}, aliquot={}, '
'step={}, increment={}'.format(ln, aliquot,
run_spec.step,
run_spec.increment))
a = db.add_analysis(lab,
user=dbuser,
uuid=run_spec.uuid,
endtime=endtime,
aliquot=aliquot,
step=run_spec.step,
increment=run_spec.increment,
comment=run_spec.comment,
whiff_result=self.per_spec.whiff_result)
sess.flush()
run_spec.analysis_dbid = a.id
run_spec.analysis_timestamp = a.analysis_timestamp
experiment = db.get_experiment(self.dbexperiment_identifier, key='id')
if experiment is not None:
# added analysis to experiment
a.experiment_id = experiment.id
else:
self.warning('no experiment found for {}'.format(self.dbexperiment_identifier))
# save measurement
meas = self._save_measurement(db, a)
# save extraction
ext = self._db_extraction_id
if ext is not None:
dbext = db.get_extraction(ext, key='id')
a.extraction_id = dbext.id
# save sensitivity info to extraction
self._save_sensitivity(dbext, meas)
else:
self.debug('no extraction to associate with this run')
self._save_spectrometer_info(db, meas)
# add selected history
db.add_selected_histories(a)
# self._save_isotope_info(a, ss)
self._save_isotope_data(db, a)
# save ic factor
self._save_detector_intercalibration(db, a)
# save blanks
self._save_blank_info(db, a)
# save peak center
self._save_peak_center(db, a, cp)
# save monitor
self._save_monitor_info(db, a)
# save gains
self._save_gains(db, a)
if self.use_analysis_grouping:
self._save_analysis_group(db, a)
# mem_log('post pychron save')
pt = time.time() - pt
self.debug('pychron save time= {:0.3f} '.format(pt))
# file_log(pt)
self.debug('$$$$$$$$$$$$$$$ auto_save_detector_ic={}'.format(self.per_spec.auto_save_detector_ic))
if self.per_spec.auto_save_detector_ic:
try:
self._save_detector_ic_csv()
except BaseException, e:
self.debug('Failed auto saving detector ic. {}'.format(e))
def post_measurement_save(self, save_local=True):
"""
check for runid conflicts. automatically update runid if conflict
#. save to primary database (aka mainstore)
#. save detector_ic to csv if applicable
#. save to secondary database
"""
# self.debug('AutomatedRunPersister post_measurement_save deprecated')
# return
if DEBUG:
self.debug('Not measurement saving to database')
return
self.info('post measurement save')
if not self.save_enabled:
self.info('Database saving disabled')
return
# check for conflicts immediately before saving
# automatically update if there is an issue
run_spec = self.per_spec.run_spec
# conflict = self.datahub.is_conflict(run_spec)
# if conflict:
# self.debug('post measurement datastore conflict found. Automatically updating the aliquot and step')
# self.datahub.update_spec(run_spec)
cp = self._current_data_frame
ln = run_spec.labnumber
aliquot = run_spec.aliquot
if save_local:
# save to local sqlite database for backup and reference
self._local_db_save()
# save to a database
db = self.datahub.get_db('isotopedb')
if not db or not db.connected:
self.warning(
'No database instance. Not saving post measurement to isotopedb database'
)
else:
with db.session_ctx() as sess:
pt = time.time()
lab = db.get_labnumber(ln)
endtime = get_datetime().time()
self.info('analysis finished at {}'.format(endtime))
un = run_spec.username
dbuser = db.get_user(un)
if dbuser is None:
self.debug(
'user= {} does not existing. adding to database now'.
format(un))
dbuser = db.add_user(un)
self.debug('adding analysis identifier={}, aliquot={}, '
'step={}, increment={}'.format(
ln, aliquot, run_spec.step, run_spec.increment))
a = db.add_analysis(lab,
user=dbuser,
uuid=run_spec.uuid,
endtime=endtime,
aliquot=aliquot,
step=run_spec.step,
increment=run_spec.increment,
comment=run_spec.comment,
whiff_result=self.per_spec.whiff_result)
sess.flush()
run_spec.analysis_dbid = a.id
run_spec.analysis_timestamp = a.analysis_timestamp
experiment = db.get_experiment(self.dbexperiment_identifier,
key='id')
if experiment is not None:
# added analysis to experiment
a.experiment_id = experiment.id
else:
self.warning('no experiment found for {}'.format(
self.dbexperiment_identifier))
# save measurement
meas = self._save_measurement(db, a)
# save extraction
ext = self._db_extraction_id
if ext is not None:
dbext = db.get_extraction(ext, key='id')
a.extraction_id = dbext.id
# save sensitivity info to extraction
self._save_sensitivity(dbext, meas)
else:
self.debug('no extraction to associate with this run')
self._save_spectrometer_info(db, meas)
# add selected history
db.add_selected_histories(a)
# self._save_isotope_info(a, ss)
self._save_isotope_data(db, a)
# save ic factor
self._save_detector_intercalibration(db, a)
# save blanks
self._save_blank_info(db, a)
# save peak center
self._save_peak_center(db, a, cp)
# save monitor
self._save_monitor_info(db, a)
# save gains
self._save_gains(db, a)
if self.use_analysis_grouping:
self._save_analysis_group(db, a)
# mem_log('post pychron save')
pt = time.time() - pt
self.debug('pychron save time= {:0.3f} '.format(pt))
# file_log(pt)
self.debug('$$$$$$$$$$$$$$$ auto_save_detector_ic={}'.format(
self.per_spec.auto_save_detector_ic))
if self.per_spec.auto_save_detector_ic:
try:
self._save_detector_ic_csv()
except BaseException as e:
self.debug('Failed auto saving detector ic. {}'.format(e))
# don't save detector_ic runs to mass spec
# measurement of an isotope on multiple detectors likely possible with mass spec but at this point
# not worth trying.
# if self.use_secondary_database:
if self.use_massspec_database:
from pychron.experiment.datahub import check_massspec_database_save
if check_massspec_database_save(ln):
if not self.datahub.store_connect('massspec'):
# if not self.massspec_importer or not self.massspec_importer.db.connected:
self.debug('Mass Spec database is not available')
else:
self.debug('saving post measurement to secondary database')
# save to massspec
mt = time.time()
self._save_to_massspec(cp)
self.debug(
'mass spec save time= {:0.3f}'.format(time.time() -
mt))
def _easy_find_associated_analyses(self,
found=None,
use_cache=True,
progress=None):
if self.active_editor:
unks = self.active_editor.analyses
key = lambda x: x.labnumber
unks = sorted(unks, key=key)
db = self.manager.db
with db.session_ctx():
tans = []
if found is None:
uuids = []
else:
uuids = found
ngroups = len(list(groupby(unks, key=key)))
if progress is None:
progress = self.manager.open_progress(ngroups + 1)
else:
progress.increase_max(ngroups + 1)
for ln, ais in groupby(unks, key=key):
msg = 'find associated analyses for labnumber {}'.format(
ln)
self.debug(msg)
progress.change_message(msg)
ais = list(ais)
ts = [get_datetime(ai.timestamp) for ai in ais]
ref = ais[0]
ms = ref.mass_spectrometer
ed = ref.extract_device
self.debug("{} {}".format(ms, ed))
for atype in ('blank_unknown', 'blank_air',
'blank_cocktail', 'air', 'cocktail'):
for i in range(10):
td = timedelta(hours=6 * (i + 1))
lpost, hpost = min(ts) - td, max(ts) + td
ans = db.get_analyses_date_range(
lpost,
hpost,
analysis_type=atype,
mass_spectrometers=ms)
if ans:
self.debug('{} {} to {}. nanalyses={}'.format(
atype, lpost, hpost, len(ans)))
ans = [
ai for ai in ans if ai.uuid not in uuids
]
self.debug('new ans {}'.format(len(ans)))
if ans:
tans.extend(ans)
uuids.extend([ai.uuid for ai in ans])
break
progress.soft_close()
self.active_editor.set_items(tans,
is_append=True,
use_cache=use_cache,
progress=progress)
return uuids
def bulk_import_irradiation(self, irradname, creator, dry=True):
src = self.processor.db
tol_hrs = 6
self.debug('bulk import irradiation {}'.format(irradname))
oruns = []
ts, idxs = self._get_irradiation_timestamps(irradname, tol_hrs=tol_hrs)
print(ts)
repository_identifier = 'Irradiation-{}'.format(irradname)
# add project
with self.dvc.db.session_ctx():
self.dvc.db.add_project(repository_identifier, creator)
def filterfunc(x):
a = x.labnumber.irradiation_position is None
b = False
if not a:
b = x.labnumber.irradiation_position.level.irradiation.name == irradname
d = False
if x.extraction:
ed = x.extraction.extraction_device
if not ed:
d = True
else:
d = ed.name == 'Fusions CO2'
return (a or b) and d
# for ms in ('jan', 'obama'):
# monitors not run on obama
for ms in ('jan', ):
for i, ais in enumerate(array_split(ts, idxs + 1)):
if not ais.shape[0]:
self.debug('skipping {}'.format(i))
continue
low = get_datetime(ais[0]) - timedelta(hours=tol_hrs / 2.)
high = get_datetime(ais[-1]) + timedelta(hours=tol_hrs / 2.)
with src.session_ctx():
ans = src.get_analyses_date_range(
low,
high,
mass_spectrometers=(ms, ),
samples=('FC-2', 'blank_unknown', 'blank_air',
'blank_cocktail', 'air', 'cocktail'))
# runs = filter(lambda x: x.labnumber.irradiation_position is None or
# x.labnumber.irradiation_position.level.irradiation.name == irradname, ans)
runs = list(filter(filterfunc, ans))
if dry:
for ai in runs:
oruns.append(ai.record_id)
print(ms, ai.record_id)
else:
self.debug(
'================= Do Export i: {} low: {} high: {}'
.format(i, low, high))
self.debug('N runs: {}'.format(len(runs)))
self.do_export([ai.record_id for ai in runs],
repository_identifier,
creator,
monitor_mapping=('FC-2', 'Sanidine',
repository_identifier))
return oruns
def bulk_import_irradiation(self, irradname, creator, dry=True):
src = self.processor.db
tol_hrs = 6
self.debug('bulk import irradiation {}'.format(irradname))
oruns = []
ts, idxs = self._get_irradiation_timestamps(irradname, tol_hrs=tol_hrs)
print ts
repository_identifier = 'Irradiation-{}'.format(irradname)
# add project
with self.dvc.db.session_ctx():
self.dvc.db.add_project(repository_identifier, creator)
def filterfunc(x):
a = x.labnumber.irradiation_position is None
b = False
if not a:
b = x.labnumber.irradiation_position.level.irradiation.name == irradname
d = False
if x.extraction:
ed = x.extraction.extraction_device
if not ed:
d = True
else:
d = ed.name == 'Fusions CO2'
return (a or b) and d
# for ms in ('jan', 'obama'):
# monitors not run on obama
for ms in ('jan',):
for i, ais in enumerate(array_split(ts, idxs + 1)):
if not ais.shape[0]:
self.debug('skipping {}'.format(i))
continue
low = get_datetime(ais[0]) - timedelta(hours=tol_hrs / 2.)
high = get_datetime(ais[-1]) + timedelta(hours=tol_hrs / 2.)
with src.session_ctx():
ans = src.get_analyses_date_range(low, high,
mass_spectrometers=(ms,),
samples=('FC-2',
'blank_unknown', 'blank_air', 'blank_cocktail', 'air',
'cocktail'))
# runs = filter(lambda x: x.labnumber.irradiation_position is None or
# x.labnumber.irradiation_position.level.irradiation.name == irradname, ans)
runs = filter(filterfunc, ans)
if dry:
for ai in runs:
oruns.append(ai.record_id)
print ms, ai.record_id
else:
self.debug('================= Do Export i: {} low: {} high: {}'.format(i, low, high))
self.debug('N runs: {}'.format(len(runs)))
self.do_export([ai.record_id for ai in runs],
repository_identifier, creator,
monitor_mapping=('FC-2', 'Sanidine', repository_identifier))
return oruns
def post_measurement_save(self):
if DEBUG:
self.debug('Not measurement saving to database')
return
self.info('post measurement save')
# mem_log('pre post measurement save')
if not self.save_enabled:
self.info('Database saving disabled')
return
#check for conflicts immediately before saving
#automatically update if there is an issue
conflict = self.datahub.is_conflict(self.run_spec)
if conflict:
self.debug('post measurement datastore conflict found. Automatically updating the aliquot and step')
self.datahub.update_spec(self.run_spec)
cp = self._current_data_frame
ln = self.run_spec.labnumber
aliquot = self.run_spec.aliquot
# save to local sqlite database for backup and reference
# self._local_db_save()
# save to a database
db = self.datahub.mainstore.db
# if db and db.connect(force=True):
if not db or not db.connected:
self.warning('No database instanc. Not saving post measurement to primary database')
else:
with db.session_ctx() as sess:
pt = time.time()
lab = db.get_labnumber(ln)
endtime = get_datetime().time()
self.info('analysis finished at {}'.format(endtime))
un = self.run_spec.username
dbuser = db.get_user(un)
if dbuser is None:
self.debug('user= {} does not existing. adding to database now'.format(un))
dbuser = db.add_user(un)
self.debug('adding analysis identifier={}, aliquot={}, '
'step={}, increment={}'.format(ln, aliquot,
self.run_spec.step,
self.run_spec.increment))
a = db.add_analysis(lab,
user=dbuser,
uuid=self.uuid,
endtime=endtime,
aliquot=aliquot,
step=self.run_spec.step,
increment=self.run_spec.increment,
comment=self.run_spec.comment,
whiff_result=self.whiff_result)
sess.flush()
self.run_spec.analysis_dbid = a.id
self.run_spec.analysis_timestamp = a.analysis_timestamp
experiment = db.get_experiment(self.experiment_identifier, key='id')
if experiment is not None:
# added analysis to experiment
a.experiment_id = experiment.id
else:
self.warning('no experiment found for {}'.format(self.experiment_identifier))
# save measurement
meas = self._save_measurement(db, a)
# save extraction
ext = self._db_extraction_id
if ext is not None:
dbext = db.get_extraction(ext, key='id')
a.extraction_id = dbext.id
# save sensitivity info to extraction
self._save_sensitivity(dbext, meas)
else:
self.debug('no extraction to associate with this run')
self._save_spectrometer_info(db, meas)
# add selected history
db.add_selected_histories(a)
# self._save_isotope_info(a, ss)
self._save_isotope_data(db, a)
# save ic factor
self._save_detector_intercalibration(db, a)
# save blanks
self._save_blank_info(db, a)
# save peak center
self._save_peak_center(db, a, cp)
# save monitor
self._save_monitor_info(db, a)
# save gains
self._save_gains(db, a)
if self.use_analysis_grouping:
self._save_analysis_group(db, a)
mem_log('post pychron save')
pt = time.time() - pt
self.debug('pychron save time= {:0.3f} '.format(pt))
file_log(pt)
# don't save detector_ic runs to mass spec
# measurement of an isotope on multiple detectors likely possible with mass spec but at this point
# not worth trying.
if self.use_secondary_database and check_secondary_database_save(ln):#not self.run_spec.analysis_type in ('detector_ic',):
if not self.datahub.secondary_connect():
# if not self.massspec_importer or not self.massspec_importer.db.connected:
self.debug('Secondary database is not available')
else:
self.debug('saving post measurement to secondary database')
# save to massspec
mt = time.time()
self._save_to_massspec(cp)
self.debug('mass spec save time= {:0.3f}'.format(time.time() - mt))
mem_log('post mass spec save')
def pre_extraction_save(self):
d = get_datetime()
self.runtime = d.time()
self.rundate = d.date()
self.info('Analysis started at {}'.format(self.runtime))