def init(self):
conn = dict(
host=os.environ.get('ARGONSERVER_HOST'),
username='******', # os.environ.get('ARGONSERVER_DB_USER'),
password=os.environ.get('ARGONSERVER_DB_PWD'),
kind='mysql')
use_local = False
if use_local:
meta_name = 'MetaDataDev'
dest_conn = dict(
host='localhost',
username=os.environ.get('LOCALHOST_DB_USER'),
password=os.environ.get('LOCALHOST_DB_PWD'),
kind='mysql',
# echo=True,
name='pychrondvc_dev')
else:
meta_name = 'NMGRLMetaData'
dest_conn = conn.copy()
dest_conn['name'] = 'pychrondvc'
self.dvc = DVC(bind=False,
organization='NMGRLData',
meta_repo_name=meta_name)
paths.meta_root = os.path.join(paths.dvc_dir, self.dvc.meta_repo_name)
self.dvc.db.trait_set(**dest_conn)
if not self.dvc.initialize():
self.warning_dialog('Failed to initialize DVC')
return
self.dvc.meta_repo.smart_pull(quiet=self.quiet)
self.persister = DVCPersister(dvc=self.dvc,
stage_files=False,
bind=False)
proc = IsotopeDatabaseManager(bind=False, connect=False)
use_local_src = False
if use_local_src:
conn = dict(
host='localhost',
username=os.environ.get('LOCALHOST_DB_USER'),
password=os.environ.get('LOCALHOST_DB_PWD'),
kind='mysql',
# echo=True,
name='pychrondata')
else:
conn['name'] = 'pychrondata'
proc.db.trait_set(**conn)
src = proc.db
src.connect()
self.processor = proc
def test_save_persister():
paths.build('_dev')
dvc = DVC(bind=False)
dvc.db.connect()
per = DVCPersister(dvc=dvc)
run_spec = AutomatedRunSpec()
run_spec.labnumber = '10001'
run_spec.project = 'Test'
arar = ArArAge()
arar.isotopes['Ar40'] = Isotope(xs=[1, 2, 3], ys=[1, 2, 3],
name='Ar40', detector='H1')
sd = {}
dd = {'H1': 100}
gd = {'H1': 1.021}
per_spec = PersistenceSpec(run_spec=run_spec,
arar_age=arar,
spec_dict=sd,
defl_dict=dd,
gains=gd,
positions=[1, ],
experiment_queue_name='testexp.txt',
measurement_name='jan_unknown.py',
extraction_name='foo.py')
per.per_spec_save(per_spec)
def train(self):
dvc = DVC(bind=False)
ans = dvc.get_analyses_date_range()
isos = []
klasses = []
for dbai in ans:
ai = dvc.make_analyses((dbai,))
for iso in ai.isotopes.values():
klass = self._get_klass(iso)
if klass is not None:
isos.append(iso)
klasses.append(klass)
self.clf.add_isotopes(isos, klasses)
def train(self):
dvc = DVC(bind=False)
ans = dvc.get_analyses_date_range()
isos = []
klasses = []
for dbai in ans:
ai = dvc.make_analyses((dbai, ))
for iso in ai.isotopes.values():
klass = self._get_klass(iso)
if klass is not None:
isos.append(iso)
klasses.append(klass)
self.clf.add_isotopes(isos, klasses)
def train(self):
dvc = DVC(bind=False, organization='NMGRLData')
dvc.db.trait_set(name='pychronmeta1',
username=os.environ.get('ARGONSERVER_DB_USER'),
password=os.environ.get('ARGONSERVER_DB_PWD'),
kind='mysql',
host=os.environ.get('ARGONSERVER_HOST'))
dvc.connect()
isos = []
klasses = []
uuids = UUIDS
with dvc.session_ctx():
for uuid in uuids:
broke = False
dbai = dvc.get_analysis_uuid(uuid)
ai = dvc.make_analyses((dbai, ))[0]
ai.load_raw_data()
for iso in ai.isotopes.values():
klass = self._get_klass(iso)
if klass is not None:
isos.append(iso)
klasses.append(klass)
else:
broke = True
break
if broke:
break
if isos:
clf = IsotopeClassifier()
clf.add_isotopes(isos, klasses)
clf.dump()
def get_db():
conn = dict(host=os.environ.get('ARGONSERVER_HOST'),
username='******', # os.environ.get('ARGONSERVER_DB_USER'),
password=os.environ.get('ARGONSERVER_DB_PWD'),
kind='mysql')
paths.build('~/PychronDev')
meta_name = 'NMGRLMetaData'
dvc = DVC(bind=False,
organization='NMGRLData',
meta_repo_name=meta_name)
paths.meta_root = os.path.join(paths.dvc_dir, dvc.meta_repo_name)
dvc.db.trait_set(**conn)
if not dvc.initialize():
warning(None, 'Failed to initialize DVC')
return
dvc.meta_repo.smart_pull()
return dvc
def _setup_dvc(self, dvc_obj):
self.dvc = dvc = DVC(bind=False)
dvc.db.trait_set(**dvc_obj['connection'])
dvc.trait_set(**dvc_obj['traits'])
paths.dvc_dir = dvc_obj['root']
paths.repository_dataset_dir = os.path.join(paths.dvc_dir,
'repositories')
dvc.initialize()
def get_dvc():
conn = dict(
host=os.environ.get('ARGONSERVER_HOST'),
username='******', # os.environ.get('ARGONSERVER_DB_USER'),
password=os.environ.get('ARGONSERVER_DB_PWD'),
name='pychrondvc',
kind='mysql')
paths.build('~/PychronDev')
meta_name = 'NMGRLMetaData'
dvc = DVC(bind=False, organization='NMGRLData', meta_repo_name=meta_name)
paths.meta_root = os.path.join(paths.dvc_dir, dvc.meta_repo_name)
dvc.db.trait_set(**conn)
if not dvc.initialize():
warning(None, 'Failed to initialize DVC')
return
dvc.meta_repo.smart_pull()
return dvc
def _service_offers_default(self):
# p = {'dvc': self.dvc_factory()}
# self.debug('DDDDD {}'.format(p))
so = self.service_offer_factory(
protocol=IsotopeClassifier,
factory=IsotopeClassifier,
properties={'dvc': DVC(application=self.application)})
return [
so,
]
def train(self):
dvc = DVC(bind=False,
organization='NMGRLData')
dvc.db.trait_set(name='pychronmeta1',
username=os.environ.get('ARGONSERVER_DB_USER'),
password=os.environ.get('ARGONSERVER_DB_PWD'),
kind='mysql',
host=os.environ.get('ARGONSERVER_HOST'))
dvc.connect()
isos = []
klasses = []
uuids = UUIDS
with dvc.session_ctx():
for uuid in uuids:
broke = False
dbai = dvc.get_analysis_uuid(uuid)
ai = dvc.make_analyses((dbai,))[0]
ai.load_raw_data()
for iso in ai.isotopes.values():
klass = self._get_klass(iso)
if klass is not None:
isos.append(iso)
klasses.append(klass)
else:
broke = True
break
if broke:
break
if isos:
clf = IsotopeClassifier()
clf.add_isotopes(isos, klasses)
clf.dump()
def init(self):
conn = dict(host=os.environ.get('ARGONSERVER_HOST'),
username='******',#os.environ.get('ARGONSERVER_DB_USER'),
password=os.environ.get('ARGONSERVER_DB_PWD'),
kind='mysql')
use_local = False
if use_local:
meta_name = 'MetaDataDev'
dest_conn = dict(host='localhost',
username=os.environ.get('LOCALHOST_DB_USER'),
password=os.environ.get('LOCALHOST_DB_PWD'),
kind='mysql',
# echo=True,
name='pychrondvc_dev')
else:
meta_name = 'NMGRLMetaData'
dest_conn = conn.copy()
dest_conn['name'] = 'pychrondvc'
self.dvc = DVC(bind=False,
organization='NMGRLData',
meta_repo_name=meta_name)
paths.meta_root = os.path.join(paths.dvc_dir, self.dvc.meta_repo_name)
self.dvc.db.trait_set(**dest_conn)
if not self.dvc.initialize():
self.warning_dialog('Failed to initialize DVC')
return
self.dvc.meta_repo.smart_pull(quiet=self.quiet)
self.persister = DVCPersister(dvc=self.dvc, stage_files=False)
proc = IsotopeDatabaseManager(bind=False, connect=False)
use_local_src = False
if use_local_src:
conn = dict(host='localhost',
username=os.environ.get('LOCALHOST_DB_USER'),
password=os.environ.get('LOCALHOST_DB_PWD'),
kind='mysql',
# echo=True,
name='pychrondata')
else:
conn['name'] = 'pychrondata'
proc.db.trait_set(**conn)
src = proc.db
src.connect()
self.processor = proc
import matplotlib.pyplot as plt
from pychron.core.ui import set_qt
set_qt()
# ============= standard library imports ========================
# ============= local library imports ==========================
from pychron.dvc.dvc import DVC
"""
proof of concept test see gist https://gist.github.com/jirhiker/3ba3e1e358ef933fa9b5
"""
dvc = DVC(clear_db=True, auto_add=True)
DP = dvc.db.path
units_map = {'K': 1e3, 'M': 1e6, 'G': 1e9}
def get_dir_size(path):
return subprocess.check_output(['du', '-sh',
path]).split()[0].decode('utf-8')
def report_irrad_stats(wfile, nirrads, nlevels, npositions):
if nirrads == 1 and nlevels == 1:
print '{:<10s}{:<10s}{:<10s}{:<10s}{:<10s}'.format(
'irrad', 'level', 'pos', 'sizedb', 'sizegit')
wfile.write('{}\n'.format(','.join(
def dvc_factory(self):
d = DVC()
# d.initialize()
return d
class IsoDBTransfer(Loggable):
"""
transfer analyses from an isotope_db database to a dvc database
"""
dvc = Instance(DVC)
processor = Instance(IsotopeDatabaseManager)
persister = Instance(DVCPersister)
quiet = False
def init(self):
conn = dict(
host=os.environ.get('ARGONSERVER_HOST'),
username='******', # os.environ.get('ARGONSERVER_DB_USER'),
password=os.environ.get('ARGONSERVER_DB_PWD'),
kind='mysql')
use_local = False
if use_local:
meta_name = 'MetaDataDev'
dest_conn = dict(
host='localhost',
username=os.environ.get('LOCALHOST_DB_USER'),
password=os.environ.get('LOCALHOST_DB_PWD'),
kind='mysql',
# echo=True,
name='pychrondvc_dev')
else:
meta_name = 'NMGRLMetaData'
dest_conn = conn.copy()
dest_conn['name'] = 'pychrondvc'
self.dvc = DVC(bind=False,
organization='NMGRLData',
meta_repo_name=meta_name)
paths.meta_root = os.path.join(paths.dvc_dir, self.dvc.meta_repo_name)
self.dvc.db.trait_set(**dest_conn)
if not self.dvc.initialize():
self.warning_dialog('Failed to initialize DVC')
return
self.dvc.meta_repo.smart_pull(quiet=self.quiet)
self.persister = DVCPersister(dvc=self.dvc,
stage_files=False,
bind=False)
proc = IsotopeDatabaseManager(bind=False, connect=False)
use_local_src = False
if use_local_src:
conn = dict(
host='localhost',
username=os.environ.get('LOCALHOST_DB_USER'),
password=os.environ.get('LOCALHOST_DB_PWD'),
kind='mysql',
# echo=True,
name='pychrondata')
else:
conn['name'] = 'pychrondata'
proc.db.trait_set(**conn)
src = proc.db
src.connect()
self.processor = proc
def copy_productions(self):
src = self.processor.db
dvc = self.dvc
dest = dvc.db
with src.session_ctx():
pr = src.get_irradiation_productions()
for p in pr:
# copy to database
pname = p.name.replace(' ', '_')
if not dest.get_production(pname):
dest.add_production(pname)
# copy to meta
dvc.copy_production(p)
def set_spectrometer_files(self, repository):
set_spectrometer_files(
self.processor.db, self.dvc.db, repository,
os.path.join(paths.repository_dataset_dir, repository))
def bulk_import_irradiations(self, irradiations, creator, dry=True):
# for i in xrange(251, 277):
# for i in xrange(258, 259):
# for i in (258, 259, 260, 261,):
# for i in (262, 263, 264, 265):
# for i in (266, 267, 268, 269):
# for i in (270, 271, 272, 273):
for i in irradiations:
irradname = 'NM-{}'.format(i)
runs = self.bulk_import_irradiation(irradname, creator, dry=dry)
# if runs:
# with open('/Users/ross/Sandbox/bulkimport/irradiation_runs.txt', 'a') as wfile:
# for o in runs:
# wfile.write('{}\n'.format(o))
def bulk_import_irradiation(self, irradname, creator, dry=True):
src = self.processor.db
tol_hrs = 6
self.debug('bulk import irradiation {}'.format(irradname))
oruns = []
ts, idxs = self._get_irradiation_timestamps(irradname, tol_hrs=tol_hrs)
print(ts)
repository_identifier = 'Irradiation-{}'.format(irradname)
# add project
with self.dvc.db.session_ctx():
self.dvc.db.add_project(repository_identifier, creator)
def filterfunc(x):
a = x.labnumber.irradiation_position is None
b = False
if not a:
b = x.labnumber.irradiation_position.level.irradiation.name == irradname
d = False
if x.extraction:
ed = x.extraction.extraction_device
if not ed:
d = True
else:
d = ed.name == 'Fusions CO2'
return (a or b) and d
# for ms in ('jan', 'obama'):
# monitors not run on obama
for ms in ('jan', ):
for i, ais in enumerate(array_split(ts, idxs + 1)):
if not ais.shape[0]:
self.debug('skipping {}'.format(i))
continue
low = get_datetime(ais[0]) - timedelta(hours=tol_hrs / 2.)
high = get_datetime(ais[-1]) + timedelta(hours=tol_hrs / 2.)
with src.session_ctx():
ans = src.get_analyses_date_range(
low,
high,
mass_spectrometers=(ms, ),
samples=('FC-2', 'blank_unknown', 'blank_air',
'blank_cocktail', 'air', 'cocktail'))
# runs = filter(lambda x: x.labnumber.irradiation_position is None or
# x.labnumber.irradiation_position.level.irradiation.name == irradname, ans)
runs = list(filter(filterfunc, ans))
if dry:
for ai in runs:
oruns.append(ai.record_id)
print(ms, ai.record_id)
else:
self.debug(
'================= Do Export i: {} low: {} high: {}'
.format(i, low, high))
self.debug('N runs: {}'.format(len(runs)))
self.do_export([ai.record_id for ai in runs],
repository_identifier,
creator,
monitor_mapping=('FC-2', 'Sanidine',
repository_identifier))
return oruns
def bulk_import_project(self,
project,
principal_investigator,
source_name=None,
dry=True):
src = self.processor.db
tol_hrs = 6
self.debug('bulk import project={}, pi={}'.format(
project, principal_investigator))
oruns = []
repository_identifier = project
# def filterfunc(x):
# a = x.labnumber.irradiation_position is None
# b = False
# if not a:
# b = x.labnumber.irradiation_position.level.irradiation.name == irradname
#
# d = False
# if x.extraction:
# ed = x.extraction.extraction_device
# if not ed:
# d = True
# else:
# d = ed.name == 'Fusions CO2'
#
# return (a or b) and d
if source_name is None:
source_name = project
for ms in ('jan', 'obama', 'felix'):
ts, idxs = self._get_project_timestamps(source_name,
ms,
tol_hrs=tol_hrs)
for i, ais in enumerate(array_split(ts, idxs + 1)):
if not ais.shape[0]:
self.debug('skipping {}'.format(i))
continue
low = get_datetime(ais[0]) - timedelta(hours=tol_hrs / 2.)
high = get_datetime(ais[-1]) + timedelta(hours=tol_hrs / 2.)
print('========{}, {}, {}'.format(ms, low, high))
with src.session_ctx():
runs = src.get_analyses_date_range(
low,
high,
# labnumber=(63630, 63632, 63634, 63636, 63638, 63646, 63648),
projects=('REFERENCES', ),
# projects=('REFERENCES', source_name),
mass_spectrometers=(ms, ))
if dry:
for ai in runs:
oruns.append(ai.record_id)
print(ai.measurement.mass_spectrometer.name, ai.record_id, ai.labnumber.sample.name, \
ai.analysis_timestamp)
else:
self.debug(
'================= Do Export i: {} low: {} high: {}'
.format(i, low, high))
self.debug('N runs: {}'.format(len(runs)))
self.do_export([ai.record_id for ai in runs],
repository_identifier,
principal_investigator)
return oruns
def find_project_overlaps(self, projects):
tol_hrs = 6
src = self.processor.db
pdict = {}
for p in projects:
for ms in ('jan', 'obama'):
ts, idxs = self._get_project_timestamps(p, ms, tol_hrs=tol_hrs)
for i, ais in enumerate(array_split(ts, idxs + 1)):
if not ais.shape[0]:
self.debug('skipping {}'.format(i))
continue
low = get_datetime(ais[0]) - timedelta(hours=tol_hrs / 2.)
high = get_datetime(
ais[-1]) + timedelta(hours=tol_hrs / 2.)
print('========{}, {}, {}'.format(ms, low, high))
with src.session_ctx():
runs = src.get_analyses_date_range(
low,
high,
projects=('REFERENCES', ),
mass_spectrometers=(ms, ))
pdict[p] = [ai.record_id for ai in runs]
for p in projects:
for o in projects:
if p == o:
continue
pruns = pdict[p]
oruns = pdict[o]
for ai in pruns:
if ai in oruns:
print(p, o, ai)
def import_date_range(self, low, high, spectrometer, repository_identifier,
creator):
src = self.processor.db
with src.session_ctx():
runs = src.get_analyses_date_range(low,
high,
mass_spectrometers=spectrometer)
ais = [ai.record_id for ai in runs]
self.do_export(ais, repository_identifier, creator)
def do_export_runlist(self):
path = '/Users/ross/Downloads/Nabro_data_2.csv'
with open(path, 'r') as rfile:
next(rfile)
runs = [line.split(',')[:2] for line in rfile]
runs = sorted(runs,
key=lambda x: x[1]
if x[1] in ('Blank', 'air', 'cocktail', 'FC-2') else '0')
for sample, rs in groupby(
runs,
key=lambda x: x[1]
if x[1] in ('Blank', 'air', 'cocktail', 'FC-2') else '0'):
rs = [r[0] for r in rs]
if sample in ('cocktail', ):
# for ri in rs:
# print('asd', ri)
self.do_export(rs, 'Nabro', 'Iverson,N')
# if sample in ('Blank', 'cocktail'):
# self.do_export([r[0] for r in rs], 'Nabro', 'Iverson,N')
# elif sample == 'FC-2':
# if sample == 'FC-2':
# self.do_export([r[0] for r in rs], 'Irradiation-NM-257', 'NMGRL')
# self.do_export(runs, '')
def do_export(self,
runs,
repository_identifier,
creator,
create_repo=False,
monitor_mapping=None):
# self._init_src_dest()
src = self.processor.db
dest = self.dvc.db
with src.session_ctx():
key = lambda x: x.split('-')[0]
runs = sorted(runs, key=key)
with dest.session_ctx():
repo = self._add_repository(dest, repository_identifier,
creator, create_repo)
self.persister.active_repository = repo
self.dvc.current_repository = repo
total = len(runs)
j = 0
for ln, ans in groupby(runs, key=key):
ans = list(ans)
n = len(ans)
for i, a in enumerate(ans):
with dest.session_ctx() as sess:
st = time.time()
try:
if self._transfer_analysis(
a,
repository_identifier,
monitor_mapping=monitor_mapping):
j += 1
self.debug(
'{}/{} transfer time {:0.3f}'.format(
j, total,
time.time() - st))
except BaseException as e:
import traceback
traceback.print_exc()
self.warning('failed transfering {}. {}'.format(
a, e))
# private
def _get_project_timestamps(self, project, mass_spectrometer, tol_hrs=6):
src = self.processor.db
return get_project_timestamps(src, project, mass_spectrometer, tol_hrs)
def _get_irradiation_timestamps(self, irradname, tol_hrs=6):
src = self.processor.db
return get_irradiation_timestamps(src, irradname, tol_hrs)
def _add_repository(self, dest, repository_identifier, creator,
create_repo):
repository_identifier = format_repository_identifier(
repository_identifier)
# sys.exit()
proot = os.path.join(paths.repository_dataset_dir,
repository_identifier)
if not os.path.isdir(proot):
# create new local repo
os.mkdir(proot)
repo = GitRepoManager()
repo.open_repo(proot)
repo.add_ignore('.DS_Store')
self.repo_man = repo
if create_repo:
# add repo to central location
create_github_repo(repository_identifier)
url = 'https://github.com/{}/{}.git'.format(
ORG, repository_identifier)
self.debug('Create repo at github. url={}'.format(url))
repo.create_remote(url)
else:
repo = GitRepoManager()
repo.open_repo(proot)
dbexp = dest.get_repository(repository_identifier)
if not dbexp:
dest.add_repository(repository_identifier, creator)
return repo
def _transfer_meta(self, dest, dban, monitor_mapping):
pi = 'Iverson,N'
self.debug('transfer meta {}'.format(monitor_mapping))
dblab = dban.labnumber
dbsam = dblab.sample
material_name = dbsam.material.name
project = dbsam.project.name
project_name = project.replace('/', '_').replace('\\', '_')
sample_name = dbsam.name
# if monitor_mapping is None:
# sample_name, material, project = monitor_mapping
#
# if material_name == 'bi':
# material_name = 'Biotite'
sam = dest.get_sample(sample_name, project_name, pi, material_name)
if not sam:
if not dest.get_material(material_name):
self.debug('add material {}'.format(material_name))
dest.add_material(material_name)
dest.flush()
if not dest.get_project(project_name, pi):
self.debug('add project {}'.format(project_name))
dest.add_project(project_name, pi)
dest.flush()
self.debug('add sample {}'.format(sample_name))
sam = dest.add_sample(sample_name, project_name, pi, material_name)
dest.flush()
dbirradpos = dblab.irradiation_position
if not dbirradpos:
irradname = 'NoIrradiation'
levelname = 'A'
holder = 'Grid'
pos = None
identifier = dblab.identifier
doses = []
prod = None
prodname = 'NoIrradiation'
# geom = make_geom([(0, 0, 0.0175),
# (1, 0, 0.0175),
# (2, 0, 0.0175),
# (3, 0, 0.0175),
# (4, 0, 0.0175),
#
# (0, 1, 0.0175),
# (1, 1, 0.0175),
# (2, 1, 0.0175),
# (3, 1, 0.0175),
# (4, 1, 0.0175),
#
# (0, 2, 0.0175),
# (1, 2, 0.0175),
# (2, 2, 0.0175),
# (3, 2, 0.0175),
# (4, 2, 0.0175),
#
# (0, 3, 0.0175),
# (1, 3, 0.0175),
# (2, 3, 0.0175),
# (3, 3, 0.0175),
# (4, 3, 0.0175),
#
# (0, 4, 0.0175),
# (1, 4, 0.0175),
# (2, 4, 0.0175),
# (3, 4, 0.0175),
# (4, 4, 0.0175)
# ])
else:
dblevel = dbirradpos.level
dbirrad = dblevel.irradiation
dbchron = dbirrad.chronology
irradname = dbirrad.name
levelname = dblevel.name
holder = dblevel.holder.name if dblevel.holder else ''
# geom = dblevel.holder.geometry if dblevel.holder else ''
prodname = dblevel.production.name if dblevel.production else ''
prodname = prodname.replace(' ', '_')
pos = dbirradpos.position
doses = dbchron.get_doses()
meta_repo = self.dvc.meta_repo
# save db irradiation
if not dest.get_irradiation(irradname):
self.debug('Add irradiation {}'.format(irradname))
self.dvc.add_irradiation(irradname, doses)
dest.flush()
# meta_repo.add_irradiation(irradname)
# meta_repo.add_chronology(irradname, doses, add=False)
# meta_repo.commit('added irradiation {}'.format(irradname))
# save production name to db
# if not dest.get_production(prodname):
# self.debug('Add production {}'.format(prodname))
# dest.add_production(prodname)
# dest.flush()
# meta_repo.add_production(irradname, prodname, prod, add=False)
# meta_repo.commit('added production {}'.format(prodname))
# save db level
if not dest.get_irradiation_level(irradname, levelname):
self.debug('Add level irrad:{} level:{}'.format(
irradname, levelname))
dest.add_irradiation_level(levelname, irradname, holder, prodname)
dest.flush()
# meta_repo.add_irradiation_holder(holder, geom, add=False)
# meta_repo.add_level(irradname, levelname, add=False)
# meta_repo.update_level_production(irradname, levelname, prodname)
# meta_repo.commit('added empty level {}{}'.format(irradname, levelname))
if pos is None:
pos = self._get_irradpos(dest, irradname, levelname, identifier)
# save db irradiation position
if not dest.get_irradiation_position(irradname, levelname, pos):
self.debug('Add position irrad:{} level:{} pos:{}'.format(
irradname, levelname, pos))
p = meta_repo.get_level_path(irradname, levelname)
with open(p, 'r') as rfile:
yd = json.load(rfile)
dd = dest.add_irradiation_position(irradname, levelname, pos)
dd.identifier = idn = dblab.identifier
dd.sample = sam
dest.flush()
try:
f = dban.labnumber.selected_flux_history.flux
j, e = f.j, f.j_err
except AttributeError:
j, e = 0, 0
ps = yd['positions']
ps.append({
'j': j,
'j_err': e,
'position': pos,
'analyses': [],
'identifier': idn,
'decay_constants': {}
})
yd['positions'] = ps
dvc_dump(yd, p)
dest.commit()
def _transfer_analysis(self,
rec,
exp,
overwrite=True,
monitor_mapping=None):
dest = self.dvc.db
proc = self.processor
src = proc.db
# args = rec.split('-')
# idn = '-'.join(args[:-1])
# t = args[-1]
# try:
# aliquot = int(t)
# step = None
# except ValueError:
# aliquot = int(t[:-1])
# step = t[-1]
m = IDENTIFIER_REGEX.match(rec)
if not m:
m = SPECIAL_IDENTIFIER_REGEX.match(rec)
if not m:
self.warning('invalid runid {}'.format(rec))
return
else:
idn = m.group('identifier')
aliquot = m.group('aliquot')
try:
step = m.group('step') or None
except IndexError:
step = None
if idn == '4359':
idn = 'c-01-j'
elif idn == '4358':
idn = 'c-01-o'
# check if analysis already exists. skip if it does
if dest.get_analysis_runid(idn, aliquot, step):
self.warning('{} already exists'.format(
make_runid(idn, aliquot, step)))
return
dban = src.get_analysis_runid(idn, aliquot, step)
iv = IsotopeRecordView()
iv.uuid = dban.uuid
self.debug('make analysis idn:{}, aliquot:{} step:{}'.format(
idn, aliquot, step))
# try:
an = proc.make_analysis(iv,
unpack=True,
use_cache=False,
use_progress=False)
# except BaseException as e:
# self.warning('Failed to make {}'.format(make_runid(idn, aliquot, step)))
# self.warning('exception: {}'.format(e))
# return
self._transfer_meta(dest, dban, monitor_mapping)
# return
dblab = dban.labnumber
if dblab.irradiation_position:
irrad = dblab.irradiation_position.level.irradiation.name
level = dblab.irradiation_position.level.name
irradpos = dblab.irradiation_position.position
else:
irrad = 'NoIrradiation'
level = 'A'
irradpos = self._get_irradpos(dest, irrad, level, dblab.identifier)
# irrad, level, irradpos = '', '', 0
extraction = dban.extraction
ms = dban.measurement.mass_spectrometer.name
if not dest.get_mass_spectrometer(ms):
self.debug('adding mass spectrometer {}'.format(ms))
dest.add_mass_spectrometer(ms)
dest.commit()
ed = extraction.extraction_device.name if extraction.extraction_device else None
if not ed:
ed = 'No Extract Device'
if not dest.get_extraction_device(ed):
self.debug('adding extract device {}'.format(ed))
dest.add_extraction_device(ed)
dest.commit()
if step is None:
inc = -1
else:
inc = alpha_to_int(step)
username = ''
if dban.user:
username = dban.user.name
if not dest.get_user(username):
self.debug('adding user. username:{}'.format(username))
dest.add_user(username)
dest.commit()
if monitor_mapping:
sample_name, material_name, project_name = monitor_mapping
else:
dbsam = dblab.sample
sample_name = dbsam.name
material_name = dbsam.material.name
project_name = format_repository_identifier(dbsam.project.name)
rs = AutomatedRunSpec(labnumber=idn,
username=username,
material=material_name,
project=project_name,
sample=sample_name,
irradiation=irrad,
irradiation_level=level,
irradiation_position=irradpos,
repository_identifier=exp,
mass_spectrometer=ms,
uuid=dban.uuid,
_step=inc,
comment=dban.comment.decode('utf-8') or '',
aliquot=int(aliquot),
extract_device=ed,
duration=extraction.extract_duration,
cleanup=extraction.cleanup_duration,
beam_diameter=extraction.beam_diameter,
extract_units=extraction.extract_units or '',
extract_value=extraction.extract_value,
pattern=extraction.pattern or '',
weight=extraction.weight,
ramp_duration=extraction.ramp_duration or 0,
ramp_rate=extraction.ramp_rate or 0,
collection_version='0.1:0.1',
queue_conditionals_name='',
tray='')
meas = dban.measurement
# get spectrometer parameters
# gains
gains = {}
gain_history = dban.gain_history
if gain_history:
gains = {d.detector.name: d.value for d in gain_history.gains}
# deflections
deflections = {d.detector.name: d.deflection for d in meas.deflections}
# source
src = {
k: getattr(meas.spectrometer_parameters, k)
for k in QTEGRA_SOURCE_KEYS
}
ps = PersistenceSpec(
run_spec=rs,
tag=an.tag.name,
isotope_group=an,
timestamp=dban.analysis_timestamp,
defl_dict=deflections,
gains=gains,
spec_dict=src,
use_repository_association=True,
positions=[p.position for p in extraction.positions])
self.debug('transfer analysis with persister')
self.persister.per_spec_save(ps,
commit=False,
commit_tag='Database Transfer')
return True
def _get_irradpos(self, dest, irradname, levelname, identifier):
dl = dest.get_irradiation_level(irradname, levelname)
pos = 1
if dl.positions:
for p in dl.positions:
if p.identifier == identifier:
pos = p.position
break
else:
pos = dl.positions[-1].position + 1
return pos
class IsoDBTransfer(Loggable):
"""
transfer analyses from an isotope_db database to a dvc database
"""
dvc = Instance(DVC)
processor = Instance(IsotopeDatabaseManager)
persister = Instance(DVCPersister)
quiet = False
def init(self):
conn = dict(host=os.environ.get('ARGONSERVER_HOST'),
username=os.environ.get('ARGONSERVER_DB_USER'),
password=os.environ.get('ARGONSERVER_DB_PWD'),
kind='mysql')
self.dvc = DVC(bind=False,
organization='NMGRLData',
meta_repo_name='MetaData')
paths.meta_root = os.path.join(paths.dvc_dir, self.dvc.meta_repo_name)
use_local = True
if use_local:
dest_conn = dict(host='localhost',
username=os.environ.get('LOCALHOST_DB_USER'),
password=os.environ.get('LOCALHOST_DB_PWD'),
kind='mysql',
# echo=True,
name='pychrondvc_dev')
else:
dest_conn = conn.copy()
dest_conn['name'] = 'pychrondvc'
self.dvc.db.trait_set(**dest_conn)
if not self.dvc.initialize():
self.warning_dialog('Failed to initialize DVC')
return
self.dvc.meta_repo.smart_pull(quiet=self.quiet)
self.persister = DVCPersister(dvc=self.dvc, stage_files=False)
proc = IsotopeDatabaseManager(bind=False, connect=False)
use_local_src = True
if use_local_src:
conn = dict(host='localhost',
username=os.environ.get('LOCALHOST_DB_USER'),
password=os.environ.get('LOCALHOST_DB_PWD'),
kind='mysql',
# echo=True,
name='pychrondata')
else:
conn['name'] = 'pychrondata'
proc.db.trait_set(**conn)
src = proc.db
src.connect()
self.processor = proc
def copy_productions(self):
src = self.processor.db
dvc = self.dvc
dest = dvc.db
with src.session_ctx():
pr = src.get_irradiation_productions()
for p in pr:
# copy to database
pname = p.name.replace(' ', '_')
if not dest.get_production(pname):
dest.add_production(pname)
# copy to meta
dvc.copy_production(p)
def set_spectrometer_files(self, repository):
set_spectrometer_files(self.processor.db,
self.dvc.db,
repository,
os.path.join(paths.repository_dataset_dir, repository))
def bulk_import_irradiations(self, irradiations, creator, dry=True):
# for i in xrange(251, 277):
# for i in xrange(258, 259):
# for i in (258, 259, 260, 261,):
# for i in (262, 263, 264, 265):
# for i in (266, 267, 268, 269):
# for i in (270, 271, 272, 273):
for i in irradiations:
irradname = 'NM-{}'.format(i)
runs = self.bulk_import_irradiation(irradname, creator, dry=dry)
# if runs:
# with open('/Users/ross/Sandbox/bulkimport/irradiation_runs.txt', 'a') as wfile:
# for o in runs:
# wfile.write('{}\n'.format(o))
def bulk_import_irradiation(self, irradname, creator, dry=True):
src = self.processor.db
tol_hrs = 6
self.debug('bulk import irradiation {}'.format(irradname))
oruns = []
ts, idxs = self._get_irradiation_timestamps(irradname, tol_hrs=tol_hrs)
print ts
repository_identifier = 'Irradiation-{}'.format(irradname)
# add project
with self.dvc.db.session_ctx():
self.dvc.db.add_project(repository_identifier, creator)
def filterfunc(x):
a = x.labnumber.irradiation_position is None
b = False
if not a:
b = x.labnumber.irradiation_position.level.irradiation.name == irradname
d = False
if x.extraction:
ed = x.extraction.extraction_device
if not ed:
d = True
else:
d = ed.name == 'Fusions CO2'
return (a or b) and d
# for ms in ('jan', 'obama'):
# monitors not run on obama
for ms in ('jan',):
for i, ais in enumerate(array_split(ts, idxs + 1)):
if not ais.shape[0]:
self.debug('skipping {}'.format(i))
continue
low = get_datetime(ais[0]) - timedelta(hours=tol_hrs / 2.)
high = get_datetime(ais[-1]) + timedelta(hours=tol_hrs / 2.)
with src.session_ctx():
ans = src.get_analyses_date_range(low, high,
mass_spectrometers=(ms,),
samples=('FC-2',
'blank_unknown', 'blank_air', 'blank_cocktail', 'air',
'cocktail'))
# runs = filter(lambda x: x.labnumber.irradiation_position is None or
# x.labnumber.irradiation_position.level.irradiation.name == irradname, ans)
runs = filter(filterfunc, ans)
if dry:
for ai in runs:
oruns.append(ai.record_id)
print ms, ai.record_id
else:
self.debug('================= Do Export i: {} low: {} high: {}'.format(i, low, high))
self.debug('N runs: {}'.format(len(runs)))
self.do_export([ai.record_id for ai in runs],
repository_identifier, creator,
monitor_mapping=('FC-2', 'Sanidine', repository_identifier))
return oruns
def bulk_import_project(self, project, principal_investigator, dry=True):
src = self.processor.db
tol_hrs = 6
self.debug('bulk import project={}, pi={}'.format(project, principal_investigator))
oruns = []
repository_identifier = project
# def filterfunc(x):
# a = x.labnumber.irradiation_position is None
# b = False
# if not a:
# b = x.labnumber.irradiation_position.level.irradiation.name == irradname
#
# d = False
# if x.extraction:
# ed = x.extraction.extraction_device
# if not ed:
# d = True
# else:
# d = ed.name == 'Fusions CO2'
#
# return (a or b) and d
#
for ms in ('jan', 'obama'):
ts, idxs = self._get_project_timestamps(project, ms, tol_hrs=tol_hrs)
for i, ais in enumerate(array_split(ts, idxs + 1)):
if not ais.shape[0]:
self.debug('skipping {}'.format(i))
continue
low = get_datetime(ais[0]) - timedelta(hours=tol_hrs / 2.)
high = get_datetime(ais[-1]) + timedelta(hours=tol_hrs / 2.)
print '========{}, {}, {}'.format(ms, low, high)
with src.session_ctx():
runs = src.get_analyses_date_range(low, high,
projects=('REFERENCES', project),
mass_spectrometers=(ms,))
if dry:
for ai in runs:
oruns.append(ai.record_id)
print ai.measurement.mass_spectrometer.name, ai.record_id, ai.labnumber.sample.name, \
ai.analysis_timestamp
else:
self.debug('================= Do Export i: {} low: {} high: {}'.format(i, low, high))
self.debug('N runs: {}'.format(len(runs)))
self.do_export([ai.record_id for ai in runs], repository_identifier, principal_investigator)
return oruns
def find_project_overlaps(self, projects):
tol_hrs = 6
src = self.processor.db
pdict = {}
for p in projects:
for ms in ('jan', 'obama'):
ts, idxs = self._get_project_timestamps(p, ms, tol_hrs=tol_hrs)
for i, ais in enumerate(array_split(ts, idxs + 1)):
if not ais.shape[0]:
self.debug('skipping {}'.format(i))
continue
low = get_datetime(ais[0]) - timedelta(hours=tol_hrs / 2.)
high = get_datetime(ais[-1]) + timedelta(hours=tol_hrs / 2.)
print '========{}, {}, {}'.format(ms, low, high)
with src.session_ctx():
runs = src.get_analyses_date_range(low, high,
projects=('REFERENCES',),
mass_spectrometers=(ms,))
pdict[p] = [ai.record_id for ai in runs]
for p in projects:
for o in projects:
if p == o:
continue
pruns = pdict[p]
oruns = pdict[o]
for ai in pruns:
if ai in oruns:
print p, o, ai
def import_date_range(self, low, high, spectrometer, repository_identifier, creator):
src = self.processor.db
with src.session_ctx():
runs = src.get_analyses_date_range(low, high, mass_spectrometers=spectrometer)
ais = [ai.record_id for ai in runs]
self.do_export(ais, repository_identifier, creator)
def do_export(self, runs, repository_identifier, creator, create_repo=False, monitor_mapping=None):
# self._init_src_dest()
src = self.processor.db
dest = self.dvc.db
with src.session_ctx():
key = lambda x: x.split('-')[0]
runs = sorted(runs, key=key)
with dest.session_ctx():
repo = self._add_repository(dest, repository_identifier, creator, create_repo)
self.persister.active_repository = repo
self.dvc.current_repository = repo
total = len(runs)
j = 0
for ln, ans in groupby(runs, key=key):
ans = list(ans)
n = len(ans)
for i, a in enumerate(ans):
with dest.session_ctx() as sess:
st = time.time()
try:
if self._transfer_analysis(a, repository_identifier, monitor_mapping=monitor_mapping):
j += 1
self.debug('{}/{} transfer time {:0.3f}'.format(j, total, time.time() - st))
except BaseException, e:
import traceback
traceback.print_exc()
self.warning('failed transfering {}. {}'.format(a, e))
def _dvc_factory(self):
d = DVC(application=self.application)
# d.initialize()
return d
class IsoDBTransfer(Loggable):
"""
transfer analyses from an isotope_db database to a dvc database
"""
dvc = Instance(DVC)
processor = Instance(IsotopeDatabaseManager)
persister = Instance(DVCPersister)
quiet = False
def init(self):
conn = dict(host=os.environ.get('ARGONSERVER_HOST'),
username='******',#os.environ.get('ARGONSERVER_DB_USER'),
password=os.environ.get('ARGONSERVER_DB_PWD'),
kind='mysql')
use_local = False
if use_local:
meta_name = 'MetaDataDev'
dest_conn = dict(host='localhost',
username=os.environ.get('LOCALHOST_DB_USER'),
password=os.environ.get('LOCALHOST_DB_PWD'),
kind='mysql',
# echo=True,
name='pychrondvc_dev')
else:
meta_name = 'NMGRLMetaData'
dest_conn = conn.copy()
dest_conn['name'] = 'pychrondvc'
self.dvc = DVC(bind=False,
organization='NMGRLData',
meta_repo_name=meta_name)
paths.meta_root = os.path.join(paths.dvc_dir, self.dvc.meta_repo_name)
self.dvc.db.trait_set(**dest_conn)
if not self.dvc.initialize():
self.warning_dialog('Failed to initialize DVC')
return
self.dvc.meta_repo.smart_pull(quiet=self.quiet)
self.persister = DVCPersister(dvc=self.dvc, stage_files=False)
proc = IsotopeDatabaseManager(bind=False, connect=False)
use_local_src = False
if use_local_src:
conn = dict(host='localhost',
username=os.environ.get('LOCALHOST_DB_USER'),
password=os.environ.get('LOCALHOST_DB_PWD'),
kind='mysql',
# echo=True,
name='pychrondata')
else:
conn['name'] = 'pychrondata'
proc.db.trait_set(**conn)
src = proc.db
src.connect()
self.processor = proc
def copy_productions(self):
src = self.processor.db
dvc = self.dvc
dest = dvc.db
with src.session_ctx():
pr = src.get_irradiation_productions()
for p in pr:
# copy to database
pname = p.name.replace(' ', '_')
if not dest.get_production(pname):
dest.add_production(pname)
# copy to meta
dvc.copy_production(p)
def set_spectrometer_files(self, repository):
set_spectrometer_files(self.processor.db,
self.dvc.db,
repository,
os.path.join(paths.repository_dataset_dir, repository))
def bulk_import_irradiations(self, irradiations, creator, dry=True):
# for i in xrange(251, 277):
# for i in xrange(258, 259):
# for i in (258, 259, 260, 261,):
# for i in (262, 263, 264, 265):
# for i in (266, 267, 268, 269):
# for i in (270, 271, 272, 273):
for i in irradiations:
irradname = 'NM-{}'.format(i)
runs = self.bulk_import_irradiation(irradname, creator, dry=dry)
# if runs:
# with open('/Users/ross/Sandbox/bulkimport/irradiation_runs.txt', 'a') as wfile:
# for o in runs:
# wfile.write('{}\n'.format(o))
def bulk_import_irradiation(self, irradname, creator, dry=True):
src = self.processor.db
tol_hrs = 6
self.debug('bulk import irradiation {}'.format(irradname))
oruns = []
ts, idxs = self._get_irradiation_timestamps(irradname, tol_hrs=tol_hrs)
print(ts)
repository_identifier = 'Irradiation-{}'.format(irradname)
# add project
with self.dvc.db.session_ctx():
self.dvc.db.add_project(repository_identifier, creator)
def filterfunc(x):
a = x.labnumber.irradiation_position is None
b = False
if not a:
b = x.labnumber.irradiation_position.level.irradiation.name == irradname
d = False
if x.extraction:
ed = x.extraction.extraction_device
if not ed:
d = True
else:
d = ed.name == 'Fusions CO2'
return (a or b) and d
# for ms in ('jan', 'obama'):
# monitors not run on obama
for ms in ('jan',):
for i, ais in enumerate(array_split(ts, idxs + 1)):
if not ais.shape[0]:
self.debug('skipping {}'.format(i))
continue
low = get_datetime(ais[0]) - timedelta(hours=tol_hrs / 2.)
high = get_datetime(ais[-1]) + timedelta(hours=tol_hrs / 2.)
with src.session_ctx():
ans = src.get_analyses_date_range(low, high,
mass_spectrometers=(ms,),
samples=('FC-2',
'blank_unknown', 'blank_air', 'blank_cocktail', 'air',
'cocktail'))
# runs = filter(lambda x: x.labnumber.irradiation_position is None or
# x.labnumber.irradiation_position.level.irradiation.name == irradname, ans)
runs = list(filter(filterfunc, ans))
if dry:
for ai in runs:
oruns.append(ai.record_id)
print(ms, ai.record_id)
else:
self.debug('================= Do Export i: {} low: {} high: {}'.format(i, low, high))
self.debug('N runs: {}'.format(len(runs)))
self.do_export([ai.record_id for ai in runs],
repository_identifier, creator,
monitor_mapping=('FC-2', 'Sanidine', repository_identifier))
return oruns
def bulk_import_project(self, project, principal_investigator, source_name=None, dry=True):
src = self.processor.db
tol_hrs = 6
self.debug('bulk import project={}, pi={}'.format(project, principal_investigator))
oruns = []
repository_identifier = project
# def filterfunc(x):
# a = x.labnumber.irradiation_position is None
# b = False
# if not a:
# b = x.labnumber.irradiation_position.level.irradiation.name == irradname
#
# d = False
# if x.extraction:
# ed = x.extraction.extraction_device
# if not ed:
# d = True
# else:
# d = ed.name == 'Fusions CO2'
#
# return (a or b) and d
if source_name is None:
source_name = project
for ms in ('jan', 'obama', 'felix'):
ts, idxs = self._get_project_timestamps(source_name, ms, tol_hrs=tol_hrs)
for i, ais in enumerate(array_split(ts, idxs + 1)):
if not ais.shape[0]:
self.debug('skipping {}'.format(i))
continue
low = get_datetime(ais[0]) - timedelta(hours=tol_hrs / 2.)
high = get_datetime(ais[-1]) + timedelta(hours=tol_hrs / 2.)
print('========{}, {}, {}'.format(ms, low, high))
with src.session_ctx():
runs = src.get_analyses_date_range(low, high,
# labnumber=(63630, 63632, 63634, 63636, 63638, 63646, 63648),
projects=('REFERENCES', ),
# projects=('REFERENCES', source_name),
mass_spectrometers=(ms,))
if dry:
for ai in runs:
oruns.append(ai.record_id)
print(ai.measurement.mass_spectrometer.name, ai.record_id, ai.labnumber.sample.name, \
ai.analysis_timestamp)
else:
self.debug('================= Do Export i: {} low: {} high: {}'.format(i, low, high))
self.debug('N runs: {}'.format(len(runs)))
self.do_export([ai.record_id for ai in runs], repository_identifier, principal_investigator)
return oruns
def find_project_overlaps(self, projects):
tol_hrs = 6
src = self.processor.db
pdict = {}
for p in projects:
for ms in ('jan', 'obama'):
ts, idxs = self._get_project_timestamps(p, ms, tol_hrs=tol_hrs)
for i, ais in enumerate(array_split(ts, idxs + 1)):
if not ais.shape[0]:
self.debug('skipping {}'.format(i))
continue
low = get_datetime(ais[0]) - timedelta(hours=tol_hrs / 2.)
high = get_datetime(ais[-1]) + timedelta(hours=tol_hrs / 2.)
print('========{}, {}, {}'.format(ms, low, high))
with src.session_ctx():
runs = src.get_analyses_date_range(low, high,
projects=('REFERENCES',),
mass_spectrometers=(ms,))
pdict[p] = [ai.record_id for ai in runs]
for p in projects:
for o in projects:
if p == o:
continue
pruns = pdict[p]
oruns = pdict[o]
for ai in pruns:
if ai in oruns:
print(p, o, ai)
def import_date_range(self, low, high, spectrometer, repository_identifier, creator):
src = self.processor.db
with src.session_ctx():
runs = src.get_analyses_date_range(low, high, mass_spectrometers=spectrometer)
ais = [ai.record_id for ai in runs]
self.do_export(ais, repository_identifier, creator)
def do_export(self, runs, repository_identifier, creator, create_repo=False, monitor_mapping=None):
# self._init_src_dest()
src = self.processor.db
dest = self.dvc.db
with src.session_ctx():
key = lambda x: x.split('-')[0]
runs = sorted(runs, key=key)
with dest.session_ctx():
repo = self._add_repository(dest, repository_identifier, creator, create_repo)
self.persister.active_repository = repo
self.dvc.current_repository = repo
total = len(runs)
j = 0
for ln, ans in groupby(runs, key=key):
ans = list(ans)
n = len(ans)
for i, a in enumerate(ans):
with dest.session_ctx() as sess:
st = time.time()
try:
if self._transfer_analysis(a, repository_identifier, monitor_mapping=monitor_mapping):
j += 1
self.debug('{}/{} transfer time {:0.3f}'.format(j, total, time.time() - st))
except BaseException as e:
import traceback
traceback.print_exc()
self.warning('failed transfering {}. {}'.format(a, e))
# private
def _get_project_timestamps(self, project, mass_spectrometer, tol_hrs=6):
src = self.processor.db
return get_project_timestamps(src, project, mass_spectrometer, tol_hrs)
def _get_irradiation_timestamps(self, irradname, tol_hrs=6):
src = self.processor.db
return get_irradiation_timestamps(src, irradname, tol_hrs)
def _add_repository(self, dest, repository_identifier, creator, create_repo):
repository_identifier = format_repository_identifier(repository_identifier)
# sys.exit()
proot = os.path.join(paths.repository_dataset_dir, repository_identifier)
if not os.path.isdir(proot):
# create new local repo
os.mkdir(proot)
repo = GitRepoManager()
repo.open_repo(proot)
repo.add_ignore('.DS_Store')
self.repo_man = repo
if create_repo:
# add repo to central location
create_github_repo(repository_identifier)
url = 'https://github.com/{}/{}.git'.format(ORG, repository_identifier)
self.debug('Create repo at github. url={}'.format(url))
repo.create_remote(url)
else:
repo = GitRepoManager()
repo.open_repo(proot)
dbexp = dest.get_repository(repository_identifier)
if not dbexp:
dest.add_repository(repository_identifier, creator)
return repo
def _transfer_meta(self, dest, dban, monitor_mapping):
pi = 'Mcintosh,W'
self.debug('transfer meta {}'.format(monitor_mapping))
dblab = dban.labnumber
dbsam = dblab.sample
material_name = dbsam.material.name
project = dbsam.project.name
project_name = project.replace('/', '_').replace('\\', '_')
sample_name = dbsam.name
# if monitor_mapping is None:
# sample_name, material, project = monitor_mapping
#
# if material_name == 'bi':
# material_name = 'Biotite'
sam = dest.get_sample(sample_name, project_name, pi, material_name)
if not sam:
if not dest.get_material(material_name):
self.debug('add material {}'.format(material_name))
dest.add_material(material_name)
dest.flush()
if not dest.get_project(project_name, pi):
self.debug('add project {}'.format(project_name))
dest.add_project(project_name, pi)
dest.flush()
self.debug('add sample {}'.format(sample_name))
sam = dest.add_sample(sample_name, project_name, pi, material_name)
dest.flush()
dbirradpos = dblab.irradiation_position
if not dbirradpos:
irradname = 'NoIrradiation'
levelname = 'A'
holder = 'Grid'
pos = None
identifier = dblab.identifier
doses = []
prod = None
prodname = 'NoIrradiation'
# geom = make_geom([(0, 0, 0.0175),
# (1, 0, 0.0175),
# (2, 0, 0.0175),
# (3, 0, 0.0175),
# (4, 0, 0.0175),
#
# (0, 1, 0.0175),
# (1, 1, 0.0175),
# (2, 1, 0.0175),
# (3, 1, 0.0175),
# (4, 1, 0.0175),
#
# (0, 2, 0.0175),
# (1, 2, 0.0175),
# (2, 2, 0.0175),
# (3, 2, 0.0175),
# (4, 2, 0.0175),
#
# (0, 3, 0.0175),
# (1, 3, 0.0175),
# (2, 3, 0.0175),
# (3, 3, 0.0175),
# (4, 3, 0.0175),
#
# (0, 4, 0.0175),
# (1, 4, 0.0175),
# (2, 4, 0.0175),
# (3, 4, 0.0175),
# (4, 4, 0.0175)
# ])
else:
dblevel = dbirradpos.level
dbirrad = dblevel.irradiation
dbchron = dbirrad.chronology
irradname = dbirrad.name
levelname = dblevel.name
holder = dblevel.holder.name if dblevel.holder else ''
# geom = dblevel.holder.geometry if dblevel.holder else ''
prodname = dblevel.production.name if dblevel.production else ''
prodname = prodname.replace(' ', '_')
pos = dbirradpos.position
doses = dbchron.get_doses()
meta_repo = self.dvc.meta_repo
# save db irradiation
if not dest.get_irradiation(irradname):
self.debug('Add irradiation {}'.format(irradname))
self.dvc.add_irradiation(irradname, doses)
dest.flush()
# meta_repo.add_irradiation(irradname)
# meta_repo.add_chronology(irradname, doses, add=False)
# meta_repo.commit('added irradiation {}'.format(irradname))
# save production name to db
if not dest.get_production(prodname):
self.debug('Add production {}'.format(prodname))
dest.add_production(prodname)
dest.flush()
# meta_repo.add_production(irradname, prodname, prod, add=False)
# meta_repo.commit('added production {}'.format(prodname))
# save db level
if not dest.get_irradiation_level(irradname, levelname):
self.debug('Add level irrad:{} level:{}'.format(irradname, levelname))
dest.add_irradiation_level(levelname, irradname, holder, prodname)
dest.flush()
# meta_repo.add_irradiation_holder(holder, geom, add=False)
meta_repo.add_level(irradname, levelname, add=False)
meta_repo.update_level_production(irradname, levelname, prodname)
# meta_repo.commit('added empty level {}{}'.format(irradname, levelname))
if pos is None:
pos = self._get_irradpos(dest, irradname, levelname, identifier)
# save db irradiation position
if not dest.get_irradiation_position(irradname, levelname, pos):
self.debug('Add position irrad:{} level:{} pos:{}'.format(irradname, levelname, pos))
p = meta_repo.get_level_path(irradname, levelname)
with open(p, 'r') as rfile:
yd = json.load(rfile)
dd = dest.add_irradiation_position(irradname, levelname, pos)
dd.identifier = idn = dblab.identifier
dd.sample = sam
dest.flush()
try:
f = dban.labnumber.selected_flux_history.flux
j, e = f.j, f.j_err
except AttributeError:
j, e = 0, 0
ps = yd['positions']
ps.append({'j': j, 'j_err': e, 'position': pos, 'analyses': [],
'identifier': idn, 'decay_constants': {}})
yd['positions'] = ps
dvc_dump(yd, p)
dest.commit()
def _transfer_analysis(self, rec, exp, overwrite=True, monitor_mapping=None):
dest = self.dvc.db
proc = self.processor
src = proc.db
# args = rec.split('-')
# idn = '-'.join(args[:-1])
# t = args[-1]
# try:
# aliquot = int(t)
# step = None
# except ValueError:
# aliquot = int(t[:-1])
# step = t[-1]
m = IDENTIFIER_REGEX.match(rec)
if not m:
m = SPECIAL_IDENTIFIER_REGEX.match(rec)
if not m:
self.warning('invalid runid {}'.format(rec))
return
else:
idn = m.group('identifier')
aliquot = m.group('aliquot')
try:
step = m.group('step') or None
except IndexError:
step = None
if idn == '4359':
idn = 'c-01-j'
elif idn == '4358':
idn = 'c-01-o'
# check if analysis already exists. skip if it does
if dest.get_analysis_runid(idn, aliquot, step):
self.warning('{} already exists'.format(make_runid(idn, aliquot, step)))
return
dban = src.get_analysis_runid(idn, aliquot, step)
iv = IsotopeRecordView()
iv.uuid = dban.uuid
self.debug('make analysis idn:{}, aliquot:{} step:{}'.format(idn, aliquot, step))
# try:
an = proc.make_analysis(iv, unpack=True, use_cache=False, use_progress=False)
# except BaseException as e:
# self.warning('Failed to make {}'.format(make_runid(idn, aliquot, step)))
# self.warning('exception: {}'.format(e))
# return
self._transfer_meta(dest, dban, monitor_mapping)
# return
dblab = dban.labnumber
if dblab.irradiation_position:
irrad = dblab.irradiation_position.level.irradiation.name
level = dblab.irradiation_position.level.name
irradpos = dblab.irradiation_position.position
else:
irrad = 'NoIrradiation'
level = 'A'
irradpos = self._get_irradpos(dest, irrad, level, dblab.identifier)
# irrad, level, irradpos = '', '', 0
extraction = dban.extraction
ms = dban.measurement.mass_spectrometer.name
if not dest.get_mass_spectrometer(ms):
self.debug('adding mass spectrometer {}'.format(ms))
dest.add_mass_spectrometer(ms)
dest.commit()
ed = extraction.extraction_device.name if extraction.extraction_device else None
if not ed:
ed = 'No Extract Device'
if not dest.get_extraction_device(ed):
self.debug('adding extract device {}'.format(ed))
dest.add_extraction_device(ed)
dest.commit()
if step is None:
inc = -1
else:
inc = ALPHAS.index(step)
username = ''
if dban.user:
username = dban.user.name
if not dest.get_user(username):
self.debug('adding user. username:{}'.format(username))
dest.add_user(username)
dest.commit()
if monitor_mapping:
sample_name, material_name, project_name = monitor_mapping
else:
dbsam = dblab.sample
sample_name = dbsam.name
material_name = dbsam.material.name
project_name = format_repository_identifier(dbsam.project.name)
rs = AutomatedRunSpec(labnumber=idn,
username=username,
material=material_name,
project=project_name,
sample=sample_name,
irradiation=irrad,
irradiation_level=level,
irradiation_position=irradpos,
repository_identifier=exp,
mass_spectrometer=ms,
uuid=dban.uuid,
_step=inc,
comment=dban.comment.decode('utf-8') or '',
aliquot=int(aliquot),
extract_device=ed,
duration=extraction.extract_duration,
cleanup=extraction.cleanup_duration,
beam_diameter=extraction.beam_diameter,
extract_units=extraction.extract_units or '',
extract_value=extraction.extract_value,
pattern=extraction.pattern or '',
weight=extraction.weight,
ramp_duration=extraction.ramp_duration or 0,
ramp_rate=extraction.ramp_rate or 0,
collection_version='0.1:0.1',
queue_conditionals_name='',
tray='')
meas = dban.measurement
# get spectrometer parameters
# gains
gains = {}
gain_history = dban.gain_history
if gain_history:
gains = {d.detector.name: d.value for d in gain_history.gains}
# deflections
deflections = {d.detector.name: d.deflection for d in meas.deflections}
# source
src = {k: getattr(meas.spectrometer_parameters, k) for k in QTEGRA_SOURCE_KEYS}
ps = PersistenceSpec(run_spec=rs,
tag=an.tag.name,
isotope_group=an,
timestamp=dban.analysis_timestamp,
defl_dict=deflections,
gains=gains,
spec_dict=src,
use_repository_association=True,
positions=[p.position for p in extraction.positions])
self.debug('transfer analysis with persister')
self.persister.per_spec_save(ps, commit=False, commit_tag='Database Transfer')
return True
def _get_irradpos(self, dest, irradname, levelname, identifier):
dl = dest.get_irradiation_level(irradname, levelname)
pos = 1
if dl.positions:
for p in dl.positions:
if p.identifier == identifier:
pos = p.position
break
else:
pos = dl.positions[-1].position + 1
return pos
