def _add_repository(self, dest, repository_identifier, creator,
create_repo):
repository_identifier = format_repository_identifier(
repository_identifier)
# sys.exit()
proot = os.path.join(paths.repository_dataset_dir,
repository_identifier)
if not os.path.isdir(proot):
# create new local repo
os.mkdir(proot)
repo = GitRepoManager()
repo.open_repo(proot)
repo.add_ignore('.DS_Store')
self.repo_man = repo
if create_repo:
# add repo to central location
create_github_repo(repository_identifier)
url = 'https://github.com/{}/{}.git'.format(
ORG, repository_identifier)
self.debug('Create repo at github. url={}'.format(url))
repo.create_remote(url)
else:
repo = GitRepoManager()
repo.open_repo(proot)
dbexp = dest.get_repository(repository_identifier)
if not dbexp:
dest.add_repository(repository_identifier, creator)
return repo
def _add_repository(self, dest, repository_identifier, creator, create_repo):
repository_identifier = format_repository_identifier(repository_identifier)
# sys.exit()
proot = os.path.join(paths.repository_dataset_dir, repository_identifier)
if not os.path.isdir(proot):
# create new local repo
os.mkdir(proot)
repo = GitRepoManager()
repo.open_repo(proot)
repo.add_ignore('.DS_Store')
self.repo_man = repo
if create_repo:
# add repo to central location
create_github_repo(repository_identifier)
url = 'https://github.com/{}/{}.git'.format(ORG, repository_identifier)
self.debug('Create repo at github. url={}'.format(url))
repo.create_remote(url)
else:
repo = GitRepoManager()
repo.open_repo(proot)
dbexp = dest.get_repository(repository_identifier)
if not dbexp:
dest.add_repository(repository_identifier, creator)
return repo
def _transfer_analysis(self,
rec,
exp,
overwrite=True,
monitor_mapping=None):
dest = self.dvc.db
proc = self.processor
src = proc.db
# args = rec.split('-')
# idn = '-'.join(args[:-1])
# t = args[-1]
# try:
# aliquot = int(t)
# step = None
# except ValueError:
# aliquot = int(t[:-1])
# step = t[-1]
m = IDENTIFIER_REGEX.match(rec)
if not m:
m = SPECIAL_IDENTIFIER_REGEX.match(rec)
if not m:
self.warning('invalid runid {}'.format(rec))
return
else:
idn = m.group('identifier')
aliquot = m.group('aliquot')
try:
step = m.group('step') or None
except IndexError:
step = None
if idn == '4359':
idn = 'c-01-j'
elif idn == '4358':
idn = 'c-01-o'
# check if analysis already exists. skip if it does
if dest.get_analysis_runid(idn, aliquot, step):
self.warning('{} already exists'.format(
make_runid(idn, aliquot, step)))
return
dban = src.get_analysis_runid(idn, aliquot, step)
iv = IsotopeRecordView()
iv.uuid = dban.uuid
self.debug('make analysis idn:{}, aliquot:{} step:{}'.format(
idn, aliquot, step))
# try:
an = proc.make_analysis(iv,
unpack=True,
use_cache=False,
use_progress=False)
# except BaseException as e:
# self.warning('Failed to make {}'.format(make_runid(idn, aliquot, step)))
# self.warning('exception: {}'.format(e))
# return
self._transfer_meta(dest, dban, monitor_mapping)
# return
dblab = dban.labnumber
if dblab.irradiation_position:
irrad = dblab.irradiation_position.level.irradiation.name
level = dblab.irradiation_position.level.name
irradpos = dblab.irradiation_position.position
else:
irrad = 'NoIrradiation'
level = 'A'
irradpos = self._get_irradpos(dest, irrad, level, dblab.identifier)
# irrad, level, irradpos = '', '', 0
extraction = dban.extraction
ms = dban.measurement.mass_spectrometer.name
if not dest.get_mass_spectrometer(ms):
self.debug('adding mass spectrometer {}'.format(ms))
dest.add_mass_spectrometer(ms)
dest.commit()
ed = extraction.extraction_device.name if extraction.extraction_device else None
if not ed:
ed = 'No Extract Device'
if not dest.get_extraction_device(ed):
self.debug('adding extract device {}'.format(ed))
dest.add_extraction_device(ed)
dest.commit()
if step is None:
inc = -1
else:
inc = alpha_to_int(step)
username = ''
if dban.user:
username = dban.user.name
if not dest.get_user(username):
self.debug('adding user. username:{}'.format(username))
dest.add_user(username)
dest.commit()
if monitor_mapping:
sample_name, material_name, project_name = monitor_mapping
else:
dbsam = dblab.sample
sample_name = dbsam.name
material_name = dbsam.material.name
project_name = format_repository_identifier(dbsam.project.name)
rs = AutomatedRunSpec(labnumber=idn,
username=username,
material=material_name,
project=project_name,
sample=sample_name,
irradiation=irrad,
irradiation_level=level,
irradiation_position=irradpos,
repository_identifier=exp,
mass_spectrometer=ms,
uuid=dban.uuid,
_step=inc,
comment=dban.comment.decode('utf-8') or '',
aliquot=int(aliquot),
extract_device=ed,
duration=extraction.extract_duration,
cleanup=extraction.cleanup_duration,
beam_diameter=extraction.beam_diameter,
extract_units=extraction.extract_units or '',
extract_value=extraction.extract_value,
pattern=extraction.pattern or '',
weight=extraction.weight,
ramp_duration=extraction.ramp_duration or 0,
ramp_rate=extraction.ramp_rate or 0,
collection_version='0.1:0.1',
queue_conditionals_name='',
tray='')
meas = dban.measurement
# get spectrometer parameters
# gains
gains = {}
gain_history = dban.gain_history
if gain_history:
gains = {d.detector.name: d.value for d in gain_history.gains}
# deflections
deflections = {d.detector.name: d.deflection for d in meas.deflections}
# source
src = {
k: getattr(meas.spectrometer_parameters, k)
for k in QTEGRA_SOURCE_KEYS
}
ps = PersistenceSpec(
run_spec=rs,
tag=an.tag.name,
isotope_group=an,
timestamp=dban.analysis_timestamp,
defl_dict=deflections,
gains=gains,
spec_dict=src,
use_repository_association=True,
positions=[p.position for p in extraction.positions])
self.debug('transfer analysis with persister')
self.persister.per_spec_save(ps,
commit=False,
commit_tag='Database Transfer')
return True
def _transfer_analysis(self, rec, exp, overwrite=True, monitor_mapping=None):
dest = self.dvc.db
proc = self.processor
src = proc.db
# args = rec.split('-')
# idn = '-'.join(args[:-1])
# t = args[-1]
# try:
# aliquot = int(t)
# step = None
# except ValueError:
# aliquot = int(t[:-1])
# step = t[-1]
m = IDENTIFIER_REGEX.match(rec)
if not m:
m = SPECIAL_IDENTIFIER_REGEX.match(rec)
if not m:
self.warning('invalid runid {}'.format(rec))
return
else:
idn = m.group('identifier')
aliquot = m.group('aliquot')
try:
step = m.group('step') or None
except IndexError:
step = None
if idn == '4359':
idn = 'c-01-j'
elif idn == '4358':
idn = 'c-01-o'
# check if analysis already exists. skip if it does
if dest.get_analysis_runid(idn, aliquot, step):
self.warning('{} already exists'.format(make_runid(idn, aliquot, step)))
return
dban = src.get_analysis_runid(idn, aliquot, step)
iv = IsotopeRecordView()
iv.uuid = dban.uuid
self.debug('make analysis idn:{}, aliquot:{} step:{}'.format(idn, aliquot, step))
try:
an = proc.make_analysis(iv, unpack=True, use_cache=False, use_progress=False)
except:
self.warning('Failed to make {}'.format(make_runid(idn, aliquot, step)))
return
self._transfer_meta(dest, dban, monitor_mapping)
# return
dblab = dban.labnumber
if dblab.irradiation_position:
irrad = dblab.irradiation_position.level.irradiation.name
level = dblab.irradiation_position.level.name
irradpos = dblab.irradiation_position.position
else:
irrad = 'NoIrradiation'
level = 'A'
irradpos = self._get_irradpos(dest, irrad, level, dblab.identifier)
# irrad, level, irradpos = '', '', 0
extraction = dban.extraction
ms = dban.measurement.mass_spectrometer.name
if not dest.get_mass_spectrometer(ms):
self.debug('adding mass spectrometer {}'.format(ms))
dest.add_mass_spectrometer(ms)
dest.commit()
ed = extraction.extraction_device.name if extraction.extraction_device else None
if not ed:
ed = 'No Extract Device'
if not dest.get_extraction_device(ed):
self.debug('adding extract device {}'.format(ed))
dest.add_extraction_device(ed)
dest.commit()
if step is None:
inc = -1
else:
inc = ALPHAS.index(step)
username = ''
if dban.user:
username = dban.user.name
if not dest.get_user(username):
self.debug('adding user. username:{}'.format(username))
dest.add_user(username)
dest.commit()
if monitor_mapping:
sample_name, material_name, project_name = monitor_mapping
else:
dbsam = dblab.sample
sample_name = dbsam.name
material_name = dbsam.material.name
project_name = format_repository_identifier(dbsam.project.name)
rs = AutomatedRunSpec(labnumber=idn,
username=username,
material=material_name,
project=project_name,
sample=sample_name,
irradiation=irrad,
irradiation_level=level,
irradiation_position=irradpos,
repository_identifier=exp,
mass_spectrometer=ms,
uuid=dban.uuid,
_step=inc,
comment=dban.comment or '',
aliquot=int(aliquot),
extract_device=ed,
duration=extraction.extract_duration,
cleanup=extraction.cleanup_duration,
beam_diameter=extraction.beam_diameter,
extract_units=extraction.extract_units or '',
extract_value=extraction.extract_value,
pattern=extraction.pattern or '',
weight=extraction.weight,
ramp_duration=extraction.ramp_duration or 0,
ramp_rate=extraction.ramp_rate or 0,
collection_version='0.1:0.1',
queue_conditionals_name='',
tray='')
meas = dban.measurement
# get spectrometer parameters
# gains
gains = {}
gain_history = dban.gain_history
if gain_history:
gains = {d.detector.name: d.value for d in gain_history.gains}
# deflections
deflections = {d.detector.name: d.deflection for d in meas.deflections}
# source
src = {k: getattr(meas.spectrometer_parameters, k) for k in QTEGRA_SOURCE_KEYS}
ps = PersistenceSpec(run_spec=rs,
tag=an.tag.name,
arar_age=an,
timestamp=dban.analysis_timestamp,
defl_dict=deflections,
gains=gains,
spec_dict=src,
use_repository_association=True,
positions=[p.position for p in extraction.positions])
self.debug('transfer analysis with persister')
self.persister.per_spec_save(ps, commit=False, msg_prefix='Database Transfer')
return True
