def do_acc(pid, job):
parsed = parse_pid(pid)
lid = parsed[LID]
ts_label = parsed[TS_LABEL]
roots = get_data_roots(session, ts_label) # get raw data roots
fileset = parsed_pid2fileset(parsed, roots)
fileset[LID] = lid
session.expire_all() # don't be stale!
acc = Accession(session,ts_label)#,fast=True)
# FIXME fast=True disables checksumming
client.update(pid,ttl=3600) # allow 1hr for accession
ret = acc.add_fileset(fileset)
if ret=='ADDED':
schedule_products(pid, client)
session.commit()
client.wakeup()
elif ret=='FAILED':
raise Exception('accession failed')
def acc_wakeup(wakeup_key):
"""- wake up and expire the session
- acquire a mutex on the acquisition key
- query for the instrument
- run the copy job
- schedule the accession job
- wakeup accession workers"""
# figure out if this wakeup matters to us
if not is_acc_key(wakeup_key):
return
time_series = get_time_series(wakeup_key)
# attempt to acquire mutex. if fails, that's fine,
# that means batch accession is already underway
try:
then = time.time()
count = 0
with Mutex(wakeup_key,ttl=45) as mutex:
session.expire_all() # don't be stale!
accession = Accession(session, time_series)
logging.warn('START BATCH %s' % time_series)
for fs in accession.list_filesets(): # FIXME debug, do all
lid = fs[LID]
if accession.bin_exists(lid):
continue # don't schedule accession if bin exists
pid = canonicalize(URL_PREFIX, time_series, fs[LID])
count += 1
if count % 100 == 0:
logging.warn('batch %s: scheduled %d bins' % (time_series, count))
schedule_accession(client,pid)
elapsed = time.time() - then
if elapsed > 25: # don't send heartbeats too often
mutex.heartbeat() # retain mutex
then = time.time()
client.wakeup() # wakeup workers
logging.warn('END BATCH %s: %d bins scheduled' % (time_series,count))
client.wakeup()
except Busy:
logging.warn('BATCH not waking up')
pass
def acq_wakeup(wakeup_key):
"""- wake up and expire the session
- acquire a mutex on the acquisition key
- query for the instrument
- run the copy job
- schedule the accession job
- wakeup accession workers"""
# figure out if this wakeup matters to us
if not is_acq_key(wakeup_key):
return
instrument_name = get_instrument_name(wakeup_key)
# attempt to acquire mutex. if fails, that's fine,
# that means acquisition is aready underway
try:
with Mutex(wakeup_key,ttl=45) as mutex:
# get the instrument info
session.expire_all() # don't be stale!
instrument = session.query(Instrument).\
filter(Instrument.name==instrument_name).\
first()
if instrument is None:
logging.warn('ERROR cannot find instrument named "%s"' % instrument_name)
return
ts_label = instrument.time_series.label
logging.warn('%s: starting acquisition cycle' % instrument_name)
def callback(lid):
logging.warn('%s: copied %s from %s' % (ts_label, lid, instrument_name))
mutex.heartbeat() # still alive
# schedule an accession job and wake up accession workers
pid = '%s%s/%s' % (URL_PREFIX, ts_label, lid)
schedule_accession(client,pid)
client.wakeup()
logging.warn('%s: scheduled accession for %s' % (ts_label, lid))
copy_work(instrument, callback=callback)
logging.warn('%s: acquisition cycle complete' % ts_label)
except Busy:
pass
