def doit(ts_label):
acc = Accession(session, ts_label)
ts = acc.get_time_series()
if ts is None:
raise ValueError('No time series named %s' % ts_label)
logging.warn('Starting batch accession for %s' % ts_label)
acc.add_all_filesets()
logging.warn('Finished batch accession for %s' % ts_label)
def do_acc(pid, job):
parsed = parse_pid(pid)
lid = parsed[LID]
ts_label = parsed[TS_LABEL]
roots = get_data_roots(session, ts_label) # get raw data roots
fileset = parsed_pid2fileset(parsed, roots)
fileset[LID] = lid
session.expire_all() # don't be stale!
acc = Accession(session,ts_label)#,fast=True)
# FIXME fast=True disables checksumming
client.update(pid,ttl=3600) # allow 1hr for accession
ret = acc.add_fileset(fileset)
if ret=='ADDED':
schedule_products(pid, client)
session.commit()
client.wakeup()
elif ret=='FAILED':
raise Exception('accession failed')
def acc_wakeup(wakeup_key):
"""- wake up and expire the session
- acquire a mutex on the acquisition key
- query for the instrument
- run the copy job
- schedule the accession job
- wakeup accession workers"""
# figure out if this wakeup matters to us
if not is_acc_key(wakeup_key):
return
time_series = get_time_series(wakeup_key)
# attempt to acquire mutex. if fails, that's fine,
# that means batch accession is already underway
try:
then = time.time()
count = 0
with Mutex(wakeup_key,ttl=45) as mutex:
session.expire_all() # don't be stale!
accession = Accession(session, time_series)
logging.warn('START BATCH %s' % time_series)
for fs in accession.list_filesets(): # FIXME debug, do all
lid = fs[LID]
if accession.bin_exists(lid):
continue # don't schedule accession if bin exists
pid = canonicalize(URL_PREFIX, time_series, fs[LID])
count += 1
if count % 100 == 0:
logging.warn('batch %s: scheduled %d bins' % (time_series, count))
schedule_accession(client,pid)
elapsed = time.time() - then
if elapsed > 25: # don't send heartbeats too often
mutex.heartbeat() # retain mutex
then = time.time()
client.wakeup() # wakeup workers
logging.warn('END BATCH %s: %d bins scheduled' % (time_series,count))
client.wakeup()
except Busy:
logging.warn('BATCH not waking up')
pass