def do_acc(pid, job): parsed = parse_pid(pid) lid = parsed[LID] ts_label = parsed[TS_LABEL] roots = get_data_roots(session, ts_label) # get raw data roots fileset = parsed_pid2fileset(parsed, roots) fileset[LID] = lid session.expire_all() # don't be stale! acc = Accession(session,ts_label)#,fast=True) # FIXME fast=True disables checksumming client.update(pid,ttl=3600) # allow 1hr for accession ret = acc.add_fileset(fileset) if ret=='ADDED': schedule_products(pid, client) session.commit() client.wakeup() elif ret=='FAILED': raise Exception('accession failed')
def acc_wakeup(wakeup_key): """- wake up and expire the session - acquire a mutex on the acquisition key - query for the instrument - run the copy job - schedule the accession job - wakeup accession workers""" # figure out if this wakeup matters to us if not is_acc_key(wakeup_key): return time_series = get_time_series(wakeup_key) # attempt to acquire mutex. if fails, that's fine, # that means batch accession is already underway try: then = time.time() count = 0 with Mutex(wakeup_key,ttl=45) as mutex: session.expire_all() # don't be stale! accession = Accession(session, time_series) logging.warn('START BATCH %s' % time_series) for fs in accession.list_filesets(): # FIXME debug, do all lid = fs[LID] if accession.bin_exists(lid): continue # don't schedule accession if bin exists pid = canonicalize(URL_PREFIX, time_series, fs[LID]) count += 1 if count % 100 == 0: logging.warn('batch %s: scheduled %d bins' % (time_series, count)) schedule_accession(client,pid) elapsed = time.time() - then if elapsed > 25: # don't send heartbeats too often mutex.heartbeat() # retain mutex then = time.time() client.wakeup() # wakeup workers logging.warn('END BATCH %s: %d bins scheduled' % (time_series,count)) client.wakeup() except Busy: logging.warn('BATCH not waking up') pass
def acq_wakeup(wakeup_key): """- wake up and expire the session - acquire a mutex on the acquisition key - query for the instrument - run the copy job - schedule the accession job - wakeup accession workers""" # figure out if this wakeup matters to us if not is_acq_key(wakeup_key): return instrument_name = get_instrument_name(wakeup_key) # attempt to acquire mutex. if fails, that's fine, # that means acquisition is aready underway try: with Mutex(wakeup_key,ttl=45) as mutex: # get the instrument info session.expire_all() # don't be stale! instrument = session.query(Instrument).\ filter(Instrument.name==instrument_name).\ first() if instrument is None: logging.warn('ERROR cannot find instrument named "%s"' % instrument_name) return ts_label = instrument.time_series.label logging.warn('%s: starting acquisition cycle' % instrument_name) def callback(lid): logging.warn('%s: copied %s from %s' % (ts_label, lid, instrument_name)) mutex.heartbeat() # still alive # schedule an accession job and wake up accession workers pid = '%s%s/%s' % (URL_PREFIX, ts_label, lid) schedule_accession(client,pid) client.wakeup() logging.warn('%s: scheduled accession for %s' % (ts_label, lid)) copy_work(instrument, callback=callback) logging.warn('%s: acquisition cycle complete' % ts_label) except Busy: pass