Example #1
0
def doit(ts_label):
    acc = Accession(session, ts_label)
    ts = acc.get_time_series()
    if ts is None:
        raise ValueError('No time series named %s' % ts_label)
    logging.warn('Starting batch accession for %s' % ts_label)
    acc.add_all_filesets()
    logging.warn('Finished batch accession for %s' % ts_label)
Example #2
0
def do_acc(pid, job):
    parsed = parse_pid(pid)
    lid = parsed[LID]
    ts_label = parsed[TS_LABEL]
    roots = get_data_roots(session, ts_label) # get raw data roots
    fileset = parsed_pid2fileset(parsed, roots)
    fileset[LID] = lid
    session.expire_all() # don't be stale!
    acc = Accession(session,ts_label)#,fast=True)
    # FIXME fast=True disables checksumming
    client.update(pid,ttl=3600) # allow 1hr for accession
    ret = acc.add_fileset(fileset)
    if ret=='ADDED':
        schedule_products(pid, client)
        session.commit()
        client.wakeup()
    elif ret=='FAILED':
        raise Exception('accession failed')
Example #3
0
def acc_wakeup(wakeup_key):
    """- wake up and expire the session
    - acquire a mutex on the acquisition key
    - query for the instrument
    - run the copy job
    - schedule the accession job
    - wakeup accession workers"""
    # figure out if this wakeup matters to us
    if not is_acc_key(wakeup_key):
        return
    time_series = get_time_series(wakeup_key)
    # attempt to acquire mutex. if fails, that's fine,
    # that means batch accession is already underway
    try:
        then = time.time()
        count = 0
        with Mutex(wakeup_key,ttl=45) as mutex:
            session.expire_all() # don't be stale!
            accession = Accession(session, time_series)
            logging.warn('START BATCH %s' % time_series)
            for fs in accession.list_filesets(): # FIXME debug, do all
                lid = fs[LID]
                if accession.bin_exists(lid):
                    continue # don't schedule accession if bin exists
                pid = canonicalize(URL_PREFIX, time_series, fs[LID])
                count += 1
                if count % 100 == 0:
                    logging.warn('batch %s: scheduled %d bins' % (time_series, count))
                schedule_accession(client,pid)
                elapsed = time.time() - then
                if elapsed > 25: # don't send heartbeats too often
                    mutex.heartbeat() # retain mutex
                    then = time.time()
                client.wakeup() # wakeup workers
            logging.warn('END BATCH %s: %d bins scheduled' % (time_series,count))
            client.wakeup()
    except Busy:
        logging.warn('BATCH not waking up')
        pass