) p.add_argument('conn_name', metavar='CONNECTION-NAME', help='Which Librarian to talk to; as in ~/.hl_client.cfg.') p.add_argument('file_name', metavar='PATH', help='The name of the file to locate.') args = p.parse_args() def die(fmt, *args): if not len(args): text = str(fmt) else: text = fmt % args print('error:', text, file=sys.stderr) sys.exit(1) # Let's do it. # In case the user has provided directory components: file_name = os.path.basename(args.file_name) client = hera_librarian.LibrarianClient(args.conn_name) try: result = client.locate_file_instance(file_name) except hera_librarian.RPCError as e: die('couldn\'t locate file: %s', e) print('%(store_ssh_host)s:%(full_path_on_store)s' % result)
if not len(args): text = str(fmt) else: text = fmt % args print('error:', text, file=sys.stderr) sys.exit(1) # Argument validation is pretty simple known_staging_store = None known_staging_subdir = None if args.pre_staged is not None: known_staging_store, known_staging_subdir = args.pre_staged.split(':', 1) # Let's do it. file_name = os.path.basename( args.file_name) # in case the user has spelled out a path client = hera_librarian.LibrarianClient(args.source_conn_name) try: client.launch_file_copy(file_name, args.dest_conn_name, remote_store_path=args.dest, known_staging_store=known_staging_store, known_staging_subdir=known_staging_subdir) except hera_librarian.RPCError as e: die('launch failed: %s', e)
def main(args): o = optparse.OptionParser() o.set_usage('load_observations_librarian.py') o.set_description(__doc__) o.add_option( '--connection', help= 'the name of the Librarian connection to use (as in ~/.hl_client.cfg)') o.add_option('--config_file', help='RTP configuration file default=RTP/etc/still.cfg', default='etc/still.cfg') o.add_option( '--source', help= 'Only load files originating from the named "source" (default "%default")', default='correlator') opts, args = o.parse_args(args) # Some boilerplate to set up the database interface ... spawner = SpawnerClass() workflow = WorkFlow() spawner.config_file = os.path.join(basedir, opts.config_file) process_client_config_file(spawner, workflow) dbi = get_dbi_from_config(spawner.config_file) dbi.test_db() # Get the list of potentially-relevant files from the Librarian. lc = hera_librarian.LibrarianClient(opts.connection) try: listing = lc.describe_session_without_event(opts.source, rtp_ingested_key) except hera_librarian.RPCError as e: print('RPC to librarian failed: %s' % e.message) sys.exit(1) if not listing['any_matching']: print('No new sessions.') return # For each record we get a dict of at least # # date -- the start Julian Date of the observation # pol -- the polarization of the data ("xx" or "yy") # store_path -- the path of a file instance *within* a store # path_prefix -- the store's path prefix, used to construct full paths # host -- the hostname of the store # length -- the duration of the observation in days # # This is a pretty good start ... because of course the Librarian's API # call has been engineered to give us what we need. def augment_record(r): return { 'obsnum': os.path.basename( r['store_path']), # NOTE: this is actually free text 'date': r['date'], 'date_type': 'julian', 'pol': r['pol'], 'host': r['host'], 'filename': os.path.join(r['path_prefix'], r['store_path']), 'path_prefix': r['path_prefix'], 'outputhost': '', 'status': initial_status, 'length': r['length'], } obsinfos = [augment_record(r) for r in listing['info']] # Now we need to fill in the "neighbor" information. pols = set(oi['pol'] for oi in obsinfos) for pol in pols: soi = sorted((oi for oi in obsinfos if oi['pol'] == pol), key=lambda oi: oi['date']) for i in xrange(len(soi)): oi_this = soi[i] if i > 0: oi_prev = soi[i - 1] if (oi_this['date'] - oi_prev['date']) < (1.2 * oi_this['length']): oi_this['neighbor_low'] = oi_prev['date'] if i < len(soi) - 1: oi_next = soi[i + 1] if (oi_next['date'] - oi_this['date']) < (1.2 * oi_this['length']): oi_this['neighbor_high'] = oi_next['date'] # Now that we've computed everything, avoid duplicating files that we # already know about. We shouldn't end up ever trying to submit # duplicates, but in practice ... from sqlalchemy.orm.exc import NoResultFound def not_already_seen(oi): try: obs = dbi.get_obs(oi['obsnum']) return False except NoResultFound: return True n_before = len(obsinfos) obsinfos = [oi for oi in obsinfos if not_already_seen(oi)] if len(obsinfos) != n_before: print('Dropping %d already-ingested records.' % (n_before - len(obsinfos))) if not len(obsinfos): print('Nothing to add.') return # Try ingesting into the RTP. try: print('Attempting to add %d observations to the still ...' % len(obsinfos)) dbi.add_observations(obsinfos, initial_status) except Exception as e: print('addition failed! here\'s what was attempted:', file=sys.stderr) print('', file=sys.stderr) for oi in obsinfos: print(oi, file=sys.stderr) raise # Add events to the Librarian indicating that these files were # successfully ingested into the RTP. for oi in obsinfos: lc.create_file_event(os.path.basename(oi['filename']), rtp_ingested_key)
conn_name, path, event_type = args[:3] payload = {} for arg in args[3:]: bits = arg.split('=', 1) if len(bits) != 2: die('argument %r must take the form "key=value"', arg) # We parse each "value" as JSON ... and then re-encode it as JSON when # talking to the server. So this is mostly about sanity checking. key, text_val = bits try: value = json.loads(text_val) except ValueError: die('value %r for keyword %r does not parse as JSON', text_val, key) payload[key] = value path = os.path.basename(path) # in case user provided a real filesystem path # Let's do it. client = hera_librarian.LibrarianClient(conn_name) try: client.create_file_event(path, event_type, **payload) except hera_librarian.RPCError as e: die('event creation failed: %s', e)