def main():

    parser = argparse.ArgumentParser(description='MWA - Add observations to Workflow Manager')

    parser.add_argument('--config_file', dest='config_file', required=False,
                        help="Specify the complete path to the config file, by default we'll use etc/still.cfg")
    parser.add_argument('-o', dest='obsnums', required=False, nargs='+',
                        help="List of obervations seperated by spaces")
    parser.add_argument('--generate', dest='generate', required=False,
                        help="Generates a list of unprocessed obsnums, note that projID and filecount have not been checked")

    parser.set_defaults(config_file="%setc/still.cfg" % basedir)

    args, unknown = parser.parse_known_args()

    sg = SpawnerClass()
    wf = WorkFlow()

    sg.config_file = args.config_file
    process_client_config_file(sg, wf)
    dbi = get_dbi_from_config(args.config_file)
    dbi.test_db()  # Testing the database to make sure we made a connection, its fun..
    if args.generate:
        ingest_addtional_opsids(sg)
    else:
        for obsid in args.obsnums:
            print("Obsid: %s") % obsid
            dbi.add_observation(obsid, obsid, "GPS", None, None, None, outputhost=None, length=None, status='NEW')
def main():

    parser = argparse.ArgumentParser(description="MWA - Add observations to Workflow Manager")

    parser.add_argument(
        "--config_file",
        dest="config_file",
        required=False,
        help="Specify the complete path to the config file, by default we'll use etc/still.cfg",
    )
    parser.add_argument("-o", dest="obsnums", required=False, nargs="+", help="List of obervations seperated by spaces")

    parser.set_defaults(config_file="%setc/still.cfg" % basedir)

    args, unknown = parser.parse_known_args()

    sg = SpawnerClass()
    wf = WorkFlow()

    sg.config_file = args.config_file
    process_client_config_file(sg, wf)
    dbi = get_dbi_from_config(args.config_file)
    dbi.test_db()  # Testing the database to make sure we made a connection, its fun..
    for obsid in args.obsnums:
        print("Obsid: %s") % obsid
        dbi.add_observation(obsid, obsid, "GPS", None, None, None, outputhost=None, length=None, status="NEW")
Example #3
0
 def setUp(self):
     self.files = [1, 2, 3]
     self.still = 0
     self.task = 'UVC'  # Jon : change me : HARDWF
     self.sg = SpawnerClass()
     self.sg.config_file = "still_test_paper.cfg"
     self.wf = WorkFlow()
     process_client_config_file(self.sg, self.wf)
Example #4
0
 def setUp(self):
     self.files = [1, 2, 3]
     self.still = 0
     self.task = 'UVC'  # Jon : change me : HARDWF
     self.sg = SpawnerClass()
     self.sg.config_file = "still_test_paper.cfg"
     self.wf = WorkFlow()
     process_client_config_file(self.sg, self.wf)
Example #5
0
 def setUp(self):
     self.ntimes = 10
     self.npols = 4
     self.dbi = PopulatedDataBaseInterface(self.ntimes, self.npols, test=True)
     self.files = self.dbi.list_observations()
     self.sg = SpawnerClass()
     self.sg.config_file = "still_test_paper.cfg"
     self.wf = WorkFlow()
     process_client_config_file(self.sg, self.wf)
     self.task_clients = [TaskClient(self.dbi, 'localhost', self.wf, port=TEST_PORT)]
 def setUp(self):
     self.ntimes = 10
     self.npols = 4
     self.dbi = PopulatedDataBaseInterface(self.ntimes, self.npols, test=True)
     self.files = self.dbi.list_observations()
     self.sg = SpawnerClass()
     self.sg.config_file = "still_test_paper.cfg"
     self.wf = WorkFlow()
     process_client_config_file(self.sg, self.wf)
     self.task_clients = [TaskClient(self.dbi, "localhost", self.wf, port=TEST_PORT)]
def main():

    parser = argparse.ArgumentParser(
        description='MWA - Add observations to Workflow Manager')

    parser.add_argument(
        '--config_file',
        dest='config_file',
        required=False,
        help=
        "Specify the complete path to the config file, by default we'll use etc/still.cfg"
    )
    parser.add_argument('-o',
                        dest='obsnums',
                        required=False,
                        nargs='+',
                        help="List of obervations seperated by spaces")
    parser.add_argument(
        '--generate',
        dest='generate',
        required=False,
        help=
        "Generates a list of unprocessed obsnums, note that projID and filecount have not been checked"
    )

    parser.set_defaults(config_file="%setc/still.cfg" % basedir)

    args, unknown = parser.parse_known_args()

    sg = SpawnerClass()
    wf = WorkFlow()

    sg.config_file = args.config_file
    process_client_config_file(sg, wf)
    dbi = get_dbi_from_config(args.config_file)
    dbi.test_db(
    )  # Testing the database to make sure we made a connection, its fun..
    if args.generate:
        ingest_addtional_opsids(sg)
    else:
        for obsid in args.obsnums:
            print("Obsid: %s") % obsid
            dbi.add_observation(obsid,
                                obsid,
                                "GPS",
                                None,
                                None,
                                None,
                                outputhost=None,
                                length=None,
                                status='NEW')
Example #8
0
    def setUp(self):
        self.nfiles = 10
        dbi = FakeDataBaseInterface(self.nfiles)
        self.dbi = dbi
        self.sg = SpawnerClass()
        self.sg.config_file = "still_test_paper.cfg"
        self.wf = WorkFlow()
        process_client_config_file(self.sg, self.wf)

        class FakeAction(sch.Action):
            def run_remote_task(self):
                dbi.files[self.filename] = self.task
        self.FakeAction = FakeAction
        self.task_clients = TaskClient(dbi, 'localhost', self.wf, port=TEST_PORT)
def main (args):
    o = optparse.OptionParser ()
    o.set_usage ('load_observations_librarian.py')
    o.set_description (__doc__)
    o.add_option('--connection', help='the name of the Librarian connection to use (specified in .hl_client.cfg)')
    opts, args = o.parse_args (args)

    # Some boilerplate to set up the database interface ...
    spawner = SpawnerClass()
    workflow = WorkFlow()
    spawner.config_file = os.path.join (basedir, 'etc/still.cfg')
    process_client_config_file (spawner, workflow)
    dbi = get_dbi_from_config (spawner.config_file)
    dbi.test_db ()

    # Get the list of potentially-relevant files from the Librarian.

    lc = hera_librarian.LibrarianClient (opts.connection)
    try:
        listing = lc.list_files_without_history_item (librarian_source,
                                                      rtp_processed_key)
    except hera_librarian.RPCFailedError as e:
        print ('RPC to librarian failed: %s' % e.message)
        sys.exit (1)

    try:
        files = listing['files']
        len (files)
    except Exception as e:
        print ('unexpected response from librarian: %s' % e)
        sys.exit (1)

    if not len (files):
        print ('No new files.')
        return

    # for each file we should have a dict of at least:
    #
    # name              -- something like 2456892/zen.2456892.49664.xx.uv
    # obsid             -- the obsid associated with this file
    # create_time       -- the Unix timestamp that the file was sent to the Librarian
    # size              -- file size in bytes
    # type              -- file "type" stored in the Librarian; "uv" for UV data
    # md5               -- the MD5 of the file contents; XXX may be calculated weirdly by Librarian
    # store_ssh_prefix  -- the Librarian "ssh_prefix" of the file's storage location
    # store_path_prefix -- the Librarian "path_prefix" of the file's storage location
    #
    # We start by extracting a few useful pieces of meta-information:

    for f in files:
        f['jd'] = float (path_to_jd (f['name']))
        f['pol'] = path_to_pol (f['name'])

        # Extract the hostname of the store on which this file is stored from
        # its store's ssh_prefix. The prefix will look like "user@host",
        # but the "user@" part might not be present.
        f['store_host'] = f['store_ssh_prefix'].split ('@', 1)[-1]

        # Meanwhile, the RTP system expects the filenames to be absolute paths.
        f['name'] = f['store_path_prefix'] + '/' + f['name']

    # If at all possible, get a default observation length from the
    # separations between observations, in case we have any funky datasets
    # without the length embedded. We are somewhat recklessly assuming that
    # even if this batch of datasets spans different nights, they all will
    # have the same DJD; this doesn't seem too unreasonable.

    pols = list (set (f['pol'] for f in files))
    bestjds = []

    for pol in pols:
        jds = np.sort ([f['jd'] for f in files if f['pol'] == pol])
        if len (jds) > len (bestjds):
            bestjds = jds

    default_djd = None

    if len (bestjds) > 2:
        default_djd = np.median (np.diff (bestjds))
        print ('Inferring default djd = %.5f days' % default_djd)

        for f in files:
            f['djd'] = default_djd

    # Buuut let's get djd straight from the data if at all possible. If there
    # are any files for which we have no idea about the djd, we can't add
    # them. For everything else, no we can compute the 'obsnum' magic number
    # (which is not the same as obsid!).

    for f in files:
        djd = try_get_file_djd (f)
        if djd is not None:
            f['djd'] = djd

    files = [f for f in files if f.get ('djd') is not None]

    for f in files:
        f['obsnum'] = str (jdpol2obsnum (f['jd'], f['pol'], f['djd']))

    # Now let's fill in the "neighbor" information. XXX: if we only get, say,
    # a random subset of observations from one night, this information will be
    # grievously incomplete! I don't see a way around that given the way that
    # this aspect of things is handled at the moment.

    for pol in pols:
        sfiles = sorted ((f for f in files if f['pol'] == pol), key=lambda f: f['jd'])

        for i in xrange (len (sfiles)):
            f_this = sfiles[i]

            if i > 0:
                f_prev = sfiles[i - 1]

                if (f_this['jd'] - f_prev['jd']) < (1.2 * f_this['djd']):
                    f_this['neighbor_low'] = f_prev['jd']

            if i < len (sfiles) - 1:
                f_next = sfiles[i + 1]

                if (f_next['jd'] - f_this['jd']) < (1.2 * f_this['djd']):
                    f_this['neighbor_high'] = f_next['jd']

    # Now that we've computed everything, avoid duplicating files that we
    # already know about.

    from sqlalchemy.orm.exc import NoResultFound

    def not_already_seen (filerec):
        try:
            obs = dbi.get_obs (filerec['obsnum'])
            print (repr (obs))
            return False
        except NoResultFound:
            return True

    n_before = len (files)
    files = [f for f in files if not_already_seen (f)]
    if len (files) != n_before:
        print ('Dropping %d already-ingested files.' % (n_before - len (files)))

    if not len (files):
        print ('Nothing to add.')
        return

    # Let's go for it.

    try:
        print ('Attempting to add %d observations to the still ...' % len (files))
        dbi.add_observations ([augmented_file_to_obsinfo (f) for f in files], initial_status)
    except Exception as e:
        print ('addition failed! here\'s what was attempted:', file=sys.stderr)
        for f in files:
            print ('', file=sys.stderr)
            print (augmented_file_to_obsinfo (f), file=sys.stderr)
        raise
Example #10
0
def main(args):
    o = optparse.OptionParser()
    o.set_usage('load_observations_librarian.py')
    o.set_description(__doc__)
    o.add_option(
        '--connection',
        help=
        'the name of the Librarian connection to use (as in ~/.hl_client.cfg)')
    o.add_option('--config_file',
                 help='RTP configuration file default=RTP/etc/still.cfg',
                 default='etc/still.cfg')
    o.add_option(
        '--source',
        help=
        'Only load files originating from the named "source" (default "%default")',
        default='correlator')
    opts, args = o.parse_args(args)

    # Some boilerplate to set up the database interface ...
    spawner = SpawnerClass()
    workflow = WorkFlow()
    spawner.config_file = os.path.join(basedir, opts.config_file)
    process_client_config_file(spawner, workflow)
    dbi = get_dbi_from_config(spawner.config_file)
    dbi.test_db()

    # Get the list of potentially-relevant files from the Librarian.

    lc = hera_librarian.LibrarianClient(opts.connection)
    try:
        listing = lc.describe_session_without_event(opts.source,
                                                    rtp_ingested_key)
    except hera_librarian.RPCError as e:
        print('RPC to librarian failed: %s' % e.message)
        sys.exit(1)

    if not listing['any_matching']:
        print('No new sessions.')
        return

    # For each record we get a dict of at least
    #
    #   date        -- the start Julian Date of the observation
    #   pol         -- the polarization of the data ("xx" or "yy")
    #   store_path  -- the path of a file instance *within* a store
    #   path_prefix -- the store's path prefix, used to construct full paths
    #   host        -- the hostname of the store
    #   length      -- the duration of the observation in days
    #
    # This is a pretty good start ... because of course the Librarian's API
    # call has been engineered to give us what we need.

    def augment_record(r):
        return {
            'obsnum': os.path.basename(
                r['store_path']),  # NOTE: this is actually free text
            'date': r['date'],
            'date_type': 'julian',
            'pol': r['pol'],
            'host': r['host'],
            'filename': os.path.join(r['path_prefix'], r['store_path']),
            'path_prefix': r['path_prefix'],
            'outputhost': '',
            'status': initial_status,
            'length': r['length'],
        }

    obsinfos = [augment_record(r) for r in listing['info']]

    # Now we need to fill in the "neighbor" information.

    pols = set(oi['pol'] for oi in obsinfos)

    for pol in pols:
        soi = sorted((oi for oi in obsinfos if oi['pol'] == pol),
                     key=lambda oi: oi['date'])

        for i in xrange(len(soi)):
            oi_this = soi[i]

            if i > 0:
                oi_prev = soi[i - 1]

                if (oi_this['date'] - oi_prev['date']) < (1.2 *
                                                          oi_this['length']):
                    oi_this['neighbor_low'] = oi_prev['date']

            if i < len(soi) - 1:
                oi_next = soi[i + 1]

                if (oi_next['date'] - oi_this['date']) < (1.2 *
                                                          oi_this['length']):
                    oi_this['neighbor_high'] = oi_next['date']

    # Now that we've computed everything, avoid duplicating files that we
    # already know about. We shouldn't end up ever trying to submit
    # duplicates, but in practice ...

    from sqlalchemy.orm.exc import NoResultFound

    def not_already_seen(oi):
        try:
            obs = dbi.get_obs(oi['obsnum'])
            return False
        except NoResultFound:
            return True

    n_before = len(obsinfos)
    obsinfos = [oi for oi in obsinfos if not_already_seen(oi)]
    if len(obsinfos) != n_before:
        print('Dropping %d already-ingested records.' %
              (n_before - len(obsinfos)))

    if not len(obsinfos):
        print('Nothing to add.')
        return

    # Try ingesting into the RTP.

    try:
        print('Attempting to add %d observations to the still ...' %
              len(obsinfos))
        dbi.add_observations(obsinfos, initial_status)
    except Exception as e:
        print('addition failed! here\'s what was attempted:', file=sys.stderr)
        print('', file=sys.stderr)
        for oi in obsinfos:
            print(oi, file=sys.stderr)
        raise

    # Add events to the Librarian indicating that these files were
    # successfully ingested into the RTP.

    for oi in obsinfos:
        lc.create_file_event(os.path.basename(oi['filename']),
                             rtp_ingested_key)
Example #11
0
parser.add_argument('--config_file', dest='config_file', required=False,
                    help="Specify the complete path to the config file")

parser.add_argument('jds', nargs='+',type=str,metavar='JD',
                    help="List of integer julian dates to reset.")

parser.set_defaults(config_file="%setc/still.cfg" % basedir)

args = parser.parse_args()

sg = SpawnerClass()
wf = WorkFlow()

sg.config_file = args.config_file
process_client_config_file(sg, wf)
if args.status == '':
    args.status = wf.workflow_actions[0]

# connect to the database
dbi = StillDataBaseInterface(sg.dbhost, sg.dbport, sg.dbtype, sg.dbname, sg.dbuser, sg.dbpasswd, test=False)

# Setup logging
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger('reset_observations.py')

if args.debug:
    logger.setLevel(logging.DEBUG)
else:
    logger.setLevel(logging.INFO)
Example #12
0
parser.add_argument('jds',
                    nargs='+',
                    type=str,
                    metavar='JD',
                    help="List of integer julian dates to reset.")

parser.set_defaults(config_file="%setc/still.cfg" % basedir)

args = parser.parse_args()

sg = SpawnerClass()
wf = WorkFlow()

sg.config_file = args.config_file
process_client_config_file(sg, wf)
if args.status == '':
    args.status = wf.workflow_actions[0]

# connect to the database
dbi = StillDataBaseInterface(sg.dbhost,
                             sg.dbport,
                             sg.dbtype,
                             sg.dbname,
                             sg.dbuser,
                             sg.dbpasswd,
                             test=False)

# Setup logging
logging.basicConfig(
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
Example #13
0
 def setUp(self):
     self.dbi = FakeDataBaseInterface()
     self.sg = SpawnerClass()
     self.sg.config_file = "still_test_paper.cfg"
     self.wf = WorkFlow()
     process_client_config_file(self.sg, self.wf)
Example #14
0
 def setUp(self):
     self.dbi = FakeDataBaseInterface()
     self.sg = SpawnerClass()
     self.sg.config_file = "still_test_paper.cfg"
     self.wf = WorkFlow()
     process_client_config_file(self.sg, self.wf)
def main (args):
    o = optparse.OptionParser ()
    o.set_usage ('load_observations_librarian.py')
    o.set_description (__doc__)
    o.add_option('--connection', help='the name of the Librarian connection to use (as in ~/.hl_client.cfg)')
    o.add_option('--config_file',help='RTP configuration file default=RTP/etc/still.cfg',default='etc/still.cfg')
    o.add_option('--source', help='Only load files originating from the named "source" (default "%default")',
                 default='correlator')
    opts, args = o.parse_args (args)

    # Some boilerplate to set up the database interface ...
    spawner = SpawnerClass()
    workflow = WorkFlow()
    spawner.config_file = os.path.join (basedir, opts.config_file)
    process_client_config_file (spawner, workflow)
    dbi = get_dbi_from_config (spawner.config_file)
    dbi.test_db ()

    # Get the list of potentially-relevant files from the Librarian.

    lc = hera_librarian.LibrarianClient (opts.connection)
    try:
        listing = lc.describe_session_without_event (opts.source,
                                                     rtp_ingested_key)
    except hera_librarian.RPCError as e:
        print ('RPC to librarian failed: %s' % e.message)
        sys.exit (1)

    if not listing['any_matching']:
        print ('No new sessions.')
        return

    # For each record we get a dict of at least
    #
    #   date        -- the start Julian Date of the observation
    #   pol         -- the polarization of the data ("xx" or "yy")
    #   store_path  -- the path of a file instance *within* a store
    #   path_prefix -- the store's path prefix, used to construct full paths
    #   host        -- the hostname of the store
    #   length      -- the duration of the observation in days
    #
    # This is a pretty good start ... because of course the Librarian's API
    # call has been engineered to give us what we need.

    def augment_record (r):
        return {
            'obsnum': os.path.basename (r['store_path']), # NOTE: this is actually free text
            'date': r['date'],
            'date_type': 'julian',
            'pol': r['pol'],
            'host': r['host'],
            'filename': os.path.join (r['path_prefix'], r['store_path']),
            'path_prefix': r['path_prefix'],
            'outputhost': '',
            'status': initial_status,
            'length': r['length'],
        }

    obsinfos = [augment_record (r) for r in listing['info']]

    # Now we need to fill in the "neighbor" information.

    pols = set (oi['pol'] for oi in obsinfos)

    for pol in pols:
        soi = sorted ((oi for oi in obsinfos if oi['pol'] == pol), key=lambda oi: oi['date'])

        for i in xrange (len (soi)):
            oi_this = soi[i]

            if i > 0:
                oi_prev = soi[i - 1]

                if (oi_this['date'] - oi_prev['date']) < (1.2 * oi_this['length']):
                    oi_this['neighbor_low'] = oi_prev['date']

            if i < len (soi) - 1:
                oi_next = soi[i + 1]

                if (oi_next['date'] - oi_this['date']) < (1.2 * oi_this['length']):
                    oi_this['neighbor_high'] = oi_next['date']

    # Now that we've computed everything, avoid duplicating files that we
    # already know about. We shouldn't end up ever trying to submit
    # duplicates, but in practice ...

    from sqlalchemy.orm.exc import NoResultFound

    def not_already_seen (oi):
        try:
            obs = dbi.get_obs (oi['obsnum'])
            return False
        except NoResultFound:
            return True

    n_before = len (obsinfos)
    obsinfos = [oi for oi in obsinfos if not_already_seen (oi)]
    if len (obsinfos) != n_before:
        print ('Dropping %d already-ingested records.' % (n_before - len (obsinfos)))

    if not len (obsinfos):
        print ('Nothing to add.')
        return

    # Try ingesting into the RTP.

    try:
        print ('Attempting to add %d observations to the still ...' % len (obsinfos))
        dbi.add_observations (obsinfos, initial_status)
    except Exception as e:
        print ('addition failed! here\'s what was attempted:', file=sys.stderr)
        print ('', file=sys.stderr)
        for oi in obsinfos:
            print (oi, file=sys.stderr)
        raise

    # Add events to the Librarian indicating that these files were
    # successfully ingested into the RTP.

    for oi in obsinfos:
        lc.create_file_event (os.path.basename (oi['filename']), rtp_ingested_key)