def main(processor_config_path, job_config_path, to_archive): """ Main function to run processing chain for sequence files :type processor_config_path: str :param processor_config_path: processor configuration file path :type job_config_path: str :param job_config_path: job configuration file path :type to_archive: bool :param to_archive: switch for if to add processed data to data archive """ processor_config = read_config_file(processor_config_path) job_config = read_config_file(job_config_path) # Configure logging name = __name__ if "job_name" in job_config["Job"].keys(): name = job_config["Job"]["job_name"] logger = configure_logging(config=job_config, name=name) # Define context context = Context( processor_config=processor_config, job_config=job_config, logger=logger ) context.set_config_value("to_archive", to_archive) # Determine target sequences target_sequences = get_target_sequences(context, to_archive) # Run processor sp = SequenceProcessor(context=context) target_sequences_passed = 0 target_sequences_total = len(target_sequences) if target_sequences_total == 0: msg = "No sequences to process" else: for target_sequence in target_sequences: context.logger.info("Processing sequence: " + target_sequence) try: sp.process_sequence(target_sequence) target_sequences_passed += 1 context.logger.info("Complete") except Exception as e: logger.error("Failed: " + repr(e)) logger.debug(traceback.format_exc()) context.anomaly_db.add_x_anomaly() msg = str(target_sequences_passed) + "/" + str(target_sequences_total) + \ " sequences successfully processed" return msg
def main(scheduler_config_path): """ Main function to schedule automated hypernets_processor jobs :type scheduler_config_path: str :param scheduler_config_path: path of scheduler config file """ scheduler_config = read_config_file(scheduler_config_path) logger = configure_logging(config=scheduler_config, name=__name__) scheduler_config = unpack_scheduler_config(scheduler_config) jobs_list = read_jobs_list( scheduler_config["Processor Schedule"]["jobs_list"]) # schedule jobs processor_sch = Scheduler(logger=logger) for job_config_path in jobs_list: # define scheduler job config scheduler_job_config = dict() # read job config file to set job name job_config = read_config_file(job_config_path) if ("Job" in job_config.keys()) and ("job_name" in job_config["Job"].keys()): scheduler_job_config["name"] = job_config["Job"]["job_name"] else: scheduler_job_config["name"] = job_config_path del job_config scheduler_job_config["seconds"] = scheduler_config[ "Processor Schedule"]["seconds"] scheduler_job_config["minutes"] = scheduler_config[ "Processor Schedule"]["minutes"] scheduler_job_config["hours"] = scheduler_config["Processor Schedule"][ "hours"] scheduler_job_config["parallel"] = scheduler_config[ "Processor Schedule"]["parallel"] # schedule job processor_sch.schedule(processor_main, scheduler_job_config=scheduler_job_config, job_config_path=job_config_path, processor_config_path=PROCESSOR_CONFIG_PATH, to_archive=True) # run scheduled jobs processor_sch.run( start_time=scheduler_config["Processor Schedule"]["start_time"]) return None
def main(settings): """ Main function to run routine to setup hypernets_processor :type settings: dict :param settings: user defined configuration values """ # Set default config values for chosen network config_path = PROCESSOR_CONFIG_PATH if settings["network_defaults"]: if settings["network"] == "l": def_config_path = PROCESSOR_LAND_DEFAULTS_CONFIG_PATH elif settings["network"] == "w": def_config_path = PROCESSOR_WATER_DEFAULTS_CONFIG_PATH else: raise KeyError("Invalid network name (" + settings["network"] + ")") config_path = [PROCESSOR_CONFIG_PATH, def_config_path] processor_config = read_config_file(config_path) # Create directories (resets with existing if unchanged) os.makedirs(settings["working_directory"], exist_ok=True) processor_config["Processor"]["processor_working_directory"] = settings[ "working_directory"] os.makedirs(settings["archive_directory"], exist_ok=True) processor_config["Output"]["archive_directory"] = settings[ "archive_directory"] # Create databases for db_fmt in DB_DICT_DEFS.keys(): url = settings[db_fmt + "_db_url"] if url is not None: new_db = open_database(url, db_format=db_fmt) new_db.close() processor_config["Databases"][db_fmt + "_db_url"] = url # Write updated config file with open(PROCESSOR_CONFIG_PATH, 'w') as f: processor_config.write(f) # Set scheduler log file scheduler_config = read_config_file(SCHEDULER_CONFIG_PATH) if not os.path.exists(settings["log_path"]): open(settings["log_path"], 'a').close() scheduler_config["Log"]["log_path"] = settings["log_path"] with open(SCHEDULER_CONFIG_PATH, 'w') as f: scheduler_config.write(f) return None
def test_read_config_file(self): fname = "file.config" create_config_file(fname) config = read_config_file(fname) self.assertEqual(type(config), RawConfigParser) self.assertEqual(config["Section"]["entry"], "value") os.remove(fname)
def setup_test_processor_config( archive_directory=None, metadata_db_url=None, archive_db_url=None, anomaly_db_url=None, ): """ Creates processor_config for testing :type archive_directory: str :param archive_directory: (optional) data archive directory, set if provided else default value used :type metadata_db_url: str :param metadata_db_url: (optional) metadata db url, set if provided else default value used :type archive_db_url: str :param archive_db_url: (optional) archive db url, set if provided else default value used :type anomaly_db_url: str :param anomaly_db_url: (optional) anomaly db url, set if provided else default value used :return: test processor configuration information :rtype: configparser.RawConfigParser """ processor_config = read_config_file(PROCESSOR_CONFIG_PATH) processor_config["Processor"]["version"] = "0.0" processor_config["Databases"]["metadata_db_url"] = ( metadata_db_url if metadata_db_url is not None else "sqlite:///metadata.db") processor_config["Databases"]["anomaly_db_url"] = ( anomaly_db_url if anomaly_db_url is not None else "sqlite:///anomaly.db") processor_config["Databases"]["archive_db_url"] = ( archive_db_url if archive_db_url is not None else "sqlite:///archive.db") processor_config["Output"]["archive_directory"] = ( archive_directory if archive_directory is not None else "out") processor_config["Processing Options"] = {"write_l1a": "False"} return processor_config
def main(settings): """ Main function for job init :type settings: dict :param settings: user defined configuration values """ # Create config from template job_config = read_config_file(JOB_CONFIG_TEMPLATE_PATH) job_config["Job"]["job_name"] = settings["job_name"] job_config["Job"]["site_id"] = settings["site_id"] # Create directories (resets with existing if unchanged) os.makedirs(settings["job_working_directory"], exist_ok=True) job_config["Job"]["job_working_directory"] = os.path.abspath( settings["job_working_directory"]) os.makedirs(settings["raw_data_directory"], exist_ok=True) job_config["Input"]["raw_data_directory"] = os.path.abspath( settings["raw_data_directory"]) # Set job log file log_path = os.path.join(settings["job_working_directory"], settings["job_name"] + ".log") if not os.path.exists(log_path): open(log_path, 'a').close() job_config["Log"]["log_path"] = os.path.abspath(log_path) # Write config job_config_path = os.path.abspath( os.path.join(settings["job_working_directory"], settings["job_name"] + ".config")) with open(job_config_path, 'w') as f: job_config.write(f) # Add to scheduler if settings["add_to_scheduler"]: with open(JOBS_FILE_PATH, "a") as f: if os.path.getsize(JOBS_FILE_PATH) > 0: f.write("\n" + job_config_path) else: f.write(job_config_path)
def setup_test_job_config(raw_data_directory=None): """ Creates processor_config for testing :type raw_data_directory: str :param raw_data_directory: (optional) raw data directory, set if provided else default value used :return: test job configuration information :rtype: configparser.RawConfigParser """ job_config = read_config_file(JOB_CONFIG_TEMPLATE_PATH) job_config["Job"]["network"] = "l" job_config["Job"]["site_id"] = "test" job_config["Input"]["raw_data_directory"] = ( raw_data_directory if raw_data_directory is not None else "data") return job_config
def cli(): """ Command line interface for hypernets_processor setup """ processor_config = read_config_file(PROCESSOR_CONFIG_PATH) context = Context(processor_config) settings = dict() # Determine network settings["network"] = determine_set_value("network", context, options=["l", "w"], return_existing=True) settings["network_defaults"] = cli_input_yn( "set network default config values (overwrites existing)") # Determine archive directory to set settings["archive_directory"] = os.path.abspath( determine_set_value("archive_directory", context, return_existing=True)) home_directory = os.path.expanduser("~") settings["working_directory"] = os.path.abspath( determine_set_value("processor_working_directory", context, default=os.path.join(home_directory, ".hypernets"), return_existing=True)) for db_fmt in DB_DICT_DEFS.keys(): settings[db_fmt + "_db_url"] = determine_set_value( db_fmt + "_db_url", context, default="sqlite:///" + os.path.join(settings["working_directory"], db_fmt + ".db"), ) settings["log_path"] = os.path.join(settings["working_directory"], "scheduler.log") main(settings)
# COMPUTE WATER LEAVING RADIANCE LWN, REFLECTANCE RHOW_NOSC FOR EACH Lu SCAN! # wind=RhymerHypstar(context).retrieve_wind(L1c) # lw_all, rhow_all, rhow_nosc_all, epsilon, fresnel_coeff = RhymerHypstar(context).fresnelrefl_qc_simil(L1c, wind) # print(lw_all) # print(rhow_all) # print(fresnel_coeff) # L1c= # average all scans to series # L1d # AVERAGE LWN, RHOW and RHOW_NOSC # L2a # print(L1b) # # L2a=surf.process(L1c,"LandNetworkProtocol") self.context.logger.info("all done!") print("all done!") return None if __name__ == "__main__": this_directory_path = os.path.abspath(os.path.dirname(__file__)) processor_config_path= os.path.join(this_directory_path,"etc/processor_2.config") job_config_path= os.path.join(this_directory_path,"etc/job.config") processor_config = read_config_file(processor_config_path) job_config = read_config_file(job_config_path) hp = HypernetsProcessor(job_config=job_config,processor_config=processor_config) hp.context.set_config_value("processor_directory",this_directory_path) hp.run() pass
def cli(): """ Command line interface to sequence_processor_main for ad-hoc job processing """ # If job config specified use if parsed_args.job_config: tmp_job = False job_config_path = parsed_args.job_config # Else build and write temporary job config from command line arguments else: tmp_job = True processor_defaults = PROCESSOR_WATER_DEFAULTS_CONFIG_PATH if parsed_args.network == "land": processor_defaults = PROCESSOR_LAND_DEFAULTS_CONFIG_PATH job_config = read_config_file( [JOB_CONFIG_TEMPLATE_PATH, processor_defaults]) if parsed_args.input_directory is not None: job_config["Input"]["raw_data_directory"] = os.path.abspath( parsed_args.input_directory) else: print("-i required") sys.exit(2) if parsed_args.output_directory is not None: job_config["Output"]["archive_directory"] = os.path.abspath( parsed_args.output_directory) else: print("-o required") sys.exit(2) if parsed_args.write_all: for key in job_config["Output"].keys(): if key[:5] == "write": job_config["Output"][key] = "True" job_config["Log"]["log_path"] = os.path.abspath( parsed_args.log) if parsed_args.log is not None else "" job_config["Log"]["verbose"] = str( parsed_args.verbose) if parsed_args.verbose is not None else "" job_config["Log"]["quiet"] = str( parsed_args.quiet) if parsed_args.verbose is not None else "" job_config["Job"]["job_name"] = "run_" + dt.now().strftime( "%Y%m%dT%H%M%S") home_directory = os.path.expanduser("~") job_config["Job"]["job_working_directory"] = os.path.join( home_directory, ".hypernets", "tmp") job_config_path = os.path.join( job_config["Job"]["job_working_directory"], job_config["Job"]["job_name"] + ".config") os.makedirs(job_config["Job"]["job_working_directory"], exist_ok=True) with open(job_config_path, "w") as f: job_config.write(f) # run main main(processor_config_path=PROCESSOR_CONFIG_PATH, job_config_path=job_config_path, to_archive=False) if tmp_job: os.remove(job_config_path) return None