def main(processor_config_path, job_config_path, to_archive):
"""
Main function to run processing chain for sequence files
:type processor_config_path: str
:param processor_config_path: processor configuration file path
:type job_config_path: str
:param job_config_path: job configuration file path
:type to_archive: bool
:param to_archive: switch for if to add processed data to data archive
"""
processor_config = read_config_file(processor_config_path)
job_config = read_config_file(job_config_path)
# Configure logging
name = __name__
if "job_name" in job_config["Job"].keys():
name = job_config["Job"]["job_name"]
logger = configure_logging(config=job_config, name=name)
# Define context
context = Context(
processor_config=processor_config, job_config=job_config, logger=logger
)
context.set_config_value("to_archive", to_archive)
# Determine target sequences
target_sequences = get_target_sequences(context, to_archive)
# Run processor
sp = SequenceProcessor(context=context)
target_sequences_passed = 0
target_sequences_total = len(target_sequences)
if target_sequences_total == 0:
msg = "No sequences to process"
else:
for target_sequence in target_sequences:
context.logger.info("Processing sequence: " + target_sequence)
try:
sp.process_sequence(target_sequence)
target_sequences_passed += 1
context.logger.info("Complete")
except Exception as e:
logger.error("Failed: " + repr(e))
logger.debug(traceback.format_exc())
context.anomaly_db.add_x_anomaly()
msg = str(target_sequences_passed) + "/" + str(target_sequences_total) + \
" sequences successfully processed"
return msg
def main(scheduler_config_path):
"""
Main function to schedule automated hypernets_processor jobs
:type scheduler_config_path: str
:param scheduler_config_path: path of scheduler config file
"""
scheduler_config = read_config_file(scheduler_config_path)
logger = configure_logging(config=scheduler_config, name=__name__)
scheduler_config = unpack_scheduler_config(scheduler_config)
jobs_list = read_jobs_list(
scheduler_config["Processor Schedule"]["jobs_list"])
# schedule jobs
processor_sch = Scheduler(logger=logger)
for job_config_path in jobs_list:
# define scheduler job config
scheduler_job_config = dict()
# read job config file to set job name
job_config = read_config_file(job_config_path)
if ("Job" in job_config.keys()) and ("job_name"
in job_config["Job"].keys()):
scheduler_job_config["name"] = job_config["Job"]["job_name"]
else:
scheduler_job_config["name"] = job_config_path
del job_config
scheduler_job_config["seconds"] = scheduler_config[
"Processor Schedule"]["seconds"]
scheduler_job_config["minutes"] = scheduler_config[
"Processor Schedule"]["minutes"]
scheduler_job_config["hours"] = scheduler_config["Processor Schedule"][
"hours"]
scheduler_job_config["parallel"] = scheduler_config[
"Processor Schedule"]["parallel"]
# schedule job
processor_sch.schedule(processor_main,
scheduler_job_config=scheduler_job_config,
job_config_path=job_config_path,
processor_config_path=PROCESSOR_CONFIG_PATH,
to_archive=True)
# run scheduled jobs
processor_sch.run(
start_time=scheduler_config["Processor Schedule"]["start_time"])
return None
def main(settings):
"""
Main function to run routine to setup hypernets_processor
:type settings: dict
:param settings: user defined configuration values
"""
# Set default config values for chosen network
config_path = PROCESSOR_CONFIG_PATH
if settings["network_defaults"]:
if settings["network"] == "l":
def_config_path = PROCESSOR_LAND_DEFAULTS_CONFIG_PATH
elif settings["network"] == "w":
def_config_path = PROCESSOR_WATER_DEFAULTS_CONFIG_PATH
else:
raise KeyError("Invalid network name (" + settings["network"] +
")")
config_path = [PROCESSOR_CONFIG_PATH, def_config_path]
processor_config = read_config_file(config_path)
# Create directories (resets with existing if unchanged)
os.makedirs(settings["working_directory"], exist_ok=True)
processor_config["Processor"]["processor_working_directory"] = settings[
"working_directory"]
os.makedirs(settings["archive_directory"], exist_ok=True)
processor_config["Output"]["archive_directory"] = settings[
"archive_directory"]
# Create databases
for db_fmt in DB_DICT_DEFS.keys():
url = settings[db_fmt + "_db_url"]
if url is not None:
new_db = open_database(url, db_format=db_fmt)
new_db.close()
processor_config["Databases"][db_fmt + "_db_url"] = url
# Write updated config file
with open(PROCESSOR_CONFIG_PATH, 'w') as f:
processor_config.write(f)
# Set scheduler log file
scheduler_config = read_config_file(SCHEDULER_CONFIG_PATH)
if not os.path.exists(settings["log_path"]):
open(settings["log_path"], 'a').close()
scheduler_config["Log"]["log_path"] = settings["log_path"]
with open(SCHEDULER_CONFIG_PATH, 'w') as f:
scheduler_config.write(f)
return None
def test_read_config_file(self):
fname = "file.config"
create_config_file(fname)
config = read_config_file(fname)
self.assertEqual(type(config), RawConfigParser)
self.assertEqual(config["Section"]["entry"], "value")
os.remove(fname)
def setup_test_processor_config(
archive_directory=None,
metadata_db_url=None,
archive_db_url=None,
anomaly_db_url=None,
):
"""
Creates processor_config for testing
:type archive_directory: str
:param archive_directory: (optional) data archive directory, set if provided else default value used
:type metadata_db_url: str
:param metadata_db_url: (optional) metadata db url, set if provided else default value used
:type archive_db_url: str
:param archive_db_url: (optional) archive db url, set if provided else default value used
:type anomaly_db_url: str
:param anomaly_db_url: (optional) anomaly db url, set if provided else default value used
:return: test processor configuration information
:rtype: configparser.RawConfigParser
"""
processor_config = read_config_file(PROCESSOR_CONFIG_PATH)
processor_config["Processor"]["version"] = "0.0"
processor_config["Databases"]["metadata_db_url"] = (
metadata_db_url
if metadata_db_url is not None else "sqlite:///metadata.db")
processor_config["Databases"]["anomaly_db_url"] = (
anomaly_db_url
if anomaly_db_url is not None else "sqlite:///anomaly.db")
processor_config["Databases"]["archive_db_url"] = (
archive_db_url
if archive_db_url is not None else "sqlite:///archive.db")
processor_config["Output"]["archive_directory"] = (
archive_directory if archive_directory is not None else "out")
processor_config["Processing Options"] = {"write_l1a": "False"}
return processor_config
def main(settings):
"""
Main function for job init
:type settings: dict
:param settings: user defined configuration values
"""
# Create config from template
job_config = read_config_file(JOB_CONFIG_TEMPLATE_PATH)
job_config["Job"]["job_name"] = settings["job_name"]
job_config["Job"]["site_id"] = settings["site_id"]
# Create directories (resets with existing if unchanged)
os.makedirs(settings["job_working_directory"], exist_ok=True)
job_config["Job"]["job_working_directory"] = os.path.abspath(
settings["job_working_directory"])
os.makedirs(settings["raw_data_directory"], exist_ok=True)
job_config["Input"]["raw_data_directory"] = os.path.abspath(
settings["raw_data_directory"])
# Set job log file
log_path = os.path.join(settings["job_working_directory"],
settings["job_name"] + ".log")
if not os.path.exists(log_path):
open(log_path, 'a').close()
job_config["Log"]["log_path"] = os.path.abspath(log_path)
# Write config
job_config_path = os.path.abspath(
os.path.join(settings["job_working_directory"],
settings["job_name"] + ".config"))
with open(job_config_path, 'w') as f:
job_config.write(f)
# Add to scheduler
if settings["add_to_scheduler"]:
with open(JOBS_FILE_PATH, "a") as f:
if os.path.getsize(JOBS_FILE_PATH) > 0:
f.write("\n" + job_config_path)
else:
f.write(job_config_path)
def setup_test_job_config(raw_data_directory=None):
"""
Creates processor_config for testing
:type raw_data_directory: str
:param raw_data_directory: (optional) raw data directory, set if provided else default value used
:return: test job configuration information
:rtype: configparser.RawConfigParser
"""
job_config = read_config_file(JOB_CONFIG_TEMPLATE_PATH)
job_config["Job"]["network"] = "l"
job_config["Job"]["site_id"] = "test"
job_config["Input"]["raw_data_directory"] = (
raw_data_directory if raw_data_directory is not None else "data")
return job_config
def cli():
"""
Command line interface for hypernets_processor setup
"""
processor_config = read_config_file(PROCESSOR_CONFIG_PATH)
context = Context(processor_config)
settings = dict()
# Determine network
settings["network"] = determine_set_value("network",
context,
options=["l", "w"],
return_existing=True)
settings["network_defaults"] = cli_input_yn(
"set network default config values (overwrites existing)")
# Determine archive directory to set
settings["archive_directory"] = os.path.abspath(
determine_set_value("archive_directory", context,
return_existing=True))
home_directory = os.path.expanduser("~")
settings["working_directory"] = os.path.abspath(
determine_set_value("processor_working_directory",
context,
default=os.path.join(home_directory, ".hypernets"),
return_existing=True))
for db_fmt in DB_DICT_DEFS.keys():
settings[db_fmt + "_db_url"] = determine_set_value(
db_fmt + "_db_url",
context,
default="sqlite:///" +
os.path.join(settings["working_directory"], db_fmt + ".db"),
)
settings["log_path"] = os.path.join(settings["working_directory"],
"scheduler.log")
main(settings)
# COMPUTE WATER LEAVING RADIANCE LWN, REFLECTANCE RHOW_NOSC FOR EACH Lu SCAN!
# wind=RhymerHypstar(context).retrieve_wind(L1c)
# lw_all, rhow_all, rhow_nosc_all, epsilon, fresnel_coeff = RhymerHypstar(context).fresnelrefl_qc_simil(L1c, wind)
# print(lw_all)
# print(rhow_all)
# print(fresnel_coeff)
# L1c=
# average all scans to series
# L1d
# AVERAGE LWN, RHOW and RHOW_NOSC
# L2a
# print(L1b)
# # L2a=surf.process(L1c,"LandNetworkProtocol")
self.context.logger.info("all done!")
print("all done!")
return None
if __name__ == "__main__":
this_directory_path = os.path.abspath(os.path.dirname(__file__))
processor_config_path= os.path.join(this_directory_path,"etc/processor_2.config")
job_config_path= os.path.join(this_directory_path,"etc/job.config")
processor_config = read_config_file(processor_config_path)
job_config = read_config_file(job_config_path)
hp = HypernetsProcessor(job_config=job_config,processor_config=processor_config)
hp.context.set_config_value("processor_directory",this_directory_path)
hp.run()
pass
def cli():
"""
Command line interface to sequence_processor_main for ad-hoc job processing
"""
# If job config specified use
if parsed_args.job_config:
tmp_job = False
job_config_path = parsed_args.job_config
# Else build and write temporary job config from command line arguments
else:
tmp_job = True
processor_defaults = PROCESSOR_WATER_DEFAULTS_CONFIG_PATH
if parsed_args.network == "land":
processor_defaults = PROCESSOR_LAND_DEFAULTS_CONFIG_PATH
job_config = read_config_file(
[JOB_CONFIG_TEMPLATE_PATH, processor_defaults])
if parsed_args.input_directory is not None:
job_config["Input"]["raw_data_directory"] = os.path.abspath(
parsed_args.input_directory)
else:
print("-i required")
sys.exit(2)
if parsed_args.output_directory is not None:
job_config["Output"]["archive_directory"] = os.path.abspath(
parsed_args.output_directory)
else:
print("-o required")
sys.exit(2)
if parsed_args.write_all:
for key in job_config["Output"].keys():
if key[:5] == "write":
job_config["Output"][key] = "True"
job_config["Log"]["log_path"] = os.path.abspath(
parsed_args.log) if parsed_args.log is not None else ""
job_config["Log"]["verbose"] = str(
parsed_args.verbose) if parsed_args.verbose is not None else ""
job_config["Log"]["quiet"] = str(
parsed_args.quiet) if parsed_args.verbose is not None else ""
job_config["Job"]["job_name"] = "run_" + dt.now().strftime(
"%Y%m%dT%H%M%S")
home_directory = os.path.expanduser("~")
job_config["Job"]["job_working_directory"] = os.path.join(
home_directory, ".hypernets", "tmp")
job_config_path = os.path.join(
job_config["Job"]["job_working_directory"],
job_config["Job"]["job_name"] + ".config")
os.makedirs(job_config["Job"]["job_working_directory"], exist_ok=True)
with open(job_config_path, "w") as f:
job_config.write(f)
# run main
main(processor_config_path=PROCESSOR_CONFIG_PATH,
job_config_path=job_config_path,
to_archive=False)
if tmp_job:
os.remove(job_config_path)
return None
