def concatenate(
from_date: "datedoy", to_date: "datedoy", tech: "pipeline", stage: "option", writers: "option", id: "option"
):
log.init(log_level="info")
# Get options
writer_names = writers.replace(",", " ").split()
dataset_id = util.read_option_value("--dset_id", default="last")
dataset_id = "last" if dataset_id == "last" else int(dataset_id)
dataset_name = util.read_option_value("--dset_name", default="")
session = util.read_option_value("--session", default="")
dset_vars = dict(
tech=tech,
stage=stage,
session=session,
dataset_name=session,
dataset_id=dataset_id,
session_name=id + "_concatenated",
)
dset = concatenate_datasets(from_date, to_date, dset_vars)
if dset.num_obs == 0:
log.fatal(f"No data to read period from {from_date} to {to_date}.")
dset.write()
# Loop over writers
for writer in writer_names:
write(writer, dset=dset)
def concatenate(from_date: "datedoy", to_date: "datedoy", pipeline: "pipeline",
stage: "option"):
log.init(log_level="info")
# Get options
label = util.read_option_value("--label", default="None")
# TODO: label = "last" if label == "last" else label
id_ = util.read_option_value("--id", default="")
only_for_rundate = True if util.check_options(
"--only_for_rundate") else False
session = util.read_option_value("--session", default="")
station = util.read_option_value("--station", default="")
writers = util.read_option_value("--writers",
default="").replace(",", " ").split()
# Update configuration of Where analysis
config.where.update_from_options(_clean_sys_argv(pipeline))
dset_vars = dict(pipeline=pipeline,
stage=stage,
session=session,
station=station,
label=label,
id=id_)
dset_vars = config.create_file_vars(rundate=from_date, **dset_vars)
dset = _concatenate_datasets(from_date, to_date, dset_vars,
only_for_rundate)
if dset.num_obs == 0:
log.fatal(f"No data to read period from {from_date} to {to_date}.")
dset.write()
# Loop over writers
for writer in writers:
write(writer, dset=dset)
def plot(rundate: "datedoy", tech: "pipeline", stage: "option",
writers: "option"):
log.init(log_level="info")
# Get options
writer_names = writers.replace(",", " ").split()
dataset_id = util.read_option_value("--dset_id", default="last")
dataset_id = "last" if dataset_id == "last" else int(dataset_id)
identifier = util.read_option_value("--id", default="")
identifier = f"-{identifier}" if identifier else ""
session = util.read_option_value("--session", default="")
dset = data.Dataset(rundate=rundate,
tech=tech,
stage=stage,
id=identifier,
dataset_name=session,
dataset_id=dataset_id)
path_hdf5 = files.path("dataset_hdf5", file_vars=dset.vars)
path_json = files.path("dataset_json", file_vars=dset.vars)
log.info(f"Read Where dataset files {path_hdf5} and {path_json}.")
if dset.num_obs == 0:
log.fatal(f"No data to read for date {rundate}.")
# Loop over writers
for writer in writer_names:
write(writer, dset=dset)
def main(date: "datedoy", pipeline: "pipeline", ids: "option"):
log.init(log_level="info")
# Additional required options
identifiers = [id_.strip() for id_ in ids.split(",")]
difference_by = util.read_option_value("--difference_by").replace(
",", " ").split()
stage = util.read_option_value("--stage")
# Get optional options
dataset_id = util.read_option_value("--dset_id", default="last")
dataset_id = "last" if dataset_id == "last" else int(dataset_id)
dataset_name = util.read_option_value("--dset_name", default="")
writer_names = util.read_option_value("--writers",
default="").replace(",",
" ").split()
session = util.read_option_value("--session", default="")
station = util.read_option_value("--station", default="")
# Get datasets
dset = data.Dataset(
rundate=date,
pipeline=pipeline,
stage=stage,
session=session,
station=station,
label=label,
id="-" + identifiers[0],
)
dset_other = data.Dataset(
rundate=date,
pipeline=pipeline,
stage=stage,
session=session,
station=station,
label=label,
id="-" + identifiers[1],
)
if dset.num_obs == 0:
log.warn(
f"Nothing to differentiate. Dataset '{identifiers[0]}' is empty.")
return 1
if dset_other.num_obs == 0:
log.warn(
f"Nothing to differentiate. Dataset '{identifiers[1]}' is empty.")
return 1
# Differentiate dataset
dset_diff = dset.difference(dset_other, index_by=','.join(difference_by))
dset_diff.write_as(stage="difference")
# Loop over writers
for writer in writer_names:
write(writer, dset_diff)
def concatenate(from_date: "datedoy", to_date: "datedoy", tech: "pipeline",
stage: "option"):
log.init(log_level="info")
# Get options
dataset_id = util.read_option_value("--dset_id", default="last")
dataset_id = "last" if dataset_id == "last" else int(dataset_id)
dataset_name = util.read_option_value("--dset_name", default="")
id = util.read_option_value("--id", default="")
only_for_rundate = True if util.check_options(
"--only_for_rundate") else False
session = util.read_option_value("--session", default="")
writer_names = util.read_option_value("--writers",
default="").replace(",",
" ").split()
# Update configuration of Where analysis
config.where.update_from_options(_clean_sys_argv(tech))
dset_vars = dict(
tech=tech,
stage=stage,
session=session,
dataset_name=session,
dataset_id=dataset_id,
session_name=id + "_concatenated",
)
dset = _concatenate_datasets(from_date, to_date, dset_vars,
only_for_rundate)
if dset.num_obs == 0:
log.fatal(f"No data to read period from {from_date} to {to_date}.")
dset.write()
# Loop over writers
for writer in writer_names:
write(writer, dset=dset)
def main(date: "datedoy", pipeline: "pipeline", items: "option",
specifier: "option"):
log.init(log_level="info")
dsets = dict()
# Additional options
stage = util.read_option_value("--stage")
writer_names = util.read_option_value("--writers").replace(",",
" ").split()
items_ = [s.strip() for s in items.split(",")]
# Get optional options
label = util.read_option_value("--label", default="None")
# TODO label = "last" if label == "last" else label
station = util.read_option_value("--station", default="")
id_ = util.read_option_value("--id", default="")
# Get dataset variables
dset_vars = dict(pipeline=pipeline,
stage=stage,
station=station,
label=label,
id=id_)
dset_vars = config.create_file_vars(rundate=date, **dset_vars)
# Read datasets for given specifier
if specifier == "id":
for id_ in items_:
dset = dataset.Dataset().read(rundate=date,
pipeline=pipeline,
stage=stage,
label=label,
id=id_,
station=station)
if dset.num_obs == 0:
log.warn(f"Dataset '{id_}' is empty.")
continue
dset_vars[
"id"] = id_ #TODO: Better solution for handling of dataset variables?
dset.vars.update(
dset_vars
) # Necessary for example for getting correct file path in used writers.
dsets.update({id_: dset})
elif specifier == "station":
for station in items_:
dset = dataset.Dataset().read(rundate=date,
pipeline=pipeline,
stage=stage,
label=label,
id=id_,
station=station)
if dset.num_obs == 0:
log.warn(f"Dataset '{station}' is empty.")
continue
dset_vars[
"station"] = station #TODO: Better solution for handling of dataset variables?
dset.vars.update(
dset_vars
) # Necessary for example for getting correct file path in used writers.
dsets.update({station: dset})
elif specifier == "stage":
for stage in items_:
dset = dataset.Dataset().read(rundate=date,
pipeline=pipeline,
stage=stage,
label=label,
id=id_,
station=station)
if dset.num_obs == 0:
log.warn(f"Dataset '{stage}' is empty.")
continue
dset_vars[
"stage"] = stage #TODO: Better solution for handling of dataset variables?
dset.vars.update(
dset_vars
) # Necessary for example for getting correct file path in used writers.
dsets.update({stage: dset})
else:
log.fatal(
f"Specifier {specifier} is not defined. It should be either 'id', 'station' or 'stage'."
)
if len(dsets) == 0:
log.fatal(f"All given datasets are empty [{', '.join(dsets.keys())}].")
elif len(dsets) == 1:
log.warn(
f"Nothing to compare. Only dataset '{list(dsets.keys())[0]}' is available."
)
# Loop over writers
for writer in writer_names:
write(writer, dset=dsets)
def main(date: "datedoy", tech: "pipeline", items: "option",
specifier: "option"):
log.init(log_level="info")
dsets = dict()
# Additional options
stage = util.read_option_value("--stage")
writer_names = util.read_option_value("--writers").replace(",",
" ").split()
items_ = [s.strip() for s in items.split(",")]
# Get optional options
dataset_id = util.read_option_value("--dset_id", default="last")
dataset_id = "last" if dataset_id == "last" else int(dataset_id)
dataset_name = util.read_option_value("--dset_name", default="")
session = util.read_option_value("--session", default="")
id_ = "-" + util.read_option_value(
"--id", default="") if util.read_option_value("--id",
default="") else ""
# Read datasets for given specifier
if specifier == "id":
for id_ in items_:
dset = data.Dataset(rundate=date,
tech=tech,
stage=stage,
dataset_name=dataset_name,
dataset_id=dataset_id,
id="-" + id_)
if dset.num_obs == 0:
log.warn(f"Dataset '{id_}' is empty.")
continue
dsets.update({id_: dset})
elif specifier == "session":
for session in items_:
dset = data.Dataset(rundate=date,
tech=tech,
stage=stage,
dataset_name=session,
dataset_id=dataset_id,
id=id_)
if dset.num_obs == 0:
log.warn(f"Dataset '{session}' is empty.")
continue
dsets.update({session: dset})
elif specifier == "stage":
for stage in items_:
dset = data.Dataset(rundate=date,
tech=tech,
stage=stage,
dataset_name=dataset_name,
dataset_id=dataset_id,
id=id_)
if dset.num_obs == 0:
log.warn(f"Dataset '{stage}' is empty.")
continue
dsets.update({stage: dset})
else:
log.fatal(
f"Specifier {specifier} is not defined. It should be either 'id', 'session' or 'stage'."
)
if len(dsets) == 0:
log.fatal(f"All given datasets are empty [{', '.join(dsets.keys())}].")
elif len(dsets) == 1:
log.warn(
f"Nothing to compare. Only dataset '{list(dsets.keys())[0]}' is available."
)
# Loop over writers
for writer in writer_names:
write(writer, dset=dsets)
def compare(date: "datedoy", pipeline: "pipeline", items: "option",
specifier: "option"):
log.init(log_level="info")
dsets = dict()
# Additional options
stage = util.read_option_value("--stage")
writer_names = util.read_option_value("--writers").replace(",",
" ").split()
items_ = [s.strip() for s in items.split(",")]
# Get optional options
label = util.read_option_value("--label", default="None")
# TODO label = "last" if label == "last" else label
station = util.read_option_value("--station", default="")
id_ = util.read_option_value("--id", default="")
# Update configuration of Where analysis
config.where.update_from_options(_clean_sys_argv(pipeline))
# Get dataset variables
dset_vars = config.create_file_vars(rundate=date, pipeline=pipeline)
# Read datasets for given specifier
if specifier == "id":
for id_ in items_:
try:
dset = dataset.Dataset().read(rundate=date,
pipeline=pipeline,
stage=stage,
label=label,
id=id_,
station=station)
except OSError:
log.warn(f"No data to read for Dataset id '{id_}'.")
continue
dset.vars.update(dset_vars)
dset.vars["id"] = id_
dsets.update({id_: dset})
elif specifier == "station":
for station in items_:
try:
dset = dataset.Dataset().read(rundate=date,
pipeline=pipeline,
stage=stage,
label=label,
id=id_,
station=station)
except OSError:
log.warn(f"No data to read for Dataset station '{station}'.")
continue
dset.vars.update(dset_vars)
dset.vars["station"] = station
dsets.update({station: dset})
elif specifier == "stage":
for stage in items_:
try:
dset = dataset.Dataset().read(rundate=date,
pipeline=pipeline,
stage=stage,
label=label,
id=id_,
station=station)
except OSError:
log.warn(f"No data to read for Dataset stage '{stage}'.")
continue
dset.vars.update(dset_vars)
dset.vars["stage"] = stage
dsets.update({stage: dset})
else:
log.fatal(
f"Specifier {specifier} is not defined. It should be either 'id', 'station' or 'stage'."
)
if len(dsets) == 0:
log.fatal(f"All given datasets are empty [{', '.join(dsets.keys())}].")
elif len(dsets) == 1:
log.warn(
f"Nothing to compare. Only dataset '{list(dsets.keys())[0]}' is available."
)
# Loop over writers
for writer in writer_names:
write(writer, dset=dsets)