def main():
parser = ArgumentParser()
parser = process_args(parser)
parser = logging_args(parser)
args = parser.parse_args()
setup_logging(args.log_conf, args.log_filename, args.error_email,
args.log_level, "crmprtd")
utc = pytz.utc
args.start_date = utc.localize(
verify_date(args.start_date, datetime.min, "start date"))
args.end_date = utc.localize(
verify_date(args.end_date, datetime.max, "end date"))
process(
args.connection_string,
args.sample_size,
args.network,
args.start_date,
args.end_date,
args.diag,
args.infer,
)
def test_verify_date_exception(datestring):
default = datetime.datetime.now()
warning = (
f"Parameter '{datestring}' is undefined or unparseable. Using the "
f"default '{default:%Y-%m-%d %H:%M:%S}'")
with pytest.warns(UserWarning, match=warning):
assert verify_date(datestring, default, "") == default
def main(partner):
"""Main download function to use for download scripts for the EC_SWOB
provincial partners (e.g. bc-env-snow, bc-env-aq, bc-forestry and
bc-tran).
Args:
partner (str): The partner abbreviation found in the SWOB URL
(e.g. bc-tran)
Returns:
No return value. Produces side-effect of sending downloaded
XML files to STDOUT
"""
desc = globals()["__doc__"]
parser = ArgumentParser(description=desc)
parser = logging_args(parser)
parser.add_argument(
"-d",
"--date",
help=("Alternate date to use for downloading "
"(interpreted with dateutil.parser.parse)."
"Defaults to now."),
)
args = parser.parse_args()
setup_logging(
args.log_conf,
args.log_filename,
args.error_email,
args.log_level,
"crmprtd.{}".format(partner),
)
if not args.date:
dl_date = datetime.datetime.now()
else:
dl_date = verify_date(args.date, datetime.datetime.now(), "date")
download(
"https://dd.weather.gc.ca/observations/swob-ml/partners/{}/".format(
partner),
dl_date,
)
def download(time, frequency, province, language, baseurl):
log.info("Starting EC rtd")
try:
# Determine time parameter
deltat = timedelta(1 / 24.0) if frequency == "hourly" else timedelta(1)
if time:
time = verify_date(time, datetime.utcnow() - deltat, "time")
log.info("Starting manual run " f"using timestamp {time}")
else:
# go back a day
time = datetime.utcnow() - deltat
log.info("Starting automatic run " f"using timestamp {time}")
# Construct and download the xml
url = makeurl(frequency, province, language, time, baseurl)
scheme, _ = url.split(":", 1)
https_download(url, scheme, log)
except IOError:
log.exception("Unable to download or open xml data")
sys.exit(1)
def test_verify_date(datestring, default):
assert verify_date(datestring, default,
"") == datetime.datetime(2020, 1, 1)
def main(): # pragma: no cover
desc = globals()["__doc__"]
parser = ArgumentParser(description=desc)
parser = logging_args(parser)
parser.add_argument("-u",
"--username",
default="pcic",
help=("Username for the ftp server "))
parser.add_argument(
"-f",
"--ftp_server",
default="sftp2.bchydro.com",
help=("Full uri to BC Hydro's ftp "
"server"),
)
parser.add_argument(
"-F",
"--ftp_dir",
default=("pcic"),
help=("FTP Directory containing BC hydro's "
"data files"),
)
parser.add_argument(
"-S",
"--ssh_private_key",
required=True,
help=("Path to file with SSH private key"),
)
end = datetime.now()
start = end - relativedelta.relativedelta(months=1)
parser.add_argument(
"-s",
"--start_date",
help=("Optional start time to use for downloading "
"(interpreted with dateutil.parser.parse). "
"Defaults to one month prior to now."),
)
parser.add_argument(
"-e",
"--end_date",
help=("Optional end time to use for downloading "
"(interpreted with dateutil.parser.parse). "
"Defaults to now."),
)
args = parser.parse_args()
setup_logging(
args.log_conf,
args.log_filename,
args.error_email,
args.log_level,
"crmprtd.bc_hydro",
)
args.start_date = verify_date(args.start_date, start, "start date")
args.end_date = verify_date(args.end_date, end, "end date")
download(
args.username,
args.ssh_private_key,
args.ftp_server,
args.ftp_dir,
args.start_date,
args.end_date,
)