def get_file_list(self):
logger.debug("fetching files")
backfill = timedelta(days=int(tutil.get_env_var("GINA_BACKFILL_DAYS")))
end_date = datetime.utcnow() + timedelta(days=1)
start_date = end_date - backfill
url = GINA_URL
url += "&start_date=" + start_date.strftime("%Y-%m-%d")
url += "&end_date=" + end_date.strftime("%Y-%m-%d")
url += "&sensors[]=viirs"
url += "&processing_levels[]=level1"
url += "&facilities[]=" + tutil.get_env_var("VIIRS_FACILITY")
url += "&satellites[]=" + SATELLITE
logger.debug("URL: %s", url)
buf = BytesIO()
c = pycurl.Curl()
c.setopt(c.URL, url)
c.setopt(c.WRITEFUNCTION, buf.write)
c.perform()
files = []
for file in json.loads(buf.getvalue()):
files.append(Viirs(file["url"], file["md5sum"]))
buf.close()
logger.info("Found %s files", len(files))
return files
def publish_product(filename, pngimg, volcview_args):
user = tutil.get_env_var("VOLCVIEW_USER")
passwd = tutil.get_env_var("VOLCVIEW_PASSWD")
headers = {"username": user, "password": passwd}
files = {"file": (filename, pngimg)}
for endpoint in tutil.get_env_var("VV_ENDPOINTS").split(","):
pngimg.seek(0)
url = endpoint + "/imageApi/uploadImage"
print("publishing image to {}".format(url))
print("data {}".format(volcview_args))
try:
response = requests.post(
url,
headers=headers,
data=volcview_args,
files=files,
timeout=POST_TIMEOUT,
verify=False,
)
print("server said: {}".format(response.text))
image_size = len(pngimg.getbuffer())
print(f"image size {image_size}")
except requests.exceptions.RequestException as e:
print(e)
print("returning {}".format(response))
return response
def report_error(missing, extra):
mailhost = tutil.get_env_var("MAILHOST", "NULL")
sender = tutil.get_env_var("SENDER", "NULL")
recipients = tutil.get_env_var("RECIPIENTS", "NULL")
message = "From: {}\nTo: {}\nSubject: AQMS ComCat error\n\n"
message = message.format(sender, recipients)
if missing:
logger.info("Found missing events")
message += "Events not in ComCat:\n"
for evid in missing:
message += "\t{}\n".format(evid)
message += "\n"
if extra:
logger.info("Found extra events")
message += "Events that should be removed from ComCat:\n"
for evid in extra:
message += "\t{}\n".format(evid)
message += "\n"
if mailhost != "NULL":
try:
smtpObj = smtplib.SMTP(mailhost)
smtpObj.sendmail(sender, recipients.split(","), message)
except smtplib.SMTPException:
pass
else:
print(message)
def main():
"""Where it all begins."""
global logger
logger = tutil.setup_logging("camfetchers errors")
check_version()
if 'CF_TIMEOUT' in os.environ:
timeout = float(os.environ['CF_TIMEOUT'])
logger.debug("Setting timeout to %.2f" % timeout)
socket.setdefaulttimeout(timeout)
with IMAP4_SSL(tutil.get_env_var('IMAPSERVER')) as M:
try:
M.login(tutil.get_env_var('CF_USER'),
tutil.get_env_var('CF_PASSWD'))
except IMAP4.error:
tutil.exit_with_error("Login failed.")
for cam in tutil.get_env_var('CF_CAMS').split(':'):
rv, data = M.select(cam)
if rv == 'OK':
logger.debug("Processing mailbox %s", cam)
process_mailbox(M, cam)
else:
msg = "Received non-OK response opening mailbox %s, " \
+ "lets skip this one. (%s)"
logger.error(msg.format(cam, rv))
logger.debug("That's all for now, bye.")
logging.shutdown()
def get_aqms_events():
avouser = tutil.get_env_var("AVOUSER")
avopass = tutil.get_env_var("AVOPASS")
logger.debug("Requesting AQMS events")
end_time = datetime.now()
start_time = end_time - timedelta(days=LOOKBACK)
aqms_args = {
"from": start_time.strftime("%Y/%m/%d/%H:%M:%S"),
"to": end_time.strftime("%Y/%m/%d/%H:%M:%S"),
"review": "F",
"format": "summary",
"selectFlag": "selected",
"result": "display",
}
response = None
try:
response = requests.get(AQMS_URL,
params=aqms_args,
verify=CERT,
auth=(avouser, avopass))
except requests.exceptions.SSLError:
response = requests.get(AQMS_URL,
params=aqms_args,
auth=(avouser, avopass))
evids = []
for event in response.text.splitlines()[2:]:
evid = event[100:108]
if evid.isdigit():
evids.append(evid)
else:
print("NOT EVENT: " + evid)
return evids
def fetch_file(c, out_file, resume):
tmp_dir = tutil.get_env_var("FF_TMP_DIR", default=".")
tmp_file = "{}.tmp".format(os.path.basename(out_file))
tmp_path = pathlib.Path(tmp_dir) / tmp_file
if os.path.exists(tmp_path) and resume:
range = "{}-".format(os.path.getsize(tmp_path))
logger.info("Resuming download of %s for bytes %s", tmp_path, range)
c.setopt(c.RANGE, range)
mode = "ab"
else:
mode = "wb"
try:
with open(tmp_path, mode, buffering=0) as f:
c.setopt(c.WRITEDATA, f)
c.perform()
make_out_dir(os.path.dirname(out_file))
os.rename(tmp_path, out_file)
except pycurl.error as e:
minor_errors = tutil.get_env_var("PYCURL_MINOR_ERRORS").split(",")
minor_errors = [int(i) for i in minor_errors]
if e.args[0] in minor_errors:
logger.info("Error retrieving %s: %s", out_file, e)
else:
logger.exception(
f"Error retrieving {out_file}. {e.args[0]} not in {minor_errors}"
)
return True
return False
def poll_queue(config):
tmp_dir = tutil.get_env_var("FF_TMP_DIR", default=".")
tmp_file = "{}.lock".format(config["name"])
lock_file = pathlib.Path(tmp_dir) / tmp_file
lock = Lock(lock_file)
gotlock, pid = lock.lock_pid()
if not gotlock:
logger.info("Queue {} locked, skipping".format(config["name"]))
return
try:
day = datetime.utcnow().date()
dataloggers = config["dataloggers"]
while dataloggers:
day -= timedelta(1)
dataloggers = poll_loggers(dataloggers, day)
finally:
logger.info("All done with queue %s.", config["name"])
for handler in logger.handlers:
handler.flush()
logger.info("flushed handlers for queue %s.", config["name"])
if gotlock:
try:
lock.unlock()
except AttributeError:
pass
logger.info("All done with queue %s.", config["name"])
def main():
global logger
logger = tutil.setup_logging("filefetcher errors")
global global_args
global_args = arg_parse()
msg = (
"Python interpreter is too old. I need at least {} "
+ "for EmailMessage.iter_attachments() support."
)
tutil.enforce_version(REQ_VERSION, msg.format(REQ_VERSION))
try:
config_file = pathlib.Path(tutil.get_env_var(CONFIG_FILE_ENV))
config = tutil.parse_config(config_file)
except KeyError:
msg = "Environment variable %s unset, exiting.".format(CONFIG_FILE_ENV)
tutil.exit_with_error(msg)
queues = process_queues(config)
logger.debug("Queues: %s", queues)
tmpl = jinjatmpl(EMAIL_TEMPLATE)
logger.debug(tmpl)
email = tmpl.render(queues=queues, style=STYLE, ad_hoc=global_args.span)
send_email(email)
logger.debug("That's all for now, bye.")
logging.shutdown()
def main():
# let ctrl-c work as it should.
signal.signal(signal.SIGINT, signal.SIG_DFL)
context = zmq.Context()
logger.debug("starting update_subscriber")
update_subscriber = UpdateSubscriber(context)
update_subscriber.start()
logger.debug("started update_subscriber")
task_client = context.socket(zmq.REQ)
task_client.connect(TASK_SERVER)
desired_products = tutil.get_env_var("VIIRS_PRODUCTS")
desired_products = desired_products.split(",")
while True:
logger.debug("beating heart")
Path(HEARTBEAT_FILE).touch()
if update_subscriber.task_waiting:
request = {"desired products": desired_products}
task_client.send_json(request)
msg_bytes = task_client.recv()
if msg_bytes:
process_message(msg_bytes)
else:
logger.debug("No job received")
time.sleep(1)
logger.debug("tomp says 4")
else:
logger.debug("Queue empty")
time.sleep(5)
def find_sectors(self):
"""Identify sectors with at least some coverage by the provided scene.
Returns
-------
list
area_id of each sector with some coverage.
"""
data = self.message.data
overpass = Pass(
data["platform_name"],
self.scene.start_time,
self.scene.end_time,
instrument="viirs",
)
logger.debug(f"Created overpass {overpass}")
logger.debug(f"args: {data['platform_name']} :: "
"{self.scene.start_time} :: {self.scene.end_time}")
sectors = []
coverage_threashold = float(
tutil.get_env_var("COVERAGE_THRESHOLD", 0.1))
for sector_def in parse_area_file(AREA_DEF):
logger.debug("Checking coverage for %s", sector_def.area_id)
coverage = overpass.area_coverage(sector_def)
logger.debug("{} coverage: {}".format(sector_def.area_id,
coverage))
if coverage > coverage_threashold:
sectors.append(sector_def)
return sectors
def main():
# let ctrl-c work as it should.
signal.signal(signal.SIGINT, signal.SIG_DFL)
global logger
logger = tutil.setup_logging("imageshepherd errors")
multiprocessing_logging.install_mp_handler()
logger.info("Launching imageshepherd. Lets go!")
global global_config
global_config = None
while global_config is None:
try:
config_file = tutil.get_env_var(CONFIG_FILE_ENV)
global_config = tutil.parse_config(config_file)
except FileNotFoundError:
error = "Config file %s not found. ".format(CONFIG_FILE_ENV)
error += "Lets wait a minute and look again."
logger.info(error)
time.sleep(60)
time.sleep(1)
device = start_proxy()
if "watchers" in global_config:
start_watchers(global_config["watchers"])
start_shippers()
start_fetchers(global_config["sources"])
logger.info("Waiting for proxy to die")
device.join()
def send_email(html):
msg = MIMEMultipart("alternative")
day = datetime.utcnow().date() - timedelta(2)
msg["Subject"] = day.strftime("GPS retrieval %x")
if global_args.span > 0:
msg["Subject"] += " - {} day span".format(global_args.span)
msg["From"] = tutil.get_env_var("LOG_SENDER")
msg["To"] = tutil.get_env_var("REPORT_RECIPIENT")
msg.attach(MIMEText(html, "html"))
try:
s = smtplib.SMTP(tutil.get_env_var("MAILHOST"))
s.sendmail(
tutil.get_env_var("LOG_SENDER"), global_args.recipient, msg.as_string()
)
except OSError as e:
logger.exception(e)
def get_archive_dir(cam, image_time):
archive_dir = pathlib.Path(tutil.get_env_var('CF_OUT_DIR'))
archive_dir /= time.strftime(cam + "/images/archive/%Y/%m/%d/%H",
image_time)
if not os.path.exists(archive_dir):
logger.info("Creating directory: %s", archive_dir)
os.makedirs(archive_dir)
return archive_dir
def main():
"""Where it all begins."""
global logger
logger = tutil.setup_logging()
for cam in tutil.get_env_var('CF_CAMS').split(":"):
update_cam(cam)
logger.debug("That's all for now, bye.")
logging.shutdown()
def bootstrap_config():
"""
Retrieve and parse my config file.
Returns
-------
Dict
My configuration.
"""
bootstrap = {
'name': 'bootstrap',
'type': 'remotefile',
'source': tutil.get_env_var('CU_CONFIG_URL'),
'target': tutil.get_env_var('CU_LOCAL_CONFIG', CONFIG_PATH),
'user': tutil.get_env_var('CU_USER', None),
'passwd': tutil.get_env_var('CU_PASSWORD', None, secret=True)
}
update_config(bootstrap)
return parse_config(CONFIG_PATH)
def __init__(self):
self.tmp_path = os.path.join(BASE_DIR, "tmp")
self.connection_count = int(tutil.get_env_var("NUM_GINA_CONNECTIONS"))
self.file_store_type = tutil.get_env_var("VIIRS_FILE_STORE_TYPE")
if self.file_store_type == "S3":
self.file_store = avoviirscollector.viirs_s3_store
elif self.file_store_type == "local":
self.file_store = avoviirscollector.viirs_filesystem_store
else:
tutil.exit_with_error(
"Unknown VIIRS_FILE_STORE_TYPE env var: {}".format(
self.file_store_type))
# We should ignore SIGPIPE when using pycurl.NOSIGNAL - see
# the libcurl tutorial for more info.
try:
signal.signal(signal.SIGPIPE, signal.SIG_IGN)
except ImportError:
pass
self.hostname = socket.gethostname()
def arg_parse():
description = "I create and email a daily report of GPS download health."
parser = argparse.ArgumentParser(description=description)
parser.add_argument(
"-s", "--span", help="How many days back should I look?", type=int, default=-1
)
parser.add_argument(
"-r",
"--recipient",
help="Who should I email?",
default=tutil.get_env_var("REPORT_RECIPIENT"),
)
return parser.parse_args()
def main():
# let ctrl-c work as it should.
signal.signal(signal.SIGINT, signal.SIG_DFL)
global logger
logger = tutil.setup_logging("webrelaypoker errors")
multiprocessing_logging.install_mp_handler()
config = tutil.parse_config(tutil.get_env_var(CONFIG_FILE_ENV))
procs = poke_relays(config["relays"])
for proc in procs:
proc.join()
logger.debug("That's all for now, bye.")
logging.shutdown()
def parse_config():
config_file = pathlib.Path(tutil.get_env_var(CONFIG_FILE_ENV))
yaml = ruamel.yaml.YAML()
try:
global_config = yaml.load(config_file)
except ruamel.yaml.parser.ParserError as e1:
logger.error("Cannot parse config file")
tutil.exit_with_error(e1)
except OSError as e:
if e.errno == errno.EEXIST:
logger.error("Cannot read config file %s", config_file)
tutil.exit_with_error(e)
else:
raise
return global_config
def create_curl(datalogger, url):
c = pycurl.Curl()
c.setopt(c.VERBOSE, True)
if "userpwd" in datalogger:
userpwd = tutil.get_env_var(datalogger["userpwd"], secret=True)
logger.debug("Setting userpw to whatever is in $%s",
datalogger["userpwd"])
c.setopt(pycurl.USERPWD, userpwd)
if "recvSpeed" in datalogger:
setRecvSpeed(c, datalogger["recvSpeed"])
if "port" in datalogger:
c.setopt(pycurl.PORT, datalogger["port"])
last_update = datetime.now()
def progress(download_t, download_d, upload_t, upload_d):
nonlocal last_update
now = datetime.now()
if now > last_update + MAX_UPDATE_FREQ:
download_d_str = humanize.naturalsize(download_d, format="%.2f")
download_t_str = humanize.naturalsize(download_t, format="%.2f")
logger.debug("Downloaded %s of %s from %s", download_d_str,
download_t_str, url)
last_update = now
return 0
if "low_speed_limit" in datalogger:
logger.info(
"Setting low speed limit to %db/s over %ds",
datalogger["low_speed_limit"],
datalogger["low_speed_time"],
)
c.setopt(c.LOW_SPEED_LIMIT, datalogger["low_speed_limit"])
c.setopt(c.LOW_SPEED_TIME, datalogger["low_speed_time"])
c.setopt(c.NOPROGRESS, False)
c.setopt(c.XFERINFOFUNCTION, progress)
c.setopt(c.URL, url)
return c
def main():
logger = tutil.setup_logging("filefetcher - errors")
tmp_dir = tutil.get_env_var("FF_TMP_DIR")
for filename in os.listdir(tmp_dir):
if filename.endswith(".lock"):
with open(os.path.join(tmp_dir, filename)) as file:
pid = int(file.read())
try:
process = psutil.Process(pid)
except psutil.NoSuchProcess:
continue
create_time = process.create_time()
create_time = datetime.fromtimestamp(create_time)
process_age = datetime.now() - create_time
if process_age > MAX_RUN_TIME:
logger.info("Killing process %s, has been running for %s",
pid, process_age)
process.terminate()
print("pid: {} age: {}".format(pid, process_age))
# I waive copyright and related rights in the this work worldwide
# through the CC0 1.0 Universal public domain dedication.
# https://creativecommons.org/publicdomain/zero/1.0/legalcode
# Author(s):
# Tom Parker <*****@*****.**>
""" Store VIIRS files in a S3 bucket
"""
import tomputils.util as tutil
import re
import boto3
import botocore.exceptions
from avoviirscollector import logger, SATELLITE
BUCKET_NAME = tutil.get_env_var("S3_BUCKET", "UNSET")
VERIFY = tutil.get_env_var("VERIFY")
if VERIFY == "True":
VERIFY = True
elif VERIFY == "False":
VERIFY = False
def list_files(orbit):
files = []
client = boto3.client("s3", verify=VERIFY)
paginator = client.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=BUCKET_NAME,
Prefix=f"{SATELLITE}/{orbit}/"):
if page["KeyCount"] == 0:
continue
def get_image_dir(cam):
base_dir = pathlib.Path(tutil.get_env_var('CF_OUT_DIR'))
image_dir = base_dir / cam / "images"
logger.debug("image_dir: %s", image_dir)
return image_dir
# -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Purpose: fetch viirs data
# Author: Tom Parker
#
# -----------------------------------------------------------------------------
"""
avoviirscollector
=================
Fetch viirs data at AVO
:license:
CC0 1.0 Universal
http://creativecommons.org/publicdomain/zero/1.0/
"""
from avoviirscollector.version import __version__
import tomputils.util as tutil
logger = tutil.setup_logging("mirror_gina errors")
BASE_DIR = tutil.get_env_var("VIIRS_BASE_DIR", "unset")
SATELLITE = tutil.get_env_var("VIIRS_SATELLITE", "unset")
CHANNELS = tutil.get_env_var("VIIRS_CHANNELS", "unset").split("|")
__all__ = ["__version__"]
