def get_file_list(self): logger.debug("fetching files") backfill = timedelta(days=int(tutil.get_env_var("GINA_BACKFILL_DAYS"))) end_date = datetime.utcnow() + timedelta(days=1) start_date = end_date - backfill url = GINA_URL url += "&start_date=" + start_date.strftime("%Y-%m-%d") url += "&end_date=" + end_date.strftime("%Y-%m-%d") url += "&sensors[]=viirs" url += "&processing_levels[]=level1" url += "&facilities[]=" + tutil.get_env_var("VIIRS_FACILITY") url += "&satellites[]=" + SATELLITE logger.debug("URL: %s", url) buf = BytesIO() c = pycurl.Curl() c.setopt(c.URL, url) c.setopt(c.WRITEFUNCTION, buf.write) c.perform() files = [] for file in json.loads(buf.getvalue()): files.append(Viirs(file["url"], file["md5sum"])) buf.close() logger.info("Found %s files", len(files)) return files
def publish_product(filename, pngimg, volcview_args): user = tutil.get_env_var("VOLCVIEW_USER") passwd = tutil.get_env_var("VOLCVIEW_PASSWD") headers = {"username": user, "password": passwd} files = {"file": (filename, pngimg)} for endpoint in tutil.get_env_var("VV_ENDPOINTS").split(","): pngimg.seek(0) url = endpoint + "/imageApi/uploadImage" print("publishing image to {}".format(url)) print("data {}".format(volcview_args)) try: response = requests.post( url, headers=headers, data=volcview_args, files=files, timeout=POST_TIMEOUT, verify=False, ) print("server said: {}".format(response.text)) image_size = len(pngimg.getbuffer()) print(f"image size {image_size}") except requests.exceptions.RequestException as e: print(e) print("returning {}".format(response)) return response
def report_error(missing, extra): mailhost = tutil.get_env_var("MAILHOST", "NULL") sender = tutil.get_env_var("SENDER", "NULL") recipients = tutil.get_env_var("RECIPIENTS", "NULL") message = "From: {}\nTo: {}\nSubject: AQMS ComCat error\n\n" message = message.format(sender, recipients) if missing: logger.info("Found missing events") message += "Events not in ComCat:\n" for evid in missing: message += "\t{}\n".format(evid) message += "\n" if extra: logger.info("Found extra events") message += "Events that should be removed from ComCat:\n" for evid in extra: message += "\t{}\n".format(evid) message += "\n" if mailhost != "NULL": try: smtpObj = smtplib.SMTP(mailhost) smtpObj.sendmail(sender, recipients.split(","), message) except smtplib.SMTPException: pass else: print(message)
def main(): """Where it all begins.""" global logger logger = tutil.setup_logging("camfetchers errors") check_version() if 'CF_TIMEOUT' in os.environ: timeout = float(os.environ['CF_TIMEOUT']) logger.debug("Setting timeout to %.2f" % timeout) socket.setdefaulttimeout(timeout) with IMAP4_SSL(tutil.get_env_var('IMAPSERVER')) as M: try: M.login(tutil.get_env_var('CF_USER'), tutil.get_env_var('CF_PASSWD')) except IMAP4.error: tutil.exit_with_error("Login failed.") for cam in tutil.get_env_var('CF_CAMS').split(':'): rv, data = M.select(cam) if rv == 'OK': logger.debug("Processing mailbox %s", cam) process_mailbox(M, cam) else: msg = "Received non-OK response opening mailbox %s, " \ + "lets skip this one. (%s)" logger.error(msg.format(cam, rv)) logger.debug("That's all for now, bye.") logging.shutdown()
def get_aqms_events(): avouser = tutil.get_env_var("AVOUSER") avopass = tutil.get_env_var("AVOPASS") logger.debug("Requesting AQMS events") end_time = datetime.now() start_time = end_time - timedelta(days=LOOKBACK) aqms_args = { "from": start_time.strftime("%Y/%m/%d/%H:%M:%S"), "to": end_time.strftime("%Y/%m/%d/%H:%M:%S"), "review": "F", "format": "summary", "selectFlag": "selected", "result": "display", } response = None try: response = requests.get(AQMS_URL, params=aqms_args, verify=CERT, auth=(avouser, avopass)) except requests.exceptions.SSLError: response = requests.get(AQMS_URL, params=aqms_args, auth=(avouser, avopass)) evids = [] for event in response.text.splitlines()[2:]: evid = event[100:108] if evid.isdigit(): evids.append(evid) else: print("NOT EVENT: " + evid) return evids
def fetch_file(c, out_file, resume): tmp_dir = tutil.get_env_var("FF_TMP_DIR", default=".") tmp_file = "{}.tmp".format(os.path.basename(out_file)) tmp_path = pathlib.Path(tmp_dir) / tmp_file if os.path.exists(tmp_path) and resume: range = "{}-".format(os.path.getsize(tmp_path)) logger.info("Resuming download of %s for bytes %s", tmp_path, range) c.setopt(c.RANGE, range) mode = "ab" else: mode = "wb" try: with open(tmp_path, mode, buffering=0) as f: c.setopt(c.WRITEDATA, f) c.perform() make_out_dir(os.path.dirname(out_file)) os.rename(tmp_path, out_file) except pycurl.error as e: minor_errors = tutil.get_env_var("PYCURL_MINOR_ERRORS").split(",") minor_errors = [int(i) for i in minor_errors] if e.args[0] in minor_errors: logger.info("Error retrieving %s: %s", out_file, e) else: logger.exception( f"Error retrieving {out_file}. {e.args[0]} not in {minor_errors}" ) return True return False
def poll_queue(config): tmp_dir = tutil.get_env_var("FF_TMP_DIR", default=".") tmp_file = "{}.lock".format(config["name"]) lock_file = pathlib.Path(tmp_dir) / tmp_file lock = Lock(lock_file) gotlock, pid = lock.lock_pid() if not gotlock: logger.info("Queue {} locked, skipping".format(config["name"])) return try: day = datetime.utcnow().date() dataloggers = config["dataloggers"] while dataloggers: day -= timedelta(1) dataloggers = poll_loggers(dataloggers, day) finally: logger.info("All done with queue %s.", config["name"]) for handler in logger.handlers: handler.flush() logger.info("flushed handlers for queue %s.", config["name"]) if gotlock: try: lock.unlock() except AttributeError: pass logger.info("All done with queue %s.", config["name"])
def main(): global logger logger = tutil.setup_logging("filefetcher errors") global global_args global_args = arg_parse() msg = ( "Python interpreter is too old. I need at least {} " + "for EmailMessage.iter_attachments() support." ) tutil.enforce_version(REQ_VERSION, msg.format(REQ_VERSION)) try: config_file = pathlib.Path(tutil.get_env_var(CONFIG_FILE_ENV)) config = tutil.parse_config(config_file) except KeyError: msg = "Environment variable %s unset, exiting.".format(CONFIG_FILE_ENV) tutil.exit_with_error(msg) queues = process_queues(config) logger.debug("Queues: %s", queues) tmpl = jinjatmpl(EMAIL_TEMPLATE) logger.debug(tmpl) email = tmpl.render(queues=queues, style=STYLE, ad_hoc=global_args.span) send_email(email) logger.debug("That's all for now, bye.") logging.shutdown()
def main(): # let ctrl-c work as it should. signal.signal(signal.SIGINT, signal.SIG_DFL) context = zmq.Context() logger.debug("starting update_subscriber") update_subscriber = UpdateSubscriber(context) update_subscriber.start() logger.debug("started update_subscriber") task_client = context.socket(zmq.REQ) task_client.connect(TASK_SERVER) desired_products = tutil.get_env_var("VIIRS_PRODUCTS") desired_products = desired_products.split(",") while True: logger.debug("beating heart") Path(HEARTBEAT_FILE).touch() if update_subscriber.task_waiting: request = {"desired products": desired_products} task_client.send_json(request) msg_bytes = task_client.recv() if msg_bytes: process_message(msg_bytes) else: logger.debug("No job received") time.sleep(1) logger.debug("tomp says 4") else: logger.debug("Queue empty") time.sleep(5)
def find_sectors(self): """Identify sectors with at least some coverage by the provided scene. Returns ------- list area_id of each sector with some coverage. """ data = self.message.data overpass = Pass( data["platform_name"], self.scene.start_time, self.scene.end_time, instrument="viirs", ) logger.debug(f"Created overpass {overpass}") logger.debug(f"args: {data['platform_name']} :: " "{self.scene.start_time} :: {self.scene.end_time}") sectors = [] coverage_threashold = float( tutil.get_env_var("COVERAGE_THRESHOLD", 0.1)) for sector_def in parse_area_file(AREA_DEF): logger.debug("Checking coverage for %s", sector_def.area_id) coverage = overpass.area_coverage(sector_def) logger.debug("{} coverage: {}".format(sector_def.area_id, coverage)) if coverage > coverage_threashold: sectors.append(sector_def) return sectors
def main(): # let ctrl-c work as it should. signal.signal(signal.SIGINT, signal.SIG_DFL) global logger logger = tutil.setup_logging("imageshepherd errors") multiprocessing_logging.install_mp_handler() logger.info("Launching imageshepherd. Lets go!") global global_config global_config = None while global_config is None: try: config_file = tutil.get_env_var(CONFIG_FILE_ENV) global_config = tutil.parse_config(config_file) except FileNotFoundError: error = "Config file %s not found. ".format(CONFIG_FILE_ENV) error += "Lets wait a minute and look again." logger.info(error) time.sleep(60) time.sleep(1) device = start_proxy() if "watchers" in global_config: start_watchers(global_config["watchers"]) start_shippers() start_fetchers(global_config["sources"]) logger.info("Waiting for proxy to die") device.join()
def send_email(html): msg = MIMEMultipart("alternative") day = datetime.utcnow().date() - timedelta(2) msg["Subject"] = day.strftime("GPS retrieval %x") if global_args.span > 0: msg["Subject"] += " - {} day span".format(global_args.span) msg["From"] = tutil.get_env_var("LOG_SENDER") msg["To"] = tutil.get_env_var("REPORT_RECIPIENT") msg.attach(MIMEText(html, "html")) try: s = smtplib.SMTP(tutil.get_env_var("MAILHOST")) s.sendmail( tutil.get_env_var("LOG_SENDER"), global_args.recipient, msg.as_string() ) except OSError as e: logger.exception(e)
def get_archive_dir(cam, image_time): archive_dir = pathlib.Path(tutil.get_env_var('CF_OUT_DIR')) archive_dir /= time.strftime(cam + "/images/archive/%Y/%m/%d/%H", image_time) if not os.path.exists(archive_dir): logger.info("Creating directory: %s", archive_dir) os.makedirs(archive_dir) return archive_dir
def main(): """Where it all begins.""" global logger logger = tutil.setup_logging() for cam in tutil.get_env_var('CF_CAMS').split(":"): update_cam(cam) logger.debug("That's all for now, bye.") logging.shutdown()
def bootstrap_config(): """ Retrieve and parse my config file. Returns ------- Dict My configuration. """ bootstrap = { 'name': 'bootstrap', 'type': 'remotefile', 'source': tutil.get_env_var('CU_CONFIG_URL'), 'target': tutil.get_env_var('CU_LOCAL_CONFIG', CONFIG_PATH), 'user': tutil.get_env_var('CU_USER', None), 'passwd': tutil.get_env_var('CU_PASSWORD', None, secret=True) } update_config(bootstrap) return parse_config(CONFIG_PATH)
def __init__(self): self.tmp_path = os.path.join(BASE_DIR, "tmp") self.connection_count = int(tutil.get_env_var("NUM_GINA_CONNECTIONS")) self.file_store_type = tutil.get_env_var("VIIRS_FILE_STORE_TYPE") if self.file_store_type == "S3": self.file_store = avoviirscollector.viirs_s3_store elif self.file_store_type == "local": self.file_store = avoviirscollector.viirs_filesystem_store else: tutil.exit_with_error( "Unknown VIIRS_FILE_STORE_TYPE env var: {}".format( self.file_store_type)) # We should ignore SIGPIPE when using pycurl.NOSIGNAL - see # the libcurl tutorial for more info. try: signal.signal(signal.SIGPIPE, signal.SIG_IGN) except ImportError: pass self.hostname = socket.gethostname()
def arg_parse(): description = "I create and email a daily report of GPS download health." parser = argparse.ArgumentParser(description=description) parser.add_argument( "-s", "--span", help="How many days back should I look?", type=int, default=-1 ) parser.add_argument( "-r", "--recipient", help="Who should I email?", default=tutil.get_env_var("REPORT_RECIPIENT"), ) return parser.parse_args()
def main(): # let ctrl-c work as it should. signal.signal(signal.SIGINT, signal.SIG_DFL) global logger logger = tutil.setup_logging("webrelaypoker errors") multiprocessing_logging.install_mp_handler() config = tutil.parse_config(tutil.get_env_var(CONFIG_FILE_ENV)) procs = poke_relays(config["relays"]) for proc in procs: proc.join() logger.debug("That's all for now, bye.") logging.shutdown()
def parse_config(): config_file = pathlib.Path(tutil.get_env_var(CONFIG_FILE_ENV)) yaml = ruamel.yaml.YAML() try: global_config = yaml.load(config_file) except ruamel.yaml.parser.ParserError as e1: logger.error("Cannot parse config file") tutil.exit_with_error(e1) except OSError as e: if e.errno == errno.EEXIST: logger.error("Cannot read config file %s", config_file) tutil.exit_with_error(e) else: raise return global_config
def create_curl(datalogger, url): c = pycurl.Curl() c.setopt(c.VERBOSE, True) if "userpwd" in datalogger: userpwd = tutil.get_env_var(datalogger["userpwd"], secret=True) logger.debug("Setting userpw to whatever is in $%s", datalogger["userpwd"]) c.setopt(pycurl.USERPWD, userpwd) if "recvSpeed" in datalogger: setRecvSpeed(c, datalogger["recvSpeed"]) if "port" in datalogger: c.setopt(pycurl.PORT, datalogger["port"]) last_update = datetime.now() def progress(download_t, download_d, upload_t, upload_d): nonlocal last_update now = datetime.now() if now > last_update + MAX_UPDATE_FREQ: download_d_str = humanize.naturalsize(download_d, format="%.2f") download_t_str = humanize.naturalsize(download_t, format="%.2f") logger.debug("Downloaded %s of %s from %s", download_d_str, download_t_str, url) last_update = now return 0 if "low_speed_limit" in datalogger: logger.info( "Setting low speed limit to %db/s over %ds", datalogger["low_speed_limit"], datalogger["low_speed_time"], ) c.setopt(c.LOW_SPEED_LIMIT, datalogger["low_speed_limit"]) c.setopt(c.LOW_SPEED_TIME, datalogger["low_speed_time"]) c.setopt(c.NOPROGRESS, False) c.setopt(c.XFERINFOFUNCTION, progress) c.setopt(c.URL, url) return c
def main(): logger = tutil.setup_logging("filefetcher - errors") tmp_dir = tutil.get_env_var("FF_TMP_DIR") for filename in os.listdir(tmp_dir): if filename.endswith(".lock"): with open(os.path.join(tmp_dir, filename)) as file: pid = int(file.read()) try: process = psutil.Process(pid) except psutil.NoSuchProcess: continue create_time = process.create_time() create_time = datetime.fromtimestamp(create_time) process_age = datetime.now() - create_time if process_age > MAX_RUN_TIME: logger.info("Killing process %s, has been running for %s", pid, process_age) process.terminate() print("pid: {} age: {}".format(pid, process_age))
# I waive copyright and related rights in the this work worldwide # through the CC0 1.0 Universal public domain dedication. # https://creativecommons.org/publicdomain/zero/1.0/legalcode # Author(s): # Tom Parker <*****@*****.**> """ Store VIIRS files in a S3 bucket """ import tomputils.util as tutil import re import boto3 import botocore.exceptions from avoviirscollector import logger, SATELLITE BUCKET_NAME = tutil.get_env_var("S3_BUCKET", "UNSET") VERIFY = tutil.get_env_var("VERIFY") if VERIFY == "True": VERIFY = True elif VERIFY == "False": VERIFY = False def list_files(orbit): files = [] client = boto3.client("s3", verify=VERIFY) paginator = client.get_paginator("list_objects_v2") for page in paginator.paginate(Bucket=BUCKET_NAME, Prefix=f"{SATELLITE}/{orbit}/"): if page["KeyCount"] == 0: continue
def get_image_dir(cam): base_dir = pathlib.Path(tutil.get_env_var('CF_OUT_DIR')) image_dir = base_dir / cam / "images" logger.debug("image_dir: %s", image_dir) return image_dir
# -*- coding: utf-8 -*- # ----------------------------------------------------------------------------- # Purpose: fetch viirs data # Author: Tom Parker # # ----------------------------------------------------------------------------- """ avoviirscollector ================= Fetch viirs data at AVO :license: CC0 1.0 Universal http://creativecommons.org/publicdomain/zero/1.0/ """ from avoviirscollector.version import __version__ import tomputils.util as tutil logger = tutil.setup_logging("mirror_gina errors") BASE_DIR = tutil.get_env_var("VIIRS_BASE_DIR", "unset") SATELLITE = tutil.get_env_var("VIIRS_SATELLITE", "unset") CHANNELS = tutil.get_env_var("VIIRS_CHANNELS", "unset").split("|") __all__ = ["__version__"]