def main(): """Where it all begins.""" global logger logger = tutil.setup_logging("camfetchers errors") check_version() if 'CF_TIMEOUT' in os.environ: timeout = float(os.environ['CF_TIMEOUT']) logger.debug("Setting timeout to %.2f" % timeout) socket.setdefaulttimeout(timeout) with IMAP4_SSL(tutil.get_env_var('IMAPSERVER')) as M: try: M.login(tutil.get_env_var('CF_USER'), tutil.get_env_var('CF_PASSWD')) except IMAP4.error: tutil.exit_with_error("Login failed.") for cam in tutil.get_env_var('CF_CAMS').split(':'): rv, data = M.select(cam) if rv == 'OK': logger.debug("Processing mailbox %s", cam) process_mailbox(M, cam) else: msg = "Received non-OK response opening mailbox %s, " \ + "lets skip this one. (%s)" logger.error(msg.format(cam, rv)) logger.debug("That's all for now, bye.") logging.shutdown()
def main(): # let ctrl-c work as it should. signal.signal(signal.SIGINT, signal.SIG_DFL) global logger logger = tutil.setup_logging("imageshepherd errors") multiprocessing_logging.install_mp_handler() logger.info("Launching imageshepherd. Lets go!") global global_config global_config = None while global_config is None: try: config_file = tutil.get_env_var(CONFIG_FILE_ENV) global_config = tutil.parse_config(config_file) except FileNotFoundError: error = "Config file %s not found. ".format(CONFIG_FILE_ENV) error += "Lets wait a minute and look again." logger.info(error) time.sleep(60) time.sleep(1) device = start_proxy() if "watchers" in global_config: start_watchers(global_config["watchers"]) start_shippers() start_fetchers(global_config["sources"]) logger.info("Waiting for proxy to die") device.join()
def main(): # let ctrl-c work as it should. signal.signal(signal.SIGINT, signal.SIG_DFL) global logger logger = tutil.setup_logging("filefetcher errors") multiprocessing_logging.install_mp_handler() msg = ("Python interpreter is too old. I need at least {} " + "for EmailMessage.iter_attachments() support.") tutil.enforce_version(REQ_VERSION, msg.format(REQ_VERSION)) global args args = _arg_parse() try: global global_config global_config = parse_config() except KeyError: msg = "Environment variable %s unset, exiting.".format(CONFIG_FILE_ENV) tutil.exit_with_error(msg) procs = poll_queues() for proc in procs: proc.join() logger.debug("That's all for now, bye.") logging.shutdown()
def main(): # let ctrl-c work as it should. signal.signal(signal.SIGINT, signal.SIG_DFL) global logger logger = tutil.setup_logging("msg_broker errors") global msgs_lock msgs_lock = threading.Lock() logger.debug("Current libzmq version is %s" % zmq.zmq_version()) logger.debug("Current pyzmq version is %s" % zmq.__version__) context = zmq.Context() msgs = collections.OrderedDict() client = ClientTask(msgs) client.start() logger.info("client started") tasker = Tasker(context, msgs) tasker.start() logger.info("tasker started") updater = Updater(context, msgs) updater.start() logger.info("updater started") client.join() tasker.join() updater.join()
def main(): global logger logger = tutil.setup_logging("filefetcher errors") global global_args global_args = arg_parse() msg = ( "Python interpreter is too old. I need at least {} " + "for EmailMessage.iter_attachments() support." ) tutil.enforce_version(REQ_VERSION, msg.format(REQ_VERSION)) try: config_file = pathlib.Path(tutil.get_env_var(CONFIG_FILE_ENV)) config = tutil.parse_config(config_file) except KeyError: msg = "Environment variable %s unset, exiting.".format(CONFIG_FILE_ENV) tutil.exit_with_error(msg) queues = process_queues(config) logger.debug("Queues: %s", queues) tmpl = jinjatmpl(EMAIL_TEMPLATE) logger.debug(tmpl) email = tmpl.render(queues=queues, style=STYLE, ad_hoc=global_args.span) send_email(email) logger.debug("That's all for now, bye.") logging.shutdown()
def main(): global logger logger = tutil.setup_logging("1FPS") if 'PYLOGLEVEL' in os.environ: level = logging.getLevelName(os.getenv('PYLOGLEVEL', 'DEBUG')) logger.setLevel(level) args = parser.parse_args() logger.info('Starting') global config config = parse_config(args.config) global endtm endtm = starttm + timedelta(seconds=config['1fps']['time']) global timeout timeout = int(config['1fps']['interval']) * 2 try: tmppath.mkdir() os.chdir(str(tmppath)) except FileExistsError as e: logger.error('Temp path already exists. Is another process using it?') logger.error(e) logger.info('Exiting because of error') exit(0) global loop loop = task.LoopingCall(get_image) loopDeferred = loop.start(config['1fps']['interval']) loopDeferred.addCallback(images_to_video_to_share) loopDeferred.addErrback(loop_failed) reactor.run() cleanup() logger.info('Finished') logging.shutdown()
def __init__(self, config, proxy_frontend, context=None): global logger logger = tutil.setup_logging("watcher errors") self.config = config self.context = context or zmq.Context().instance() self.socket = self.context.socket(zmq.SUB) self.socket.connect(proxy_frontend)
def __init__(self, config, proxy_backend, context=None): global logger logger = tutil.setup_logging("fetcher errors") self.config = config self.context = context or zmq.Context().instance() self.socket = self.context.socket(zmq.PUB) logger.debug("Connecting to proxy on {}".format(proxy_backend)) self.socket.connect(proxy_backend)
def main(): """Where it all begins.""" global logger logger = tutil.setup_logging() for cam in tutil.get_env_var('CF_CAMS').split(":"): update_cam(cam) logger.debug("That's all for now, bye.") logging.shutdown()
def main(): # let ctrl-c work as it should. signal.signal(signal.SIGINT, signal.SIG_DFL) global logger logger = tutil.setup_logging("webrelaypoker errors") multiprocessing_logging.install_mp_handler() config = tutil.parse_config(tutil.get_env_var(CONFIG_FILE_ENV)) procs = poke_relays(config["relays"]) for proc in procs: proc.join() logger.debug("That's all for now, bye.") logging.shutdown()
def main(): # let ctrl-c work as it should. signal.signal(signal.SIGINT, signal.SIG_DFL) global logger if 'CU_CONTEXT_NAME' in os.environ: context_name = os.environ['CU_CONTEXT_NAME'] subject = "{} config file changed".format(context_name) else: subject = "Config file changed" logger = tutil.setup_logging(subject) my_config = bootstrap_config() for config in my_config['configs']: update_config(config) logger.debug("That's all for now, bye.") logging.shutdown()
def main(): # let ctrl-c work as it should. signal.signal(signal.SIGINT, signal.SIG_DFL) global logger logger = tutil.setup_logging("msg_pub errors") context = zmq.Context() socket = context.socket(zmq.PUB) socket.setsockopt(zmq.TCP_KEEPALIVE, 1) socket.setsockopt(zmq.TCP_KEEPALIVE_IDLE, 60) socket.setsockopt(zmq.TCP_KEEPALIVE_CNT, 20) socket.setsockopt(zmq.TCP_KEEPALIVE_INTVL, 60) socket.bind("tcp://*:29092") logger.debug("Listening for subscribers.") with Subscribe("", "", True) as sub: for msg in sub.recv(): msg_string = msg.encode() logger.debug("sending msg: %s", msg_string) socket.send(bytes(msg_string, "UTF-8"))
def main(): logger = tutil.setup_logging("filefetcher - errors") tmp_dir = tutil.get_env_var("FF_TMP_DIR") for filename in os.listdir(tmp_dir): if filename.endswith(".lock"): with open(os.path.join(tmp_dir, filename)) as file: pid = int(file.read()) try: process = psutil.Process(pid) except psutil.NoSuchProcess: continue create_time = process.create_time() create_time = datetime.fromtimestamp(create_time) process_age = datetime.now() - create_time if process_age > MAX_RUN_TIME: logger.info("Killing process %s, has been running for %s", pid, process_age) process.terminate() print("pid: {} age: {}".format(pid, process_age))
# -*- coding: utf-8 -*- # ----------------------------------------------------------------------------- # Purpose: pull GPS images # Author: Tom Parker # # ----------------------------------------------------------------------------- """ filefetcher ========= Pull daily GPS files :license: CC0 1.0 Universal http://creativecommons.org/publicdomain/zero/1.0/ """ import tomputils.util as tutil from filefetcher.version import __version__ logger = tutil.setup_logging("filefetcher - errors") __all__ = ["__version__"]
# -*- coding: utf-8 -*- # ----------------------------------------------------------------------------- # Purpose: process VIIRS data # Author: Tom Parker # # ----------------------------------------------------------------------------- """ avoviirsprocessor ================= Process VIIRS data at AVO :license: CC0 1.0 Universal http://creativecommons.org/publicdomain/zero/1.0/ """ import tomputils.util as tutil from avoviirsprocessor.version import __version__ logger = tutil.setup_logging("avoviirsprocessor errors") __all__ = ["__version__"]
# -*- coding: utf-8 -*- # ----------------------------------------------------------------------------- # Purpose: fetch viirs data # Author: Tom Parker # # ----------------------------------------------------------------------------- """ avoviirscollector ================= Fetch viirs data at AVO :license: CC0 1.0 Universal http://creativecommons.org/publicdomain/zero/1.0/ """ from avoviirscollector.version import __version__ import tomputils.util as tutil logger = tutil.setup_logging("mirror_gina errors") BASE_DIR = tutil.get_env_var("VIIRS_BASE_DIR", "unset") SATELLITE = tutil.get_env_var("VIIRS_SATELLITE", "unset") CHANNELS = tutil.get_env_var("VIIRS_CHANNELS", "unset").split("|") __all__ = ["__version__"]
# -*- coding: utf-8 -*- # ----------------------------------------------------------------------------- # Purpose: work with webcam images # Author: Tom Parker # # ----------------------------------------------------------------------------- """ camcommander ========= Tools to handle webcam images. :license: CC0 1.0 Universal http://creativecommons.org/publicdomain/zero/1.0/ """ import tomputils.util as tutil from camcommander.version import __version__ logger = tutil.setup_logging("camcommander - errors") __all__ = ["__version__"]
#!/usr/bin/env python import argparse from posttroll.message import Message from avoviirsprocessor.coreprocessors import * # NOQA import threading import zmq import tomputils.util as tutil import collections from datetime import timedelta import time DATA_PUBLISHER = "tcp://viirscollector:29092" logger = tutil.setup_logging("pass_plotter errors") ORBIT_SLACK = timedelta(minutes=30) message_q_lock = threading.Lock() def _arg_parse(): description = "Reprocesses a serialized message in a file." parser = argparse.ArgumentParser(description=description) parser.add_argument("message", help="path to serialized message") return parser.parse_args() class DataSubscriber(threading.Thread): def __init__(self, context, message_q): threading.Thread.__init__(self)
# -*- coding: utf-8 -*- # ----------------------------------------------------------------------------- # Purpose: keep an eye on AQMS # Author: Tom Parker # # ----------------------------------------------------------------------------- """ aqmswatcher ================= Keep an eye on AQMS. :license: CC0 1.0 Universal http://creativecommons.org/publicdomain/zero/1.0/ """ import tomputils.util as tutil logger = tutil.setup_logging("aqmswatcher errors") __version__ = "0.1.11"
if not os.path.exists(archiveloc): os.mkdir(archiveloc) logger.info(f'Copying to {archiveloc}') shutil.copy2(localfile, archiveloc) # Now parse local file, moving from microseconds to milliseconds # Then copy to the lamp directory modfile = f'{localfile[:-4]}mod.dat' convert_times(localfile, modfile) logger.info(f'Copying to {lamp}\n') shutil.copy2(modfile, lamp) os.remove(modfile) os.remove(localfile) except Exception as e: logger.error(e) if __name__ == '__main__': global logger logger = tutil.setup_logging("GetGravity") if 'PYLOGLEVEL' in os.environ: level = logging.getLevelName(os.getenv('PYLOGLEVEL', 'DEBUG')) logger.setLevel(level) logger.info('Starting') args = parser.parse_args() sites = load_config(args.config) datalogger_to_valve_and_archive(sites) logger.info('Finished') logging.shutdown()