def validate(config_file):
"""
Validate a YAML config file.
Parameters
----------
config_file : str
Local path to config file.
Returns
-------
Bool
True if file contains valid YAML
"""
if config_file.endswith('yaml'):
try:
yaml = ruamel.yaml.YAML()
yaml.load(config_file)
except ruamel.yaml.YAMLError as e1:
logger.error("Cannot parse YAML config file")
tutil.exit_with_error(e1)
else:
return True
def main():
global logger
logger = tutil.setup_logging("filefetcher errors")
global global_args
global_args = arg_parse()
msg = (
"Python interpreter is too old. I need at least {} "
+ "for EmailMessage.iter_attachments() support."
)
tutil.enforce_version(REQ_VERSION, msg.format(REQ_VERSION))
try:
config_file = pathlib.Path(tutil.get_env_var(CONFIG_FILE_ENV))
config = tutil.parse_config(config_file)
except KeyError:
msg = "Environment variable %s unset, exiting.".format(CONFIG_FILE_ENV)
tutil.exit_with_error(msg)
queues = process_queues(config)
logger.debug("Queues: %s", queues)
tmpl = jinjatmpl(EMAIL_TEMPLATE)
logger.debug(tmpl)
email = tmpl.render(queues=queues, style=STYLE, ad_hoc=global_args.span)
send_email(email)
logger.debug("That's all for now, bye.")
logging.shutdown()
def main():
# let ctrl-c work as it should.
signal.signal(signal.SIGINT, signal.SIG_DFL)
global logger
logger = tutil.setup_logging("filefetcher errors")
multiprocessing_logging.install_mp_handler()
msg = ("Python interpreter is too old. I need at least {} " +
"for EmailMessage.iter_attachments() support.")
tutil.enforce_version(REQ_VERSION, msg.format(REQ_VERSION))
global args
args = _arg_parse()
try:
global global_config
global_config = parse_config()
except KeyError:
msg = "Environment variable %s unset, exiting.".format(CONFIG_FILE_ENV)
tutil.exit_with_error(msg)
procs = poll_queues()
for proc in procs:
proc.join()
logger.debug("That's all for now, bye.")
logging.shutdown()
def update_remotefile(config):
"""
Update a config file stored at a URL.
Parameters
----------
config : dict
This files config stanza.
"""
config_str = None
try:
r = requests.get(config['source'],
auth=(config['user'], config['passwd']))
r.raise_for_status()
config_str = r.text
except requests.exceptions.RequestException as e:
logger.error("Cannot retrieve config file from %s", config['url'])
tutil.exit_with_error(e)
tmp_config = '/tmp/config.tmp'
with open(tmp_config, 'w') as f:
f.write(config_str)
config['type'] = 'localfile'
config['source'] = tmp_config
update_localfile(config)
os.remove(tmp_config)
def parse_config(config_path):
"""
Parse my YAML config file.
Parameters
----------
config_path : str
Path to my YAML config file.
Returns
-------
Dict
My configuration.
"""
logger.debug("Parsing config %s", config_path)
config_file = pathlib.Path(config_path)
yaml = ruamel.yaml.YAML()
config = None
try:
config = yaml.load(config_file)
except ruamel.yaml.parser.ParserError as e1:
logger.error("Cannot parse config file")
tutil.exit_with_error(e1)
except OSError as e:
if e.errno == errno.EEXIST:
logger.error("Cannot read config file %s", config_file)
tutil.exit_with_error(e)
else:
raise
return config
def main():
# let ctrl-c work as it should.
signal.signal(signal.SIGINT, signal.SIG_DFL)
# exit quickly if queue is already running
(gotlock, lock) = aquire_lock()
if not gotlock:
tutil.exit_with_error(
"Queue {} locked, skipping".format(SATELLITE + "-".join(CHANNELS)))
return
try:
mirror_gina = MirrorGina()
mirror_gina.fetch_files()
finally:
logger.info("All done with queue.")
if gotlock:
try:
lock.unlock()
except AttributeError:
pass
logger.debug("That's all for now, bye.")
logging.shutdown()
def main():
"""Where it all begins."""
global logger
logger = tutil.setup_logging("camfetchers errors")
check_version()
if 'CF_TIMEOUT' in os.environ:
timeout = float(os.environ['CF_TIMEOUT'])
logger.debug("Setting timeout to %.2f" % timeout)
socket.setdefaulttimeout(timeout)
with IMAP4_SSL(tutil.get_env_var('IMAPSERVER')) as M:
try:
M.login(tutil.get_env_var('CF_USER'),
tutil.get_env_var('CF_PASSWD'))
except IMAP4.error:
tutil.exit_with_error("Login failed.")
for cam in tutil.get_env_var('CF_CAMS').split(':'):
rv, data = M.select(cam)
if rv == 'OK':
logger.debug("Processing mailbox %s", cam)
process_mailbox(M, cam)
else:
msg = "Received non-OK response opening mailbox %s, " \
+ "lets skip this one. (%s)"
logger.error(msg.format(cam, rv))
logger.debug("That's all for now, bye.")
logging.shutdown()
def watcher_factory(config, proxy_frontend):
if config["type"] == "console":
msg = "Creating %s watcher %s."
logger.debug(msg.format(config["name"], config["type"]))
return ConsoleWatcher(config, proxy_frontend)
else:
error_msg = "Unkown watcher type %s for source %s"
tutil.exit_with_error(error_msg.format(config["type"], config["name"]))
def parse_config():
config_file = pathlib.Path(tutil.get_env_var(CONFIG_FILE_ENV))
yaml = ruamel.yaml.YAML()
try:
global_config = yaml.load(config_file)
except ruamel.yaml.parser.ParserError as e1:
logger.error("Cannot parse config file")
tutil.exit_with_error(e1)
except OSError as e:
if e.errno == errno.EEXIST:
logger.error("Cannot read config file %s", config_file)
tutil.exit_with_error(e)
else:
raise
return global_config
def __init__(self):
self.tmp_path = os.path.join(BASE_DIR, "tmp")
self.connection_count = int(tutil.get_env_var("NUM_GINA_CONNECTIONS"))
self.file_store_type = tutil.get_env_var("VIIRS_FILE_STORE_TYPE")
if self.file_store_type == "S3":
self.file_store = avoviirscollector.viirs_s3_store
elif self.file_store_type == "local":
self.file_store = avoviirscollector.viirs_filesystem_store
else:
tutil.exit_with_error(
"Unknown VIIRS_FILE_STORE_TYPE env var: {}".format(
self.file_store_type))
# We should ignore SIGPIPE when using pycurl.NOSIGNAL - see
# the libcurl tutorial for more info.
try:
signal.signal(signal.SIGPIPE, signal.SIG_IGN)
except ImportError:
pass
self.hostname = socket.gethostname()
def check_version():
if sys.version_info < REQ_VERSION:
msg = "Python interpreter is too old. I need at least 3.5 " \
+ "for EmailMessage.iter_attachments() support."
tutil.exit_with_error(msg)
def fetcher_factory(config, proxy_backend):
if config["type"] == "rsync":
return RsyncFetcher(config, proxy_backend)
else:
error_msg = "Unkown fetcher type {} for source {}"
tutil.exit_with_error(error_msg.format(config["type"], config["name"]))