def main():
"""
Call this a a test unit or use as CLI of this module
"""
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--configtemplate',
action='store_true',
help='prints the expected module configuration')
parser.add_argument(
'--configinfo',
action='store_true',
help='prints config template along with produces and consumes info')
args = parser.parse_args()
if args.configtemplate:
module_config_template()
elif args.configinfo:
module_config_info()
else:
config_manager = configmanager.ConfigManager()
config_manager.load()
global_config = config_manager.get_global_config()
print("GLOBAL CONF", global_config)
ds = dataspace.DataSpace(global_config)
def main(args=None):
'''If you pass a list of args, they will be used instead of sys.argv'''
program_options = parse_program_options(args)
conf_manager = Conf_Manager.ConfigManager(program_options)
try:
conf_manager.load()
except Exception as msg:
sys.exit("Failed to load configuration {}\n{}".format(
conf_manager.config_dir, msg))
channels = conf_manager.get_channels()
channels_required = not os.getenv('DECISIONENGINE_NO_CHANNELS')
if channels_required and not channels:
sys.exit("No channel configurations available in {}".format(
conf_manager.config_dir))
global_config = conf_manager.get_global_config()
server_address = tuple(global_config.get('server_address'))
try:
server = DecisionEngine(conf_manager, server_address, RequestHandler)
server.reaper_start(delay=global_config['dataspace'].get(
'reaper_start_delay_seconds', 1818))
if channels_required:
server.start_channels()
server.serve_forever()
except Exception as msg:
sys.exit("Server Address: {}\n".format(server_address) +
"Config Dir: {}\n".format(conf_manager.config_dir) +
"Fatal Error: {}\n".format(msg))
def _call(filename,
relative_channel_config_dir=None,
program_options=None):
monkeypatch.setenv('CONFIG_PATH', os.path.join(_this_dir, 'de'))
if relative_channel_config_dir is None:
monkeypatch.setenv('CHANNEL_CONFIG_PATH',
os.path.join(_this_dir, 'channels/no_config_files'))
else:
monkeypatch.setenv('CHANNEL_CONFIG_PATH',
os.path.join(_this_dir, relative_channel_config_dir))
manager = ConfigManager.ConfigManager(program_options)
manager.load(filename)
return manager.get_global_config()
def __init__(self, *args, **kwargs):
if not set(must_have).issubset(set(args[0].keys())):
raise RuntimeError(
'SourceProxy misconfigured. Must have %s defined' %
(must_have, ))
self.source_channel = args[0]['channel_name']
self.data_keys = args[0]['Dataproducts']
self.retries = args[0].get('retries', RETRIES)
self.retry_to = args[0].get('retry_timeout', RETRY_TO)
self.logger = de_logger.get_logger()
config_manager = configmanager.ConfigManager()
config_manager.load()
global_config = config_manager.get_global_config()
self.dataspace = dataspace.DataSpace(global_config)
def main():
"""
Call this a a test unit or use as CLI of this module
"""
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--configtemplate',
action='store_true',
help='prints the expected module configuration')
parser.add_argument(
'--configinfo',
action='store_true',
help='prints config template along with produces and consumes info')
args = parser.parse_args()
if args.configtemplate:
module_config_template()
elif args.configinfo:
module_config_info()
else:
config_manager = configmanager.ConfigManager()
config_manager.load()
global_config = config_manager.get_global_config()
print("GLOBAL CONF", global_config)
ds = dataspace.DataSpace(global_config)
data_block = datablock.DataBlock(
ds,
# '5CC840DD-88B9-45CE-9DA2-FF531289AC66',
'C56E0AAF-99D3-42A8-88A3-921E30C1879C',
1)
fm_info = AWSFOMPublisher({
"publish_to_graphite":
True,
"graphite_host":
"fifemondata.fnal.gov",
"graphite_port":
2104,
"graphite_context":
"hepcloud.aws",
"output_file":
"%s/de_data/AWS_figure_of_merit.csv" % (os.environ.get('HOME'), )
})
rc = fm_info.publish(data_block)
def main():
"""
Call this a a test unit or use as CLI of this module
"""
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--configtemplate',
action='store_true',
help='prints the expected module configuration')
parser.add_argument(
'--configinfo',
action='store_true',
help='prints config template along with produces and consumes info')
args = parser.parse_args()
if args.configtemplate:
module_config_template()
elif args.configinfo:
module_config_info()
else:
config_manager = configmanager.ConfigManager()
config_manager.load()
global_config = config_manager.get_global_config()
print "GLOBAL CONF", global_config
ds = dataspace.DataSpace(global_config)
#data_block = datablock.DataBlock(ds,
# '6D596F43-B4DB-4418-812A-79869001E72B',
# 1)
data_block = datablock.DataBlock(
ds, "AWS_Calculations_with_source_proxy",
"F70B4110-E66D-49CA-9333-4A983A679F37", 1, 109)
fm_info = FigureOfMerit()
rc = fm_info.transform(data_block)
print "INFO"
print rc
#!/usr/bin/env python3
"""
A stand-alone script purges data in database older than specified
in configuration. Configuration file has to have this bit added:
{
"dataspace" : { "retention_interval_in_days" : 365,
"datasource" : { ... }
}
}
Can be used in a cron job.
"""
import os
import pwd
import sys
import decisionengine.framework.configmanager.ConfigManager as config
import decisionengine.framework.dataspace.dataspace as dataspace
if __name__ == "__main__":
ALLOWED_USERS = ("root", "decisionengine")
username = pwd.getpwuid(os.getuid()).pw_name
if username not in ALLOWED_USERS:
sys.stderr.write(
"User '{}' is not allowed to run this script.\n".format(username))
sys.exit(1)
conf_manager = config.ConfigManager()
conf_manager.load()
reaper = dataspace.Reaper(conf_manager.get_global_config())
reaper.reap()
def handle_sighup(self, signum, frame):
self.stop_channels()
self.reload_config()
self.start_channels()
def rpc_reload_config(self):
self.reload_config()
return "OK"
def reload_config(self):
self.config_manager.reload()
if __name__ == '__main__':
try:
conf_manager = Conf_Manager.ConfigManager()
conf_manager.load()
channels = conf_manager.get_channels()
if not channels:
raise RuntimeError("No channels configured")
global_config = conf_manager.get_global_config()
server_address = global_config.get("server_address",
("localhost", 8888))
server = DecisionEngine(conf_manager, server_address, RequestHandler)
server.start_channels()
server.serve_forever()
except Exception as msg:
if not data_block:
return
for key, action_list in actions.items():
for action in action_list:
logging.getLogger().info('run publisher %s',
self.channel.publishers[action].name)
logging.getLogger().debug('run publisher %s %s',
self.channel.publishers[action].name,
data_block)
self.channel.publishers[action].worker.publish(data_block)
if __name__ == '__main__':
import os
config_manager = configmanager.ConfigManager()
config_manager.load()
global_config = config_manager.get_global_config()
print('GLOBAL CONF', global_config)
my_logger = logging.getLogger('decision_engine')
my_logger.info('Starting decision engine')
if len(sys.argv) > 1:
channel_name = sys.argv[1]
channel_conf = os.path.join(config_manager.channel_config_dir,
channel_name)
with open(os.path.abspath(channel_conf), 'r') as f:
channels = {}
channel_name = channel_name.split('.')[0]
code = 'channels[channel_name]=' + ''.join(f.readlines())