def run():
"""
Main function for command-line interface.
"""
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
args = ArgParser().get_args()
if args.model == 'version':
print("MyTardis Client v%s" % VERSION)
sys.exit(0)
if args.verbose and (not hasattr(args, 'json') or not args.json):
print("MyTardis Client v%s" % VERSION)
config_path = DEFAULT_CONFIG_PATH
if not os.path.exists(config_path) or \
args.model == 'config':
ConfigController(config_path).configure(args)
if args.model == 'config':
sys.exit(0)
config = Config(config_path)
config.validate()
logging.config.fileConfig(config.logging_config_path,
disable_existing_loggers=False)
logging.getLogger("dogpile.core.dogpile").setLevel(logging.WARNING)
logging.getLogger("requests").setLevel(logging.WARNING)
logger = logging.getLogger(__name__)
logger.info("MyTardis Client v%s", VERSION)
if args.verbose and (not hasattr(args, 'json') or not args.json):
print("General config: %s" % config_path)
print("Logging config: %s" % config.logging_config_path)
print("MyTardis URL: %s" % config.url)
print("Username: %s" % config.username)
if args.model == 'api':
ApiController().run_command(args)
elif args.model == 'facility':
FacilityController().run_command(args)
elif args.model == 'instrument':
InstrumentController().run_command(args)
elif args.model == 'experiment':
ExperimentController().run_command(args)
elif args.model == 'dataset':
DatasetController().run_command(args)
elif args.model == 'datafile':
DataFileController().run_command(args)
elif args.model == 'storagebox':
StorageBoxController().run_command(args)
elif args.model == 'schema':
SchemaController().run_command(args)
def initialize_configuration():
global config
global configfilename
configfilename = "server.conf"
config = configuration()
print "Reading configuration from " + configfilename + "..."
ret = config.read(configfilename)
config.validate()
return ret
def run():
"""
Main function for command-line interface.
"""
# pylint: disable=too-many-branches
args = ArgParser().get_args()
if args.model == 'version':
print("MyTardis Client v%s" % VERSION)
sys.exit(0)
if args.verbose and (not hasattr(args, 'json') or not args.json):
print("MyTardis Client v%s" % VERSION)
config_path = DEFAULT_CONFIG_PATH
if not os.path.exists(config_path) or \
args.model == 'config':
ConfigController(config_path).configure(args)
if args.model == 'config':
sys.exit(0)
config = Config(config_path)
config.validate()
logging.config.fileConfig(config.logging_config_path,
disable_existing_loggers=False)
logging.getLogger("requests").setLevel(logging.WARNING)
if args.version:
print("MyTardis Client v%s" % VERSION)
if args.verbose and (not hasattr(args, 'json') or not args.json):
print("General config: %s" % config_path)
print("Logging config: %s" % config.logging_config_path)
print("MyTardis URL: %s" % config.url)
print("Username: %s" % config.username)
if args.model == 'api':
ApiController().run_command(args)
elif args.model == 'facility':
FacilityController().run_command(args)
elif args.model == 'instrument':
InstrumentController().run_command(args)
elif args.model == 'experiment':
ExperimentController().run_command(args)
elif args.model == 'dataset':
DatasetController().run_command(args)
elif args.model == 'datafile':
DataFileController().run_command(args)
elif args.model == 'storagebox':
StorageBoxController().run_command(args)
elif args.model == 'schema':
SchemaController().run_command(args)
def load() -> ConfigDict:
# language=rst
""" Load and validate the configuration.
"""
config_path = _config_path()
config = ConfigDict(_load_yaml(config_path))
if 'logging' not in config:
raise ConfigError(
"No 'logging' entry in config file {}".format(config_path))
logging.config.dictConfig(config['logging'])
logger.info("Loaded configuration from '%s'",
os.path.abspath(str(config_path)))
config.validate(_config_schema())
return config
def __init__(self, config_file, error, initial_conf=None):
conf = configobj.ConfigObj(config_file, configspec=configobj.ConfigObj(self.spec), list_values=False, interpolation='Template')
conf.merge(initial_conf or {})
config.validate(config_file, conf, error)
logging.addLevelName(10000, 'NONE')
log_conf = ''
for k, v in conf['logging'].items():
log_conf += '\n[%s]\n' % k
log_conf += '\n'.join(map('='.join, v.items()))
logging.config.fileConfig(StringIO.StringIO(log_conf))
self.manager = Manager(mqtt_conf=conf['mqtt'])
def start(fqdn, dynamo_region, dynamo_host, table_root, log_config, verbosity, environment, server_config):
"""
Starts an APScheduler job to periodically reload HAProxy config as well as run the API to register/deregister
new services, target groups and backends.
:param fqdn: The fully qualified domain name of Flyby - requests coming here will go to the API endpoints
:param dynamo_region: The AWS region of the DynamoDB tables Flyby stores and reads config in
:param dynamo_host: The hostname and port to use for DynamoDB connections. Useful for local testing with
moto or DynamoDB Local.
:param table_root: The root that will be used for table names in DynamboDB. This will be prefixed to all tables
created.
:param log_config: Location of python yaml config file.
:param verbosity: Logging verbosity, defaults to INFO.
:return:
"""
logging.getLogger().setLevel(level=getattr(logging, verbosity))
dynamo_manager = DynamoTableManagement()
config = ConfigObj(infile=server_config, configspec='flyby/configspec.ini', stringify=True)
res = config.validate(Validator(), preserve_errors=True)
if res is not True:
for section, key, msg in flatten_errors(config, res):
click.echo("{}: {} in {}".format(key, msg, section))
raise click.ClickException('bad server config')
# Create the DynamoDB tables if missing, update the DynamoDB read/write capacity if required
dynamo_manager.update_capacity(dynamo_host, dynamo_region, table_root, logger, config)
if log_config:
with open(log_config, 'r') as conf:
logging.config.dictConfig(yaml.load(conf))
scheduler = BackgroundScheduler(timezone=utc)
scheduler.add_job(update, 'interval', seconds=10, args=(fqdn,))
scheduler.start()
if environment == "development":
app.run(host='0.0.0.0')
else:
serve(app, listen='*:5000')
def init_config(configfiles=None):
"""
Initialize the config object with values from the configuration files.
:param configfiles: A list of config files (will default to DEFAULT_CONFIG_FILES module global)
:type configfiles: list
"""
global config
if configfiles is None:
configfiles = DEFAULT_CONFIG_FILES
for fname in configfiles:
if exists(fname):
# This is a huge hack, but basically because I can't pass in a dict of "default"
# values to ConfigObj constructure, I have to modify the actual "file" (well, the
# list of lines from file) to prepend the value for 'root' else interpolation fails.
# -1 for ConfigObj
with open(fname) as fp:
filelines = [line for line in fp]
partial = configobj.ConfigObj(["root=%s\n" % ROOT_DIR] + filelines, encoding='utf-8')
config.merge(partial)
validation = config.validate(Validator(), preserve_errors=True)
if validation != True:
raise ConfigurationError("configuration validation error(s) (): {0!r}".format(configobj.flatten_errors(config, validation)))
def main():
global _logger
parser = ArgumentParser(usage='prog [options]')
parser.add_argument('-c', '--config',
dest='config_file', required=True,
metavar='CONFIG_FILE', help='The full path to the config file to open')
parser.add_argument('-l', '--log-config',
default='logger.conf', dest='log_config_file',
metavar='LOG_CONFIG_FILE', help='The full path to the log config file to open')
parser.add_argument('--start-frame-time', default=-1, type=int, dest='start_frame_pos',
help='Start frame time in seconds')
parser.add_argument('--end-frame-time', default=-1, type=int, dest='end_frame_pos',
help='End frame time in seconds')
parser.add_argument('--smooth-filter-size', default=3, type=int, dest='gaussian_filter_size',
help='Gaussian filter kernel size')
parser.add_argument('--smooth-filter-sigma', default=0, type=int, dest='gaussian_filter_sigma',
help='Gaussian filter sigma')
parser.add_argument('--nthreads', default=1, type=int, dest='nthreads')
parser.add_argument('--nprocesses', default=1, type=int, dest='nprocesses',
help='Number of processes to run in parallel')
args = parser.parse_args()
# setup logger
logging.config.fileConfig(args.log_config_file)
_logger = logging.getLogger('tracker')
if args.config_file is None:
_logger.warning('Missing config file')
parser.exit(1, 'Missing config file\n')
# load config file
config, errors = load_config(args.config_file)
errors |= set(config.validate())
if len(errors) == 0:
if args.nprocesses > 1:
source = MovieFile(config.get_source(),
start_msecs=args.start_frame_pos * 1000,
end_msecs=args.end_frame_pos * 1000,
resolution=config.get_image_size())
if not source.is_opened():
_logger.error('Error opening %s' % config.get_source())
return
start_frame_pos = int(source.get_start_time_in_seconds())
end_frame_pos = int(source.get_end_time_in_seconds())
frame_interval = int((end_frame_pos - start_frame_pos) / args.nprocesses)
tracker_args = [(config, s * 1000, (s + frame_interval) * 1000,
args.gaussian_filter_size, args.gaussian_filter_sigma,
args.nthreads, _get_run_interval(s, s + frame_interval)[1]) for s in
range(start_frame_pos, end_frame_pos, frame_interval)
]
with Pool(args.nprocesses) as p:
p.starmap(_run_tracker, tracker_args)
else:
_run_tracker(config, args.start_frame_pos * 1000, args.end_frame_pos * 1000,
args.gaussian_filter_size, args.gaussian_filter_sigma, args.nthreads)
else:
_logger.error('Config load error: %r' % errors)
def __init__(self, config_file, error, initial_conf=None):
conf = configobj.ConfigObj(config_file,
configspec=configobj.ConfigObj(self.spec),
list_values=False,
interpolation='Template')
conf.merge(initial_conf or {})
config.validate(config_file, conf, error)
logging.addLevelName(10000, 'NONE')
log_conf = ''
for k, v in conf['logging'].items():
log_conf += '\n[%s]\n' % k
log_conf += '\n'.join(map('='.join, v.items()))
logging.config.fileConfig(StringIO.StringIO(log_conf))
self.manager = Manager(mqtt_conf=conf['mqtt'])
def load_instance_config(instance_name):
def b(unicode):
"""
convert unicode to bytestring
"""
return unicode.encode('utf-8')
confspec = []
confspec.append(b(u'[instance]'))
confspec.append(b(u'source-directory = string()'))
confspec.append(b(u'backup-directory = string()'))
confspec.append(b(u'target-file = string()'))
confspec.append(b(u'exchange = string(default="navitia")'))
confspec.append(b(u'synonyms_file = string(default="")'))
confspec.append(b(u'aliases_file = string(default="")'))
confspec.append(b(u'name = string()'))
confspec.append(b(u'is-free = boolean(default=False)'))
confspec.append(b(u'[database]'))
confspec.append(b(u'host = string()'))
confspec.append(b(u'dbname = string()'))
confspec.append(b(u'username = string()'))
confspec.append(b(u'password = string()'))
confspec.append(b(u'port = string(default="5432")'))
ini_file = os.path.join(
os.path.realpath(current_app.config['INSTANCES_DIR']),
instance_name + ".ini")
if not os.path.isfile(ini_file):
raise ValueError("File doesn't exists or is not a file %s" % ini_file)
config = ConfigObj(ini_file, configspec=confspec, stringify=True)
val = Validator()
res = config.validate(val, preserve_errors=True)
# validate retourne true, ou un dictionaire ...
if type(res) is dict:
error = build_error(config, res)
raise ValueError("Config is not valid: %s in %s" % (error, ini_file))
instance = InstanceConfig()
instance.source_directory = config['instance']['source-directory']
instance.backup_directory = config['instance']['backup-directory']
instance.target_file = config['instance']['target-file']
instance.exchange = config['instance']['exchange']
instance.synonyms_file = config['instance']['synonyms_file']
instance.aliases_file = config['instance']['aliases_file']
instance.name = config['instance']['name']
instance.is_free = config['instance']['is-free']
instance.pg_host = config['database']['host']
instance.pg_dbname = config['database']['dbname']
instance.pg_username = config['database']['username']
instance.pg_password = config['database']['password']
instance.pg_port = config['database']['port']
return instance
def read_config(resources, path):
""" build and validate a ConfigObj using config file found at given filesystem path """
_locate_config_files(path)
# grab contents of config file
with open(os.path.join(path, "mediarover.conf"), "r") as f:
file = f.readlines()
version_check = re.compile("__version__")
# TODO the next time the user is required to regenerate the config file this code will no longer
# be needed. Between version 1 and version 2, __version__ was moved into the __SYSTEM__ subsection
version = 0
if version_check.search(file[0]):
value = file[0]
elif version_check.search(file[-1]):
value = file[-1]
if value:
(left, sep, right) = value.partition("=")
version = right.strip(" \n")
# check if users config file is current
if version > 0:
if int(version) < int(__config_version__.get('min', __config_version__['version'])):
raise ConfigurationError("Configuration file is out of date and needs to be regenerated! See `python mediarover.py write-configs --help` for instructions")
else:
raise ConfigurationError("Out of date or corrupt configuration file! See `python mediarover.py write-configs --help` for instructions")
spec = os.path.join(resources, "config.spec")
config = ConfigObj(file, configspec=spec)
# validate config options
results = config.validate(_get_validator(), preserve_errors=True)
if results != True:
results = flatten_errors(config, results)
message = ["ERROR: Encountered the following configuration error(s):"]
for error in results:
level = 1
section = []
dict = config
for key in error[0]:
section.append(("[" * level) + key + ("]" * level))
dict = dict[key]
level += 1
message.append(" %s %s = %s" % (" ".join(section), error[1], error[2]))
raise ConfigurationError("Invalid Data in configuration file\n\n%s\n" % "\n".join(message))
return config
def load_instance_config(instance_name):
confspec = []
confspec.append('[instance]')
confspec.append('source-directory = string()')
confspec.append('backup-directory = string()')
confspec.append('tmp-file = string()')
confspec.append('target-file = string()')
confspec.append('rt-topics = string_list()')
confspec.append('exchange = string(default="navitia")')
confspec.append('synonyms_file = string(default="")')
confspec.append('aliases_file = string(default="")')
confspec.append('name = string()')
confspec.append('is-free = boolean(default=False)')
confspec.append('[database]')
confspec.append('host = string()')
confspec.append('dbname = string()')
confspec.append('username = string()')
confspec.append('password = string()')
ini_file = '%s/%s.ini' % \
(current_app.config['INSTANCES_DIR'], instance_name)
if not os.path.isfile(ini_file):
raise ValueError("File doesn't exists or is not a file %s" % ini_file)
config = ConfigObj(ini_file, configspec=confspec, stringify=True)
val = Validator()
res = config.validate(val, preserve_errors=True)
#validate retourne true, ou un dictionaire ...
if type(res) is dict:
error = build_error(config, res)
raise ValueError("Config is not valid: %s in %s" % (error, ini_file))
instance = InstanceConfig()
instance.source_directory = config['instance']['source-directory']
instance.backup_directory = config['instance']['backup-directory']
instance.tmp_file = config['instance']['tmp-file']
instance.target_file = config['instance']['target-file']
instance.rt_topics = config['instance']['rt-topics']
instance.exchange = config['instance']['exchange']
instance.synonyms_file = config['instance']['synonyms_file']
instance.aliases_file = config['instance']['aliases_file']
instance.name = config['instance']['name']
instance.is_free = config['instance']['is-free']
instance.pg_host = config['database']['host']
instance.pg_dbname = config['database']['dbname']
instance.pg_username = config['database']['username']
instance.pg_password = config['database']['password']
return instance
def _validate_config(config, validator, filename):
""" validate the given config object using the given validator """
results = config.validate(validator, preserve_errors=True)
if results != True:
results = flatten_errors(config, results)
message = ["ERROR: Encountered the following error(s) in %s:" % filename]
for error in results:
level = 1
section = []
dict = config
for key in error[0]:
section.append(("[" * level) + key + ("]" * level))
dict = dict[key]
level += 1
message.append(" %s %s = %s" % (" ".join(section), error[1], error[2]))
raise ConfigurationError("Invalid Data in configuration file\n\n%s\n" % "\n".join(message))
def LoadIni(FilePath,ConfigSpecPath=None):
config = ConfigObj(FilePath,configspec=ConfigSpecPath)
validator = Validator()
if ConfigSpecPath!=None:
result = config.validate(validator)
return config
"handlers": ["file_scheduler", "console"],
"level": logging.INFO,
},
"debug": {"handlers": ["file_debug"], "level": logging.DEBUG},
"atelier": {"handlers": ["file", "console"], "level": logging.DEBUG},
"server": {"handlers": ["file_server"], "level": logging.INFO},
},
)
)
logger = logging.getLogger("atelier")
server_logger = logging.getLogger("server")
try:
config.validate()
except Exception as e:
server_logger.error("Configuration invalide.", exc_info=e)
raise e
from server import alerts, arduino, db, scheduler, web
def remove_logging_handler(logger, handler_name):
for handler in logger.handlers:
if handler.name == handler_name:
logger.handlers.remove(handler)
return
