def __init__(self):
# check if it is already initialized
if self.instance != None:
self.config = self.instance.config
self.conf_dict = self.instance.conf_dict
self.args = self.instance.args
self.path = self.instance.path
self.snakefile = self.instance.snakefile
self.snakeroot = self.instance.snakeroot
return
# we dont need the first argument aka call to snakemake
self.sysargs = sys.argv[1:]
parser = get_argument_parser()
self.args = parser.parse_args(self.sysargs)
self.path = self.args.configfile
self.snakefile = self.args.snakefile
self.config = parse_config(self.args)
if self.path is None:
for p in ["wbuild.yaml", "config.yaml", "wBuild.yaml"]:
if os.path.exists(p):
self.path = p
break
else:
if type(self.path) is list:
self.path = self.path[0]
self.path = os.path.abspath(self.path)
# this is taken from the snakemake main file
if self.snakefile is None:
for p in SNAKEFILE_CHOICES:
if os.path.exists(p):
self.snakefile = p
break
self.snakeroot = os.path.dirname(self.snakefile)
#load defaults
self.loadDefaultConfiguration()
try:
fh = open(self.path, "r")
except IOError:
raise IOError("Can not read config. Are you sure you have enough "
"rights and config path (wbuild.yaml) is right?")
configDict = next(yaml.safe_load_all(fh))
if configDict == None:
logger.error(
"Error parsing wbuild.yaml - format is wrong. Working with defaults..."
)
else:
self.conf_dict = merge_two_dicts(self.conf_dict, configDict)
#fill Singleton
Config.instance = self
def parser_create():
"""Creates the arguments parser
:return: command line arguments
"""
parser = argparse.ArgumentParser()
parser.add_argument("-c",
"--config-file",
type=str,
help="yaml configuration file name",
required=True)
return parser.parse_args()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('source_dir', help='path of YAML data directory')
parser.add_argument('target_path', help='name of generated CSV file')
parser.add_argument('-v',
'--verbose',
action='store_true',
default=False,
help='increase output verbosity')
global args
args = parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.WARNING,
stream=sys.stdout)
paths = set()
rows = []
for source_data_path, source_data in iter_yaml_files(args.source_dir):
flat = {}
flatten(paths, flat, (), source_data)
rows.append(flat)
labels = []
paths = sorted(paths)
for path in paths:
label_fragments = []
for path_fragment in path:
if isinstance(path_fragment, str):
if label_fragments:
label_fragments.append('.')
label_fragments.append(path_fragment)
else:
assert isinstance(path_fragment, int), path_fragment
label_fragments.append('[{}]'.format(path_fragment))
labels.append(''.join(label_fragments))
with open(args.target_path, 'w') as target_file:
csv_writer = csv.writer(target_file)
csv_writer.writerow(labels)
for row in rows:
csv_writer.writerow(
[row[path] if path in row else '' for path in paths])
return 0
def parse_arguments():
"""Parse command line arguments
Args:
Returns:
Command line arguments
Raises:
"""
parser = argparse.ArgumentParser(prog='pypgmon', )
parser.add_argument(
'-l',
'--logger',
default='pypgmon/conf/logger.yml',
)
parser.add_argument(
'-t',
'--targets',
default='pypgmon/conf/targets.yml',
)
arguments = parser.parse_args()
return arguments
def main():
parser = argparse.ArgumentParser()
parser.add_argument('source_dir', help='path of source data directory')
parser.add_argument('target_dir', help='path of target directory for generated YAML files')
parser.add_argument('-v', '--verbose', action='store_true', default=False, help='increase output verbosity')
global args
args = parser.parse_args()
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARNING, stream=sys.stdout)
assert os.path.exists(args.source_dir)
if os.path.exists(args.target_dir):
for filename in os.listdir(args.target_dir):
if filename.startswith('.'):
continue
path = os.path.join(args.target_dir, filename)
if os.path.isdir(path):
shutil.rmtree(path)
else:
os.makedirs(args.target_dir)
# ACTORS
entity_type = 'actors'
source_entity_type_dir = os.path.join(args.source_dir, entity_type)
target_entity_type_dir = os.path.join(args.target_dir, entity_type)
if not os.path.exists(target_entity_type_dir):
os.makedirs(target_entity_type_dir)
for yaml_file_path, entry in iter_yaml_files(source_entity_type_dir):
yaml_file_relative_path = os.path.relpath(yaml_file_path, source_entity_type_dir)
canonical = collections.OrderedDict()
# name
# Lowercase version of this name is the unique name of the actor and the slugified version of this unique name
# is the file name for the actor.
for path in (
'civic-graph.name',
):
value = get_path(entry, path)
if value is not None:
value = value.strip()
if value:
canonical['name'] = dict(
source = get_path_source(path),
value = value,
)
break
# longDescription
# Description of the actor, for each supported language, in a map indexed by the two letter (ISO-639-1)
# language code
for language, paths in dict(
en = (
'civic-graph.description',
),
es = (
),
fr = (
),
).items():
for path in paths:
value = get_path(entry, path)
if value is not None:
value = value.strip()
if value:
canonical.setdefault('longDescription', {})[language] = dict(
source = get_path_source(path),
value = value,
)
break
# tags
sources_by_value_by_language = {}
for path, extractor in (
('civic-graph.categories', extract_from_name_id_list),
('civic-graph.type', functools.partial(extract_from_value, 'en')),
):
source = get_path_source(path)
value = get_path(entry, path)
for language, item in extractor(value):
assert isinstance(item, str), (path, language, item, value)
if language is not None and item is not None:
item = item.strip()
if item:
sources_by_value_by_language.setdefault(language, {}).setdefault(item, set()).add(source)
if sources_by_value_by_language:
for language, sources_by_value in sources_by_value_by_language.items():
canonical.setdefault('tags', {})[language] = [
dict(
sources = sorted(sources),
value = value,
)
for value, sources in sorted(sources_by_value.items())
]
# website
for path in (
'civic-graph.url',
):
value = get_path(entry, path)
if value is not None:
value = value.strip()
if value:
canonical['website'] = dict(
source = get_path_source(path),
value = value,
)
break
if canonical:
entry['canonical'] = canonical
with open(os.path.join(target_entity_type_dir, yaml_file_relative_path), 'w') as yaml_file:
yaml.dump(entry, yaml_file, allow_unicode=True, default_flow_style=False, indent=2, width=120)
# PROJECTS
entity_type = 'projects'
source_entity_type_dir = os.path.join(args.source_dir, entity_type)
target_entity_type_dir = os.path.join(args.target_dir, entity_type)
if not os.path.exists(target_entity_type_dir):
os.makedirs(target_entity_type_dir)
for yaml_file_path, entry in iter_yaml_files(source_entity_type_dir):
yaml_file_relative_path = os.path.relpath(yaml_file_path, source_entity_type_dir)
canonical = collections.OrderedDict()
# name
# Lowercase version of this name is the unique name of the project and the slugified version of this unique name
# is the file name for the project.
for path in (
'participatedb.Name',
):
value = get_path(entry, path)
if value is not None:
value = value.strip()
if value:
canonical['name'] = dict(
source = get_path_source(path),
value = value,
)
break
# longDescription
# Description of the project, for each supported language, in a map indexed by the two letter (ISO-639-1)
# language code
for language, paths in dict(
en = (
'participatedb.Description',
),
es = (
),
fr = (
),
).items():
for path in paths:
value = get_path(entry, path)
if value is not None:
value = value.strip()
if value:
canonical.setdefault('longDescription', {})[language] = dict(
source = get_path_source(path),
value = value,
)
break
# tags
sources_by_value_by_language = {}
for path, extractor in (
('participatedb.Category', functools.partial(extract_from_singletion_or_list, 'en')),
('participatedb.category', functools.partial(extract_from_value, 'en')),
):
source = get_path_source(path)
value = get_path(entry, path)
for language, item in extractor(value):
assert isinstance(item, str), (path, language, item, value)
if language is not None and item is not None:
item = item.strip()
if item:
sources_by_value_by_language.setdefault(language, {}).setdefault(item, set()).add(source)
if sources_by_value_by_language:
for language, sources_by_value in sources_by_value_by_language.items():
canonical.setdefault('tags', {})[language] = [
dict(
sources = sorted(sources),
value = value,
)
for value, sources in sorted(sources_by_value.items())
]
# tools
sources_by_value = {}
for path, extractor in (
('participatedb.Tools used', functools.partial(extract_from_list, None)),
):
source = get_path_source(path)
value = get_path(entry, path)
for language, item in extractor(value):
assert isinstance(item, str), (path, language, item, value)
if language in (None, 'en') and item is not None:
item = item.strip()
if item:
sources_by_value.setdefault(item, set()).add(source)
if sources_by_value:
canonical['tools'] = [
dict(
sources = sorted(sources),
value = value,
)
for value, sources in sorted(sources_by_value.items())
]
# website
for path in (
'participatedb.Web',
):
value = get_path(entry, path)
if value is not None:
value = value.strip()
if value:
canonical['website'] = dict(
source = get_path_source(path),
value = value,
)
break
if canonical:
entry['canonical'] = canonical
with open(os.path.join(target_entity_type_dir, yaml_file_relative_path), 'w') as yaml_file:
yaml.dump(entry, yaml_file, allow_unicode=True, default_flow_style=False, indent=2, width=120)
# TOOLS
entity_type = 'tools'
source_entity_type_dir = os.path.join(args.source_dir, entity_type)
target_entity_type_dir = os.path.join(args.target_dir, entity_type)
if not os.path.exists(target_entity_type_dir):
os.makedirs(target_entity_type_dir)
for yaml_file_path, entry in iter_yaml_files(source_entity_type_dir):
yaml_file_relative_path = os.path.relpath(yaml_file_path, source_entity_type_dir)
canonical = collections.OrderedDict()
# bugTracker
# URL of the service where bugs related to the tool can be reported
for path in (
'wikidata.bug_tracking_system.0.value',
'ogptoolbox-framacalc.URL suivi de bogues',
):
value = get_path(entry, path)
if value is not None:
value = value.strip()
if value:
canonical['bugTracker'] = dict(
source = get_path_source(path),
value = value,
)
break
# license
# Name of the license governing the tool.
for path in (
'wikidata.license_label.0.value',
'civicstack.license.name.en',
'nuit-debout.Nom de la licence',
'ogptoolbox-framacalc.Licence',
):
value = get_path(entry, path)
if value is not None:
value = value.strip()
if value:
canonical['license'] = dict(
source = get_path_source(path),
value = value,
)
break
# name
# Lowercase version of this name is the unique name of the tool and the slugified version of this unique name is
# the file name for the tool.
for path in (
'debian_appstream.Name.C',
'wikidata.label.0.value',
'civic-tech-field-guide.name',
'civicstack.name',
'tech-plateforms.Name',
'nuit-debout.Outil',
'participatedb.Name',
'harnessing-collaborative-technologies.title',
'ogptoolbox-framacalc.Nom',
):
value = get_path(entry, path)
if value is not None:
value = value.strip()
if value:
canonical['name'] = dict(
source = get_path_source(path),
value = value,
)
break
# longDescription
# Description of the tool, for each supported language, in a map indexed by the two letter (ISO-639-1) language
# code
for path, extractor in (
('wikidata.description', extract_from_wikidata),
('debian.description.en.long_description', functools.partial(extract_from_value, 'en')),
('debian.description.es.long_description', functools.partial(extract_from_value, 'es')),
('debian.description.fr.long_description', functools.partial(extract_from_value, 'fr')),
('civicstack.description', extract_from_value_by_language),
('tech-plateforms.About', functools.partial(extract_from_value, 'en')),
('participatedb.Description', functools.partial(extract_from_value, 'en')),
('harnessing-collaborative-technologies.description', functools.partial(extract_from_value, 'en')),
('nuit-debout.Détails', functools.partial(extract_from_value, 'fr')),
('ogptoolbox-framacalc.Description', functools.partial(extract_from_value, 'fr')),
):
source = get_path_source(path)
value = get_path(entry, path)
for language, item in extractor(value):
assert isinstance(item, str), (path, language, item, value)
if language is not None and item is not None:
item = item.strip()
if item:
canonical_value_by_language = canonical.setdefault('longDescription', {})
if language not in canonical_value_by_language:
canonical_value_by_language[language] = dict(
source = source,
value = item,
)
# programmingLanguages
sources_by_value = {}
for path, extractor in (
('civicstack.technology', extract_from_name_id_list),
):
source = get_path_source(path)
value = get_path(entry, path)
for language, item in extractor(value):
assert isinstance(item, str), (path, language, item, value)
if language in (None, 'en') and item is not None:
item = item.strip()
if item:
sources_by_value.setdefault(item, set()).add(source)
if sources_by_value:
canonical['programmingLanguages'] = [
dict(
sources = sorted(sources),
value = value,
)
for value, sources in sorted(sources_by_value.items())
]
# screenshot
# The URL of a screenshot displaying the tool user interface
for path in (
'debian.screenshot.large_image_url',
'wikidata.image.0.value',
"ogptoolbox-framacalc.Capture d'écran",
'harnessing-collaborative-technologies.logo_url',
):
value = get_path(entry, path)
if value is not None:
value = value.strip()
if value:
canonical['screenshot'] = dict(
source = get_path_source(path),
value = value,
)
break
# sourceCode
# URL from which the source code of the tool can be obtained.
for path in (
'wikidata.source_code_repository.0.value',
'civicstack.github',
'nuit-debout.Lien vers le code',
'ogptoolbox-framacalc.URL code source',
):
value = get_path(entry, path)
if value is not None:
value = value.strip()
if value:
canonical['sourceCode'] = dict(
source = get_path_source(path),
value = value,
)
break
# stackexchangeTag:
# Tag from http://stackexchange.org/ uniquely associated with the tool.
for path in (
'wikidata.stack_exchange_tag.0.value',
'ogptoolbox-framacalc.Tag stack exchange',
):
value = get_path(entry, path)
if value is not None:
value = value.strip()
if value:
canonical['stackexchangeTag'] = dict(
source = get_path_source(path),
value = value,
)
break
# tags
sources_by_value_by_language = {}
for path, extractor in (
('civic-tech-field-guide.category', functools.partial(extract_from_value, 'en')),
# ('civicstack.category', extract_from_name_id),
('civicstack.tags', extract_from_name_id_list),
('debian_appstream.Categories', functools.partial(extract_from_list, 'en')),
('harnessing-collaborative-technologies.category', functools.partial(extract_from_value, 'en')),
('nuit-debout.Fonction', functools.partial(extract_from_value, 'fr')),
('ogptoolbox-framacalc.Catégorie', functools.partial(extract_from_value, 'fr')),
('participatedb.Category', functools.partial(extract_from_singletion_or_list, 'en')),
('participatedb.category', functools.partial(extract_from_value, 'en')),
('tech-plateforms.CivicTech or GeneralPurpose', functools.partial(extract_from_value, 'en')),
('tech-plateforms.Functions', functools.partial(extract_from_value, 'en')),
('tech-plateforms.AppCivist Service 1', functools.partial(extract_from_value, 'en')),
('tech-plateforms.AppCivist Service 2', functools.partial(extract_from_value, 'en')),
('tech-plateforms.AppCivist Service 3', functools.partial(extract_from_value, 'en')),
('wikidata.genre_label', extract_from_wikidata),
('wikidata.instance_of_label', extract_from_wikidata),
):
source = get_path_source(path)
value = get_path(entry, path)
for language, item in extractor(value):
assert isinstance(item, str), (path, language, item, value)
if language is not None and item is not None:
item = item.strip()
if item:
sources_by_value_by_language.setdefault(language, {}).setdefault(item, set()).add(source)
if sources_by_value_by_language:
for language, sources_by_value in sources_by_value_by_language.items():
canonical.setdefault('tags', {})[language] = [
dict(
sources = sorted(sources),
value = value,
)
for value, sources in sorted(sources_by_value.items())
]
if canonical:
entry['canonical'] = canonical
with open(os.path.join(target_entity_type_dir, yaml_file_relative_path), 'w') as yaml_file:
yaml.dump(entry, yaml_file, allow_unicode=True, default_flow_style=False, indent=2, width=120)
return 0
def parse_args(cls, args=None):
"""
Parses the given arguments (formatted like sys.argv) or returns default configuration if None specified.
:param args:
:return:
"""
# ArgumentParser.add_argument(name or flags...[, action][, nargs][, const][, default][, type][, choices][, required][, help][, metavar][, dest])
# action is one of: store[_const,_true,_false], append[_const], count
# nargs is one of: N, ?(defaults to const when no args), *, +, argparse.REMAINDER
# help supports %(var)s: help='default value is %(default)s'
test_default_quit_time = 20
test_config_filename = 'test_config.yml'
default_config_filename = cls._build_config_file_path(
'default_config.yml')
parser = argparse.ArgumentParser(
description=
'''Scale Client main process. You can specify a configuration
file for generating the client's components that will run and/or manually configure them using command
line arguments (NOTE: these may overwrite parameters in the configuration file if there are conflicts,
but things like sensors, apps, etc. will be interpreted as additional components).'''
)
# Configure what components run
config_group = parser.add_mutually_exclusive_group()
config_group.add_argument(
'--config-file',
'-f',
type=str,
dest='config_filename',
default=None,
help='''file from which to read configuration (NOTE: if you don't
specify an absolute path, SCALE assumes you're referring to a relative
path within the 'config' directory). Default is %s when no
manual configurations specified. If manual configurations are in use,
no configuration file is used unless you explicitly set one.'''
% default_config_filename)
config_group.add_argument(
'--test',
'-t',
action='store_true',
help='''run client with simple test configuration found in file %s
(publishes dummy sensor data to simple logging sink).
It also quits after %d seconds if you don't specify --quit-time'''
% (test_config_filename, test_default_quit_time))
# Manually configure components
parser.add_argument(
'--sensors',
'-s',
type=str,
nargs='+',
default=None,
help='''manually specify sensors (and their configurations) to run.
Arguments should be in YAML format (JSON is a subset of YAML!)
e.g. can specify two sensors using:
--sensors '{class: "network.heartbeat_virtual_sensor.HeartbeatSensor",
interval: 5}' '{class:
"dummy.dummy_gas_virtual_sensor.DummyGasPhysicalSensor",
name: "gas0", interval: 3}'
Alternatively, you can also assign a name to your custom component, which
can be used to overwrite or modify one of the same name in your configuration
file such as the following to change the actual class definition used:
'{TempSensor: {class: "environment.usb_temperature_virtual_sensor.UsbTemperaturePhysicalSensor"}'
''')
parser.add_argument(
'--applications',
'-a',
type=str,
nargs='+',
default=None,
help=
'''manually specify applications (and their configurations) to run.
See --sensors help description for example.''')
parser.add_argument(
'--event-sinks',
'-e',
type=str,
nargs='+',
default=None,
dest='event_sinks',
help=
'''manually specify event sinks (and their configurations) to run.
See --sensors help description for example.''')
parser.add_argument(
'--networks',
'-n',
type=str,
nargs='+',
default=None,
help=
'''manually specify network components (and their configurations) to run.
See --sensors help description for example.''')
# Configure logging
parser.add_argument('--log-level',
type=str,
default='WARNING',
dest='log_level',
help='''one of debug, info, error, warning''')
parser.add_argument(
'--enable-log-modules',
dest='enable_log_modules',
nargs='+',
default=(),
help='''explicitly enable logging for the specified modules
(by default all of %s are disabled)''' %
DEFAULT_DISABLED_LOG_MODULES)
parser.add_argument(
'--disable-log-modules',
dest='disable_log_modules',
nargs='+',
default=(),
help='''explicitly disable logging for the specified modules''')
parser.add_argument(
'--format-logging',
type=str,
default=DEFAULT_LOG_FORMAT,
dest='log_format',
help=
'''formats logging as per the given argument (default=%(default)s).
NOTE: add timestamp by doing %%(asctime)s''')
# Misc config params
parser.add_argument(
'--quit-time',
'-q',
type=int,
default=None,
dest='quit_time',
help='''quit the client after specified number of seconds
(default is to never quit)''')
parser.add_argument(
'--raise-errors',
action='store_true',
dest='raise_errors',
help=
'''when constructing a component, raise the (non-import) errors to allow printing
a stack trace rather than trying to gracefully skip it and logging the error'''
)
parsed_args = parser.parse_args(args if args is not None else '')
# Correct configuration filename
if parsed_args.test:
parsed_args.config_filename = cls._build_config_file_path(
test_config_filename)
elif parsed_args.config_filename is not None:
parsed_args.config_filename = cls._build_config_file_path(
parsed_args.config_filename)
# Set default config file if no files or manual configurations are specified
elif parsed_args.config_filename is None and not any(
(parsed_args.sensors, parsed_args.applications,
parsed_args.event_sinks, parsed_args.networks)):
parsed_args.config_filename = default_config_filename
# Testing configuration quits after a time
if parsed_args.test and parsed_args.quit_time is None:
parsed_args.quit_time = test_default_quit_time
return parsed_args
def parse_args(cls, args=None):
"""
Parses the given arguments (formatted like sys.argv) or returns default configuration if None specified.
:param args:
:return:
"""
# ArgumentParser.add_argument(name or flags...[, action][, nargs][, const][, default][, type][, choices][, required][, help][, metavar][, dest])
# action is one of: store[_const,_true,_false], append[_const], count
# nargs is one of: N, ?(defaults to const when no args), *, +, argparse.REMAINDER
# help supports %(var)s: help='default value is %(default)s'
test_default_quit_time = 20
test_config_filename = 'test_config.yml'
default_config_filename = cls._build_config_file_path(
'default_config.yml')
parser = argparse.ArgumentParser(
description=
'''Scale Client main process. You can specify a configuration
file for generating the client's components that will run and/or manually configure them using command
line arguments (NOTE: these may overwrite parameters in the configuration file if there are conflicts,
but things like sensors, apps, etc. will be interpreted as additional components).'''
)
# Configure what components run
config_group = parser.add_mutually_exclusive_group()
config_group.add_argument(
'--config-file',
'-f',
type=str,
dest='config_filename',
default=None,
help='''file from which to read configuration (NOTE: if you don't
specify an absolute path, SCALE assumes you're referring to a relative
path within the 'config' directory). Default is %s when no
manual configurations specified. If manual configurations are in use,
no configuration file is used unless you explicitly set one.'''
% default_config_filename)
config_group.add_argument(
'--test',
'-t',
action='store_true',
help='''run client with simple test configuration found in file %s
(publishes dummy sensor data to simple logging sink).
It also quits after %d seconds if you don't specify --quit-time'''
% (test_config_filename, test_default_quit_time))
# Manually configure components
parser.add_argument(
'--sensors',
'-s',
type=str,
nargs='+',
default=None,
help='''manually specify sensors (and their configurations) to run.
Arguments should be in YAML format e.g. can specify two sensors using
JSON: --sensors '{class: "heartbeat_virtual_sensor.HeartbeatVirtualSensor",
dev_name: "hb0", interval: 5}' '{class:
"dummy_sensors.dummy_gas_virtual_sensor.DummyGasVirtualSensor",
dev_name: "gas0", interval: 3}'
''')
parser.add_argument(
'--applications',
'-a',
type=str,
nargs='+',
default=None,
help=
'''manually specify applications (and their configurations) to run.
See --sensors help description for example.''')
parser.add_argument(
'--event-sinks',
'-e',
type=str,
nargs='+',
default=None,
dest='event_sinks',
help=
'''manually specify event sinks (and their configurations) to run.
See --sensors help description for example.''')
parser.add_argument(
'--network',
'-n',
type=str,
nargs='+',
default=None,
help=
'''manually specify network components (and their configurations) to run.
See --sensors help description for example.''')
# Misc config params
parser.add_argument('--log-level',
type=str,
default='WARNING',
dest='log_level',
help='''one of debug, info, error, warning''')
parser.add_argument(
'--quit-time',
'-q',
type=int,
default=None,
dest='quit_time',
help='''quit the client after specified number of seconds
(default is to never quit)''')
parser.add_argument(
'--raise-errors',
action='store_true',
dest='raise_errors',
help=
'''when constructing a component, raise the (non-import) errors to allow printing
a stack trace rather than trying to gracefully skip it and logging the error'''
)
parsed_args = parser.parse_args(args)
# Correct configuration filename
if parsed_args.test:
parsed_args.config_filename = cls._build_config_file_path(
test_config_filename)
elif parsed_args.config_filename is not None:
parsed_args.config_filename = cls._build_config_file_path(
parsed_args.config_filename)
# Set default config file if no files or manual configurations are specified
elif parsed_args.config_filename is None and not any(
(parsed_args.sensors, parsed_args.applications,
parsed_args.event_sinks, parsed_args.network)):
parsed_args.config_filename = default_config_filename
# Sanity check that we support requested options fully
if parsed_args.network is not None:
raise NotImplementedError(
"--network option not yet fully supported!")
# Testing configuration quits after a time
if parsed_args.test and parsed_args.quit_time is None:
parsed_args.quit_time = test_default_quit_time
return parsed_args
def parse_args(cls, args=None):
"""
Parses the given arguments (formatted like sys.argv) or returns default configuration if None specified.
:param args:
:return:
"""
# ArgumentParser.add_argument(name or flags...[, action][, nargs][, const][, default][, type][, choices][, required][, help][, metavar][, dest])
# action is one of: store[_const,_true,_false], append[_const], count
# nargs is one of: N, ?(defaults to const when no args), *, +, argparse.REMAINDER
# help supports %(var)s: help='default value is %(default)s'
test_default_quit_time = 20
test_config_filename = 'test_config.yml'
default_config_filename = cls._build_config_file_path('default_config.yml')
parser = argparse.ArgumentParser(description='''Scale Client main process. You can specify a configuration
file for generating the client's components that will run and/or manually configure them using command
line arguments (NOTE: these may overwrite parameters in the configuration file if there are conflicts,
but things like sensors, apps, etc. will be interpreted as additional components).''')
# Configure what components run
config_group = parser.add_mutually_exclusive_group()
config_group.add_argument('--config-file', '-f', type=str, dest='config_filename',
default=None,
help='''file from which to read configuration (NOTE: if you don't
specify an absolute path, SCALE assumes you're referring to a relative
path within the 'config' directory). Default is %s when no
manual configurations specified. If manual configurations are in use,
no configuration file is used unless you explicitly set one.''' % default_config_filename)
config_group.add_argument('--test', '-t', action='store_true',
help='''run client with simple test configuration found in file %s
(publishes dummy sensor data to simple logging sink).
It also quits after %d seconds if you don't specify --quit-time'''
% (test_config_filename, test_default_quit_time))
# Manually configure components
parser.add_argument('--sensors', '-s', type=str, nargs='+', default=None,
help='''manually specify sensors (and their configurations) to run.
Arguments should be in YAML format (JSON is a subset of YAML!)
e.g. can specify two sensors using:
--sensors '{class: "network.heartbeat_virtual_sensor.HeartbeatSensor",
interval: 5}' '{class:
"dummy.dummy_gas_virtual_sensor.DummyGasPhysicalSensor",
name: "gas0", interval: 3}'
Alternatively, you can also assign a name to your custom component, which
can be used to overwrite or modify one of the same name in your configuration
file such as the following to change the actual class definition used:
'{TempSensor: {class: "environment.usb_temperature_virtual_sensor.UsbTemperaturePhysicalSensor"}'
''')
parser.add_argument('--applications', '-a', type=str, nargs='+', default=None,
help='''manually specify applications (and their configurations) to run.
See --sensors help description for example.''')
parser.add_argument('--event-sinks', '-e', type=str, nargs='+', default=None, dest='event_sinks',
help='''manually specify event sinks (and their configurations) to run.
See --sensors help description for example.''')
parser.add_argument('--networks', '-n', type=str, nargs='+', default=None,
help='''manually specify network components (and their configurations) to run.
See --sensors help description for example.''')
# Configure logging
parser.add_argument('--log-level', type=str, default='WARNING', dest='log_level',
help='''one of debug, info, error, warning''')
parser.add_argument('--enable-log-modules', dest='enable_log_modules', nargs='+', default=(),
help='''explicitly enable logging for the specified modules
(by default all of %s are disabled)''' % DEFAULT_DISABLED_LOG_MODULES)
parser.add_argument('--disable-log-modules', dest='disable_log_modules', nargs='+', default=(),
help='''explicitly disable logging for the specified modules''')
parser.add_argument('--format-logging', type=str, default=DEFAULT_LOG_FORMAT, dest='log_format',
help='''formats logging as per the given argument (default=%(default)s).
NOTE: add timestamp by doing %%(asctime)s''')
# Misc config params
parser.add_argument('--quit-time', '-q', type=int, default=None, dest='quit_time',
help='''quit the client after specified number of seconds
(default is to never quit)''')
parser.add_argument('--raise-errors', action='store_true', dest='raise_errors',
help='''when constructing a component, raise the (non-import) errors to allow printing
a stack trace rather than trying to gracefully skip it and logging the error''')
parsed_args = parser.parse_args(args if args is not None else '')
# Correct configuration filename
if parsed_args.test:
parsed_args.config_filename = cls._build_config_file_path(test_config_filename)
elif parsed_args.config_filename is not None:
parsed_args.config_filename = cls._build_config_file_path(parsed_args.config_filename)
# Set default config file if no files or manual configurations are specified
elif parsed_args.config_filename is None and not any((parsed_args.sensors, parsed_args.applications,
parsed_args.event_sinks, parsed_args.networks)):
parsed_args.config_filename = default_config_filename
# Testing configuration quits after a time
if parsed_args.test and parsed_args.quit_time is None:
parsed_args.quit_time = test_default_quit_time
return parsed_args
def main():
parser = argparse.ArgumentParser()
parser.add_argument('source_dir', help='path of YAML data directory')
parser.add_argument('target_dir',
help='name of directory containing generated CSV file')
parser.add_argument('-v',
'--verbose',
action='store_true',
default=False,
help='increase output verbosity')
global args
args = parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.WARNING,
stream=sys.stdout)
if not os.path.exists(args.target_dir):
os.makedirs(args.target_dir)
# ACTORS
actors_csv_path = os.path.join(args.target_dir, 'actors.csv')
entries = []
tagsCount = 0
for actor_path, actor in iter_yaml_files(
os.path.join(args.source_dir, 'actors')):
canonical = actor['canonical']
tags = [tag['value'] for tag in get_path(canonical, 'tags.en', [])]
entries.append(
dict(
longDescription=get_path(canonical,
'longDescription.en.value'),
name=get_path(canonical, 'name.value'),
tags=tags,
website=get_path(canonical, 'website.value'),
))
if len(tags) > tagsCount:
tagsCount = len(tags)
with open(actors_csv_path, 'w') as target_file:
csv_writer = csv.writer(target_file)
csv_writer.writerow([
'Name',
'Description',
'Website',
] + ['Tag'] * tagsCount)
entries.sort(key=lambda entry: entry['name'] or '')
for entry in entries:
if entry['name'] is None:
print('Skipping entry without name: {}'.format(entry))
continue
tags = entry['tags']
row = [
entry['name'] or '',
entry['longDescription'] or '',
entry['website'] or '',
] + tags + [''] * (tagsCount - len(tags))
csv_writer.writerow(row)
# PROJECTS
projects_csv_path = os.path.join(args.target_dir, 'projects.csv')
entries = []
tagsCount = 0
toolsCount = 0
for project_path, project in iter_yaml_files(
os.path.join(args.source_dir, 'projects')):
canonical = project['canonical']
tags = [tag['value'] for tag in get_path(canonical, 'tags.en', [])]
tools = [tool['value'] for tool in get_path(canonical, 'tools', [])]
entries.append(
dict(
longDescription=get_path(canonical,
'longDescription.en.value'),
name=get_path(canonical, 'name.value'),
tags=tags,
tools=tools,
website=get_path(canonical, 'website.value'),
))
if len(tags) > tagsCount:
tagsCount = len(tags)
if len(tools) > toolsCount:
toolsCount = len(tools)
with open(projects_csv_path, 'w') as target_file:
csv_writer = csv.writer(target_file)
csv_writer.writerow([
'Name',
'Description',
'Website',
] + ['Tag'] * tagsCount + ['Tool'] * toolsCount)
entries.sort(key=lambda entry: entry['name'] or '')
for entry in entries:
if entry['name'] is None:
print('Skipping entry without name: {}'.format(entry))
continue
tags = entry['tags']
tools = entry['tools']
row = [
entry['name'] or '',
entry['longDescription'] or '',
entry['website'] or '',
] + tags + [''] * (tagsCount - len(tags)) + tools + [''] * (
toolsCount - len(tools))
csv_writer.writerow(row)
# TOOLS
tools_csv_path = os.path.join(args.target_dir, 'tools.csv')
entries = []
tagsCount = 0
for tool_path, tool in iter_yaml_files(
os.path.join(args.source_dir, 'tools')):
canonical = tool['canonical']
tags = [tag['value'] for tag in get_path(canonical, 'tags.en', [])]
entries.append(
dict(
bugTracker=get_path(canonical, 'bugTracker.value'),
license=get_path(canonical, 'license.value'),
longDescription=get_path(canonical,
'longDescription.en.value'),
name=get_path(canonical, 'name.value'),
screenshot=get_path(canonical, 'screenshot.value'),
sourceCode=get_path(canonical, 'sourceCode.value'),
stackexchangeTag=get_path(canonical, 'stackexchangeTag.value'),
tags=tags,
))
if len(tags) > tagsCount:
tagsCount = len(tags)
with open(tools_csv_path, 'w') as target_file:
csv_writer = csv.writer(target_file)
csv_writer.writerow([
'Name',
'Description',
'License',
'Source Code URL',
'Bug Tracker URL',
'Screenshot URL',
'StackExchange Tag',
] + ['Tag'] * tagsCount)
entries.sort(key=lambda entry: entry['name'] or '')
for entry in entries:
if entry['name'] is None:
print('Skipping entry without name: {}'.format(entry))
continue
tags = entry['tags']
row = [
entry['name'] or '',
entry['longDescription'] or '',
entry['license'] or '',
entry['sourceCode'] or '',
entry['bugTracker'] or '',
entry['screenshot'] or '',
entry['stackexchangeTag'] or '',
] + tags + [''] * (tagsCount - len(tags))
csv_writer.writerow(row)
return 0
