def __init__(self, map_input: dict):
self.validated = Schema(TOP_LEVEL_SCHEMA).validate(map_input)
'sshKeyPath': setPathCheck('sshKeyPath'),
Optional('passphrase'): setType('passphrase', str),
Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'),
Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int),
Optional('useActiveGpu'): setType('useActiveGpu', bool)
},
{
'ip': setType('ip', str),
Optional('port'): setNumberRange('port', int, 1, 65535),
'username': setType('username', str),
'passwd': setType('passwd', str),
Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'),
Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int),
Optional('useActiveGpu'): setType('useActiveGpu', bool)
})]
}
LOCAL_CONFIG_SCHEMA = Schema({**common_schema, **common_trial_schema})
REMOTE_CONFIG_SCHEMA = Schema({**common_schema, **common_trial_schema, **machine_list_schema})
PAI_CONFIG_SCHEMA = Schema({**common_schema, **pai_trial_schema, **pai_config_schema})
PAI_YARN_CONFIG_SCHEMA = Schema({**common_schema, **pai_yarn_trial_schema, **pai_yarn_config_schema})
DLTS_CONFIG_SCHEMA = Schema({**common_schema, **dlts_trial_schema, **dlts_config_schema})
KUBEFLOW_CONFIG_SCHEMA = Schema({**common_schema, **kubeflow_trial_schema, **kubeflow_config_schema})
FRAMEWORKCONTROLLER_CONFIG_SCHEMA = Schema({**common_schema, **frameworkcontroller_trial_schema, **frameworkcontroller_config_schema})
def validate(self, data):
self.algo_schema.update(self.builtin_name_schema[self.algo_type])
Schema(self.algo_schema).validate(data)
self.validate_extras(data, self.algo_type)
import json
try:
from urllib.parse import urlparse
except ImportError:
from urlparse import urlparse
from kafka import KafkaProducer
from kafka import KafkaProducer
from schema import Schema, Optional
login_schema = Schema({
"type": "LOGIN_SUCCESSFUL",
"payload": {
"user_email": str,
"app_name": str,
"time": str,
Optional("environment"): str
}
})
class Notification:
def __init__(self, broker_url):
self.schema_list = {"LOGIN_SUCCESSFUL": login_schema}
self.topic = "arc-rte-notifications"
self.__producer = KafkaProducer(
bootstrap_servers=self.get_kafka_brokers(broker_url),
value_serializer=lambda v: json.dumps(v).encode("utf-8"),
)
visualise filters
'''
from docopt import docopt
from schema import Schema, And, Or, Use
s = Schema({
'--n_epoch': Use(int),
'--alpha': Use(float),
'--width1': Use(int),
'--width2': Use(int),
'--feat_map_n_1': Use(int),
'--feat_map_n_final': Use(int),
'--feat_map_n_final': Use(int),
'--k_top': Use(int),
'--dropout_rate0': Use(float),
'--dropout_rate1': Use(float),
'--dropout_rate2': Use(float),
'--activation': Use(str),
'--learn': Use(str),
'--skip': Use(int),
'--use_regular': Use(int),
'--regular_c': Use(float),
'--shuffle': Use(int),
'--pretrain': Use(str),
'--emb_size': Use(int),
'--datatype': Use(int),
})
args = docopt(__doc__)
args = s.validate(args)
print args
def __init__(self,
input_file=None,
output_file=None,
dfxml_file=None,
report_file=None,
commit=False,
ignore_patterns=[],
key=None,
rules=[]):
# Validate configuration
from schema import Schema, Optional, Or, Use, And, SchemaError
schema = Schema({
'input_file':
Use(lambda f: open(f, 'r'), error='Cannot read the input file'),
Optional('output_file'):
Or(
None,
Use(lambda f: open(f, 'w'),
error='Cannot write to the output file')),
Optional('dfxml_file'):
Or(None, Use(lambda f: open(f, 'r'),
error='Cannot read DFXML file')),
Optional('report_file'):
Or(None,
lambda f: open(f, 'w'),
error='Cannot write to the report file'),
'commit':
Or(True, False),
'ignore_patterns':
Use(lambda f: re.compile(convert_fileglob_to_re('|'.join(f))),
error='Cannot compile unified ignore regex'),
'key':
Or(None, str),
'rules':
And([(redact_rule, redact_action)], lambda f: len(f) > 0)
})
try:
kwargs = {
'input_file': input_file,
'output_file': output_file,
'dfxml_file': dfxml_file,
'report_file': report_file,
'commit': commit,
'ignore_patterns': ignore_patterns,
'key': key,
'rules': rules
}
self.conf = schema.validate(kwargs)
except SchemaError as e:
logging.warning('The redact configuration did not validate:')
exit(e)
if self.conf['commit'] and 'output_file' not in self.conf.keys():
logging.error('An output file is required when COMMIT is on.')
exit(1)
# TODO Check input and output are not same file
logging.debug('Configuration:\n%s' % self.conf)
# Print rules
logging.debug(
json.dumps(map(
lambda x, y: (x.line, x.__class__.__name__, y.__class__.
__name__, x.lgpattern
if hasattr(x, 'lgpattern') else ''),
self.conf['rules']),
indent=4))
self.input_file = self.conf['input_file']
from os import path
self.image_size = path.getsize(self.input_file.name)
self.output_file = self.conf['output_file']
self.report_file = self.conf['report_file']
self.dfxml_file = self.conf['dfxml_file']
self.commit = self.conf['commit']
self.configure_report_logger()
class Stage(object):
STAGE_FILE = "Dvcfile"
STAGE_FILE_SUFFIX = ".dvc"
PARAM_MD5 = "md5"
PARAM_CMD = "cmd"
PARAM_WDIR = "wdir"
PARAM_DEPS = "deps"
PARAM_OUTS = "outs"
PARAM_LOCKED = "locked"
SCHEMA = {
Optional(PARAM_MD5): Or(str, None),
Optional(PARAM_CMD): Or(str, None),
Optional(PARAM_WDIR): Or(str, None),
Optional(PARAM_DEPS): Or(And(list, Schema([dependency.SCHEMA])), None),
Optional(PARAM_OUTS): Or(And(list, Schema([output.SCHEMA])), None),
Optional(PARAM_LOCKED): bool,
}
TAG_REGEX = r"^(?P<path>.*)@(?P<tag>[^\\/@:]*)$"
def __init__(
self,
repo,
path=None,
cmd=None,
wdir=os.curdir,
deps=None,
outs=None,
md5=None,
locked=False,
tag=None,
state=None,
):
if deps is None:
deps = []
if outs is None:
outs = []
self.repo = repo
self.path = path
self.cmd = cmd
self.wdir = wdir
self.outs = outs
self.deps = deps
self.md5 = md5
self.locked = locked
self.tag = tag
self._state = state or {}
def __repr__(self):
return "Stage: '{path}'".format(
path=self.relpath if self.path else "No path")
@property
def relpath(self):
return os.path.relpath(self.path)
@property
def is_data_source(self):
"""Whether the stage file was created with `dvc add` or `dvc import`"""
return self.cmd is None
@staticmethod
def is_valid_filename(path):
return (
# path.endswith doesn't work for encoded unicode filenames on
# Python 2 and since Stage.STAGE_FILE_SUFFIX is ascii then it is
# not needed to decode the path from py2's str
path[-len(Stage.STAGE_FILE_SUFFIX):] == Stage.STAGE_FILE_SUFFIX
or os.path.basename(path) == Stage.STAGE_FILE)
@staticmethod
def is_stage_file(path):
return os.path.isfile(path) and Stage.is_valid_filename(path)
def changed_md5(self):
return self.md5 != self._compute_md5()
@property
def is_callback(self):
"""
A callback stage is always considered as changed,
so it runs on every `dvc repro` call.
"""
return not self.is_data_source and len(self.deps) == 0
@property
def is_import(self):
"""Whether the stage file was created with `dvc import`."""
return not self.cmd and len(self.deps) == 1 and len(self.outs) == 1
def _changed_deps(self):
if self.locked:
return False
if self.is_callback:
logger.warning(
"Dvc file '{fname}' is a 'callback' stage "
"(has a command and no dependencies) and thus always "
"considered as changed.".format(fname=self.relpath))
return True
for dep in self.deps:
status = dep.status()
if status:
logger.warning(
"Dependency '{dep}' of '{stage}' changed because it is "
"'{status}'.".format(dep=dep,
stage=self.relpath,
status=status[str(dep)]))
return True
return False
def _changed_outs(self):
for out in self.outs:
status = out.status()
if status:
logger.warning(
"Output '{out}' of '{stage}' changed because it is "
"'{status}'".format(out=out,
stage=self.relpath,
status=status[str(out)]))
return True
return False
def _changed_md5(self):
if self.changed_md5():
logger.warning("Dvc file '{}' changed.".format(self.relpath))
return True
return False
def changed(self):
ret = any(
[self._changed_deps(),
self._changed_outs(),
self._changed_md5()])
if ret:
logger.warning("Stage '{}' changed.".format(self.relpath))
else:
logger.info("Stage '{}' didn't change.".format(self.relpath))
return ret
def remove_outs(self, ignore_remove=False, force=False):
"""Used mainly for `dvc remove --outs` and :func:`Stage.reproduce`."""
for out in self.outs:
if out.persist and not force:
out.unprotect()
else:
logger.debug("Removing output '{out}' of '{stage}'.".format(
out=out, stage=self.relpath))
out.remove(ignore_remove=ignore_remove)
def unprotect_outs(self):
for out in self.outs:
out.unprotect()
def remove(self, force=False):
self.remove_outs(ignore_remove=True, force=force)
os.unlink(self.path)
def reproduce(self,
force=False,
dry=False,
interactive=False,
no_commit=False):
if not self.changed() and not force:
return None
msg = ("Going to reproduce '{stage}'. "
"Are you sure you want to continue?".format(stage=self.relpath))
if interactive and not prompt.confirm(msg):
raise DvcException("reproduction aborted by the user")
logger.info("Reproducing '{stage}'".format(stage=self.relpath))
self.run(dry=dry, no_commit=no_commit, force=force)
logger.debug("'{stage}' was reproduced".format(stage=self.relpath))
return self
@staticmethod
def validate(d, fname=None):
from dvc.utils import convert_to_unicode
try:
Schema(Stage.SCHEMA).validate(convert_to_unicode(d))
except SchemaError as exc:
raise StageFileFormatError(fname, exc)
@classmethod
def _stage_fname(cls, fname, outs, add):
if fname:
return fname
if not outs:
return cls.STAGE_FILE
out = outs[0]
path_handler = out.remote.ospath
fname = path_handler.basename(out.path) + cls.STAGE_FILE_SUFFIX
fname = Stage._expand_to_path_on_add_local(add, fname, out,
path_handler)
return fname
@staticmethod
def _expand_to_path_on_add_local(add, fname, out, path_handler):
if (add and out.is_in_repo
and not contains_symlink_up_to(out.path, out.repo.root_dir)):
fname = path_handler.join(path_handler.dirname(out.path), fname)
return fname
@staticmethod
def _check_stage_path(repo, path):
assert repo is not None
real_path = os.path.realpath(path)
if not os.path.exists(real_path):
raise StagePathNotFoundError(path)
if not os.path.isdir(real_path):
raise StagePathNotDirectoryError(path)
proj_dir = os.path.realpath(repo.root_dir) + os.path.sep
if not (real_path + os.path.sep).startswith(proj_dir):
raise StagePathOutsideError(path)
@property
def is_cached(self):
"""
Checks if this stage has been already ran and stored
"""
from dvc.remote.local import RemoteLOCAL
from dvc.remote.s3 import RemoteS3
old = Stage.load(self.repo, self.path)
if old._changed_outs():
return False
# NOTE: need to save checksums for deps in order to compare them
# with what is written in the old stage.
for dep in self.deps:
dep.save()
old_d = old.dumpd()
new_d = self.dumpd()
# NOTE: need to remove checksums from old dict in order to compare
# it to the new one, since the new one doesn't have checksums yet.
old_d.pop(self.PARAM_MD5, None)
new_d.pop(self.PARAM_MD5, None)
outs = old_d.get(self.PARAM_OUTS, [])
for out in outs:
out.pop(RemoteLOCAL.PARAM_CHECKSUM, None)
out.pop(RemoteS3.PARAM_CHECKSUM, None)
return old_d == new_d
@staticmethod
def create(
repo=None,
cmd=None,
deps=None,
outs=None,
outs_no_cache=None,
metrics=None,
metrics_no_cache=None,
fname=None,
cwd=None,
wdir=None,
locked=False,
add=False,
overwrite=True,
ignore_build_cache=False,
remove_outs=False,
validate_state=True,
outs_persist=None,
outs_persist_no_cache=None,
):
if outs is None:
outs = []
if deps is None:
deps = []
if outs_no_cache is None:
outs_no_cache = []
if metrics is None:
metrics = []
if metrics_no_cache is None:
metrics_no_cache = []
if outs_persist is None:
outs_persist = []
if outs_persist_no_cache is None:
outs_persist_no_cache = []
# Backward compatibility for `cwd` option
if wdir is None and cwd is not None:
if fname is not None and os.path.basename(fname) != fname:
raise StageFileBadNameError(
"stage file name '{fname}' may not contain subdirectories"
" if '-c|--cwd' (deprecated) is specified. Use '-w|--wdir'"
" along with '-f' to specify stage file path and working"
" directory.".format(fname=fname))
wdir = cwd
else:
wdir = os.curdir if wdir is None else wdir
stage = Stage(repo=repo, wdir=wdir, cmd=cmd, locked=locked)
Stage._fill_stage_outputs(
stage,
outs,
outs_no_cache,
metrics,
metrics_no_cache,
outs_persist,
outs_persist_no_cache,
)
stage.deps = dependency.loads_from(stage, deps)
stage._check_circular_dependency()
stage._check_duplicated_arguments()
fname = Stage._stage_fname(fname, stage.outs, add=add)
wdir = os.path.abspath(wdir)
if cwd is not None:
path = os.path.join(wdir, fname)
else:
path = os.path.abspath(fname)
Stage._check_stage_path(repo, wdir)
Stage._check_stage_path(repo, os.path.dirname(path))
stage.wdir = wdir
stage.path = path
# NOTE: remove outs before we check build cache
if remove_outs:
logger.warning("--remove-outs is deprecated."
" It is now the default behavior,"
" so there's no need to use this option anymore.")
stage.remove_outs(ignore_remove=False)
logger.warning("Build cache is ignored when using --remove-outs.")
ignore_build_cache = True
else:
stage.unprotect_outs()
if validate_state:
if os.path.exists(path):
if not ignore_build_cache and stage.is_cached:
logger.info("Stage is cached, skipping.")
return None
msg = (
"'{}' already exists. Do you wish to run the command and "
"overwrite it?".format(stage.relpath))
if not overwrite and not prompt.confirm(msg):
raise StageFileAlreadyExistsError(stage.relpath)
os.unlink(path)
return stage
@staticmethod
def _fill_stage_outputs(
stage,
outs,
outs_no_cache,
metrics,
metrics_no_cache,
outs_persist,
outs_persist_no_cache,
):
stage.outs = output.loads_from(stage, outs, use_cache=True)
stage.outs += output.loads_from(stage,
metrics,
use_cache=True,
metric=True)
stage.outs += output.loads_from(stage,
outs_persist,
use_cache=True,
persist=True)
stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False)
stage.outs += output.loads_from(stage,
metrics_no_cache,
use_cache=False,
metric=True)
stage.outs += output.loads_from(stage,
outs_persist_no_cache,
use_cache=False,
persist=True)
@staticmethod
def _check_dvc_filename(fname):
if not Stage.is_valid_filename(fname):
raise StageFileBadNameError(
"bad stage filename '{}'. Stage files should be named"
" 'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
os.path.relpath(fname), os.path.basename(fname)))
@staticmethod
def _check_file_exists(repo, fname):
if not repo.tree.exists(fname):
raise StageFileDoesNotExistError(fname)
@staticmethod
def _check_isfile(repo, fname):
if not repo.tree.isfile(fname):
raise StageFileIsNotDvcFileError(fname)
@classmethod
def _get_path_tag(cls, s):
regex = re.compile(cls.TAG_REGEX)
match = regex.match(s)
if not match:
return s, None
return match.group("path"), match.group("tag")
@staticmethod
def load(repo, fname):
fname, tag = Stage._get_path_tag(fname)
# it raises the proper exceptions by priority:
# 1. when the file doesn't exists
# 2. filename is not a dvc filename
# 3. path doesn't represent a regular file
Stage._check_file_exists(repo, fname)
Stage._check_dvc_filename(fname)
Stage._check_isfile(repo, fname)
with repo.tree.open(fname) as fd:
d = load_stage_fd(fd, fname)
# Making a deepcopy since the original structure
# looses keys in deps and outs load
state = copy.deepcopy(d)
Stage.validate(d, fname=os.path.relpath(fname))
path = os.path.abspath(fname)
stage = Stage(
repo=repo,
path=path,
wdir=os.path.abspath(
os.path.join(os.path.dirname(path),
d.get(Stage.PARAM_WDIR, "."))),
cmd=d.get(Stage.PARAM_CMD),
md5=d.get(Stage.PARAM_MD5),
locked=d.get(Stage.PARAM_LOCKED, False),
tag=tag,
state=state,
)
stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, []))
stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS, []))
return stage
def dumpd(self):
from dvc.remote.local import RemoteLOCAL
return {
key: value
for key, value in {
Stage.PARAM_MD5:
self.md5,
Stage.PARAM_CMD:
self.cmd,
Stage.PARAM_WDIR:
RemoteLOCAL.unixpath(
os.path.relpath(self.wdir, os.path.dirname(self.path))),
Stage.PARAM_LOCKED:
self.locked,
Stage.PARAM_DEPS: [d.dumpd() for d in self.deps],
Stage.PARAM_OUTS: [o.dumpd() for o in self.outs],
}.items() if value
}
def dump(self):
fname = self.path
self._check_dvc_filename(fname)
logger.info("Saving information to '{file}'.".format(
file=os.path.relpath(fname)))
d = self.dumpd()
apply_diff(d, self._state)
dump_stage_file(fname, self._state)
self.repo.scm.track_file(os.path.relpath(fname))
def _compute_md5(self):
from dvc.output.base import OutputBase
d = self.dumpd()
# NOTE: removing md5 manually in order to not affect md5s in deps/outs
if self.PARAM_MD5 in d.keys():
del d[self.PARAM_MD5]
# Ignore the wdir default value. In this case stage file w/o
# wdir has the same md5 as a file with the default value specified.
# It's important for backward compatibility with pipelines that
# didn't have WDIR in their stage files.
if d.get(self.PARAM_WDIR) == ".":
del d[self.PARAM_WDIR]
# NOTE: excluding parameters that don't affect the state of the
# pipeline. Not excluding `OutputLOCAL.PARAM_CACHE`, because if
# it has changed, we might not have that output in our cache.
m = dict_md5(
d,
exclude=[
self.PARAM_LOCKED,
OutputBase.PARAM_METRIC,
OutputBase.PARAM_TAGS,
OutputBase.PARAM_PERSIST,
],
)
logger.debug("Computed stage '{}' md5: '{}'".format(self.relpath, m))
return m
def save(self):
for dep in self.deps:
dep.save()
for out in self.outs:
out.save()
self.md5 = self._compute_md5()
@staticmethod
def _changed_entries(entries):
ret = []
for entry in entries:
if entry.checksum and entry.changed_checksum():
ret.append(entry.rel_path)
return ret
def check_can_commit(self, force):
changed_deps = self._changed_entries(self.deps)
changed_outs = self._changed_entries(self.outs)
if changed_deps or changed_outs or self.changed_md5():
msg = ("dependencies {}".format(changed_deps)
if changed_deps else "")
msg += " and " if (changed_deps and changed_outs) else ""
msg += "outputs {}".format(changed_outs) if changed_outs else ""
msg += "md5" if not (changed_deps or changed_outs) else ""
msg += " of '{}' changed. Are you sure you commit it?".format(
self.relpath)
if not force and not prompt.confirm(msg):
raise StageCommitError(
"unable to commit changed '{}'. Use `-f|--force` to "
"force.`".format(self.relpath))
self.save()
def commit(self):
for out in self.outs:
out.commit()
def _check_missing_deps(self):
missing = [dep for dep in self.deps if not dep.exists]
if any(missing):
raise MissingDep(missing)
@staticmethod
def _warn_if_fish(executable): # pragma: no cover
if (executable is None
or os.path.basename(os.path.realpath(executable)) != "fish"):
return
logger.warning(
"DVC detected that you are using fish as your default "
"shell. Be aware that it might cause problems by overwriting "
"your current environment variables with values defined "
"in '.fishrc', which might affect your command. See "
"https://github.com/iterative/dvc/issues/1307. ")
def _check_circular_dependency(self):
from dvc.exceptions import CircularDependencyError
circular_dependencies = set(d.path for d in self.deps) & set(
o.path for o in self.outs)
if circular_dependencies:
raise CircularDependencyError(circular_dependencies.pop())
def _check_duplicated_arguments(self):
from dvc.exceptions import ArgumentDuplicationError
from collections import Counter
path_counts = Counter(edge.path for edge in self.deps + self.outs)
for path, occurrence in path_counts.items():
if occurrence > 1:
raise ArgumentDuplicationError(path)
def _run(self):
self._check_missing_deps()
executable = os.getenv("SHELL") if os.name != "nt" else None
self._warn_if_fish(executable)
p = subprocess.Popen(
self.cmd,
cwd=self.wdir,
shell=True,
env=fix_env(os.environ),
executable=executable,
)
p.communicate()
if p.returncode != 0:
raise StageCmdFailedError(self)
def run(self, dry=False, resume=False, no_commit=False, force=False):
if (self.cmd or self.is_import) and not self.locked and not dry:
self.remove_outs(ignore_remove=False, force=False)
if self.locked:
logger.info("Verifying outputs in locked stage '{stage}'".format(
stage=self.relpath))
if not dry:
self.check_missing_outputs()
elif self.is_import:
logger.info("Importing '{dep}' -> '{out}'".format(
dep=self.deps[0].path, out=self.outs[0].path))
if not dry:
if self._already_cached() and not force:
self.outs[0].checkout()
else:
self.deps[0].download(self.outs[0].path_info,
resume=resume)
elif self.is_data_source:
msg = "Verifying data sources in '{}'".format(self.relpath)
logger.info(msg)
if not dry:
self.check_missing_outputs()
else:
logger.info("Running command:\n\t{}".format(self.cmd))
if not dry:
if (not force and not self.is_callback
and self._already_cached()):
self.checkout()
else:
self._run()
if not dry:
self.save()
if not no_commit:
self.commit()
def check_missing_outputs(self):
paths = [
out.path if out.scheme != "local" else out.rel_path
for out in self.outs if not out.exists
]
if paths:
raise MissingDataSource(paths)
def checkout(self, force=False, progress_callback=None):
for out in self.outs:
out.checkout(force=force,
tag=self.tag,
progress_callback=progress_callback)
@staticmethod
def _status(entries):
ret = {}
for entry in entries:
ret.update(entry.status())
return ret
def status(self):
ret = []
if not self.locked:
deps_status = self._status(self.deps)
if deps_status:
ret.append({"changed deps": deps_status})
outs_status = self._status(self.outs)
if outs_status:
ret.append({"changed outs": outs_status})
if self.changed_md5():
ret.append("changed checksum")
if self.is_callback:
ret.append("always changed")
if ret:
return {self.relpath: ret}
return {}
def _already_cached(self):
return (not self.changed_md5()
and all(not dep.changed() for dep in self.deps)
and all(not out.changed_cache() if out.
use_cache else not out.changed() for out in self.outs))
def get_all_files_number(self):
return sum(out.get_files_number() for out in self.outs)
def _file(error=None, **kwargs):
er = 'Must be a file!'
return And(_string(), Schema(osp.isfile, error=er), error=error)
def define_data_schema(read=True):
"""
Define data schema.
:param read:
Schema for reading?
:type read: bool
:return:
Data schema.
:rtype: schema.Schema
"""
cmv = _cmv(read=read)
dtc = _dtc(read=read)
cvt = _cvt(read=read)
gspv = _gspv(read=read)
gsch = _gsch(read=read)
string = _string(read=read)
positive = _positive(read=read)
greater_than_zero = _positive(
read=read,
error='should be as <float> and greater than zero!',
check=lambda x: x > 0)
between_zero_and_one = _positive(
read=read,
error='should be as <float> and between zero and one!',
check=lambda x: 0 <= x <= 1)
greater_than_one = _positive(
read=read,
error='should be as <float> and greater than one!',
check=lambda x: x >= 1)
positive_int = _positive(type=int, read=read)
greater_than_one_int = _positive(
type=int,
read=read,
error='should be as <int> and greater than one!',
check=lambda x: x >= 1)
limits = _limits(read=read)
index_dict = _index_dict(read=read)
np_array = _np_array(read=read)
np_array_sorted = _np_array_positive(
read=read,
error='cannot be parsed because it should be an '
'np.array dtype=<float> with ascending order!',
check=_is_sorted)
np_array_greater_than_minus_one = _np_array_positive(
read=read,
error='cannot be parsed because it should be an '
'np.array dtype=<float> and all values >= -1!',
check=lambda x: (x >= -1).all())
np_array_bool = _np_array(dtype=bool, read=read)
np_array_int = _np_array(dtype=int, read=read)
_bool = _type(type=bool, read=read)
function = _function(read=read)
tuplefloat2 = _type(type=And(Use(tuple), (_type(float), )),
length=2,
read=read)
tuplefloat = _type(type=And(Use(tuple), (_type(float), )), read=read)
dictstrdict = _dict(format={str: dict}, read=read)
ordictstrdict = _ordict(format={str: dict}, read=read)
parameters = _parameters(read=read)
dictstrfloat = _dict(format={str: Use(float)}, read=read)
dictarray = _dict(format={str: np_array}, read=read)
tyre_code = _tyre_code(read=read)
tyre_dimensions = _tyre_dimensions(read=read)
schema = {
_compare_str('CVT'):
cvt,
_compare_str('CMV'):
cmv,
_compare_str('CMV_Cold_Hot'):
gsch,
_compare_str('DTGS'):
dtc,
_compare_str('GSPV'):
gspv,
_compare_str('GSPV_Cold_Hot'):
gsch,
_compare_str('MVL'):
_mvl(read=read),
'engine_n_cylinders':
positive_int,
'lock_up_tc_limits':
tuplefloat2,
_convert_str('ki_factor', 'ki_multiplicative'):
greater_than_one,
'ki_additive':
positive,
'drive_battery_technology':
string,
'drive_battery_n_cells':
greater_than_one_int,
'drive_battery_n_series_cells':
greater_than_one_int,
'drive_battery_n_parallel_cells':
greater_than_one_int,
'tyre_dimensions':
tyre_dimensions,
'tyre_code':
tyre_code,
'wltp_base_model':
_dict(format=dict, read=read),
'fuel_type':
_select(types=('gasoline', 'diesel', 'LPG', 'NG', 'ethanol',
'biodiesel', 'methanol', 'propane'),
read=read),
'obd_fuel_type_code':
positive_int,
'vehicle_category':
_select(types='ABCDEFSMJ', read=read),
'vehicle_body':
_select(types=('cabriolet', 'sedan', 'hatchback', 'stationwagon',
'suv/crossover', 'mpv', 'coupé', 'bus', 'bestelwagen',
'pick-up'),
read=read),
'tyre_class':
_select(types=('C1', 'C2', 'C3'), read=read),
'tyre_category':
_select(types='ABCDEFG', read=read),
'engine_fuel_lower_heating_value':
positive,
'fuel_carbon_content':
positive,
'engine_capacity':
positive,
'engine_stroke':
positive,
'engine_max_power':
positive,
_convert_str('engine_max_speed_at_max_power', 'engine_speed_at_max_power'):
positive,
'engine_max_speed':
positive,
'engine_max_torque':
positive,
'idle_engine_speed_median':
positive,
'engine_idle_fuel_consumption':
greater_than_zero,
'final_drive_ratio':
positive,
'r_dynamic':
positive,
'n_wheel':
positive_int,
'wheel_drive_load_fraction':
between_zero_and_one,
'static_friction':
greater_than_zero,
'tyre_state':
_select(types=('new', 'worm'), read=read),
'road_state':
_select(types=('dry', 'wet', 'rainfall', 'puddles', 'ice'), read=read),
'wltp_class':
_select(types=('class1', 'class2', 'class3a', 'class3b'), read=read),
'downscale_phases':
tuplefloat,
'electrical_hybridization_degree':
_select(types=('mild', 'full', 'plugin', 'electric'), read=read),
'gear_box_type':
_select(types=('manual', 'automatic', 'cvt', 'planetary'), read=read),
'ignition_type':
_select(types=('positive', 'compression'), read=read),
'start_stop_activation_time':
positive,
'alternator_nominal_voltage':
positive,
_convert_str('battery_voltage', 'service_battery_nominal_voltage'):
positive,
_convert_str('battery_capacity', 'service_battery_capacity'):
positive,
_convert_str('state_of_charge_balance', 'service_battery_state_of_charge_balance'):
limits,
_convert_str('state_of_charge_balance_window', 'service_battery_state_of_charge_balance_window'):
limits,
_convert_str('initial_state_of_charge', 'initial_service_battery_state_of_charge'):
limits,
'idle_engine_speed_std':
positive,
'alternator_nominal_power':
positive,
'alternator_efficiency':
_limits(limits=(0, 1), read=read),
'time_cold_hot_transition':
positive,
'co2_params':
dictstrfloat,
'willans_factors':
dictstrfloat,
'phases_willans_factors':
_type(type=And(Use(tuple), (dictstrfloat, )), read=read),
'optimal_efficiency':
dictarray,
'velocity_speed_ratios':
index_dict,
'gear_box_ratios':
index_dict,
'final_drive_ratios':
index_dict,
'speed_velocity_ratios':
index_dict,
'full_load_speeds':
np_array_sorted,
'full_load_torques':
np_array,
'full_load_powers':
np_array,
'vehicle_mass':
positive,
'f0_uncorrected':
positive,
'f2':
positive,
'f0':
positive,
'correct_f0':
_bool,
'co2_emission_low':
positive,
'co2_emission_medium':
positive,
'co2_emission_high':
positive,
'co2_emission_extra_high':
positive,
_compare_str('co2_emission_UDC'):
positive,
_compare_str('co2_emission_EUDC'):
positive,
'co2_emission_value':
positive,
'declared_co2_emission_value':
positive,
'n_dyno_axes':
positive_int,
'n_wheel_drive':
positive_int,
'rcb_correction':
_bool,
'speed_distance_correction':
_bool,
'engine_is_turbo':
_bool,
'has_start_stop':
_bool,
'has_gear_box_thermal_management':
_bool,
'has_energy_recuperation':
_bool,
'is_hybrid':
_bool,
'has_roof_box':
_bool,
'has_periodically_regenerating_systems':
_bool,
'engine_has_variable_valve_actuation':
_bool,
'has_thermal_management':
_bool,
'engine_has_direct_injection':
_bool,
'has_lean_burn':
_bool,
'engine_has_cylinder_deactivation':
_bool,
'has_exhausted_gas_recirculation':
_bool,
'has_particle_filter':
_bool,
'has_selective_catalytic_reduction':
_bool,
'has_nox_storage_catalyst':
_bool,
'has_torque_converter':
_bool,
'is_cycle_hot':
_bool,
'is_serial':
_bool,
'use_dt_gear_shifting':
_bool,
_convert_str('eco_mode', 'fuel_saving_at_strategy'):
_bool,
'correct_start_stop_with_gears':
_bool,
'enable_phases_willans':
_bool,
'enable_willans':
_bool,
'has_engine_idle_coasting':
_bool,
'has_engine_off_coasting':
_bool,
'fuel_map':
dictarray,
'transition_cycle_index':
positive_int,
'alternator_charging_currents':
tuplefloat2,
'relative_electric_energy_change':
tuplefloat,
'alternator_current_model':
_alternator_current_model(read=read),
'dcdc_current_model':
_alternator_current_model(read=read),
'service_battery_status_model':
_service_battery_status_model(read=read),
'clutch_speed_model':
function,
'co2_emissions_model':
function,
'co2_error_function_on_emissions':
function,
'co2_error_function_on_phases':
function,
'motor_p0_electric_power_loss_function':
function,
'motor_p1_electric_power_loss_function':
function,
'motor_p2_electric_power_loss_function':
function,
'motor_p3_front_electric_power_loss_function':
function,
'motor_p3_rear_electric_power_loss_function':
function,
'motor_p4_front_electric_power_loss_function':
function,
'motor_p4_rear_electric_power_loss_function':
function,
'after_treatment_speed_model':
function,
'after_treatment_power_model':
function,
'clutch_window':
tuplefloat2,
'co2_params_calibrated':
parameters,
'co2_params_initial_guess':
parameters,
'drive_battery_technology_type':
string,
'cycle_type':
string,
'cycle_name':
string,
'specific_gear_shifting':
string,
'calibration_status':
_type(type=And(Use(list), [(bool, Or(parameters, None))]),
length=4,
read=read),
_convert_str('electric_load', 'service_battery_load'):
tuplefloat2,
'engine_thermostat_temperature_window':
tuplefloat2,
'engine_temperature_regression_model':
_engine_temperature_regression_model(read=read),
'engine_type':
string,
'starter_model':
function,
'drive_battery_model':
function,
'motor_p0_maximum_power_function':
function,
'motor_p1_maximum_power_function':
function,
'motor_p2_planetary_maximum_power_function':
function,
'start_stop_hybrid_params':
dictstrfloat,
'full_load_curve':
function,
'fmep_model':
_fmep_model(read=read),
'gear_box_efficiency_constants':
dictstrdict,
'gear_box_efficiency_parameters_cold_hot':
dictstrdict,
'scores':
dictstrdict,
'param_selections':
dictstrdict,
'model_selections':
dictstrdict,
'score_by_model':
dictstrdict,
'at_scores':
ordictstrdict,
'fuel_density':
positive,
'idle_engine_speed':
tuplefloat2,
'k1':
positive_int,
'k2':
positive_int,
'k5':
positive_int,
'max_gear':
positive_int,
'hybrid_modes':
np_array_int,
'road_loads':
_type(type=And(Use(tuple), (_type(float), )), length=3, read=read),
'start_stop_model':
function,
'gear_box_temperature_references':
tuplefloat2,
'torque_converter_speed_model':
function,
'phases_co2_emissions':
tuplefloat,
'bag_phases':
_bag_phases(read=read),
'phases_integration_times':
_type(type=And(Use(tuple), (And(Use(tuple), (_type(float), )), )),
read=read),
'active_cylinder_ratios':
tuplefloat,
'extended_phases_co2_emissions':
tuplefloat,
'extended_phases_integration_times':
_type(type=And(Use(tuple), (And(Use(tuple), (_type(float), )), )),
read=read),
'extended_integration_times':
tuplefloat,
'phases_fuel_consumptions':
tuplefloat,
'co2_rescaling_scores':
tuplefloat,
'accelerations':
np_array,
'alternator_currents':
np_array,
'active_cylinders':
np_array,
'alternator_powers':
np_array,
_convert_str('alternator_statuses', 'service_battery_charging_statuses'):
np_array_int,
'auxiliaries_power_losses':
np_array,
'auxiliaries_torque_loss_factors':
tuplefloat,
'auxiliaries_torque_losses':
np_array,
_convert_str('battery_currents', 'service_battery_currents'):
np_array,
'clutch_tc_powers':
np_array,
'clutch_tc_speeds_delta':
np_array,
'co2_emissions':
np_array,
'after_treatment_speeds_delta':
np_array,
'engine_coolant_temperatures':
np_array,
'engine_powers_out':
np_array,
'engine_speeds_out':
np_array,
'engine_speeds_out_hot':
np_array,
'engine_starts':
np_array_bool,
'co2_normalization_references':
np_array,
'final_drive_powers_in':
np_array,
'final_drive_speeds_in':
np_array,
'final_drive_torques_in':
np_array,
'fuel_consumptions':
np_array,
'gear_box_efficiencies':
np_array,
'gear_box_powers_in':
np_array,
'gear_box_speeds_in':
np_array,
'gear_box_temperatures':
np_array,
'gear_box_torque_losses':
np_array,
'gear_box_torques_in':
np_array,
'gear_shifts':
np_array_bool,
'gears':
np_array_int,
'identified_co2_emissions':
np_array,
'motive_powers':
np_array,
'on_engine':
np_array_bool,
'clutch_phases':
np_array_bool,
'after_treatment_warm_up_phases':
np_array_bool,
'on_idle':
np_array_bool,
_convert_str('state_of_charges', 'service_battery_state_of_charges'):
np_array,
'times':
np_array_sorted,
'velocities':
np_array_greater_than_minus_one,
_compare_str('obd_velocities'):
np_array_greater_than_minus_one,
'wheel_powers':
np_array,
'wheel_speeds':
np_array,
'wheel_torques':
np_array,
}
try:
from co2mpas_driver.co2mpas import plugin_schema
schema = plugin_schema(schema)
except ImportError:
pass
schema = {Optional(k): Or(Empty(), v) for k, v in schema.items()}
schema[Optional(str)] = Or(_type(type=float, read=read), np_array)
if not read:
def _f(x):
return x is sh.NONE
schema = {k: And(v, Or(_f, Use(str))) for k, v in schema.items()}
return Schema(schema)
def __init__(self, item, schema = {}):
self.item = item
self.schema = Schema(schema)
str,
"summary":
str,
}
_updates_package_v2 = {
"available_updates": [{
"basearch": str,
"erratum": str,
"releasever": str,
"repository": str,
"package": str
}]
}
_vulnerabilities_response = {'cve_list': [str]}
errata_schema = Schema(_errata)
repos_schema = Schema(_repos)
cves_schema = Schema(_cves)
pkgs_top_schema = Schema(_pkgs_top)
pkgs_list_schema = Schema(_pkgs_list)
updates_top_schema = Schema(_updates_top)
updates_top_repolist_schema = Schema(_updates_top_repolist)
updates_top_basearch_schema = Schema(_updates_top_basearch)
updates_top_releasever_schema = Schema(_updates_top_releasever)
updates_top_all_schema = Schema(_updates_top_all)
updates_package_schema = Schema(_updates_package)
updates_package_schema_v2 = Schema(_updates_package_v2)
vulnerabilities_schema = Schema(_vulnerabilities_response)
def compose_schema(self):
# if a child class want to extend schema is should override this method
schema = self.get_schema()
return Schema(schema)
validator_memberATE = Schema({
Optional('idAccount'):
And(int),
'username':
And(str, Regex(r'^[A-Za-z0-9]{3,20}$')),
'password':
And(Use(str), lambda e: 254 > len(e) > 5),
'name':
And(
str,
Regex(
r'^[a-zA-ZÀ-ÿ\u00f1\u00d1]+(\s*[a-zA-ZÀ-ÿ\u00f1\u00d1]*)*[a-zA-ZÀ-ÿ\u00f1\u00d1]{2,150}'
)),
'lastname':
And(
str,
Regex(
r'^[a-zA-ZÀ-ÿ\u00f1\u00d1]+(\s*[a-zA-ZÀ-ÿ\u00f1\u00d1]*)*[a-zA-ZÀ-ÿ\u00f1\u00d1]{2,150}'
)),
'dateBirth':
And(str,
Regex(r'^((19|20)\d\d)/(0?[1-9]|1[012])/(0?[1-9]|[12][0-9]|3[01])$')),
'email':
And(Use(str), Regex(r'\b[\w.%+-]+@[\w.-]+\.[a-zA-Z]{2,6}\b'),
lambda e: 255 > len(e) > 4),
'idCity':
And(int),
Optional('memberATEStatus'):
And(Use(int), lambda s: 0 < s < 4),
Optional('memberATEType'):
And(Use(int), lambda t: 0 < t < 4),
})
def schema(cls) -> Schema:
"""
[$name]: str
Name as specified in the key of the parent app.
Added by system. DO NOT specify this yourself in the YAML files.
[roles]: List[str]
A list of roles for this service. You can use arbitrary strings and get services by their
assigned roles using :func:`~riptide.config.document.app.App.get_service_by_role`.
Some roles are pre-defined and have a special meaning:
*main*:
This service is the main service for the app.
Some commands will default to this service and the proxy URL for this service is shorter.
Usually services are accessible via ``http://<project_name>--<service_name>.<proxy_url>``,
however the main service is accessible via ``http://<project_name>.<proxy_url>``.
Only one service is allowed to have this role.
*src*:
The container of this service will have access to the source code of the application.
It's working directory will be set accordingly.
*db*:
This service is the primary database. A database driver has to be set (see key ``driver``).
This service is then used by Riptide for `database management </user_docs/db.html>`.
image: str
Docker Image to use
[command]: str
Command to run inside of the container. Default's to command defined in image.
.. warning:: Avoid quotes (", ') inside of the command, as those may lead to strange side effects.
[port]: int
HTTP port that the web service is accessible under. This port will be used by the proxy server to redirect
the traffic.
If the port is not specified, the service is not accessible via proxy server.
[logging]
Logging settings. All logs will be placed inside the "_riptide/logs" directory.
[stdout]: bool
Whether or not to log the stdout stream of the container's main command. Default: false
[stderr]: bool
Whether or not to log the stderr stream of the container's main command. Default: false
[paths]
{key}: str
Additional text files to mount into the logging directory. Keys are filename's on host (without .log)
and values are the paths inside the containers.
[commands]
{key}: str
Additional commands to start inside the container. Their stdout and stderr will be logged to the file
specified by the key.
[pre_start]: List[str]
List of commands to run, before the container starts. They are run sequentially.
The startup will wait for the commands to finish. Exit codes (failures) are ignored.
Each of these commands is run in a separate container based on the service specification. Each command
is run in a "sh" shell.
[post_start]: List[str]
List of commands to run, after container starts. They are run sequentially.
The startup will wait for the commands to finish. Exit codes (failures) are ignored.
Each of these command's is run inside the service container (equivalent of ``docker exec``).
Each command is run in a “sh” shell.
[environment]
Additional environment variables
{key}: str
Key is the name of the variable, value is the value.
[working_directory]: str
Working directory for the service, either
- absolute, if an absolute path is given
- relative to the src specified in the project, if the role "src" is set.
- relative to the default working directory from the image, if the role is not set.
Defaults to ``.``.
[config]
Additional configuration files to mount. These files are NOT directly mounted.
Instead they are processed and the resulting file is mounted.
All variables and variable helpers inside the configuration file are processed.
Example configuration file (demo.ini)::
[demo]
domain={{domain()}}
project_name={{parent().parent().name}}
Resulting file that will be mounted::
[demo]
domain=projectname.riptide.local
project_name=projectname
{key}
from: str
Path to the configuration file, relative to any YAML file that was used
to load the project (including "riptide.yml" and all yaml files used inside the repository;
all are searched). Absolute paths are not allowed.
to: str
Path to store the configuration file at, relative to working directory of container or absolute.
[additional_ports]
Additional TCP and/or UDP ports that will be made available on the host system.
For details see section in
`user guide </user_docs/7_working_with_riptide.html#access-other-tcp-udp-ports>`_.
{key}
title: str
Title for this port, will be displayed in ``riptide status``
container: int
Port number inside the container
host_start: int
First port number on host that Riptide will try to reserve, if the
port is already occupied, the next one will be used. This port
will be reserved and permanently used for this service after that.
[additional_volumes]
Additional volumes to mount into the container for this command.
{key}
host: str
Path on the host system to the volume. Avoid hardcoded absolute paths.
container: str
Path inside the container (relative to src of Project or absolute).
[mode]: str
Whether to mount the volume read-write ("rw", default) or read-only ("ro").
[type]: str
Whether this volume is a "directory" (default) or a "file". Only checked if the file/dir does
not exist yet on the host system. Riptide will then create it with the appropriate type.
[driver]
The database driver configuration, set this only if the role "db" is set.
Detailed documentation can be found in a `separate section </config_docs/database_drivers.html>`_.
name: str
Name of the database driver, must be installed.
config: ???
Specification depends on the database driver.
[run_as_current_user]: bool
Whether to run as the user using riptide (True)
or image default (False).
Default: True
Riptide will always create the user and group, matching the host user and group,
inside the container on startup, regardless of this setting.
Some images don't support switching the user, set this to false then.
Please note that, if you set this to false and also specify the role 'src', you may run
into permission issues.
[allow_full_memlock]: bool
Whether to set memlock ulimit to -1:-1 (soft:hard).
This is required for some database services, such as Elasticsearch.
Note that engines might ignore this setting, if they don't support it.
Default: False
**Example Document:**
.. code-block:: yaml
service:
image: node:10
roles:
- main
- src
command: 'node server.js'
port: 1234
logging:
stdout: true
stderr: false
paths:
one: '/foo/bar'
commands:
two: 'varnishlog'
pre_start:
- "echo 'command 1'"
- "echo 'command 2'"
post_start:
- "echo 'command 3'"
- "echo 'command 4'"
environment:
SOMETHING_IMPORTANT: foo
config:
one:
from: ci/config.yml
to: app_config/config.yml
working_directory: www
additional_ports:
one:
title: MySQL Port
container: 3306
host_start: 3006
additional_volumes:
temporary_files:
host: '{{ get_tempdir() }}'
container: /tmp
"""
return Schema(
{
Optional('$ref'): str, # reference to other Service documents
Optional('$name'): str, # Added by system during processing parent app.
Optional('roles'): [str],
'image': str,
Optional('command'): str,
Optional('port'): int,
Optional('logging'): {
Optional('stdout'): bool,
Optional('stderr'): bool,
Optional('paths'): {str: str},
Optional('commands'): {str: str}
},
Optional('pre_start'): [str],
Optional('post_start'): [str],
Optional('environment'): {str: str},
Optional('config'): {
str: {
'from': str,
'$source': str, # Path to the document that "from" references. Is added durinng loading of service
'to': str
}
},
# Whether to run as the user using riptide (True) or image default (False). Default: True
# Limitation: If false and the image USER is not root,
# then a user with the id of the image USER must exist in /etc/passwd of the image.
Optional('run_as_current_user'): bool,
# DEPRECATED. Inverse of run_as_current_user if set
Optional('run_as_root'): bool,
# Whether to create the riptide user and group, mapped to current user. Default: False
Optional('dont_create_user'): bool,
Optional('working_directory'): str,
Optional('additional_ports'): {
str: {
'title': str,
'container': int,
'host_start': int
}
},
Optional('additional_volumes'): {
str: {
'host': str,
'container': str,
Optional('mode'): Or('rw', 'ro'), # default: rw - can be rw/ro.
Optional('type'): Or('directory', 'file') # default: directory
}
},
Optional('allow_full_memlock'): bool,
# db only
Optional('driver'): {
'name': str,
'config': any # defined by driver
}
}
)
'--font-size':
And(Use(int), lambda n: 1 < n, error="--font-size must be greater than 1"),
Optional('--skip-merge'):
bool,
Optional('--invert'):
bool,
'--ortho': [
Or(
None,
And(
Use(literal_eval), lambda x: len(x) == 3 and 0 <= x[0] and 0 <=
x[1] and 0 <= x[2]))
]
}
schema = Schema(schema_def)
def make_orthogonals(ortho, img_dir, padding, pixels_per_micron):
x_idx = str(ortho[0])
y_idx = str(ortho[1])
z_idx = str(ortho[2])
xy_paths = list((img_dir / "xy").glob("*_z*" + z_idx + "_c*.tif"))
xz_paths = list((img_dir / "xz").glob("*_z*" + y_idx + "_c*.tif"))
yz_paths = list((img_dir / "yz").glob("*_z*" + x_idx + "_c*.tif"))
xy_paths.sort()
xz_paths.sort()
yz_paths.sort()
## Get images
def main(argv, session):
args = docopt(__doc__, argv=argv)
# Validate args.
s = Schema({
six.text_type:
bool,
'<identifier>':
list,
'--modify':
list,
'--append':
list,
'--append-list':
list,
'--remove':
list,
'--spreadsheet':
Or(
None,
And(lambda f: os.path.exists(f),
error='<file> should be a readable file or directory.')),
'--target':
Or(None, str),
'--priority':
Or(None, Use(int, error='<priority> should be an integer.')),
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)),
file=sys.stderr)
sys.exit(1)
formats = set()
responses = []
for i, identifier in enumerate(args['<identifier>']):
item = session.get_item(identifier)
# Check existence of item.
if args['--exists']:
if item.exists:
responses.append(True)
print('{0} exists'.format(identifier))
else:
responses.append(False)
print('{0} does not exist'.format(identifier), file=sys.stderr)
if (i + 1) == len(args['<identifier>']):
if all(r is True for r in responses):
sys.exit(0)
else:
sys.exit(1)
# Modify metadata.
elif args['--modify'] or args['--append'] or args['--append-list'] \
or args['--remove']:
if args['--modify']:
metadata_args = args['--modify']
elif args['--append']:
metadata_args = args['--append']
elif args['--append-list']:
metadata_args = args['--append-list']
if args['--remove']:
metadata_args = args['--remove']
try:
metadata = get_args_dict(metadata_args)
except ValueError:
print(
"error: The value of --modify, --remove, --append or --append-list "
"is invalid. It must be formatted as: --modify=key:value",
file=sys.stderr)
sys.exit(1)
if args['--remove']:
responses.append(remove_metadata(item, metadata, args))
else:
responses.append(modify_metadata(item, metadata, args))
if (i + 1) == len(args['<identifier>']):
if all(r.status_code == 200 for r in responses):
sys.exit(0)
else:
for r in responses:
if r.status_code == 200:
continue
# We still want to exit 0 if the non-200 is a
# "no changes to xml" error.
elif 'no changes' in r.content.decode('utf-8'):
continue
else:
sys.exit(1)
# Get metadata.
elif args['--formats']:
for f in item.get_files():
formats.add(f.format)
if (i + 1) == len(args['<identifier>']):
print('\n'.join(formats))
# Dump JSON to stdout.
else:
metadata = json.dumps(item.item_metadata)
print(metadata)
# Edit metadata for items in bulk, using a spreadsheet as input.
if args['--spreadsheet']:
if not args['--priority']:
args['--priority'] = -5
with io.open(args['--spreadsheet'], 'rU', newline='',
encoding='utf-8') as csvfp:
spreadsheet = csv.DictReader(csvfp)
responses = []
for row in spreadsheet:
if not row['identifier']:
continue
item = session.get_item(row['identifier'])
if row.get('file'):
del row['file']
metadata = dict((k.lower(), v) for (k, v) in row.items() if v)
responses.append(modify_metadata(item, metadata, args))
if all(r.status_code == 200 for r in responses):
sys.exit(0)
else:
for r in responses:
if r.status_code == 200:
continue
# We still want to exit 0 if the non-200 is a
# "no changes to xml" error.
elif 'no changes' in r.content.decode('utf-8'):
continue
else:
sys.exit(1)
def main():
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
fh = logging.FileHandler('dht.log')
fh.setLevel(logging.DEBUG)
logging.basicConfig(
handlers=[ch, fh],
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.DEBUG)
logger = logging.getLogger(__name__)
logger.info('dht')
parser = argparse.ArgumentParser(
description=
'Publishes humidity and temperature data from Adafruit DHT series sensors'
)
parser.add_argument('-c',
'--config',
help='path to config file',
default='config.ini')
args = parser.parse_args()
config = configparser.ConfigParser()
config.read(args.config)
if not config.read(args.config):
logger.warn('failed to read config file, path={}'.format(args.config))
config_schema = Schema(
{
'dht': {
'type': Use(int),
'pin': Use(int),
Optional('read_interval', default=60): Use(int)
},
Optional('mqtt'): {
Optional('broker_host', default='localhost'): str,
Optional('broker_port', default='1883'): Use(int)
}
},
ignore_extra_keys=True)
config_schema.validate(config._sections)
location = config.get('mqtt', 'location', fallback='unknown')
humidity_topic = '/'.join((location, 'humidity'))
temperature_topic = '/'.join((location, 'temperature'))
# Create sensor
sensor_type = config.getint('dht', 'type')
pin = config.getint('dht', 'pin')
sensor = DhtSensor(sensor_type, pin)
def on_connect(client, userdata, flags, rc):
logger.info('connected to mqtt broker')
# Connect to MQTT broker
client = mqtt.Client()
client.on_connect = on_connect
host = config.get('mqtt', 'broker_host', fallback='localhost')
port = config.getint('mqtt', 'broker_port', fallback=1883)
client.connect(host, port)
client.loop_start()
# Publish sensor readings
read_interval = config.getint('dht', 'read_interval', fallback=60)
logger.debug('read_interval={}'.format(read_interval))
scheduler = sched.scheduler(time.time, time.sleep)
def read():
scheduler.enter(read_interval, 1, read)
reading = sensor.read_retry()
if not reading.is_valid():
return
logger.debug(
'publishing humidity={} and temperature={} for location={}'.format(
reading.humidity, reading.temperature, location))
client.publish(humidity_topic, reading.humidity)
client.publish(temperature_topic, reading.temperature)
read()
scheduler.run()
SCHEMA = Schema([
# The accounting table stores aggregated usage stats.
# The hash and sort keys are strings consisting of 'prefix:<optional id>'
#
# Accounting categories:
# - Per viewpoint: hash_key='vs:<vp_id>'
# Aggregate sizes/counts per viewpoint, keyed by the viewpoint
# id. Sort keys fall into three categories:
# - owned by: 'ow:<user_id>' only found in default viewpoint.
# - shared by: 'sb:<user_id>' in shared viewpoint, sum of all photos
# in episodes owned by 'user_id'
# - visible to: 'vt' in shared viewpoint, sum of all photos. not keyed
# by user. a given user's "shared with" stats will be 'vt - sb:<user_id>',
# but we do not want to keep per-user shared-by stats.
# - Per user: hash_key='us:<user_id>'
# Aggregate sizes/counts per user, keyed by user id. Sort keys are:
# - owned by: 'ow' sum of all photos in default viewpoint
# - shared by: 'sb' sum of all photos in shared viewpoints and episodes owned by this user
# - visible to: 'vt' sum of all photos in shared viewpoint (includes 'sb'). to get the
# real count of photos shared with this user but not shared by him, compute 'vt - sb'
#
# 'op_ids' holds a list of previously-applied operation IDs. This is an attempt to
# make increments idempotent with replays. The list is a comma-separated string of
# operation ids (sometimes suffixed with a viewpoint ID), in the order in which they were
# applied. We keep a maximum of Accounting._MAX_APPLIED_OP_IDS.
#
# Currently, all columns are used by each accounting category.
Table(ACCOUNTING,
'at',
read_units=100,
write_units=10,
columns=[
HashKeyColumn('hash_key', 'hk', 'S'),
RangeKeyColumn('sort_key', 'sk', 'S'),
Column('num_photos', 'np', 'N'),
Column('tn_size', 'ts', 'N'),
Column('med_size', 'ms', 'N'),
Column('full_size', 'fs', 'N'),
Column('orig_size', 'os', 'N'),
Column('op_ids', 'oi', 'S')
]),
# Activities are associated with a viewpoint and contain a record of
# all high-level operations which have modified the structure of the
# viewpoint in some way. For more details, see activity.py. The
# activity_id attribute is a composite of information gleaned from
# current operation: (reverse timestamp, user_id, op_id). The content
# of the activity is a JSON-encoded ACTIVITY structure, as defined in
# json_schema.py. 'update_seq' is set to the value of the viewpoint's
# 'update_seq' attribute after it was incremented during creation of
# the activity.
Table(ACTIVITY,
'ac',
read_units=100,
write_units=10,
columns=[
HashKeyColumn('viewpoint_id', 'vi', 'S'),
RangeKeyColumn('activity_id', 'ai', 'S'),
Column('user_id', 'ui', 'N', read_only=True),
Column('timestamp', 'ti', 'N', read_only=True),
Column('update_seq', 'us', 'N'),
Column('name', 'na', 'S', read_only=True),
Column('json', 'js', 'S', read_only=True)
]),
# Admin table. This table lists all users with access to admin and support functions.
# Entries are created by the otp script, with 'rights' being a set of roles (eg: 'root' or 'support').
# Admin users are not currently linked to viewfinder users.
Table(ADMIN_PERMISSIONS,
'ad',
read_units=10,
write_units=10,
columns=[
HashKeyColumn('username', 'un', 'S'),
Column('rights', 'ri', 'SS')
]),
# Timestamped information for various entities. The entity hash key should be of the form: <type>:<id>.
# eg: us:112 (for user with ID 112).
# sort_key: base64 hex encoded timestamp + type
# Type is a string representing the type of analytics entry. See db/analytics.py for details.
# Payload is an optional payload attached to the entry. Its format depends on the type of entry.
Table(ANALYTICS,
'an',
read_units=10,
write_units=10,
columns=[
HashKeyColumn('entity', 'et', 'S'),
RangeKeyColumn('sort_key', 'sk', 'S'),
Column('timestamp', 'ti', 'N'),
Column('type', 'tp', 'S'),
Column('payload', 'pl', 'S')
]),
# Key is composite of (viewpoint_id, comment_id), which sorts all
# comments by ascending timestamp within each viewpoint. 'user_id'
# is the user that created the comment. At this time, 'asset_id'
# can be:
# 1. Absent: The comment is not linked to any other asset.
# 2. Comment id: The comment is a response to another comment.
# 3. Photo id: The comment is a comment on a photo.
#
# 'timestamp' records the time that the comment was originally
# created. 'message' is the actual comment text.
IndexedTable(COMMENT,
'cm',
read_units=200,
write_units=20,
columns=[
HashKeyColumn('viewpoint_id', 'vi', 'S'),
RangeKeyColumn('comment_id', 'ci', 'S'),
Column('user_id', 'ui', 'N', read_only=True),
Column('asset_id', 'ai', 'S', read_only=True),
Column('timestamp', 'ti', 'N'),
Column('message', 'me', 'S')
]),
# Key is composite of (user_id, sort_key)
# sort_key: base64 hex encoded timestamp + contact_id
# contact_id: contact_source + ':' + hash (base64 encoded) of CONTACT data: name, given_name, family_name,
# rank, and identities_properties columns.
# contact_source: 'fb', 'gm', 'ip', or 'm' (for, respectively, Facebook, GMail, iPhone, and Manual sources)
# timestamp column should always match the timestamp encoded prefix of the sort_key.
# identities: set of canonicalized identity strings: Email:<email-address>, Phone:<phone>, Facebook:<fb-graph-id>
# These reference identities in the IDENTITY table. This column exists so that contacts can be queried by
# identity. Note: duplicates info that's contained in the identities_properties column.
# identities_properties: json formatted list of identities each with an optional label such as 'mobile', 'work',
# etc... This list preserves the order in which the identities were upload by (or fetched from) a
# contact source. These identities may not be in canonicalized form, but it must be possible to canonicalize
# them.
# labels: 'removed' indicates that the contact is in a removed state. This surfaces the removed state of
# contacts to clients through invalidation notifications. These contacts will be filtered out for down-level
# client queries.
IndexedTable(CONTACT,
'co',
read_units=50,
write_units=120,
columns=[
HashKeyColumn('user_id', 'ui', 'N'),
RangeKeyColumn('sort_key', 'sk', 'S'),
Column('timestamp', 'ti', 'N'),
Column('contact_id',
'ci',
'S',
SecondaryIndexer(),
read_only=True),
Column('contact_source', 'cs', 'S', read_only=True),
SetColumn('labels', 'lb', 'SS'),
SetColumn('identities',
'ids',
'SS',
SecondaryIndexer(),
read_only=True),
Column('name', 'na', 'S', read_only=True),
Column('given_name', 'gn', 'S', read_only=True),
Column('family_name', 'fn', 'S', read_only=True),
Column('rank', 'ra', 'N', read_only=True),
JSONColumn('identities_properties', 'ip', read_only=True)
]),
# Device information. Key is a composite of user id and a 32-bit
# integer device id (allocated via the id-allocation table). Each
# device is a source of photos. The device id comprises the first
# 32 bits of the photo id. The last 32 bits are sequentially
# allocated by the device (in the case of mobile), or via an
# atomic increment of 'id_seq' (in the case of the web).
#
# 'last_access' and 'push_token' are set on device registration
# and each time the application is launched (in the case of the
# mobile app). 'push_token' is indexed to allow device lookups in
# response to feedback from provider push-notification services.
# 'alert_user_id' is a sparse column index, used to quickly find
# all devices for a user that need to be alerted.
#
# Device ID of 0 is reserved to mean local to an individual device.
#
# Example Apple push token: "apns:oYJrenW5JsH42r1eevgq3HhC6bhXL3OP0SqHkOeo/58="
IndexedTable(DEVICE,
'de',
read_units=25,
write_units=5,
columns=[
HashKeyColumn('user_id', 'ui', 'N'),
RangeKeyColumn('device_id', 'di', 'N',
SecondaryIndexer()),
Column('timestamp', 'ti', 'N'),
Column('name', 'na', 'S'),
Column('version', 've', 'S'),
Column('platform', 'pl', 'S'),
Column('os', 'os', 'S'),
Column('last_access', 'la', 'N'),
Column('alert_user_id', 'aui', 'N', SecondaryIndexer()),
Column('push_token', 'pt', 'S', SecondaryIndexer()),
Column('language', 'lg', 'S'),
Column('country', 'co', 'S')
]),
# Key is episode-id. Episodes are indexed for full-text search on
# episode title and description, and lookup of all episodes for a user.
# Due to a rename, the Episode table is called Event in the database.
IndexedTable(EPISODE,
'ev',
read_units=200,
write_units=10,
name_in_db="Event",
columns=[
HashKeyColumn('episode_id', 'ei', 'S'),
Column('parent_ep_id',
'pa',
'S',
SecondaryIndexer(),
read_only=True),
Column('user_id',
'ui',
'N',
SecondaryIndexer(),
read_only=True),
Column('viewpoint_id',
'vi',
'S',
SecondaryIndexer(),
read_only=True),
Column('publish_timestamp', 'pu', 'N'),
Column('timestamp', 'cr', 'N'),
Column('title', 'ti', 'S'),
Column('description', 'de', 'S'),
LatLngColumn('location', 'lo'),
PlacemarkColumn('placemark', 'pl')
]),
# Sorts all viewpoints followed by a user in order of the date of
# on which the last activity was added. Viewpoints updated on the
# same day are in undefined order. Sort is in descending order, with
# viewpoints most recently updated coming first. The query_followed
# method returns results in this ordering. Note that paging may result
# in missed followed records, as updates to a viewpoint may cause the
# corresponding record to "jump ahead" in time past the current paging
# bookmark. 'date_updated' is a timestamp truncated to a day boundary.
# 'sort_key' is a concatenation of the 'date_updated' field and the
# viewpoint id.
IndexedTable(FOLLOWED,
'fd',
read_units=200,
write_units=10,
columns=[
HashKeyColumn('user_id', 'ui', 'N'),
RangeKeyColumn('sort_key', 'sk', 'S'),
Column('date_updated', 'du', 'N'),
Column('viewpoint_id', 'vi', 'S', read_only=True)
]),
# Key is a composite of (user-id, viewpoint-id). The 'labels' set
# specifies the features of the relation between the user and
# viewpoint: ('admin', 'contribute'). 'adding_user_id' contains the id
# of the user who added this follower, and 'timestamp' the time at which
# the follower was added. 'viewed_seq' is the sequence number of the last
# viewpoint update that has been 'read' by this follower. The last
# viewpoint update is tracked by the 'update_seq' attribute on Viewpoint.
IndexedTable(FOLLOWER,
'fo',
read_units=400,
write_units=10,
columns=[
HashKeyColumn('user_id', 'ui', 'N'),
RangeKeyColumn('viewpoint_id', 'vi', 'S',
SecondaryIndexer()),
Column('timestamp', 'ti', 'N'),
Column('adding_user_id', 'aui', 'N'),
SetColumn('labels', 'la', 'SS'),
Column('viewed_seq', 'vs', 'N')
]),
# Key is composite of user-id / friend-id. "colocated_shares" and
# "total_shares" are decaying stats that track the number of photo
# opportunities where sharing occurred. 'last_colocated' and
# 'last_share' are timestamps for computing decay. Friend status is
# one of {friend,blocked,muted}.
Table(FRIEND,
'fr',
read_units=50,
write_units=10,
columns=[
HashKeyColumn('user_id', 'ui', 'N'),
RangeKeyColumn('friend_id', 'fi', 'N'),
Column('name', 'na', 'S'),
Column('nickname', 'nn', 'S'),
Column('colocated_shares', 'cs', 'N'),
Column('last_colocated', 'lc', 'N'),
Column('total_shares', 'ts', 'N'),
Column('last_share', 'ls', 'N'),
Column('status', 'st', 'S')
]),
# Tracks the number of incorrect attempts that have been made to guess some
# secret, such as a password or an access code. 'guess_id' is of the form
# <type>:<id>, where <type> is one of these:
#
# url:<group-id> - Limits number of attempts that can be made to guess a
# valid ShortURL within any particular 24-hour period.
#
# pw:<user-id> - Limits number of attempts that can be made to guess a
# particular user's password within any particular 24-hour
# period.
#
# em:<user-id> - Limits number of attempts that can be made to guess
# access tokens e-mailed to a particular user within any
# particular 24-hour period.
#
# ph:<user-id> - Limits number of attempts that can be made to guess
# access tokens sent in SMS messages to a user within any
# particular 24-hour period.
#
# The 'guesses' field tracks the number of incorrect guesses that have been
# made so far. The 'expires' field stores the time at which the guesses count
# can be reset to 0.
Table(GUESS,
'gu',
read_units=50,
write_units=10,
columns=[
HashKeyColumn('guess_id', 'gi', 'S'),
Column('expires', 'ex', 'N'),
Column('guesses', 'gu', 'N')
]),
# Key is a composite of (group_key, timestamp), where group_key is the
# same key used to collect machine metrics in the metrics table. The
# intention is that for each metrics group_key, a single health report
# will be generated summarizing problems across all machines in that group.
#
# Alerts and Warnings are string sets which describe any problems detected
# from the metrics information. If no problems are detected, this record
# will be sparse.
Table(HEALTH_REPORT,
'hr',
read_units=10,
write_units=5,
columns=[
HashKeyColumn('group_key', 'gk', 'S'),
RangeKeyColumn('timestamp', 'ts', 'N'),
SetColumn('alerts', 'as', 'SS'),
SetColumn('warnings', 'ws', 'SS')
]),
# Key is ID type (e.g. op-id, photo-id, user-id, episode-id).
Table(ID_ALLOCATOR,
'ia',
read_units=10,
write_units=10,
columns=[
HashKeyColumn('id_type', 'it', 'S'),
Column('next_id', 'ni', 'N')
]),
# Key is identity. User-id is indexed to provide quick queries for the
# list of identities associated with a viewfinder account. The token
# allows access to external resources associated with the identity.
# 'last_fetch' specifies the last time that the contacts were
# fetched for this identity. 'authority' is one of ('Facebook', 'Google'
# 'Viewfinder', etc.) and identifies the trusted authentication authority.
#
# The complete set of attributes (if any) returned when an
# identity was authenticated is stored as a json-encoded dict in
# 'json_attrs'. Some of these may be taken to populate the
# demographic and informational attributes of the User table.
#
# The 'access_token' and 'refresh_token' fields store any tokens used to
# access the authority, with 'expires' tracking the lifetime of the
# token.
#
# The 'auth_throttle' field limits the number of auth email/sms messages
# that can be sent within a certain period of time.
IndexedTable(
IDENTITY,
'id',
read_units=50,
write_units=10,
columns=[
HashKeyColumn('key', 'ke', 'S'),
Column('user_id', 'ui', 'N', SecondaryIndexer()),
JSONColumn('json_attrs', 'ja'),
Column('last_fetch', 'lf', 'N'),
Column('authority', 'au', 'S'),
Column('access_token', 'at', 'S'),
Column('refresh_token', 'rt', 'S'),
Column('expires', 'ex', 'N'),
JSONColumn('auth_throttle', 'th'),
# TODO(Andy): Remove these attributes, as they are now deprecated.
Column('access_code', 'ac', 'S', SecondaryIndexer()),
Column('expire_code', 'xc', 'N'),
Column('token_guesses', 'tg', 'N'),
Column('token_guesses_time', 'gt', 'N')
]),
# A lock is acquired in order to control concurrent access to
# a resource. The 'lock_id' is a composite of the type of the
# resource and its unique id. The 'owner_id' is a string that
# uniquely identifies the holder of the lock. 'resource_data'
# is resource-specific information that is provided by the
# owner and stored with the lock. The 'expiration' is the time
# (UTC) at which the lock is assumed to have been abandoned by
# the owner and can be taken over by another owner.
#
# 'acquire_failures' tracks the number of times other agents
# tried to acquire the lock while it was held.
Table(LOCK,
'lo',
read_units=50,
write_units=10,
columns=[
HashKeyColumn('lock_id', 'li', 'S'),
Column('owner_id', 'oi', 'S'),
Column('expiration', 'ex', 'N'),
Column('acquire_failures', 'af', 'N'),
Column('resource_data', 'rd', 'S')
]),
# Metrics represent a timestamped payload of performance metrics
# from a single machine running viewfinder. The metrics key is a
# composite of (group_key, sort_key). The payload column is a serialized
# dictionary describing the performance metrics that were captured from
# the machine.
#
# The group_key for a metric is intended to organize metrics by the way
# they are queried. For instance, a group key might contain all
# metrics for all machines in an EC2 region, or a more specific division
# than that.
#
# The sort_key is a composite of the timestamp and machine id - the
# intention is that records will be queried by timestamp, while machine_id
# is simply included in the key to differentiate records with the same
# timestamp from different machines.
IndexedTable(METRIC,
'mt',
read_units=50,
write_units=10,
columns=[
HashKeyColumn('group_key', 'gk', 'S'),
RangeKeyColumn('sort_key', 'sk', 'S'),
Column('machine_id', 'mi', 'S', SecondaryIndexer()),
Column('timestamp', 'ts', 'N'),
Column('payload', 'p', 'S')
]),
# Notifications are messages to deliver to devices hosting the
# viewfinder client, whether mobile, desktop, web application or
# otherwise. Key is a composite of (user-id and allocated
# notification id--taken from user's uu_id sequence). Other
# fields record the name, id, and timestamp of the operation that
# resulted in the notification, as well as the user and device
# that started it. The badge attribute records the value of the
# "push badge" on client devices at the time that notification
# was recorded. The invalidate attribute is a JSON-encoded
# INVALIDATE structure, as defined in json_schema.py.
Table(NOTIFICATION,
'no',
read_units=50,
write_units=10,
columns=[
HashKeyColumn('user_id', 'ui', 'N'),
RangeKeyColumn('notification_id', 'ni', 'N'),
Column('name', 'na', 'S'),
Column('timestamp', 'ti', 'N'),
Column('sender_id', 'si', 'N'),
Column('sender_device_id', 'sd', 'N'),
Column('badge', 'ba', 'N'),
Column('invalidate', 'in', 'S'),
Column('op_id', 'oi', 'S'),
Column('viewpoint_id', 'vi', 'S'),
Column('update_seq', 'us', 'N'),
Column('viewed_seq', 'vs', 'N'),
Column('activity_id', 'ai', 'S')
]),
# Operations are write-ahead logs of mutating server
# requests. These requests need to be persisted so that they can
# be retried on server failure. They often involve multiple
# queries / updates to different database tables and/or rows, so a
# partially completed operation could leave the database in an
# inconsistent state. Each operation must be idempotent, as
# failing servers may cause retries. The actual operation is
# stored JSON-encoded in 'json'. This is often the original HTTP
# request, though in some cases, the JSON from the HTTP request
# is augmented with additional information, such as pre-allocated
# photo, user or device IDs.
#
# 'quarantine' indicates that if the operation fails, it
# should not prevent further operations for the same user from
# processing.
#
# 'checkpoint' stores progress information with the operation. If the
# operation is restarted, it can use this information to skip over
# steps it's already completed. The progress information is operation-
# specific and is not used in any way by the operation framework itself.
#
# 'triggered_failpoints' is used for testing operation idempotency. It
# contains the set of failpoints which have already been triggered for
# this operation and need not be triggered again.
Table(OPERATION,
'op',
read_units=50,
write_units=50,
columns=[
HashKeyColumn('user_id', 'ui', 'N'),
RangeKeyColumn('operation_id', 'oi', 'S'),
Column('device_id', 'di', 'N'),
Column('method', 'me', 'S'),
Column('json', 'js', 'S'),
Column('timestamp', 'ti', 'N'),
Column('attempts', 'at', 'N'),
Column('backoff', 'bo', 'N'),
Column('first_failure', 'ff', 'S'),
Column('last_failure', 'lf', 'S'),
Column('quarantine', 'sf', 'N'),
JSONColumn('checkpoint', 'cp'),
JSONColumn('triggered_failpoints', 'fa')
]),
# Key is photo-id. Photo id is composed of 32 bits of time in the
# high 32 bits, then 32 bits of device id, then 32 bits of
# monotonic photo id, unique to the device. The full 96 bits are
# base-64 hex encoded into 128 bits. Photos can have a parent
# photo-id, which refers back to an original photo if this is a
# copy. Copies are made when filters are applied to photos. The
# client_data string is a JSON-encoded dict of opaque
# client-supplied key-value pairs.
#
# The 'share_seq_no' attribute is incremented every time the shares
# for a photo are modified. It provides for efficient queries from
# clients meant to determine the list of friends with viewing
# privileges
#
# Sizes for tn, med, full, orig are file sizes in bytes for thumnail,
# medium, full and original images respectively.
#
# The 'new_assets' attribute is temporary and there to support rename
# of image asset files from underscore to period suffixes. It contains
# the value 'copied' if the asset files have been duplicated and
# 'deleted' if the original asset files have been verified as copied
# and removed.
# TODO(spencer): remove this once we have completely migrated the photo
# data.
#
# 'client_data' is deprecated; use USER_PHOTO instead.
IndexedTable(
PHOTO,
'ph',
read_units=400,
write_units=25,
columns=[
HashKeyColumn('photo_id', 'pi', 'S'),
Column('parent_id', 'pa', 'S', SecondaryIndexer(), read_only=True),
Column('episode_id', 'ei', 'S', read_only=True),
Column('user_id', 'ui', 'N', read_only=True),
Column('aspect_ratio', 'ar', 'N'),
Column('content_type', 'ct', 'S', read_only=True),
Column('timestamp', 'ti', 'N'),
Column('tn_md5', 'tm', 'S'),
Column('med_md5', 'mm', 'S'),
Column('orig_md5', 'om', 'S', SecondaryIndexer()),
Column('full_md5', 'fm', 'S', SecondaryIndexer()),
Column('tn_size', 'ts', 'N'),
Column('med_size', 'ms', 'N'),
Column('full_size', 'fs', 'N'),
Column('orig_size', 'os', 'N'),
LatLngColumn('location', 'lo'),
PlacemarkColumn('placemark', 'pl'),
Column('caption', 'ca', 'S',
FullTextIndexer(metaphone=Indexer.Option.YES)),
Column('link', 'li', 'S'),
Column('thumbnail_data', 'da', 'S'),
Column('share_seq', 'ss', 'N'),
JSONColumn('client_data', 'cd'), # deprecated
Column('new_assets', 'na', 'S')
]),
# Key is composite of (episode-id, photo_id). When photos are
# posted/reposted to episodes, a post relation is created. This
# allows the same photo to be included in many episodes. The
# 'labels' attribute associates a set of properties with the
# post.
IndexedTable(POST,
'po',
read_units=200,
write_units=25,
columns=[
HashKeyColumn('episode_id', 'ei', 'S'),
RangeKeyColumn('photo_id', 'sk', 'S'),
SetColumn('labels', 'la', 'SS')
]),
# Key is composite of (settings_id, group_name). 'settings_id' is the
# id of the entity to which the settings apply. For example, user account
# settings have ids like 'us:<user_id>'. 'group_name' can be used if
# a particular entity has large numbers of settings that need to be
# sub-grouped.
#
# All other columns are a union of all columns defined by all the groups
# stored in the table. The Settings class has support for only exposing
# columns that apply to a particular group, in order to avoid accidental
# use of a column belonging to another settings group.
Table(
SETTINGS,
'se',
read_units=100,
write_units=10,
columns=[
HashKeyColumn('settings_id', 'si', 'S'),
RangeKeyColumn('group_name', 'gn', 'S'),
# User account group settings.
Column('user_id', 'ui', 'N'),
Column('email_alerts', 'ea', 'S'),
Column('sms_alerts', 'sa', 'S'),
Column('push_alerts', 'pa', 'S'),
Column('marketing', 'mk', 'S'),
Column('sms_count', 'sc', 'N'),
SetColumn('storage_options', 'so', 'SS')
]),
# Key is composite of (group_id, random_key). 'group_id' partitions the URL
# space into groups, so that URL's generated for one group have no overlap
# with those for another group. The group id will be appended as the URL
# path, so it may contain '/' characters, and should be URL safe. The
# 'timestamp' column tracks the time at which the ShortURL was created.
#
# The 'json' column contains arbitrary named arguments that are associated
# with the short URL and are pased to the request handler when the short
# URL is used. The 'expires' field bounds the time during which the URL
# can be used.
Table(SHORT_URL,
'su',
read_units=25,
write_units=5,
columns=[
HashKeyColumn('group_id', 'gi', 'S'),
RangeKeyColumn('random_key', 'rk', 'S'),
Column('timestamp', 'ti', 'N'),
Column('expires', 'ex', 'N'),
JSONColumn('json', 'js')
]),
# The subscription table contains a user's current
# subscription(s). A subscription is any time-limited
# modification to a user's privileges, such as increased storage
# quota.
#
# This table contains a log of all transactions that have affected
# a user's subscriptions. In most cases only the most recent
# transaction for a given subscription_id is relevant - it is the
# most recent renewal.
#
# "product_type" is the type of subscription, such as "storage".
# Quantity is a interpreted based on the product_type; for the
# "storage" product it is a number of GB. "payment_type"
# indicates how the subscription was paid for (e.g. "itunes" or
# "referral_bonus"). The contents of "extra_info" and
# "renewal_data" depend on the payment type. "extra_info" is a
# dict of additional information related to the transaction, and
# "renewal_data" is an opaque blob that is used to renew a subscription
# when it expires.
Table(
SUBSCRIPTION,
'su',
read_units=10,
write_units=5,
columns=[
HashKeyColumn('user_id', 'ui', 'N'),
RangeKeyColumn('transaction_id', 'tr', 'S'),
Column('subscription_id', 'su', 'S', read_only=True),
# timestamps should be read-only too, once we fix
# problems with read-only floats.
Column('timestamp', 'ti', 'N'),
Column('expiration_ts', 'ex', 'N'),
Column('product_type', 'pr', 'S', read_only=True),
Column('quantity', 'qu', 'N'),
Column('payment_type', 'pt', 'S', read_only=True),
JSONColumn('extra_info', 'ei'),
Column('renewal_data', 'pd', 'S', read_only=True)
]),
# Key is user id. 'webapp_dev_id' is assigned on creation, and
# serves as a unique ID with which to formulate asset IDs in
# conjunction with the 'asset_id_seq' attribute. This provides a
# monotonically increasing sequence of episode/viewpoint/photo ids
# for uploads via the web application. The 'uu_id_seq' provides a
# similar increasing sequence of user update sequence numbers for
# a user.
#
# Facebook email is kept separately in an effort to maximize
# deliverability of Viewfinder invitations to Facebook contacts.
# The from: header of those emails must be from the email address
# registered for the Facebook user if incoming to
# <username>@facebook.com.
#
# 'last_notification' is the most recent notification id which has
# been queried by any of the user's devices. This is the watermark
# used to supply the badge for push notifications. 'badge' is set
# appropriately in response to notifications generated by account
# activity.
#
# The 'merged_with' column specifies the sink user account with
# which this user was merged. If 'merged_with' is set, this user
# account is invalid and should not be used. If at all possible,
# the request intended for this user should be re-routed to the
# 'merged_with' user.
#
# The 'signing_key' column is a Keyczar signing keyset used when
# it is desirable to sign a payload with a key that is specific to
# one particular user. The contents of the column are encrypted
# with the service-wide db crypt keyset.
#
# The 'pwd_hash' and 'salt' columns are used to securely generate
# and store an iterative SHA1 hash of the user's password + salt.
#
# For user index, range key column is a string-version of user ID.
IndexedTable(
USER,
'us',
read_units=50,
write_units=50,
columns=[
HashKeyColumn('user_id', 'ui', 'N'),
Column('private_vp_id', 'pvi', 'S'),
Column('webapp_dev_id', 'wdi', 'N'),
Column('asset_id_seq', 'ais', 'N'),
Column('uu_id_seq', 'uis', 'N'),
Column('given_name', 'fi', 'S', FullTextIndexer()),
Column('family_name', 'la', 'S', FullTextIndexer()),
Column('name', 'na', 'S', FullTextIndexer()),
Column('email', 'em', 'S', EmailIndexer()),
Column('facebook_email', 'fe', 'S'),
LatLngColumn('location', 'lo', LocationIndexer()),
Column('gender', 'ge', 'S'),
Column('locale', 'lc', 'S'),
Column('link', 'li', 'S'),
Column('phone', 'ph', 'S'),
Column('picture', 'pi', 'S'),
Column('timezone', 'ti', 'N'),
Column('last_notification', 'ln', 'N'),
Column('badge', 'ba', 'N'),
Column('merged_with', 'mw', 'N'),
SetColumn('labels', 'lb', 'SS'),
CryptColumn('signing_key', 'sk'),
CryptColumn('pwd_hash', 'pwd'),
CryptColumn('salt', 'slt'),
# Deprecated (to be removed).
Column('beta_status', 'bs', 'S')
]),
# The USER_PHOTO is associated with a PHOTO object, and
# represents user-specific information about the photo.
# Specifically, this includes mappings between the photo and a
# device's native asset library. Normally only the user/device
# who originated the photo will have a USER_PHOTO entry for it,
# but it is possible for other users to create USER_PHOTOS if
# they export a photo to their camera roll.
IndexedTable(USER_PHOTO,
'up',
read_units=400,
write_units=10,
columns=[
HashKeyColumn('user_id', 'di', 'N'),
RangeKeyColumn('photo_id', 'pi', 'S'),
SetColumn('asset_keys', 'ak', 'SS')
]),
# The USER_POST is associated with a POST object, and represents
# user-specific override of information in the POST. 'timestamp'
# records the creation time of the record, and 'labels' contains
# a set of values which describes the customizations. For example,
# the 'removed' label indicates that the post should not be shown
# in the user's personal collection.
#
# Rows in the USER_POST table are only created if the user wants
# to customize the viewpoint in some way. In the absence of a
# row, default values are assumed.
IndexedTable(USER_POST,
'uo',
read_units=400,
write_units=10,
columns=[
HashKeyColumn('user_id', 'ui', 'N'),
RangeKeyColumn('post_id', 'pi', 'S'),
Column('timestamp', 'ti', 'N'),
SetColumn('labels', 'la', 'SS')
]),
# Key is viewpoint-id. Viewpoints are a collection of episodes.
# Viewpoint title and description are indexed for full-text
# search. The viewpoint name, sort of like a twitter
# handler, is also indexed. 'type' is one of:
# ('default', 'event', 'thematic')
#
# The 'update_seq' is incremented each time a viewpoint asset
# is added, removed, or updated. Using this with the 'viewed_seq'
# attribute on Follower, clients can easily determine if there
# is any "unread" content in the viewpoint. Note that updates to
# user-specific content on Follower does not trigger the increment
# of this value. 'last_updated' is set to the creation timestamp of
# the latest activity that was added to this viewpoint.
#
# The 'cover_photo' column is a JSON-encoded dict of photo_id and
# episode_id which indicates which photo should be used as the cover
# photo for the viewpoint. An absent column or None value for this indicates
# that it's explicitly not available (no visible photos in the viewpoint).
# Default viewpoints will not have this column set.
IndexedTable(VIEWPOINT,
'vp',
read_units=400,
write_units=10,
columns=[
HashKeyColumn('viewpoint_id', 'vi', 'S'),
Column('user_id',
'ui',
'N',
SecondaryIndexer(),
read_only=True),
Column('timestamp', 'ts', 'N', read_only=True),
Column('title', 'ti', 'S',
FullTextIndexer(metaphone=Indexer.Option.YES)),
Column('description', 'de', 'S',
FullTextIndexer(metaphone=Indexer.Option.YES)),
Column('last_updated', 'lu', 'N'),
Column('name', 'na', 'S', SecondaryIndexer()),
Column('type', 'ty', 'S', read_only=True),
Column('update_seq', 'us', 'N'),
JSONColumn('cover_photo', 'cp')
]),
# The index table for all indexed terms. Maps from a string to a
# string for greatest flexibility. This requires that the various
# database objects convert from a string value if the doc-id
# actually does represent a number, such as the user-id in some of
# the indexed tables in this schema.
IndexTable(INDEX, 'S', 'S', read_units=200, write_units=50),
# For the dynamodb_client_test.
Table(TEST_RENAME,
'test',
read_units=10,
write_units=5,
name_in_db="Test",
columns=[
HashKeyColumn('test_hk', 'thk', 'S'),
RangeKeyColumn('test_rk', 'trk', 'N'),
Column('attr0', 'a0', 'N'),
Column('attr1', 'a1', 'N'),
Column('attr2', 'a2', 'S'),
Column('attr3', 'a3', 'NS'),
Column('attr4', 'a4', 'SS')
]),
])
machine_list_schima = {
Optional('machineList'): [
Or(
{
'ip': str,
'port': And(int, lambda x: 0 < x < 65535),
'username': str,
'passwd': str
}, {
'ip': str,
'port': And(int, lambda x: 0 < x < 65535),
'username': str,
'sshKeyPath': os.path.exists,
Optional('passphrase'): str
})
]
}
LOCAL_CONFIG_SCHEMA = Schema({**common_schema, **common_trial_schema})
REMOTE_CONFIG_SCHEMA = Schema({
**common_schema,
**common_trial_schema,
**machine_list_schima
})
PAI_CONFIG_SCHEMA = Schema({
**common_schema,
**pai_trial_schema,
**pai_config_schema
})
# Third-party libraries
import click
import loguru
import numpy as np
import requests
from PIL import Image
from schema import And, Optional, Or, Use, Schema, SchemaError
POINT_SCHEMA = Schema(
And(
And(Or(list, tuple, error='Coordinates should be a list or tuple.')),
And(lambda coords: len(coords) == 2,
error='Coordinates should be a tuple/list of a X and Y coordinates.'
),
And(lambda coords: coords[0] >= 0,
error=
'The first coordinate of a point, in pixels, should be positive.'),
And(lambda coords: coords[1] >= 0,
error=
'The second coordinate of a point, in pixels, should be positive.'
),
))
COORDINATES_SCHEMA = Schema(
And(
And(Or(list, tuple, error='Coordinates should be a list or tuple.')),
And(lambda rings: len(rings) >= 1,
error='Polygon should contain a single ring.'),
And(lambda polygon: len(polygon[0]) >= 4,
error='Polygon should contain at least 4 points.'),
And([[POINT_SCHEMA]])))
def DATASCHEMA(self):
from schema import Schema
from compas.data import is_float4x4
return Schema({"matrix": is_float4x4})
class SchemaPattern:
StockInfoSchema = Schema({
'code': str,
'name': str,
'ISIN_code': str,
'listed_date': str,
'category': str,
'industry': Or(str, None),
'CFI_code': str,
'type': str
})
TwseStockTradingSchema = Schema({
'code': Or(str, None),
'name': Or(str, None),
'closing_price': Or(float, None),
'change': Or(float, str, None),
'opening_price': Or(float, None),
'highest_price': Or(float, None),
'lowest_price': Or(float, None),
'_different': Or(str, None),
'trading_volume': Or(float, None),
'trade_value': Or(float, None),
'transaction': Or(float, None),
'last_best_bid_price': Or(float, None),
'last_best_ask_price': Or(float, None),
'last_best_bid_volume': Or(float, None),
'last_best_ask_volume': Or(float, None),
'price_earning_rate': Or(float, None),
})
TpexStockTradingSchema = Schema({
'code': Or(str, None),
'name': Or(str, None),
'closing_price': Or(float, None),
'change': Or(float, str, None),
'opening_price': Or(float, None),
'highest_price': Or(float, None),
'lowest_price': Or(float, None),
'trading_volume': Or(float, None),
'trade_value': Or(float, None),
'transaction': Or(float, None),
'last_best_bid_price': Or(float, None),
'last_best_ask_price': Or(float, None),
'last_best_bid_volume': Or(float, None),
'last_best_ask_volume': Or(float, None),
'issued_shares': Or(float, None),
'next_limit_up': Or(float, None),
'next_limit_down': Or(float, None),
})
StockInstitutionalInvestorsSchema = Schema({
'code': str,
'name': str,
'foreign_mainland_area_buy': float,
'foreign_mainland_area_sell': float,
'foreign_mainland_area_diff': float,
'foreign_buy': float,
'foreign_sell': float,
'foreign_diff': float,
'trust_buy': float,
'trust_sell': float,
'trust_diff': float,
'proprietary_dealers_buy': float,
'proprietary_dealers_sell': float,
'proprietary_dealers_diff': float,
'hedge_dealers_buy': float,
'hedge_dealers_sell': float,
'hedge_dealers_diff': float,
'total_diff': float,
})
TwseStockMarginTradingSchema = Schema({
'code': str,
'name': str,
'margin_purchase': float,
'margin_sells': float,
'cash_redemption': float,
'cash_balance_of_previous_day': float,
'cash_balance_of_the_day': float,
'cash_quota': float,
'short_covering': float,
'short_sale': float,
'stock_redemption': float,
'stock_balance_of_previous_day': float,
'stock_balance_of_the_day': float,
'stock_quota': float,
'offset': float,
'note': Or(str, None)
})
TpexStockMarginTradingSchema = Schema({
'code':
str,
'name':
str,
'cash_balance_of_previous_day':
float,
'margin_purchase':
float,
'margin_sells':
float,
'cash_redemption':
float,
'cash_balance_of_the_day':
float,
'cash_belong_to_securities_finance':
float,
'cash_utilization_rate':
float,
'cash_quota':
float,
'stock_balance_of_previous_day':
float,
'short_covering':
float,
'short_sale':
float,
'stock_redemption':
float,
'stock_balance_of_the_day':
float,
'stock_belong_to_securities_finance':
float,
'stock_utilization_rate':
float,
'stock_quota':
float,
'offset':
float,
'note':
Or(str, None)
})
StockShareholdingsSchema = Schema({
'date': date,
'code': str,
'index': int,
'number_of_shares': Or(str, None),
'number_of_shareholders': int,
'total_shares': int,
'percentage_over_total_shares': float
})
StockPERatioSchema = Schema({
'name': Or(str, None),
'code': Or(str, None),
'date': Or(date, None),
'per': Or(float, None),
'dividend_per_share': Or(float, None),
'dividend_year': Or(int, None),
'yield_ratio': Or(float, None),
'pbr': Or(float, None),
'fiscal_year_quarter': Or(str, None)
})
def get_options(self, argv):
"""
Parse the command-line options and set the following object properties:
:param argv: usually just sys.argv[1:]
:returns: Nothing
:ivar debug: Enable logging debug statements
:ivar verbose: Enable verbose logging
:ivar config: Dict of the config file
"""
padding = max([
len(x) for x in self.flow.keys()
]) # Find max length of flow step names for padding with white space
docstring = __doc__ % ( #'|'.join(self.flow),
','.join(self.flow.keys()), '\n'.join([
' ' + k + ' ' * (padding + 4 - len(k)) + v
for k, v in self.flow.items()
]))
args = docopt(docstring, version=__version__)
# Load in default conf values from file if specified
if args['--conf']:
with open(args['--conf']) as f:
conf_args = yaml.load(f)
else:
conf_args = {}
args = merge_args(conf_args, args)
logging.debug(args)
schema = Schema({
'SOURCE_DIR':
Or(os.path.isdir, error='Source directory does not exist'),
'TARGET_DIR':
Or(lambda x: x is None,
os.path.isdir,
error='Destination directory does not exist'),
object:
object
})
try:
args = schema.validate(args)
except SchemaError as e:
exit(e)
logging.debug(args)
if args['all'] == 0:
for f in list(self.flow):
if args[f] == 0: del self.flow[f]
logging.info("Doing flow steps: %s" % (','.join(self.flow.keys())))
self.src_dir = args['SOURCE_DIR']
self.tgt_dir = args['TARGET_DIR']
if self.tgt_dir:
assert os.path.abspath(self.src_dir) != os.path.abspath(
self.tgt_dir
), 'Target and source directories cannot be the same'
if args['--debug']:
logging.basicConfig(level=logging.DEBUG, format='%(message)s')
elif args['--verbose']:
logging.basicConfig(level=logging.INFO, format='%(message)s')
self.args = args # Just save this for posterity
# file.close()
if __name__ == '__main__':
args = docopt(__doc__)
schema = Schema({
"--help":
bool,
"<s>":
And(Use(int),
lambda n: 1 < n < 17,
error="<p> should be integer 1 < n < 17."),
"<p>":
And(Use(int),
lambda n: 1 < n < 40,
error="<p> should be integer 1 < n < 40."),
"<r>":
And(Use(int),
lambda n: 1 < n < 2,
error="<r> should be integer 1 < n < 2."),
"<a>":
And(Use(int),
lambda n: 1 < n < 60,
error="<a> should be integer 1 < n < 60."),
"--output_file":
str
})
script_path = os.path.dirname(os.path.realpath(__file__))
# load file ok known missing skeletons
with open(join(script_path, "..", "config",
"missing_skeletons.json")) as datafile:
missing_list = json.load(datafile)
schema = Schema(
{
Optional("auto_migrate_settings", default=True):
int,
"allow_duplicate_hostnames":
int,
"allow_duplicate_ips":
int,
"allow_duplicate_macs":
int,
"allow_dynamic_settings":
int,
"always_write_dhcp_entries":
int,
"anamon_enabled":
int,
"authn_pam_service":
str,
"auth_token_expiration":
int,
"bind_chroot_path":
str,
"bind_manage_ipmi":
int,
"bind_master":
str,
Optional("buildisodir", default="/var/cache/cobbler/buildiso"):
str,
"build_reporting_enabled":
int,
"build_reporting_email":
list,
"build_reporting_ignorelist":
list,
"build_reporting_sender":
str,
"build_reporting_smtp_server":
str,
"build_reporting_subject":
str,
Optional("build_reporting_to_address", default=""):
str,
"cheetah_import_whitelist":
list,
"client_use_https":
int,
"client_use_localhost":
int,
Optional("cobbler_master", default=""):
str,
"consoles":
str,
"createrepo_flags":
str,
Optional("default_deployment_method", default=""):
str,
"default_kickstart":
str,
"default_name_servers":
list,
Optional("default_name_servers_search", default=[]):
list,
"default_ownership":
list,
"default_password_crypted":
str,
"default_template_type":
str,
"default_virt_bridge":
str,
Optional("default_virt_disk_driver", default="raw"):
str,
"default_virt_file_size":
int,
"default_virt_ram":
int,
"default_virt_type":
str,
"enable_gpxe":
int,
"enable_menu":
int,
"func_auto_setup":
int,
"func_master":
str,
"http_port":
int,
Optional("isc_set_host_name", default=0):
int,
Optional("iso_template_dir", default="/etc/cobbler/iso"):
str,
Optional("kerberos_realm", default="EXAMPLE.COM"):
str,
"kernel_options":
dict,
"kernel_options_s390x":
dict,
"ldap_anonymous_bind":
int,
"ldap_base_dn":
str,
Optional("ldap_management_default_type", default="authconfig"):
str,
"ldap_port":
int,
"ldap_search_bind_dn":
str,
"ldap_search_passwd":
str,
"ldap_search_prefix":
str,
"ldap_server":
str,
"ldap_tls":
int,
"ldap_tls_cacertfile":
str,
"ldap_tls_certfile":
str,
"ldap_tls_keyfile":
str,
"manage_dhcp":
int,
"manage_dns":
int,
"manage_forward_zones":
list,
"manage_reverse_zones":
list,
"manage_genders":
int,
"manage_rsync":
int,
Optional("manage_tftp", default=1):
int,
"manage_tftpd":
int,
"mgmt_classes":
list,
"mgmt_parameters":
dict,
"next_server":
str,
"power_management_default_type":
str,
"power_template_dir":
str,
"proxy_url_ext":
str,
"proxy_url_int":
str,
"puppet_auto_setup":
int,
"puppetca_path":
str,
Optional("puppet_parameterized_classes", default=1):
int,
Optional("puppet_server", default="puppet"):
str,
Optional("puppet_version", default=2):
int,
"pxe_just_once":
int,
"pxe_template_dir":
str,
"redhat_management_permissive":
int,
"redhat_management_server":
str,
"redhat_management_key":
str,
"redhat_management_type":
str,
"register_new_installs":
int,
"remove_old_puppet_certs_automatically":
int,
"replicate_repo_rsync_options":
str,
"replicate_rsync_options":
str,
"reposync_flags":
str,
"restart_dhcp":
int,
"restart_dns":
int,
"run_install_triggers":
int,
"scm_track_enabled":
int,
"scm_track_mode":
str,
"serializer_pretty_json":
int,
"server":
str,
Optional("signature_path",
default="/var/lib/cobbler/distro_signatures.json"):
str,
Optional("signature_url",
default="https://cobbler.github.io/signatures/2.8.x/latest.json"):
str,
"sign_puppet_certs_automatically":
int,
"snippetsdir":
str,
"template_remote_kickstarts":
int,
"virt_auto_boot":
int,
"webdir":
str,
"xmlrpc_port":
int,
"yum_distro_priority":
int,
"yum_post_install_mirror":
int,
"yumdownloader_flags":
str,
},
ignore_extra_keys=False)
def validate_settings(settings_content: dict) -> dict:
"""
This function performs logical validation of our loaded YAML files.
This function will:
- Perform type validation on all values of all keys.
- Provide defaults for optional settings.
:param settings_content: The dictionary content from the YAML file.
:raises SchemaError: In case the data given is invalid.
:return: The Settings of Cobbler which can be safely used inside this instance.
"""
schema = Schema(
{
"allow_duplicate_hostnames":
bool,
"allow_duplicate_ips":
bool,
"allow_duplicate_macs":
bool,
"allow_dynamic_settings":
bool,
"always_write_dhcp_entries":
bool,
"anamon_enabled":
bool,
"auth_token_expiration":
int,
"authn_pam_service":
str,
"autoinstall_snippets_dir":
str,
"autoinstall_templates_dir":
str,
"bind_chroot_path":
str,
"bind_zonefile_path":
str,
"bind_master":
str,
"boot_loader_conf_template_dir":
str,
Optional("bootloaders_dir", default="/var/lib/cobbler/loaders"):
str,
Optional("grubconfig_dir", default="/var/lib/cobbler/grub_config"):
str,
"build_reporting_enabled":
bool,
"build_reporting_email": [str],
"build_reporting_ignorelist": [str],
"build_reporting_sender":
str,
"build_reporting_smtp_server":
str,
"build_reporting_subject":
str,
Optional("buildisodir", default="/var/cache/cobbler/buildiso"):
str,
"cheetah_import_whitelist": [str],
"client_use_https":
bool,
"client_use_localhost":
bool,
Optional("cobbler_master", default=""):
str,
Optional("convert_server_to_ip", default=False):
bool,
"createrepo_flags":
str,
"autoinstall":
str,
"default_name_servers": [str],
"default_name_servers_search": [str],
"default_ownership": [str],
"default_password_crypted":
str,
"default_template_type":
str,
"default_virt_bridge":
str,
Optional("default_virt_disk_driver", default="raw"):
str,
"default_virt_file_size":
int,
"default_virt_ram":
int,
"default_virt_type":
str,
"enable_ipxe":
bool,
"enable_menu":
bool,
"http_port":
int,
"include": [str],
Optional("iso_template_dir", default="/etc/cobbler/iso"):
str,
Optional("jinja2_includedir", default="/var/lib/cobbler/jinja2"):
str,
"kernel_options":
dict,
"ldap_anonymous_bind":
bool,
"ldap_base_dn":
str,
"ldap_port":
int,
"ldap_search_bind_dn":
str,
"ldap_search_passwd":
str,
"ldap_search_prefix":
str,
"ldap_server":
str,
"ldap_tls":
bool,
"ldap_tls_cacertfile":
str,
"ldap_tls_certfile":
str,
"ldap_tls_keyfile":
str,
Optional("bind_manage_ipmi", default=False):
bool,
# TODO: Remove following line
"manage_dhcp":
bool,
"manage_dhcp_v4":
bool,
"manage_dhcp_v6":
bool,
"manage_dns":
bool,
"manage_forward_zones": [str],
"manage_reverse_zones": [str],
Optional("manage_genders", False):
bool,
"manage_rsync":
bool,
"manage_tftpd":
bool,
"mgmt_classes": [str],
# TODO: Validate Subdict
"mgmt_parameters":
dict,
"next_server_v4":
str,
"next_server_v6":
str,
Optional("nsupdate_enabled", False):
bool,
Optional("nsupdate_log", default="/var/log/cobbler/nsupdate.log"):
str,
Optional("nsupdate_tsig_algorithm", default="hmac-sha512"):
str,
Optional("nsupdate_tsig_key", default=[]): [str],
"power_management_default_type":
str,
"proxy_url_ext":
str,
"proxy_url_int":
str,
"puppet_auto_setup":
bool,
Optional("puppet_parameterized_classes", default=True):
bool,
Optional("puppet_server", default="puppet"):
str,
Optional("puppet_version", default=2):
int,
"puppetca_path":
str,
"pxe_just_once":
bool,
"nopxe_with_triggers":
bool,
"redhat_management_permissive":
bool,
"redhat_management_server":
str,
"redhat_management_key":
str,
"register_new_installs":
bool,
"remove_old_puppet_certs_automatically":
bool,
"replicate_repo_rsync_options":
str,
"replicate_rsync_options":
str,
"reposync_flags":
str,
"reposync_rsync_flags":
str,
"restart_dhcp":
bool,
"restart_dns":
bool,
"run_install_triggers":
bool,
"scm_track_enabled":
bool,
"scm_track_mode":
str,
"scm_track_author":
str,
"scm_push_script":
str,
"serializer_pretty_json":
bool,
"server":
str,
"sign_puppet_certs_automatically":
bool,
Optional("signature_path",
default="/var/lib/cobbler/distro_signatures.json"):
str,
Optional("signature_url",
default="https://cobbler.github.io/signatures/3.0.x/latest.json"):
str,
"tftpboot_location":
str,
"virt_auto_boot":
bool,
"webdir":
str,
"webdir_whitelist": [str],
"xmlrpc_port":
int,
"yum_distro_priority":
int,
"yum_post_install_mirror":
bool,
"yumdownloader_flags":
str,
Optional("windows_enabled", default=False):
bool,
Optional("windows_template_dir", default="/etc/cobbler/windows"):
str,
Optional("samba_distro_share", default="DISTRO"):
str,
},
ignore_extra_keys=False)
return schema.validate(settings_content)
Optional('preCommand'): setType('preCommand', str)
},
{
'ip': setType('ip', str),
Optional('port'): setNumberRange('port', int, 1, 65535),
'username': setType('username', str),
'passwd': setType('passwd', str),
Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'),
Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int),
Optional('useActiveGpu'): setType('useActiveGpu', bool),
Optional('preCommand'): setType('preCommand', str)
})]
}
training_service_schema_dict = {
'adl': Schema({**common_schema, **adl_trial_schema}),
'local': Schema({**common_schema, **common_trial_schema}),
'remote': Schema({**common_schema, **common_trial_schema, **machine_list_schema, **remote_config_schema}),
'pai': Schema({**common_schema, **pai_trial_schema, **pai_config_schema}),
'paiYarn': Schema({**common_schema, **pai_yarn_trial_schema, **pai_yarn_config_schema}),
'kubeflow': Schema({**common_schema, **kubeflow_trial_schema, **kubeflow_config_schema}),
'frameworkcontroller': Schema({**common_schema, **frameworkcontroller_trial_schema, **frameworkcontroller_config_schema}),
'aml': Schema({**common_schema, **aml_trial_schema, **aml_config_schema}),
'dlts': Schema({**common_schema, **dlts_trial_schema, **dlts_config_schema}),
}
class NNIConfigSchema:
def validate(self, data):
train_service = data['trainingServicePlatform']
Schema(common_schema['trainingServicePlatform']).validate(train_service)
def validate_common_content(experiment_config):
'''Validate whether the common values in experiment_config is valid'''
if not experiment_config.get('trainingServicePlatform') or \
experiment_config.get('trainingServicePlatform') not in ['local', 'remote', 'pai', 'kubeflow', 'frameworkcontroller', 'paiYarn']:
print_error('Please set correct trainingServicePlatform!')
exit(1)
schema_dict = {
'local': LOCAL_CONFIG_SCHEMA,
'remote': REMOTE_CONFIG_SCHEMA,
'pai': PAI_CONFIG_SCHEMA,
'paiYarn': PAI_YARN_CONFIG_SCHEMA,
'kubeflow': KUBEFLOW_CONFIG_SCHEMA,
'frameworkcontroller': FRAMEWORKCONTROLLER_CONFIG_SCHEMA
}
separate_schema_dict = {
'tuner': tuner_schema_dict,
'advisor': advisor_schema_dict,
'assessor': assessor_schema_dict
}
separate_builtInName_dict = {
'tuner': 'builtinTunerName',
'advisor': 'builtinAdvisorName',
'assessor': 'builtinAssessorName'
}
try:
schema_dict.get(experiment_config['trainingServicePlatform']).validate(
experiment_config)
for separate_key in separate_schema_dict.keys():
if experiment_config.get(separate_key):
if experiment_config[separate_key].get(
separate_builtInName_dict[separate_key]):
validate = False
for key in separate_schema_dict[separate_key].keys():
if key.__contains__(experiment_config[separate_key][
separate_builtInName_dict[separate_key]]):
Schema({
**separate_schema_dict[separate_key][key]
}).validate(experiment_config[separate_key])
validate = True
break
if not validate:
print_error('%s %s error!' %
(separate_key,
separate_builtInName_dict[separate_key]))
exit(1)
else:
Schema({
**separate_schema_dict[separate_key]['customized']
}).validate(experiment_config[separate_key])
except SchemaError as error:
print_error(
'Your config file is not correct, please check your config file content!'
)
print_error(error.code)
exit(1)
#set default value
if experiment_config.get('maxExecDuration') is None:
experiment_config['maxExecDuration'] = '999d'
if experiment_config.get('maxTrialNum') is None:
experiment_config['maxTrialNum'] = 99999
if experiment_config['trainingServicePlatform'] == 'remote':
for index in range(len(experiment_config['machineList'])):
if experiment_config['machineList'][index].get('port') is None:
experiment_config['machineList'][index]['port'] = 22
def validate(self, data):
train_service = data['trainingServicePlatform']
Schema(common_schema['trainingServicePlatform']).validate(train_service)
train_service_schema = training_service_schema_dict[train_service]
train_service_schema.validate(data)
self.validate_extras(data)
PARAM_SCHEMA = {
Optional("notification_endpoint"): str,
Optional("schedule"): str,
Optional("restart_execution_on_update"): bool,
}
AWS_ACCOUNT_ID_REGEX_STR = r"\A[0-9]{12}\Z"
AWS_ACCOUNT_ID_SCHEMA = Schema(
And(
Or(int, str), Use(str),
Regex(
AWS_ACCOUNT_ID_REGEX_STR,
error=(
"The specified account id is incorrect. "
"This typically happens when you specify the account id as a "
"number, while the account id starts with a zero. If this is "
"the case, please wrap the account id in quotes to make it a "
"string. An AWS Account Id is a number of 12 digits, which "
"should start with a zero if the Account Id has a zero at "
"the start too. "
"The number shown to not match the regular expression could "
"be interpreted as an octal number due to the leading zero. "
"Therefore, it might not match the account id as specified "
"in the deployment map."))))
# CodeCommit Source
CODECOMMIT_SOURCE_PROPS = {
"account_id": AWS_ACCOUNT_ID_SCHEMA,
Optional("repository"): str,
Optional("branch"): str,
Optional("poll_for_changes"): bool,
