def sign_up():
schema = Schema({
"username": str,
"email": Regex(MAIL_REGEX, error="Mail address is invalid"),
"password": str
})
validated = schema.validate(request.json)
if User.objects(username=validated["username"]):
return jsonify({"error": "Username not available"}), 409
if User.objects(email=validated["email"]):
return jsonify(
{"error":
"There is already an account with your email address"}), 409
# Hash password with sha256
hashed_password = generate_password_hash(validated["password"])
user = User(username=validated["username"],
email=validated["email"],
password=hashed_password).save()
token = jwt.encode(
{
"username": user.username,
"email": user.email,
"password": user.password,
"created": str(user.created)
}, app.config["SECRET_KEY"])
return jsonify({
"success": True,
"user": {
"username": user.username,
"email": user.email,
"password": user.password,
"created": str(user.created)
},
"token": token.decode("UTF-8")
})
def validate_instance_parameters(params: dict):
from spotty.providers.aws.config.instance_config import VOLUME_TYPE_EBS
instance_parameters = {
'region': And(str, Regex(r'^[a-z0-9-]+$')),
Optional('availabilityZone', default=''): And(str, Regex(r'^[a-z0-9-]+$')),
Optional('subnetId', default=''): And(str, Regex(r'^subnet-[a-z0-9]+$')),
'instanceType': And(str, And(is_valid_instance_type, error='Invalid instance type.')),
Optional('onDemandInstance', default=False): bool,
Optional('amiName', default=None): And(str, len, Regex(r'^[\w\(\)\[\]\s\.\/\'@-]{3,128}$')),
Optional('amiId', default=None): And(str, len, Regex(r'^ami-[a-z0-9]+$')),
Optional('rootVolumeSize', default=0): And(Or(int, str), Use(str),
Regex(r'^\d+$', error='Incorrect value for "rootVolumeSize".'),
Use(int),
And(lambda x: x > 0,
error='"rootVolumeSize" should be greater than 0 or should '
'not be specified.'),
),
Optional('maxPrice', default=0): And(Or(float, int, str), Use(str),
Regex(r'^\d+(\.\d{1,6})?$', error='Incorrect value for "maxPrice".'),
Use(float),
And(lambda x: x > 0, error='"maxPrice" should be greater than 0 or '
'should not be specified.'),
),
Optional('managedPolicyArns', default=[]): [str],
}
volumes_checks = [
And(lambda x: len(x) < 12, error='Maximum 11 volumes are supported at the moment.'),
]
instance_checks = [
And(lambda x: not (x['onDemandInstance'] and x['maxPrice']),
error='"maxPrice" cannot be specified for on-demand instances'),
And(lambda x: not (x['amiName'] and x['amiId']),
error='"amiName" and "amiId" parameters cannot be used together'),
]
schema = get_instance_parameters_schema(instance_parameters, VOLUME_TYPE_EBS, instance_checks, volumes_checks)
return validate_config(schema, params)
def validate_ami_config(data):
schema = Schema(
{
'instance': {
'region':
And(str, len),
Optional('availabilityZone', default=''):
str,
Optional('subnetId', default=''):
str,
'instanceType':
And(str, len),
Optional('amiName', default=DEFAULT_AMI_NAME):
And(str, len, Regex(AMI_NAME_REGEX)),
Optional('keyName', default=''):
str,
Optional('onDemandInstance', default=False):
bool,
},
},
ignore_extra_keys=True)
return _validate(schema, data)
def validate_ebs_volume_parameters(params: dict):
from spotty.providers.aws.deployment.project_resources.ebs_volume import EbsVolume
schema = Schema({
Optional('volumeName', default=''):
And(str, Regex(r'^[\w-]{1,255}$')),
Optional('mountDir', default=''):
str, # all the checks happened in the base configuration
Optional('size', default=0):
And(int, lambda x: x > 0),
# TODO: add the "iops" parameter to support the "io1" EBS volume type
Optional('type', default='gp2'):
lambda x: x in ['gp2', 'sc1', 'st1', 'standard'],
Optional('deletionPolicy', default=EbsVolume.DP_CREATE_SNAPSHOT):
And(str,
lambda x: x in [
EbsVolume.DP_CREATE_SNAPSHOT, EbsVolume.DP_UPDATE_SNAPSHOT,
EbsVolume.DP_RETAIN, EbsVolume.DP_DELETE
],
error='Incorrect value for "deletionPolicy".'),
})
return validate_config(schema, params)
def validate_ebs_volume_parameters(params: dict):
from spotty.providers.aws.config.ebs_volume import EbsVolume
old_deletion_policies_map = {
'create_snapshot': EbsVolume.DP_CREATE_SNAPSHOT,
'update_snapshot': EbsVolume.DP_UPDATE_SNAPSHOT,
'retain': EbsVolume.DP_RETAIN,
'delete': EbsVolume.DP_DELETE,
}
schema = Schema({
Optional('volumeName', default=''):
And(str, Regex(r'^[\w-]{1,255}$')),
Optional('mountDir', default=''):
And(
str,
And(os.path.isabs,
error='Use absolute paths in the "mountDir" parameters'),
Use(lambda x: x.rstrip('/'))),
Optional('size', default=0):
And(int, lambda x: x > 0),
# TODO: add the "iops" parameter to support the "io1" EBS volume type
Optional('type', default='gp2'):
lambda x: x in ['gp2', 'sc1', 'st1', 'standard'],
Optional('deletionPolicy', default=EbsVolume.DP_RETAIN):
And(
str,
lambda x: x in [
EbsVolume.DP_CREATE_SNAPSHOT, EbsVolume.DP_UPDATE_SNAPSHOT,
EbsVolume.DP_RETAIN, EbsVolume.DP_DELETE
] + list(old_deletion_policies_map.keys()),
Use(lambda x: old_deletion_policies_map.get(x, x)),
error='Incorrect value for "deletionPolicy".',
),
})
return validate_config(schema, params)
from schema import Optional as SchemaOptional
from schema import Regex, Schema
from corehq.motech.dhis2.const import (
DHIS2_EVENT_STATUSES,
DHIS2_PROGRAM_STATUSES,
)
id_schema = Regex(r"^[A-Za-z0-9]+$")
# DHIS2 accepts date values, but returns datetime values for dates:
date_schema = Regex(r"^\d{4}-\d{2}-\d{2}(:?T\d{2}:\d{2}:\d{2}.\d{3})?$")
datetime_schema = Regex(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}$")
enrollment_status_schema = Regex(f"^({'|'.join(DHIS2_PROGRAM_STATUSES)})$")
event_status_schema = Regex(f"^({'|'.join(DHIS2_EVENT_STATUSES)})$")
def get_attribute_schema() -> dict:
return {
"attribute": id_schema,
SchemaOptional("code"): str,
SchemaOptional("created"): datetime_schema,
SchemaOptional("displayName"): str,
SchemaOptional("lastUpdated"): datetime_schema,
SchemaOptional("storedBy"): str,
"value": object,
SchemaOptional("valueType"): str,
}
def get_event_schema() -> dict:
"""
from schema import Schema, And, Or, Use, Optional, Regex
DATE_ISO = "^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])(T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\.[0-9]+)?(Z|[+-](?:2[0-3]|[01][0-9]):[0-5][0-9])?$)*"
URL_FORMAT = "^http(s)*://"
CONTEXT = "https://schema.org(/)*"
organization_schema = Schema({
"@context": Regex(CONTEXT),
"@type": Or("Corporation", "Organization"),
"name": str,
Optional("alternateName"): str,
Optional("url"): Regex(URL_FORMAT),
Optional("logo"): Regex(URL_FORMAT),
Optional("sameAs"): Or(str, [str]),
Optional(str): object
})
person_schema = Schema({
"@context":
Regex(CONTEXT),
"@type":
"Person",
"name":
str,
Optional("url"):
Regex(URL_FORMAT),
Optional("image"):
Regex(URL_FORMAT),
Optional("sameAs"):
Or(Regex(URL_FORMAT), [Regex(URL_FORMAT)]),
Optional("jobTitle"):
def image_schema(file):
valid_arches = [
'x86_64',
'ppc64le',
]
valid_distgit_namespaces = [
'apbs',
'containers',
'rpms',
]
valid_streams = support.get_valid_streams_for(file)
valid_member_references = support.get_valid_member_references_for(file)
valid_modes = [
'auto',
'disabled',
'wip',
]
valid_odcs_modes = [
'auto',
'manual',
]
return Schema({
Optional('additional_tags'): [
And(str, len),
],
Optional('arches'): [Or(*valid_arches)],
Optional('base_only'): True,
Optional('container_yaml'): {
'go': {
'modules': [
{
'module': And(str, len),
Optional('path'): str,
},
],
},
},
Optional('content'): {
'source': {
Optional('alias'): And(str, len),
Optional('dockerfile'): And(str, len),
Optional('git'): {
'branch': {
Optional('fallback'): And(str, len),
Optional('stage'): And(str, len),
'target': And(str, len),
},
'url': And(str, len, Regex(GIT_SSH_URL_REGEX)),
},
Optional('modifications'): [modification],
Optional('path'): str,
},
},
Optional('dependents'): [And(str, len)],
Optional('distgit'): {
Optional('namespace'): Or(*valid_distgit_namespaces),
Optional('component'): And(str, len),
Optional('branch'): And(str, len),
},
Optional('enabled_repos'): [
And(str, len),
],
Optional('non_shipping_repos'): [
And(str, len),
],
Optional('non_shipping_rpms'): [
And(str, len),
],
'from': {
Optional('builder'): [
{
Optional('stream'): Or(*valid_streams),
Optional('member'): Or(*valid_member_references),
Optional('image'): And(str, len),
},
],
Optional('image'):
And(str, len),
Optional('stream'):
Or(*valid_streams),
Optional('member'):
Or(*valid_member_references),
},
Optional('labels'): {
Optional('License'): And(str, len),
Optional('io.k8s.description'): And(str, len),
Optional('io.k8s.display-name'): And(str, len),
Optional('io.openshift.tags'): And(str, len),
Optional('vendor'): And(str, len),
},
Optional('image_build_method'): And(str, len),
Optional('mode'): Or(*valid_modes),
'name': And(str, len),
Optional('odcs'): {
'packages': {
Optional('exclude'): [
And(str, len),
],
Optional('list'): [
And(str, len),
],
'mode': Or(*valid_odcs_modes),
},
},
Optional('no_oit_comments'): bool,
Optional('owners'): [
And(str, len),
],
Optional('push'): {
Optional('repos'): [
And(str, len),
],
Optional('additional_tags'): [
And(str, len),
],
Optional('late'): bool,
},
Optional('required'): bool,
Optional('update-csv'): {
'manifests-dir': And(str, len),
'bundle-dir': And(str, len),
'registry': And(str, len),
Optional('channel'): And(str, len),
},
Optional('wait_for'): And(str, len),
Optional('maintainer'): {
Optional('product'): And(str, len),
'component': And(str, len),
Optional('subcomponent'): And(str, len),
},
Optional('for_payload'): bool,
})
data = {
'username': '******',
'password': '******',
'email': '*****@*****.**',
'nick_name': '12121'
}
schema = {
'username':
And(And(str, error='用户名参数字符类型错误'),
And(lambda x: 4 <= len(x) <= 16, error='用户名字符长度错误')),
'password':
And(And(str, error='密码参数字符类型错误'),
And(lambda x: 4 <= len(x) <= 16, error='密码字符长度错误')),
'email':
And(
And(str, error='邮箱参数字符类型错误'),
And(
Regex(r'^\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$',
error='邮箱正则匹配错误'))),
Optional('nick_name'):
And(And(str, error='昵称参数字符类型错误'),
And(lambda x: len(x) <= 64, error='昵称字符长度错误'))
}
try:
result = Schema(schema).validate(data)
print(result)
except SchemaError as e:
print(e.code)
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import os
from schema import Schema, And, Use, Optional, Regex, Or
common_schema = {
'authorName':
str,
'experimentName':
str,
Optional('description'):
str,
'trialConcurrency':
And(int, lambda n: 1 <= n <= 999999),
Optional('maxExecDuration'):
Regex(r'^[1-9][0-9]*[s|m|h|d]$'),
Optional('maxTrialNum'):
And(int, lambda x: 1 <= x <= 99999),
'trainingServicePlatform':
And(str, lambda x: x in ['remote', 'local', 'pai', 'kubeflow']),
Optional('searchSpacePath'):
os.path.exists,
Optional('multiPhase'):
bool,
Optional('multiThread'):
bool,
Optional('nniManagerIp'):
str,
'useAnnotation':
bool,
Optional('advisor'):
Schema(self.algo_schema).validate(data)
self.validate_extras(data, self.algo_type)
common_schema = {
'authorName':
setType('authorName', str),
'experimentName':
setType('experimentName', str),
Optional('description'):
setType('description', str),
'trialConcurrency':
setNumberRange('trialConcurrency', int, 1, 99999),
Optional('maxExecDuration'):
And(
Regex(r'^[1-9][0-9]*[s|m|h|d]$',
error='ERROR: maxExecDuration format is [digit]{s,m,h,d}')),
Optional('maxTrialNum'):
setNumberRange('maxTrialNum', int, 1, 99999),
'trainingServicePlatform':
setChoice('trainingServicePlatform', 'remote', 'local', 'pai', 'kubeflow',
'frameworkcontroller', 'paiYarn', 'dlts', 'aml', 'adl',
'hybrid'),
Optional('searchSpacePath'):
And(os.path.exists, error=SCHEMA_PATH_ERROR % 'searchSpacePath'),
Optional('multiPhase'):
setType('multiPhase', bool),
Optional('multiThread'):
setType('multiThread', bool),
Optional('nniManagerIp'):
setType('nniManagerIp', str),
Optional('logDir'):
class Config(object):
APPNAME = 'dvc'
APPAUTHOR = 'iterative'
# NOTE: used internally in RemoteLOCAL to know config
# location, that url should resolved relative to.
PRIVATE_CWD = '_cwd'
CONFIG = 'config'
CONFIG_LOCAL = 'config.local'
SECTION_CORE = 'core'
SECTION_CORE_LOGLEVEL = 'loglevel'
SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel)
SECTION_CORE_REMOTE = 'remote'
SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool))
SECTION_CORE_INTERACTIVE = 'interactive'
SECTION_CORE_ANALYTICS = 'analytics'
SECTION_CORE_ANALYTICS_SCHEMA = And(str, is_bool, Use(to_bool))
SECTION_CACHE = 'cache'
SECTION_CACHE_DIR = 'dir'
SECTION_CACHE_TYPE = 'type'
SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
SECTION_CACHE_PROTECTED = 'protected'
SECTION_CACHE_LOCAL = 'local'
SECTION_CACHE_S3 = 's3'
SECTION_CACHE_GS = 'gs'
SECTION_CACHE_SSH = 'ssh'
SECTION_CACHE_HDFS = 'hdfs'
SECTION_CACHE_AZURE = 'azure'
SECTION_CACHE_SCHEMA = {
Optional(SECTION_CACHE_LOCAL):
str,
Optional(SECTION_CACHE_S3):
str,
Optional(SECTION_CACHE_GS):
str,
Optional(SECTION_CACHE_HDFS):
str,
Optional(SECTION_CACHE_SSH):
str,
Optional(SECTION_CACHE_AZURE):
str,
Optional(SECTION_CACHE_DIR):
str,
Optional(SECTION_CACHE_TYPE, default=None):
SECTION_CACHE_TYPE_SCHEMA,
Optional(SECTION_CACHE_PROTECTED, default=False):
And(str, is_bool, Use(to_bool)),
Optional(PRIVATE_CWD):
str,
}
# backward compatibility
SECTION_CORE_CLOUD = 'cloud'
SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud)
SECTION_CORE_STORAGEPATH = 'storagepath'
SECTION_CORE_SCHEMA = {
Optional(SECTION_CORE_LOGLEVEL, default='info'):
And(str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA),
Optional(SECTION_CORE_REMOTE, default=''):
And(str, Use(str.lower)),
Optional(SECTION_CORE_INTERACTIVE, default=False):
SECTION_CORE_INTERACTIVE_SCHEMA,
Optional(SECTION_CORE_ANALYTICS, default=True):
SECTION_CORE_ANALYTICS_SCHEMA,
# backward compatibility
Optional(SECTION_CORE_CLOUD, default=''):
SECTION_CORE_CLOUD_SCHEMA,
Optional(SECTION_CORE_STORAGEPATH, default=''):
str,
}
# backward compatibility
SECTION_AWS = 'aws'
SECTION_AWS_STORAGEPATH = 'storagepath'
SECTION_AWS_CREDENTIALPATH = 'credentialpath'
SECTION_AWS_ENDPOINT_URL = 'endpointurl'
SECTION_AWS_REGION = 'region'
SECTION_AWS_PROFILE = 'profile'
SECTION_AWS_USE_SSL = 'use_ssl'
SECTION_AWS_SCHEMA = {
SECTION_AWS_STORAGEPATH:
str,
Optional(SECTION_AWS_REGION):
str,
Optional(SECTION_AWS_PROFILE):
str,
Optional(SECTION_AWS_CREDENTIALPATH):
str,
Optional(SECTION_AWS_ENDPOINT_URL):
str,
Optional(SECTION_AWS_USE_SSL, default=True):
And(str, is_bool, Use(to_bool)),
}
# backward compatibility
SECTION_GCP = 'gcp'
SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH
SECTION_GCP_PROJECTNAME = 'projectname'
SECTION_GCP_SCHEMA = {
SECTION_GCP_STORAGEPATH: str,
Optional(SECTION_GCP_PROJECTNAME): str,
}
# backward compatibility
SECTION_LOCAL = 'local'
SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH
SECTION_LOCAL_SCHEMA = {
SECTION_LOCAL_STORAGEPATH: str,
}
SECTION_AZURE_CONNECTION_STRING = 'connection_string'
SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
SECTION_REMOTE_FMT = 'remote "{}"'
SECTION_REMOTE_URL = 'url'
SECTION_REMOTE_USER = '******'
SECTION_REMOTE_PORT = 'port'
SECTION_REMOTE_KEY_FILE = 'keyfile'
SECTION_REMOTE_TIMEOUT = 'timeout'
SECTION_REMOTE_PASSWORD = '******'
SECTION_REMOTE_ASK_PASSWORD = '******'
SECTION_REMOTE_SCHEMA = {
SECTION_REMOTE_URL:
str,
Optional(SECTION_AWS_REGION):
str,
Optional(SECTION_AWS_PROFILE):
str,
Optional(SECTION_AWS_CREDENTIALPATH):
str,
Optional(SECTION_AWS_ENDPOINT_URL):
str,
Optional(SECTION_AWS_USE_SSL, default=True):
And(str, is_bool, Use(to_bool)),
Optional(SECTION_GCP_PROJECTNAME):
str,
Optional(SECTION_CACHE_TYPE):
SECTION_CACHE_TYPE_SCHEMA,
Optional(SECTION_CACHE_PROTECTED, default=False):
And(str, is_bool, Use(to_bool)),
Optional(SECTION_REMOTE_USER):
str,
Optional(SECTION_REMOTE_PORT):
Use(int),
Optional(SECTION_REMOTE_KEY_FILE):
str,
Optional(SECTION_REMOTE_TIMEOUT):
Use(int),
Optional(SECTION_REMOTE_PASSWORD):
str,
Optional(SECTION_REMOTE_ASK_PASSWORD):
And(str, is_bool, Use(to_bool)),
Optional(SECTION_AZURE_CONNECTION_STRING):
str,
Optional(PRIVATE_CWD):
str,
}
SECTION_STATE = 'state'
SECTION_STATE_ROW_LIMIT = 'row_limit'
SECTION_STATE_ROW_CLEANUP_QUOTA = 'row_cleanup_quota'
SECTION_STATE_SCHEMA = {
Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole),
Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent),
}
SCHEMA = {
Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
# backward compatibility
Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA,
Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
}
def __init__(self, dvc_dir=None, validate=True):
self.system_config_file = os.path.join(self.get_system_config_dir(),
self.CONFIG)
self.global_config_file = os.path.join(self.get_global_config_dir(),
self.CONFIG)
if dvc_dir is not None:
self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
self.config_file = os.path.join(dvc_dir, self.CONFIG)
self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
else:
self.dvc_dir = None
self.config_file = None
self.config_local_file = None
self.load(validate=validate)
@staticmethod
def get_global_config_dir():
from appdirs import user_config_dir
return user_config_dir(appname=Config.APPNAME,
appauthor=Config.APPAUTHOR)
@staticmethod
def get_system_config_dir():
from appdirs import site_config_dir
return site_config_dir(appname=Config.APPNAME,
appauthor=Config.APPAUTHOR)
@staticmethod
def init(dvc_dir):
config_file = os.path.join(dvc_dir, Config.CONFIG)
open(config_file, 'w+').close()
return Config(dvc_dir)
def _load(self):
self._system_config = configobj.ConfigObj(self.system_config_file)
self._global_config = configobj.ConfigObj(self.global_config_file)
if self.config_file is not None:
self._project_config = configobj.ConfigObj(self.config_file)
else:
self._project_config = configobj.ConfigObj()
if self.config_local_file is not None:
self._local_config = configobj.ConfigObj(self.config_local_file)
else:
self._local_config = configobj.ConfigObj()
self._config = None
def _load_config(self, path):
config = configobj.ConfigObj(path)
config = self._lower(config)
self._resolve_paths(config, path)
return config
@staticmethod
def _resolve_path(path, config_file):
assert os.path.isabs(config_file)
config_dir = os.path.dirname(config_file)
return os.path.abspath(os.path.join(config_dir, path))
def _resolve_cache_path(self, config, fname):
cache = config.get(self.SECTION_CACHE)
if cache is None:
return
cache_dir = cache.get(self.SECTION_CACHE_DIR)
if cache_dir is None:
return
cache[self.PRIVATE_CWD] = os.path.dirname(fname)
def _resolve_paths(self, config, fname):
self._resolve_cache_path(config, fname)
for name, section in config.items():
if self.SECTION_REMOTE_URL not in section.keys():
continue
section[self.PRIVATE_CWD] = os.path.dirname(fname)
def load(self, validate=True):
self._load()
try:
self._config = self._load_config(self.system_config_file)
user = self._load_config(self.global_config_file)
config = self._load_config(self.config_file)
local = self._load_config(self.config_local_file)
# NOTE: schema doesn't support ConfigObj.Section validation, so we
# need to convert our config to dict before passing it to
for c in [user, config, local]:
self._config = self._merge(self._config, c)
if validate:
self._config = Schema(self.SCHEMA).validate(self._config)
# NOTE: now converting back to ConfigObj
self._config = configobj.ConfigObj(self._config,
write_empty_values=True)
self._config.filename = self.config_file
self._resolve_paths(self._config, self.config_file)
except Exception as ex:
raise ConfigError(ex)
def _get_key(self, d, name, add=False):
for k in d.keys():
if k.lower() == name.lower():
return k
if add:
d[name] = {}
return name
return None
def save(self, config=None):
if config is not None:
clist = [config]
else:
clist = [
self._system_config, self._global_config, self._project_config,
self._local_config
]
for conf in clist:
if conf.filename is None:
continue
try:
logger.debug("Writing '{}'.".format(conf.filename))
dname = os.path.dirname(os.path.abspath(conf.filename))
try:
os.makedirs(dname)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
conf.write()
except Exception as exc:
msg = "failed to write config '{}'".format(conf.filename)
raise ConfigError(msg, exc)
def unset(self, config, section, opt=None):
if section not in config.keys():
raise ConfigError("section '{}' doesn't exist".format(section))
if opt is None:
del config[section]
return
if opt not in config[section].keys():
raise ConfigError("option '{}.{}' doesn't exist".format(
section, opt))
del config[section][opt]
if len(config[section]) == 0:
del config[section]
def set(self, config, section, opt, value):
if section not in config.keys():
config[section] = {}
config[section][opt] = value
def show(self, config, section, opt):
if section not in config.keys():
raise ConfigError("section '{}' doesn't exist".format(section))
if opt not in config[section].keys():
raise ConfigError("option '{}.{}' doesn't exist".format(
section, opt))
logger.info(config[section][opt])
@staticmethod
def _merge(first, second):
res = {}
sections = list(first.keys()) + list(second.keys())
for section in sections:
f = first.get(section, {}).copy()
s = second.get(section, {}).copy()
f.update(s)
res[section] = f
return res
@staticmethod
def _lower(config):
new_config = {}
for s_key, s_value in config.items():
new_s = {}
for key, value in s_value.items():
new_s[key.lower()] = value
new_config[s_key.lower()] = new_s
return new_config
def validate_instance_parameters(params: dict):
from spotty.providers.aws.config.ebs_volume import EbsVolume
instance_parameters = {
'region':
And(str, Regex(r'^[a-z0-9-]+$')),
Optional('availabilityZone', default=''):
And(str, Regex(r'^[a-z0-9-]+$')),
Optional('subnetId', default=''):
And(str, Regex(r'^subnet-[a-z0-9]+$')),
'instanceType':
str,
Optional('spotInstance', default=False):
bool,
Optional('amiName', default=None):
And(str, len, Regex(r'^[\w\(\)\[\]\s\.\/\'@-]{3,128}$')),
Optional('amiId', default=None):
And(str, len, Regex(r'^ami-[a-z0-9]+$')),
Optional('rootVolumeSize', default=0):
And(
Or(int, str),
Use(str),
Regex(r'^\d+$', error='Incorrect value for "rootVolumeSize".'),
Use(int),
And(lambda x: x > 0,
error='"rootVolumeSize" should be greater than 0 or should '
'not be specified.'),
),
Optional('ports', default=[]): [And(int, lambda x: 0 < x < 65536)],
Optional('maxPrice', default=0):
And(
Or(float, int, str),
Use(str),
Regex(r'^\d+(\.\d{1,6})?$',
error='Incorrect value for "maxPrice".'),
Use(float),
And(lambda x: x > 0,
error='"maxPrice" should be greater than 0 or '
'should not be specified.'),
),
Optional('managedPolicyArns', default=[]): [str],
Optional('instanceProfileArn', default=None):
str,
}
volumes_checks = [
And(lambda x: len(x) < 12,
error='Maximum 11 volumes are supported at the moment.'),
And(lambda x: not has_prefix([(volume['parameters']['mountDir'] + '/')
for volume in x if volume['parameters'].
get('mountDir')]),
error='Mount directories cannot be prefixes for each other.'),
]
instance_checks = [
And(lambda x: not (x['maxPrice'] and not x['spotInstance']),
error='"maxPrice" can be specified only for spot instances.'),
And(lambda x: not (x['amiName'] and x['amiId']),
error='"amiName" and "amiId" parameters cannot be used together.'),
]
schema = get_instance_parameters_schema(instance_parameters,
EbsVolume.TYPE_NAME,
instance_checks, volumes_checks)
return validate_config(schema, params)
#-*- coding: utf-8 -*-
import os
import re
from pyquery import PyQuery as pq
from pykl import gdom, pyhttp, pyutils, pyfile
from schema import Schema, SchemaError, Regex, And, Use, Optional
from flask import request
from mrq.dashboard.utils import jsonify
from functools import update_wrapper
from urlparse import urlparse, parse_qs
HttpUrlSchema = And(
basestring, len,
Regex(
r'^(http|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?$'
))
IPSchema = And(
basestring, len,
Regex(
r'^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$'
))
def TaskSchemaWrapper(*args, **kwgs):
schema = Schema(*args, **kwgs)
def _wrapper(func):
func.params_schema = schema
return func
def _validate_PO(self, **kw):
"""
validate required fields are present and validate types
uses schema to validate and return validated kw
"""
# following vars are used for validation
#ISO-8601 format for date and datetime.
rdatetime = r'^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?(([+-]\d\d:\d\d)|Z)?$'
toleranceList = ('AllowOverRun', 'AllowUnderrun',
'AllowOverrunOrUnderrun', 'ExactOnly')
uomList = [
'BX', 'CA', 'DZ', 'EA', 'KT', 'PR', 'PK', 'RL', 'ST', 'SL', 'TH'
]
conact_schema = Schema({
Optional("attentionTo"):
And(lambda s: var_check(s, 35),
error='"attentionTo" should evaluate to varchar(35)'),
Optional("companyName"):
And(lambda s: var_check(s, 35),
error='"companyName" should evaluate to varchar(35)'),
Optional("address1"):
And(lambda s: var_check(s, 35),
error='"address1" should evaluate to varchar(35)'),
Optional("address2"):
And(lambda s: var_check(s, 35),
error='"address2" should evaluate to varchar(35)'),
Optional("address3"):
And(lambda s: var_check(s, 35),
error='"address3" should evaluate to varchar(35)'),
Optional("city"):
And(lambda s: var_check(s, 30),
error='"city" should evaluate to varchar(30)'),
Optional("region"):
And(lambda s: var_check(s, 3),
error='"region" should evaluate to varchar(3)'),
Optional("postalCode"):
And(lambda s: var_check(s, 10),
error='"postalCode" should evaluate to varchar(10)'),
Optional("country"):
And(lambda s: var_check(s, 2),
error='"country" should evaluate to varchar(2)'),
Optional("email"):
And(lambda s: var_check(s, 128),
error='"email" should evaluate to varchar(128)'),
Optional("phone"):
And(lambda s: var_check(s, 32),
error='"phone" should evaluate to varchar(32)'),
Optional("comments"): str
})
quantity_schema = Schema({
"uom": And(str, lambda s: s in uomList),
"value": And(Use(to_decimal_4), validDecimal_12_4)
})
third_party_schema = Schema({
"accountName":
And(lambda s: var_check(s, 64),
error='"accountName" should evaluate to varchar(64)'),
"accountNumber":
And(lambda s: var_check(s, 64),
error='"accountNumber" should evaluate to varchar(64)'),
"ContactDetails": conact_schema
})
# schema used for validating PO version 1.0.0
v1_0_0_schema = Schema({
"wsVersion": And(str, len),
"id": And(str, len),
Optional("password"): str,
"PO": {
"orderType":
And(str, lambda s: s in ('Blank', 'Sample', 'Simple', 'Configured')),
"orderNumber":
And(lambda s: var_check(s, 64),
error='"orderNumber" should evaluate to varchar(64)'),
"orderDate":
And(Const(Use(fromisoformat)),
Regex(r'{}'.format(rdatetime))),
Optional("lastModified"):
And(Const(Use(fromisoformat)),
Regex(r'{}'.format(rdatetime))),
"totalAmount": And(Use(to_decimal_4), validDecimal_12_4),
Optional("paymentTerms"): str,
"rush": Use(xml_bool),
"currency":
And(lambda s: var_check(s, 3),
error='"currency" should evaluate to varchar(3)'),
Optional("DigitalProof"): {
"DigitalProofAddressArray": [{
"DigitalProofAddress": {
"type":
And(lambda s: var_check(s, 64),
error='"type" should evaluate to varchar(64)'),
"email":
And(lambda s: var_check(s, 128),
error='"email" should evaluate to varchar(128)'
),
"lineItemGroupingId": int
}
}],
"required": Use(xml_bool)
},
Optional("OrderContactArray"): [{
"Contact": {
Optional("accountName"):
And(lambda s: var_check(s, 64),
error='"accountName" should evaluate to varchar(64)'),
Optional("accountNumber"):
And(lambda s: var_check(s, 64),
error='"accountNumber" should evaluate to varchar(64)'),
"contactType":
And(str, lambda s: s in [
'Art', 'Bill', 'Expeditor', 'Order', 'Sales',
'Ship', 'Sold'
]),
"ContactDetails": conact_schema
}
}],
"ShipmentArray": [{
"Shipment": {
Optional("shipReferences"):
[lambda s: var_check(s, 64)],
Optional("comments"): str,
Optional("ThirdPartyAccount"): third_party_schema,
"allowConsolidation": Use(xml_bool),
"blindShip": Use(xml_bool),
"packingListRequired": Use(xml_bool),
"FreightDetails": {
"carrier":
And(lambda s: var_check(s, 64),
error='"carrier" should evaluate to varchar(64)'
),
"service":
And(lambda s: var_check(s, 64),
error='"service" should evaluate to varchar(64)'
)
},
"ShipTo": {
"customerPickup": Use(xml_bool),
"ContactDetails": conact_schema,
"shipmentId": int
}
}
}],
"LineItemArray": [{
"LineItem": {
"lineNumber": int,
Optional("lineReferenceId"):
And(lambda s: var_check(s, 64),
error='"lineReferenceId" should evaluate to varchar(64)'),
"description": str,
"lineType": And(str, lambda s: s in ['New', 'Repeat', 'Reference']),
Optional("Quantity"): quantity_schema,
Optional("fobid"):
And(lambda s: var_check(s, 64),
error='"fobid" should evaluate to varchar(64)'),
"ToleranceDetails": {
Optional('uom'): And(str, lambda s: s in uomList),
Optional("value"): And(Use(to_decimal_4), validDecimal_12_4),
"tolerance": And(str, lambda s: s in toleranceList)
},
"allowPartialShipments": Use(xml_bool),
Optional("unitPrice"): And(Use(to_decimal_4), validDecimal_12_4),
"lineItemTotal": And(Use(to_decimal_4), validDecimal_12_4),
Optional("requestedShipDate"):
And(Const(Use(fromisoformat)),
Regex(r'{}'.format(rdatetime))),
Optional("requestedInHands"):
And(Const(Use(fromisoformat)),
Regex(r'{}'.format(rdatetime))),
Optional("referenceSalesQuote"):
And(lambda s: var_check(s, 64),
error='"referenceSalesQuote" should evaluate to varchar(64)'),
Optional("Program"): {
Optional("id"):
And(lambda s: var_check(s, 64),
error='"id" should evaluate to varchar(64)'),
Optional("name"):
And(lambda s: var_check(s, 64),
error='"name" should evaluate to varchar(64)')
},
Optional("endCustomerSalesOrder"):
And(lambda s: var_check(s, 64),
error= '"endCustomerSalesOrder" should evaluate to varchar(64)'),
Optional("productId"):
And(lambda s: var_check(s, 64),
error='"productId" should evaluate to varchar(64)'),
Optional("customerProductId"):
And(lambda s: var_check(s, 64),
error='"customerProductId" should evaluate to varchar(64)'),
Optional("lineItemGroupingId"): int,
Optional("PartArray"): [{
"Part": {
Optional("partGroup"):
And(lambda s: var_check(s, 64),
error='"partGroup" should evaluate to varchar(64)'),
"partId":
And(lambda s: var_check(s, 64),
error='"partId" should evaluate to varchar(64)'),
Optional("customerPartId"):
And(lambda s: var_check(s, 64),
error='"customerPartId" should evaluate to varchar(64)'),
"customerSupplied": Use(xml_bool),
Optional("description"): str,
"Quantity": quantity_schema,
Optional("locationLinkId"): [int],
Optional("unitPrice"): And(Use(to_decimal_4), validDecimal_12_4),
Optional("extendedPrice"): And(Use(to_decimal_4), validDecimal_12_4),
Optional("ShipmentLinkArray"): [{
"ShipmentLink": {
"Quantity": quantity_schema,
"shipmentId": int
}
}]
}
}],
Optional("Configuration"): {
Optional("ChargeArray"): [{
"Charge": {
Optional("chargeName"):
And(lambda s: var_check(s, 128),
error='"chargeName" should evaluate to varchar(128)'),
Optional("description"): str,
Optional("extendedPrice"): And(Use(to_decimal_4), validDecimal_12_4),
Optional("unitprice"): And(Use(to_decimal_4), validDecimal_12_4),
"chargeId":
And(lambda s: var_check(s, 64),
error='"chargeId" should evaluate to varchar(64)'),
"chargeType":
And(str, lambda s: s in
['Freight', 'Order', 'Run', 'Setup']),
"Quantity": quantity_schema
}
}],
Optional("LocationArray"): [{
"Location": {
Optional("locationName"):
And(lambda s: var_check(s, 128),
error='"locationName" should evaluate to varchar(128)'),
"DecorationArray": [{
"Decoration": {
Optional("decorationName"):
And(lambda s: var_check(s, 128),
error='"decorationName" should evaluate to varchar(128)'),
"Artwork": {
Optional("instructions"): str,
Optional("refArtworkId"):
And(lambda s: var_check(s, 64),
error='"refArtworkId" should evaluate to varchar(64)'),
Optional("totalStitchCount"): int,
Optional("ArtworkFileArray"): [{
"ArtworkFile": {
"artworkType":
And(str, lambda s: s in [
'ProductionReady', 'VirtualProof',
'SupplierArtTemplate', 'NonProductionReady'
]),
"fileLocation":
And(lambda s: var_check(s, 1024),
error='"fileLocation" should evaluate to varchar(1024)'),
"fileName":
And(lambda s: var_check(s, 256),
error='"fileName" should evaluate to varchar(256)'),
"transportMechanism":
And(str, lambda s: s in [
'Email', 'Url', 'Ftp', 'ArtworkToFollow'
]),
}
}],
Optional("description"): str,
Optional("Dimensions"): {
Optional("diameter"):
And(Use(to_decimal_4),validDecimal_12_4),
Optional("height"):
And(Use(to_decimal_4),validDecimal_12_4),
Optional('uom'):
And(str, lambda s: s in uomList),
Optional("width"):
And(Use(to_decimal_4),validDecimal_12_4),
"useMaxLocationDimensions": Use(xml_bool),
"geometry":
And(str, lambda s: s in [
'Circle',
'Other',
'Rectangle'
])
},
Optional("Layers"): {
"colorSystem":
And(
str, lambda s: s in [
'Cmyk', 'Other',
'Pms', 'Rgb',
'Thread'
]),
"LayerOrStopArray": [{
"LayerOrStop":{
"color":
And(lambda s: var_check(s, 64),
error='Layer "color" should evaluate to varchar(64)'),
"nameOrNumber":
And(lambda s: var_check(s, 64),
error='"nameOrNumber" should evaluate to varchar(64)'),
"description": str
}}
]
},
Optional("TypesetArray"): [{
"Typeset":{
Optional("fontSize"): Use(Decimal),
Optional("font"):
And(lambda s: var_check(s, 64),
error='"font" should evaluate to varchar(64)'),
"sequenceNumber": int,
"value":
And(lambda s: var_check(s, 1024),
error='Typset "value" should evaluate to varchar(1024)')
}
}]
},
"decorationId": int
}
}],
"locationLinkId": int,
"locationId": int
}
}],
Optional("referenceNumberType"):
And(
str, lambda s: s in [
'PurchaseOrder', 'SalesOrder',
'JobOrWorkOrder'
]),
Optional("referenceNumber"):
And(lambda s: var_check(s, 64),
error='"referenceNumber" should evaluate to varchar(64)'),
"preProductionProof": Use(xml_bool),
}
}
}],
"termsAndConditions": str,
Optional("salesChannel"):
And(lambda s: var_check(s, 64),
error='"salesChannel" should evaluate to varchar(64)'),
Optional("promoCode"):
And(lambda s: var_check(s, 64),
error='"promoCode" should evaluate to varchar(64)'),
Optional("TaxInformationArray"): [{
"TaxInformation": {
"taxJurisdiction":
And(lambda s: var_check(s, 64),
error='"taxJurisdiction" should evaluate to varchar(64)'),
"taxExempt": Use(xml_bool),
"taxId":
And(lambda s: var_check(s, 64),
error='"taxId" should evaluate to varchar(64)'),
"taxType":
And(lambda s: var_check(s, 64),
error='"taxType" should evaluate to varchar(64)'),
Optional("taxAmount"): Use(Decimal)
}
}]
}
})
# run the validation
v1_0_0_schema.validate(kw)
Optional("restart_execution_on_update"): bool,
}
AWS_ACCOUNT_ID_REGEX_STR = r"\A[0-9]{12}\Z"
AWS_ACCOUNT_ID_SCHEMA = Schema(
And(
Or(int, str),
Use(str),
Regex(
AWS_ACCOUNT_ID_REGEX_STR,
error=(
"The specified account id is incorrect. "
"This typically happens when you specify the account id as a "
"number, while the account id starts with a zero. If this is "
"the case, please wrap the account id in quotes to make it a "
"string. An AWS Account Id is a number of 12 digits, which "
"should start with a zero if the Account Id has a zero at "
"the start too. "
"The number shown to not match the regular expression could "
"be interpreted as an octal number due to the leading zero. "
"Therefore, it might not match the account id as specified "
"in the deployment map."
)
)
)
)
# CodeCommit Source
CODECOMMIT_SOURCE_PROPS = {
"account_id": AWS_ACCOUNT_ID_SCHEMA,
Optional("repository"): str,
Optional("branch"): str,
class Config(object): # pylint: disable=too-many-instance-attributes
"""Class that manages configuration files for a dvc repo.
Args:
dvc_dir (str): optional path to `.dvc` directory, that is used to
access repo-specific configs like .dvc/config and
.dvc/config.local.
validate (bool): optional flag to tell dvc if it should validate the
config or just load it as is. 'True' by default.
Raises:
ConfigError: thrown when config has an invalid format.
"""
APPNAME = "dvc"
APPAUTHOR = "iterative"
# NOTE: used internally in RemoteLOCAL to know config
# location, that url should resolved relative to.
PRIVATE_CWD = "_cwd"
CONFIG = "config"
CONFIG_LOCAL = "config.local"
BOOL_SCHEMA = And(str, is_bool, Use(to_bool))
SECTION_CORE = "core"
SECTION_CORE_LOGLEVEL = "loglevel"
SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel)
SECTION_CORE_REMOTE = "remote"
SECTION_CORE_INTERACTIVE_SCHEMA = BOOL_SCHEMA
SECTION_CORE_INTERACTIVE = "interactive"
SECTION_CORE_ANALYTICS = "analytics"
SECTION_CORE_ANALYTICS_SCHEMA = BOOL_SCHEMA
SECTION_CACHE = "cache"
SECTION_CACHE_DIR = "dir"
SECTION_CACHE_TYPE = "type"
SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
SECTION_CACHE_PROTECTED = "protected"
SECTION_CACHE_LOCAL = "local"
SECTION_CACHE_S3 = "s3"
SECTION_CACHE_GS = "gs"
SECTION_CACHE_SSH = "ssh"
SECTION_CACHE_HDFS = "hdfs"
SECTION_CACHE_AZURE = "azure"
SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning"
SECTION_CACHE_SCHEMA = {
Optional(SECTION_CACHE_LOCAL): str,
Optional(SECTION_CACHE_S3): str,
Optional(SECTION_CACHE_GS): str,
Optional(SECTION_CACHE_HDFS): str,
Optional(SECTION_CACHE_SSH): str,
Optional(SECTION_CACHE_AZURE): str,
Optional(SECTION_CACHE_DIR): str,
Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
Optional(PRIVATE_CWD): str,
Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): BOOL_SCHEMA,
}
# backward compatibility
SECTION_CORE_CLOUD = "cloud"
SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud)
SECTION_CORE_STORAGEPATH = "storagepath"
SECTION_CORE_SCHEMA = {
Optional(SECTION_CORE_LOGLEVEL):
And(str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA),
Optional(SECTION_CORE_REMOTE, default=""):
And(str, Use(str.lower)),
Optional(SECTION_CORE_INTERACTIVE, default=False):
SECTION_CORE_INTERACTIVE_SCHEMA,
Optional(SECTION_CORE_ANALYTICS, default=True):
SECTION_CORE_ANALYTICS_SCHEMA,
# backward compatibility
Optional(SECTION_CORE_CLOUD, default=""):
SECTION_CORE_CLOUD_SCHEMA,
Optional(SECTION_CORE_STORAGEPATH, default=""):
str,
}
# backward compatibility
SECTION_AWS = "aws"
SECTION_AWS_STORAGEPATH = "storagepath"
SECTION_AWS_CREDENTIALPATH = "credentialpath"
SECTION_AWS_ENDPOINT_URL = "endpointurl"
SECTION_AWS_LIST_OBJECTS = "listobjects"
SECTION_AWS_REGION = "region"
SECTION_AWS_PROFILE = "profile"
SECTION_AWS_USE_SSL = "use_ssl"
SECTION_AWS_SCHEMA = {
SECTION_AWS_STORAGEPATH: str,
Optional(SECTION_AWS_REGION): str,
Optional(SECTION_AWS_PROFILE): str,
Optional(SECTION_AWS_CREDENTIALPATH): str,
Optional(SECTION_AWS_ENDPOINT_URL): str,
Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
}
# backward compatibility
SECTION_GCP = "gcp"
SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH
SECTION_GCP_CREDENTIALPATH = SECTION_AWS_CREDENTIALPATH
SECTION_GCP_PROJECTNAME = "projectname"
SECTION_GCP_SCHEMA = {
SECTION_GCP_STORAGEPATH: str,
Optional(SECTION_GCP_PROJECTNAME): str,
}
# backward compatibility
SECTION_LOCAL = "local"
SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH
SECTION_LOCAL_SCHEMA = {SECTION_LOCAL_STORAGEPATH: str}
SECTION_AZURE_CONNECTION_STRING = "connection_string"
# Alibabacloud oss options
SECTION_OSS_ACCESS_KEY_ID = "oss_key_id"
SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret"
SECTION_OSS_ENDPOINT = "oss_endpoint"
SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
SECTION_REMOTE_FMT = 'remote "{}"'
SECTION_REMOTE_URL = "url"
SECTION_REMOTE_USER = "******"
SECTION_REMOTE_PORT = "port"
SECTION_REMOTE_KEY_FILE = "keyfile"
SECTION_REMOTE_TIMEOUT = "timeout"
SECTION_REMOTE_PASSWORD = "******"
SECTION_REMOTE_ASK_PASSWORD = "******"
SECTION_REMOTE_SCHEMA = {
SECTION_REMOTE_URL: str,
Optional(SECTION_AWS_REGION): str,
Optional(SECTION_AWS_PROFILE): str,
Optional(SECTION_AWS_CREDENTIALPATH): str,
Optional(SECTION_AWS_ENDPOINT_URL): str,
Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
Optional(SECTION_GCP_PROJECTNAME): str,
Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
Optional(SECTION_REMOTE_USER): str,
Optional(SECTION_REMOTE_PORT): Use(int),
Optional(SECTION_REMOTE_KEY_FILE): str,
Optional(SECTION_REMOTE_TIMEOUT): Use(int),
Optional(SECTION_REMOTE_PASSWORD): str,
Optional(SECTION_REMOTE_ASK_PASSWORD): BOOL_SCHEMA,
Optional(SECTION_AZURE_CONNECTION_STRING): str,
Optional(SECTION_OSS_ACCESS_KEY_ID): str,
Optional(SECTION_OSS_ACCESS_KEY_SECRET): str,
Optional(SECTION_OSS_ENDPOINT): str,
Optional(PRIVATE_CWD): str,
}
SECTION_STATE = "state"
SECTION_STATE_ROW_LIMIT = "row_limit"
SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota"
SECTION_STATE_SCHEMA = {
Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole),
Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent),
}
SCHEMA = {
Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
# backward compatibility
Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA,
Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
}
def __init__(self, dvc_dir=None, validate=True):
self.system_config_file = os.path.join(self.get_system_config_dir(),
self.CONFIG)
self.global_config_file = os.path.join(self.get_global_config_dir(),
self.CONFIG)
if dvc_dir is not None:
self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
self.config_file = os.path.join(dvc_dir, self.CONFIG)
self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
else:
self.dvc_dir = None
self.config_file = None
self.config_local_file = None
self._system_config = None
self._global_config = None
self._repo_config = None
self._local_config = None
self.config = None
self.load(validate=validate)
@staticmethod
def get_global_config_dir():
"""Returns global config location. E.g. ~/.config/dvc/config.
Returns:
str: path to the global config directory.
"""
from appdirs import user_config_dir
return user_config_dir(appname=Config.APPNAME,
appauthor=Config.APPAUTHOR)
@staticmethod
def get_system_config_dir():
"""Returns system config location. E.g. /etc/dvc.conf.
Returns:
str: path to the system config directory.
"""
from appdirs import site_config_dir
return site_config_dir(appname=Config.APPNAME,
appauthor=Config.APPAUTHOR)
@staticmethod
def init(dvc_dir):
"""Initializes dvc config.
Args:
dvc_dir (str): path to .dvc directory.
Returns:
dvc.config.Config: config object.
"""
config_file = os.path.join(dvc_dir, Config.CONFIG)
open(config_file, "w+").close()
return Config(dvc_dir)
def _load(self):
self._system_config = configobj.ConfigObj(self.system_config_file)
self._global_config = configobj.ConfigObj(self.global_config_file)
if self.config_file is not None:
self._repo_config = configobj.ConfigObj(self.config_file)
else:
self._repo_config = configobj.ConfigObj()
if self.config_local_file is not None:
self._local_config = configobj.ConfigObj(self.config_local_file)
else:
self._local_config = configobj.ConfigObj()
self.config = None
def _load_config(self, path):
config = configobj.ConfigObj(path)
config = self._lower(config)
self._resolve_paths(config, path)
return config
@staticmethod
def _resolve_path(path, config_file):
assert os.path.isabs(config_file)
config_dir = os.path.dirname(config_file)
return os.path.abspath(os.path.join(config_dir, path))
def _resolve_cache_path(self, config, fname):
cache = config.get(self.SECTION_CACHE)
if cache is None:
return
cache_dir = cache.get(self.SECTION_CACHE_DIR)
if cache_dir is None:
return
cache[self.PRIVATE_CWD] = os.path.dirname(fname)
def _resolve_paths(self, config, fname):
if fname is None:
return
self._resolve_cache_path(config, fname)
for section in config.values():
if self.SECTION_REMOTE_URL not in section.keys():
continue
section[self.PRIVATE_CWD] = os.path.dirname(fname)
def load(self, validate=True):
"""Loads config from all the config files.
Args:
validate (bool): optional flag to tell dvc if it should validate
the config or just load it as is. 'True' by default.
Raises:
dvc.config.ConfigError: thrown if config has invalid format.
"""
self._load()
try:
self.config = self._load_config(self.system_config_file)
user = self._load_config(self.global_config_file)
config = self._load_config(self.config_file)
local = self._load_config(self.config_local_file)
# NOTE: schema doesn't support ConfigObj.Section validation, so we
# need to convert our config to dict before passing it to
for conf in [user, config, local]:
self.config = self._merge(self.config, conf)
if validate:
self.config = Schema(self.SCHEMA).validate(self.config)
# NOTE: now converting back to ConfigObj
self.config = configobj.ConfigObj(self.config,
write_empty_values=True)
self.config.filename = self.config_file
self._resolve_paths(self.config, self.config_file)
except Exception as ex:
raise ConfigError(ex)
@staticmethod
def _get_key(conf, name, add=False):
for k in conf.keys():
if k.lower() == name.lower():
return k
if add:
conf[name] = {}
return name
return None
def save(self, config=None):
"""Saves config to config files.
Args:
config (configobj.ConfigObj): optional config object to save.
Raises:
dvc.config.ConfigError: thrown if failed to write config file.
"""
if config is not None:
clist = [config]
else:
clist = [
self._system_config,
self._global_config,
self._repo_config,
self._local_config,
]
for conf in clist:
if conf.filename is None:
continue
try:
logger.debug("Writing '{}'.".format(conf.filename))
dname = os.path.dirname(os.path.abspath(conf.filename))
try:
os.makedirs(dname)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
conf.write()
except Exception as exc:
msg = "failed to write config '{}'".format(conf.filename)
raise ConfigError(msg, exc)
def get_remote_settings(self, name):
import posixpath
"""
Args:
name (str): The name of the remote that we want to retrieve
Returns:
dict: The content beneath the given remote name.
Example:
>>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
>>> get_remote_settings("server")
{'url': 'ssh://localhost/'}
"""
settings = self.config.get(self.SECTION_REMOTE_FMT.format(
name.lower()))
if settings is None:
raise ConfigError(
"unable to find remote section '{}'".format(name))
parsed = urlparse(settings["url"])
# Support for cross referenced remotes.
# This will merge the settings, giving priority to the outer reference.
# For example, having:
#
# dvc remote add server ssh://localhost
# dvc remote modify server user root
# dvc remote modify server ask_password true
#
# dvc remote add images remote://server/tmp/pictures
# dvc remote modify images user alice
# dvc remote modify images ask_password false
# dvc remote modify images password asdf1234
#
# Results on a config dictionary like:
#
# {
# "url": "ssh://localhost/tmp/pictures",
# "user": "******",
# "password": "******",
# "ask_password": False,
# }
#
if parsed.scheme == "remote":
reference = self.get_remote_settings(parsed.netloc)
url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
merged = reference.copy()
merged.update(settings)
merged["url"] = url
return merged
return settings
@staticmethod
def unset(config, section, opt=None):
"""Unsets specified option and/or section in the config.
Args:
config (configobj.ConfigObj): config to work on.
section (str): section name.
opt (str): optional option name.
"""
if section not in config.keys():
raise ConfigError("section '{}' doesn't exist".format(section))
if opt is None:
del config[section]
return
if opt not in config[section].keys():
raise ConfigError("option '{}.{}' doesn't exist".format(
section, opt))
del config[section][opt]
if not config[section]:
del config[section]
@staticmethod
def set(config, section, opt, value):
"""Sets specified option in the config.
Args:
config (configobj.ConfigObj): config to work on.
section (str): section name.
opt (str): option name.
value: value to set option to.
"""
if section not in config.keys():
config[section] = {}
config[section][opt] = value
@staticmethod
def show(config, section, opt):
"""Prints option value from the config.
Args:
config (configobj.ConfigObj): config to work on.
section (str): section name.
opt (str): option name.
"""
if section not in config.keys():
raise ConfigError("section '{}' doesn't exist".format(section))
if opt not in config[section].keys():
raise ConfigError("option '{}.{}' doesn't exist".format(
section, opt))
logger.info(config[section][opt])
@staticmethod
def _merge(first, second):
res = {}
sections = list(first.keys()) + list(second.keys())
for section in sections:
first_copy = first.get(section, {}).copy()
second_copy = second.get(section, {}).copy()
first_copy.update(second_copy)
res[section] = first_copy
return res
@staticmethod
def _lower(config):
new_config = {}
for s_key, s_value in config.items():
new_s = {}
for key, value in s_value.items():
new_s[key.lower()] = str(value)
new_config[s_key.lower()] = new_s
return new_config
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from schema import And, Optional, Or, Regex, Schema
NAME_ID_VALIDATION_REGEX = Regex(r"^[A-Za-z0-9_. ()-]+$")
RESOURCE_TYPE_REGEX = Regex(
r"^AWS\.(ACM\.Certificate|CloudFormation\.Stack|CloudTrail\.Meta|CloudTrail|CloudWatch"
r"\.LogGroup|Config\.Recorder\.Meta|Config\.Recorder|DynamoDB\.Table|EC2\.AMI|EC2\.Instance"
r"|EC2\.NetworkACL|EC2\.SecurityGroup|EC2\.Volume|EC2\.VPC|ECS\.Cluster|EKS\.Cluster|ELBV2"
r"\.ApplicationLoadBalancer|GuardDuty\.Detector\.Meta|GuardDuty\.Detector|IAM\.Group|IAM"
r"\.Policy|IAM\.Role|IAM\.RootUser|IAM\.User|KMS\.Key|Lambda\.Function|PasswordPolicy|RDS"
r"\.Instance|Redshift\.Cluster|S3\.Bucket|WAF\.Regional\.WebACL|WAF\.WebACL)$"
)
LOG_TYPE_REGEX = Regex(
r"^(Apache\.AccessCombined|Apache\.AccessCommon|AWS\.ALB|AWS\.AuroraMySQLAudit|AWS"
r"\.CloudTrail|AWS\.CloudTrailDigest|AWS\.CloudTrailInsight|AWS\.CloudWatchEvents|AWS"
r"\.GuardDuty|AWS\.S3ServerAccess|AWS\.VPCDns|AWS\.VPCFlow|Box\.Event|CiscoUmbrella"
r"\.CloudFirewall|CiscoUmbrella\.DNS|CiscoUmbrella\.IP|CiscoUmbrella\.Proxy|Cloudflare"
r"\.Firewall|Cloudflare\.HttpRequest|Cloudflare\.Spectrum|Crowdstrike\.AIDMaster|Crowdstrike"
r"\.DNSRequest|Crowdstrike\.GroupIdentity|Crowdstrike\.ManagedAssets|Crowdstrike"
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import os
from schema import Schema, And, Use, Optional, Regex, Or
common_schema = {
'authorName': str,
'experimentName': str,
Optional('description'): str,
'trialConcurrency': And(int, lambda n: 1 <=n <= 999999),
Optional('maxExecDuration'): Regex(r'^[1-9][0-9]*[s|m|h|d]$'),
Optional('maxTrialNum'): And(int, lambda x: 1 <= x <= 99999),
'trainingServicePlatform': And(str, lambda x: x in ['remote', 'local', 'pai', 'kubeflow', 'frameworkcontroller']),
Optional('searchSpacePath'): os.path.exists,
Optional('multiPhase'): bool,
Optional('multiThread'): bool,
Optional('nniManagerIp'): str,
Optional('logDir'): os.path.isdir,
Optional('logLevel'): Or('trace', 'debug', 'info', 'warning', 'error', 'fatal'),
'useAnnotation': bool,
Optional('advisor'): Or({
'builtinAdvisorName': Or('Hyperband'),
'classArgs': {
'optimize_mode': Or('maximize', 'minimize'),
Optional('R'): int,
Optional('eta'): int
rpm_schema = Schema({
'content': {
Optional('build'): {
'use_source_tito_config': bool,
'tito_target': And(str, len),
'push_release_commit': bool,
},
Optional('source'): {
Optional('alias'): And(str, len),
Optional('git'): {
'branch': {
Optional('fallback'): And(str, len),
Optional('stage'): And(str, len),
'target': And(str, len),
},
'url': And(str, len, Regex(GIT_SSH_URL_REGEX)),
},
'specfile': Regex(r'.+\.spec$'),
Optional('modifications'): [modification],
},
},
Optional('distgit'): {
Optional('branch'): And(str, len),
},
Optional('enabled_repos'): [
And(str, len),
],
Optional('mode'): Or(*valid_modes),
'name': And(str, len),
'owners': [And(str, len)],
Optional('maintainer'): {
def validate_basic_config(data):
container = And(
{
Optional('name', default=DEFAULT_CONTAINER_NAME):
And(str, Regex(r'^[\w-]+$')),
'projectDir':
And(
str,
And(os.path.isabs,
error=
'Use an absolute path when specifying the project directory'
), Use(lambda x: x.rstrip('/'))),
Optional('image', default=''):
And(str, len),
Optional('file', default=''):
And(
str, # TODO: a proper regex that the filename is valid
Regex(r'^[\w\.\/@-]*$', error='Invalid name for a Dockerfile'),
And(lambda x: not x.endswith('/'),
error='Invalid name for a Dockerfile'),
And(lambda x: not os.path.isabs(x),
error='Path to the Dockerfile should be relative to the '
'project\'s root directory.'),
),
Optional('runAsHostUser', default=False):
bool,
Optional('volumeMounts', default=[]): (And(
[{
'name':
And(Or(int, str), Use(str), Regex(r'^[\w-]+$')),
'mountPath':
And(
str,
And(os.path.isabs,
error=
'Use an absolute path when specifying a mount directory'
),
Use(lambda x: x.rstrip('/')),
),
}],
And(lambda x: is_unique_value(x, 'name'),
error='Each volume mount must have a unique name.'),
And(lambda x: not has_prefix([(volume['mountPath'] + '/')
for volume in x]),
error=
'Volume mount paths cannot be prefixes for each other.'),
)),
Optional('workingDir', default=''):
And(
str,
And(os.path.isabs,
error='Use an absolute path when specifying a '
'working directory'), Use(lambda x: x.rstrip('/'))),
Optional('env', default={}): {
And(str, Regex(r'^[a-zA-Z_]+[a-zA-Z0-9_]*$')): str,
},
Optional('hostNetwork', default=False):
bool,
Optional('ports', default=[]): [{
'containerPort':
And(int, lambda x: 0 < x < 65536),
Optional('hostPort', default=None):
And(int, lambda x: 0 < x < 65536),
}],
Optional('commands', default=''):
str,
# TODO: allow to use only certain runtime parameters
Optional('runtimeParameters', default=[]):
And([str], Use(lambda x: [p.strip() for p in x])),
},
And(lambda x: x['image'] or x['file'],
error='Either "image" or "file" should be specified.'),
And(lambda x: not (x['image'] and x['file']),
error='"image" and "file" cannot be specified together.'),
And(lambda x: not (x['hostNetwork'] and x['ports']),
error=
'Published ports and the host network mode cannot be used together.'
),
)
schema = Schema({
'project': {
'name':
And(str, Regex(r'^[a-zA-Z0-9][a-zA-Z0-9-]{,26}[a-zA-Z0-9]$')),
Optional('syncFilters', default=[]):
And(
[
And(
{
Optional('exclude'): [
And(
str, len,
And(lambda x: '**' not in x,
error=
'Use single asterisks ("*") in sync filters'
))
],
Optional('include'): [
And(
str, len,
And(lambda x: '**' not in x,
error=
'Use single asterisks ("*") in sync filters'
))
],
},
And(lambda x: x,
error=
'Either "exclude" or "include" filter should be specified.'
),
And(lambda x: not ('exclude' in x and 'include' in x),
error=
'"exclude" and "include" filters should be specified as different list items.'
),
)
],
error='"project.syncFilters" field must be a list.',
)
},
WrongKey('container',
error='Use "containers" field instead of "container".'):
object,
Optional('containers', default=[]):
And(
[container],
And(lambda x: is_unique_value(x, 'name'),
error='Each container must have a unique name.'),
error='"containers" field must be a list.',
),
WrongKey('instance',
error='Use "instances" field instead of "instance".'):
object,
'instances':
And(
[{
'name': And(Or(int, str), Use(str), Regex(r'^[\w-]+$')),
'provider': str,
Optional('parameters', default={}): {
And(str, Regex(r'^[\w]+$')): object,
}
}],
And(lambda x: len(x),
error=
'At least one instance must be specified in the configuration file.'
),
And(lambda x: is_unique_value(x, 'name'),
error='Each instance must have a unique name.'),
),
Optional('scripts', default={}): {
And(str, Regex(r'^[\w-]+$')): And(str, len),
},
})
return validate_config(schema, data)
class Config(object): # pylint: disable=too-many-instance-attributes
"""Class that manages configuration files for a dvc repo.
Args:
dvc_dir (str): optional path to `.dvc` directory, that is used to
access repo-specific configs like .dvc/config and
.dvc/config.local.
validate (bool): optional flag to tell dvc if it should validate the
config or just load it as is. 'True' by default.
Raises:
ConfigError: thrown when config has an invalid format.
"""
APPNAME = "dvc"
APPAUTHOR = "iterative"
# NOTE: used internally in RemoteLOCAL to know config
# location, that url should resolved relative to.
PRIVATE_CWD = "_cwd"
CONFIG = "config"
CONFIG_LOCAL = "config.local"
LEVEL_LOCAL = 0
LEVEL_REPO = 1
LEVEL_GLOBAL = 2
LEVEL_SYSTEM = 3
BOOL_SCHEMA = And(str, is_bool, Use(to_bool))
SECTION_CORE = "core"
SECTION_CORE_LOGLEVEL = "loglevel"
SECTION_CORE_LOGLEVEL_SCHEMA = And(
Use(str.lower), Choices("info", "debug", "warning", "error"))
SECTION_CORE_REMOTE = "remote"
SECTION_CORE_INTERACTIVE_SCHEMA = BOOL_SCHEMA
SECTION_CORE_INTERACTIVE = "interactive"
SECTION_CORE_ANALYTICS = "analytics"
SECTION_CORE_ANALYTICS_SCHEMA = BOOL_SCHEMA
SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs"
SECTION_CORE_CHECKSUM_JOBS_SCHEMA = And(Use(int), lambda x: x > 0)
SECTION_CACHE = "cache"
SECTION_CACHE_DIR = "dir"
SECTION_CACHE_TYPE = "type"
SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
SECTION_CACHE_PROTECTED = "protected"
SECTION_CACHE_SHARED = "shared"
SECTION_CACHE_SHARED_SCHEMA = And(Use(str.lower), Choices("group"))
SECTION_CACHE_LOCAL = "local"
SECTION_CACHE_S3 = "s3"
SECTION_CACHE_GS = "gs"
SECTION_CACHE_SSH = "ssh"
SECTION_CACHE_HDFS = "hdfs"
SECTION_CACHE_AZURE = "azure"
SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning"
SECTION_CACHE_SCHEMA = {
Optional(SECTION_CACHE_LOCAL): str,
Optional(SECTION_CACHE_S3): str,
Optional(SECTION_CACHE_GS): str,
Optional(SECTION_CACHE_HDFS): str,
Optional(SECTION_CACHE_SSH): str,
Optional(SECTION_CACHE_AZURE): str,
Optional(SECTION_CACHE_DIR): str,
Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
Optional(SECTION_CACHE_SHARED): SECTION_CACHE_SHARED_SCHEMA,
Optional(PRIVATE_CWD): str,
Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): BOOL_SCHEMA,
}
SECTION_CORE_SCHEMA = {
Optional(SECTION_CORE_LOGLEVEL):
And(str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA),
Optional(SECTION_CORE_REMOTE, default=""):
And(str, Use(str.lower)),
Optional(SECTION_CORE_INTERACTIVE, default=False):
SECTION_CORE_INTERACTIVE_SCHEMA,
Optional(SECTION_CORE_ANALYTICS, default=True):
SECTION_CORE_ANALYTICS_SCHEMA,
Optional(SECTION_CORE_CHECKSUM_JOBS, default=None):
SECTION_CORE_CHECKSUM_JOBS_SCHEMA,
}
# backward compatibility
SECTION_AWS = "aws"
SECTION_AWS_STORAGEPATH = "storagepath"
SECTION_AWS_CREDENTIALPATH = "credentialpath"
SECTION_AWS_ENDPOINT_URL = "endpointurl"
SECTION_AWS_LIST_OBJECTS = "listobjects"
SECTION_AWS_REGION = "region"
SECTION_AWS_PROFILE = "profile"
SECTION_AWS_USE_SSL = "use_ssl"
SECTION_AWS_SSE = "sse"
SECTION_AWS_ACL = "acl"
SECTION_AWS_SCHEMA = {
SECTION_AWS_STORAGEPATH: str,
Optional(SECTION_AWS_REGION): str,
Optional(SECTION_AWS_PROFILE): str,
Optional(SECTION_AWS_CREDENTIALPATH): str,
Optional(SECTION_AWS_ENDPOINT_URL): str,
Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
Optional(SECTION_AWS_SSE): str,
Optional(SECTION_AWS_ACL): str,
}
# backward compatibility
SECTION_GCP = "gcp"
SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH
SECTION_GCP_CREDENTIALPATH = SECTION_AWS_CREDENTIALPATH
SECTION_GCP_PROJECTNAME = "projectname"
SECTION_GCP_SCHEMA = {
SECTION_GCP_STORAGEPATH: str,
Optional(SECTION_GCP_PROJECTNAME): str,
}
# backward compatibility
SECTION_LOCAL = "local"
SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH
SECTION_LOCAL_SCHEMA = {SECTION_LOCAL_STORAGEPATH: str}
SECTION_AZURE_CONNECTION_STRING = "connection_string"
# Alibabacloud oss options
SECTION_OSS_ACCESS_KEY_ID = "oss_key_id"
SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret"
SECTION_OSS_ENDPOINT = "oss_endpoint"
SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
SECTION_REMOTE_FMT = 'remote "{}"'
SECTION_REMOTE_URL = "url"
SECTION_REMOTE_USER = "******"
SECTION_REMOTE_PORT = "port"
SECTION_REMOTE_KEY_FILE = "keyfile"
SECTION_REMOTE_TIMEOUT = "timeout"
SECTION_REMOTE_PASSWORD = "******"
SECTION_REMOTE_ASK_PASSWORD = "******"
SECTION_REMOTE_GSS_AUTH = "gss_auth"
SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
SECTION_REMOTE_SCHEMA = {
SECTION_REMOTE_URL: str,
Optional(SECTION_AWS_REGION): str,
Optional(SECTION_AWS_PROFILE): str,
Optional(SECTION_AWS_CREDENTIALPATH): str,
Optional(SECTION_AWS_ENDPOINT_URL): str,
Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA,
Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA,
Optional(SECTION_AWS_SSE): str,
Optional(SECTION_AWS_ACL): str,
Optional(SECTION_GCP_PROJECTNAME): str,
Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA,
Optional(SECTION_REMOTE_USER): str,
Optional(SECTION_REMOTE_PORT): Use(int),
Optional(SECTION_REMOTE_KEY_FILE): str,
Optional(SECTION_REMOTE_TIMEOUT): Use(int),
Optional(SECTION_REMOTE_PASSWORD): str,
Optional(SECTION_REMOTE_ASK_PASSWORD): BOOL_SCHEMA,
Optional(SECTION_REMOTE_GSS_AUTH): BOOL_SCHEMA,
Optional(SECTION_AZURE_CONNECTION_STRING): str,
Optional(SECTION_OSS_ACCESS_KEY_ID): str,
Optional(SECTION_OSS_ACCESS_KEY_SECRET): str,
Optional(SECTION_OSS_ENDPOINT): str,
Optional(PRIVATE_CWD): str,
Optional(SECTION_REMOTE_NO_TRAVERSE, default=True): BOOL_SCHEMA,
}
SECTION_STATE = "state"
SECTION_STATE_ROW_LIMIT = "row_limit"
SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota"
SECTION_STATE_SCHEMA = {
Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole),
Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent),
}
SCHEMA = {
Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
# backward compatibility
Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA,
Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
}
def __init__(self, dvc_dir=None, validate=True):
self.dvc_dir = dvc_dir
self.validate = validate
if not dvc_dir:
try:
from dvc.repo import Repo
self.dvc_dir = os.path.join(Repo.find_dvc_dir())
except NotDvcRepoError:
self.dvc_dir = None
else:
self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
self.load()
@staticmethod
def get_global_config_dir():
"""Returns global config location. E.g. ~/.config/dvc/config.
Returns:
str: path to the global config directory.
"""
from appdirs import user_config_dir
return user_config_dir(appname=Config.APPNAME,
appauthor=Config.APPAUTHOR)
@staticmethod
def get_system_config_dir():
"""Returns system config location. E.g. /etc/dvc.conf.
Returns:
str: path to the system config directory.
"""
from appdirs import site_config_dir
return site_config_dir(appname=Config.APPNAME,
appauthor=Config.APPAUTHOR)
@staticmethod
def init(dvc_dir):
"""Initializes dvc config.
Args:
dvc_dir (str): path to .dvc directory.
Returns:
dvc.config.Config: config object.
"""
config_file = os.path.join(dvc_dir, Config.CONFIG)
open(config_file, "w+").close()
return Config(dvc_dir)
def _resolve_cache_path(self, config):
cache = config.get(self.SECTION_CACHE)
if cache is None:
return
cache_dir = cache.get(self.SECTION_CACHE_DIR)
if cache_dir is None:
return
cache[self.PRIVATE_CWD] = os.path.dirname(config.filename)
def _resolve_paths(self, config):
if config.filename is None:
return config
ret = copy.deepcopy(config)
self._resolve_cache_path(ret)
for section in ret.values():
if self.SECTION_REMOTE_URL not in section.keys():
continue
section[self.PRIVATE_CWD] = os.path.dirname(ret.filename)
return ret
def _load_configs(self):
system_config_file = os.path.join(self.get_system_config_dir(),
self.CONFIG)
global_config_file = os.path.join(self.get_global_config_dir(),
self.CONFIG)
self._system_config = configobj.ConfigObj(system_config_file)
self._global_config = configobj.ConfigObj(global_config_file)
self._repo_config = configobj.ConfigObj()
self._local_config = configobj.ConfigObj()
if not self.dvc_dir:
return
config_file = os.path.join(self.dvc_dir, self.CONFIG)
config_local_file = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
self._repo_config = configobj.ConfigObj(config_file)
self._local_config = configobj.ConfigObj(config_local_file)
@property
def config_local_file(self):
return self._local_config.filename
@property
def config_file(self):
return self._repo_config.filename
def load(self):
"""Loads config from all the config files.
Raises:
dvc.config.ConfigError: thrown if config has invalid format.
"""
self._load_configs()
self.config = configobj.ConfigObj()
for c in [
self._system_config,
self._global_config,
self._repo_config,
self._local_config,
]:
c = self._resolve_paths(c)
c = self._lower(c)
self.config.merge(c)
if not self.validate:
return
d = self.config.dict()
try:
d = Schema(self.SCHEMA).validate(d)
except SchemaError as exc:
raise ConfigError("config format error", cause=exc)
self.config = configobj.ConfigObj(d, write_empty_values=True)
def save(self, config=None):
"""Saves config to config files.
Raises:
dvc.config.ConfigError: thrown if failed to write config file.
"""
if config is not None:
clist = [config]
else:
clist = [
self._system_config,
self._global_config,
self._repo_config,
self._local_config,
]
for conf in clist:
self._save(conf)
self.load()
@staticmethod
def _save(config):
if config.filename is None:
return
logger.debug("Writing '{}'.".format(config.filename))
dname = os.path.dirname(os.path.abspath(config.filename))
try:
os.makedirs(dname)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
config.write()
def unset(self, section, opt=None, level=None, force=False):
"""Unsets specified option and/or section in the config.
Args:
section (str): section name.
opt (str): optional option name.
level (int): config level to use.
force (bool): don't error-out even if section doesn't exist. False
by default.
Raises:
dvc.config.ConfigError: thrown if section doesn't exist and
`force != True`.
"""
config = self.get_configobj(level)
if section not in config.keys():
if force:
return
raise ConfigError("section '{}' doesn't exist".format(section))
if opt:
if opt not in config[section].keys():
if force:
return
raise ConfigError("option '{}.{}' doesn't exist".format(
section, opt))
del config[section][opt]
if not config[section]:
del config[section]
else:
del config[section]
self.save(config)
def set(self, section, opt, value, level=None, force=True):
"""Sets specified option in the config.
Args:
section (str): section name.
opt (str): option name.
value: value to set option to.
level (int): config level to use.
force (bool): set option even if section already exists. True by
default.
Raises:
dvc.config.ConfigError: thrown if section already exists and
`force != True`.
"""
config = self.get_configobj(level)
if section not in config.keys():
config[section] = {}
elif not force:
raise ConfigError(
"Section '{}' already exists. Use `-f|--force` to overwrite "
"section with new value.".format(section))
config[section][opt] = value
self.save(config)
def get(self, section, opt=None, level=None):
"""Return option value from the config.
Args:
section (str): section name.
opt (str): option name.
level (int): config level to use.
Returns:
value (str, int): option value.
"""
config = self.get_configobj(level)
if section not in config.keys():
raise ConfigError("section '{}' doesn't exist".format(section))
if opt not in config[section].keys():
raise ConfigError("option '{}.{}' doesn't exist".format(
section, opt))
return config[section][opt]
@staticmethod
def _lower(config):
new_config = configobj.ConfigObj()
for s_key, s_value in config.items():
new_s = {}
for key, value in s_value.items():
new_s[key.lower()] = str(value)
new_config[s_key.lower()] = new_s
return new_config
def get_configobj(self, level):
configs = {
self.LEVEL_LOCAL: self._local_config,
self.LEVEL_REPO: self._repo_config,
self.LEVEL_GLOBAL: self._global_config,
self.LEVEL_SYSTEM: self._system_config,
}
return configs.get(level, self._repo_config)
def list_options(self, section_regex, option, level=None):
ret = {}
config = self.get_configobj(level)
for section in config.keys():
r = re.match(section_regex, section)
if r:
name = r.group("name")
value = config[section].get(option, "")
ret[name] = value
return ret
class Validator(object):
"""
Validator for the Pythonic pipeline definition.
- validates matrix, hooks, model and pipeline as main items in yaml.
- validates parallel or ordered in matrix and tasks
- support for Bash, Python, Docker, Packer and Ansible tasks
"""
SCHEMA = {
# optional matrix part of the schema
Optional(Regex(r'((matrix\(parallel\)|matrix\(ordered\)|matrix))')):
And(len, [{
'name':
And(str, len),
Optional('env'):
And(len, {Regex(r'([a-zA-Z][_a-zA-Z]*)'): And(str, len)}),
Optional('tags'):
And(len, [And(str, len)])
}]),
# optional model part of the schema
# you may have any value (any yaml structure as value)
Optional('model'):
And(len, {Regex(r'([a-z][_a-z]*)'): object}),
# optional hooks part of the schema
Optional('hooks'): {
'cleanup': {
'script': And(str, len)
}
},
# mandatory pipeline part of the schema
'pipeline':
And(
len,
[
Or(
# optional environment variables in a pipeline
{
'env':
And(len,
{Regex(r'([a-zA-Z][_a-zA-Z]*)'): And(str, len)})
},
{
Regex(r'(stage\(.+\))'):
And(
len,
[
Or(
# optional environment variables in a stage
{
'env':
And(
len, {
Regex(r'([a-zA-Z][_a-zA-Z]*)'):
And(str, len)
})
},
# optional tasks
{
Regex(r'((tasks\(parallel\)|matrix\(tasks\)|tasks))'):
And(
len,
[
# optional environment variables
{
Optional('env'):
And(
len, {
Regex(r'([a-zA-Z][_a-zA-Z]*)'):
And(str, len)
})
},
# optional shell task
{
Optional('shell'): {
'script':
And(
Or(
type(' '),
type(u' ')),
len),
Optional('title'):
And(str, len),
Optional('tags'):
And([And(str, len)],
len),
Optional('with'):
And(
len,
Or(
type(' '),
type(u' '),
[object])),
# when set store output of shell task into a variable with this name
# can be referenced in jinja templating with {{ variables.name }}
Optional('variable'):
And(
Or(
type(' '),
type(u' ')),
len,
Regex(
r'([a-zA-Z][_a-zA-Z]*)'
)),
# condition when to run the task
Optional('when',
default=''):
And(
str,
Condition.is_valid)
}
},
# optional Docker container task
{
Optional('docker(container)'):
{
'script':
And(str, len),
# docker image name and docker tag (version)
Optional('image'):
And(str, len),
# the title is printed to the logs (when given)
Optional('title'):
And(str, len),
# when set the mount will be -v $PWD:/mnt/host
Optional('mount',
default=False):
bool,
# when set using -d option (docker run)
Optional('background',
default=False):
bool,
# when set using --rm option (docker run)
Optional('remove',
default=True):
bool,
# when set using --network= option (docker run)
Optional('network',
default=''):
And(str, len),
# when set using --labels= option (docker run)
Optional('labels'):
And(
len, {
Regex(r'(UL[_A-Z]*)'):
And(str, len)
}),
# when defined used for --tags option (spline --tags=)
Optional('tags'):
And([And(str, len)],
len),
Optional('with'):
And(len, [object]),
# when set store output of docker container task into a variable with this name
# can be referenced in jinja templating with {{ variables.name }}
Optional('variable'):
And(
Or(
type(' '),
type(u' ')),
len,
Regex(
r'([a-zA-Z][_a-zA-Z]*)'
)),
# condition when to run the task
Optional('when',
default=''):
And(
str,
Condition.is_valid)
}
},
# optional Docker image task
{
Optional('docker(image)'):
{
'script':
And(str, len),
'name':
And(str, len),
'tag':
And(str, len),
Optional('unique',
default=True):
bool,
Optional('tags'):
And([And(str, len)],
len),
Optional('with'):
And(len, [object]),
# condition when to run the task
Optional('when',
default=''):
And(
str,
Condition.is_valid)
}
},
# optional Python script
{
Optional('python'): {
'script':
And(str, len),
Optional('title'):
And(str, len),
Optional('tags'):
And([And(str, len)],
len),
Optional('with'):
And(len, [object]),
# when set store output of python task into a variable with this name
# can be referenced in jinja templating with {{ variables.name }}
Optional('variable'):
And(
Or(
type(' '),
type(u' ')),
len,
Regex(
r'([a-zA-Z][_a-zA-Z]*)'
)),
# condition when to run the task
Optional('when',
default=''):
And(
str,
Condition.is_valid)
}
},
# optional Packer task
{
Optional('packer'): {
'script':
And(str, len),
Optional('tags'):
And([And(str, len)],
len),
Optional('with'):
And(len, [object]),
# condition when to run the task
Optional('when',
default=''):
And(
str,
Condition.is_valid)
}
},
# optional Ansible task
{
Optional('ansible(simple)'):
{
'script':
And(str, len),
'inventory':
And(str, len),
Optional('limit',
default=''):
And(str, len),
Optional('tags'):
And([And(str, len)],
len),
Optional('with'):
And(len, [object]),
# condition when to run the task
Optional('when',
default=''):
And(
str,
Condition.is_valid)
}
}
])
} # end of tasks
)
])
}, # end of stage
)
]) # end of pipeline
} # end of schema
@staticmethod
def validate(data):
"""
Validate data against the schema.
Args:
data(dict): data structure to validate.
Returns:
dict: data as provided and defaults where defined in schema.
"""
try:
return Schema(Validator.SCHEMA).validate(data)
except SchemaError as exception:
logging.getLogger(__name__).error(exception)
return None
class Config(object):
CONFIG = 'config'
CONFIG_LOCAL = 'config.local'
SECTION_CORE = 'core'
SECTION_CORE_LOGLEVEL = 'loglevel'
SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel)
SECTION_CORE_REMOTE = 'remote'
SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool))
SECTION_CORE_INTERACTIVE = 'interactive'
SECTION_CACHE = 'cache'
SECTION_CACHE_DIR = 'dir'
SECTION_CACHE_TYPE = 'type'
SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
SECTION_CACHE_PROTECTED = 'protected'
SECTION_CACHE_LOCAL = 'local'
SECTION_CACHE_S3 = 's3'
SECTION_CACHE_GS = 'gs'
SECTION_CACHE_SSH = 'ssh'
SECTION_CACHE_HDFS = 'hdfs'
SECTION_CACHE_AZURE = 'azure'
SECTION_CACHE_SCHEMA = {
Optional(SECTION_CACHE_LOCAL): str,
Optional(SECTION_CACHE_S3): str,
Optional(SECTION_CACHE_GS): str,
Optional(SECTION_CACHE_HDFS): str,
Optional(SECTION_CACHE_SSH): str,
Optional(SECTION_CACHE_AZURE): str,
Optional(SECTION_CACHE_DIR, default='cache'): str,
Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
Optional(SECTION_CACHE_PROTECTED,
default=False): And(str, is_bool, Use(to_bool)),
}
# backward compatibility
SECTION_CORE_CLOUD = 'cloud'
SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud)
SECTION_CORE_STORAGEPATH = 'storagepath'
SECTION_CORE_SCHEMA = {
Optional(SECTION_CORE_LOGLEVEL,
default='info'): And(str, Use(str.lower),
SECTION_CORE_LOGLEVEL_SCHEMA),
Optional(SECTION_CORE_REMOTE, default=''): And(str, Use(str.lower)),
Optional(SECTION_CORE_INTERACTIVE,
default=False): SECTION_CORE_INTERACTIVE_SCHEMA,
# backward compatibility
Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA,
Optional(SECTION_CORE_STORAGEPATH, default=''): str,
}
# backward compatibility
SECTION_AWS = 'aws'
SECTION_AWS_STORAGEPATH = 'storagepath'
SECTION_AWS_CREDENTIALPATH = 'credentialpath'
SECTION_AWS_ENDPOINT_URL = 'endpointurl'
SECTION_AWS_REGION = 'region'
SECTION_AWS_PROFILE = 'profile'
SECTION_AWS_SCHEMA = {
SECTION_AWS_STORAGEPATH: str,
Optional(SECTION_AWS_REGION): str,
Optional(SECTION_AWS_PROFILE): str,
Optional(SECTION_AWS_CREDENTIALPATH): str,
Optional(SECTION_AWS_ENDPOINT_URL): str,
}
# backward compatibility
SECTION_GCP = 'gcp'
SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH
SECTION_GCP_PROJECTNAME = 'projectname'
SECTION_GCP_SCHEMA = {
SECTION_GCP_STORAGEPATH: str,
Optional(SECTION_GCP_PROJECTNAME): str,
}
# backward compatibility
SECTION_LOCAL = 'local'
SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH
SECTION_LOCAL_SCHEMA = {
SECTION_LOCAL_STORAGEPATH: str,
}
SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
SECTION_REMOTE_FMT = 'remote "{}"'
SECTION_REMOTE_URL = 'url'
SECTION_REMOTE_USER = '******'
SECTION_REMOTE_PORT = 'port'
SECTION_REMOTE_KEY_FILE = 'keyfile'
SECTION_REMOTE_TIMEOUT = 'timeout'
SECTION_REMOTE_PASSWORD = '******'
SECTION_REMOTE_ASK_PASSWORD = '******'
SECTION_REMOTE_SCHEMA = {
SECTION_REMOTE_URL: And(supported_url, error="Unsupported URL"),
Optional(SECTION_AWS_REGION): str,
Optional(SECTION_AWS_PROFILE): str,
Optional(SECTION_AWS_CREDENTIALPATH): str,
Optional(SECTION_AWS_ENDPOINT_URL): str,
Optional(SECTION_GCP_PROJECTNAME): str,
Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
Optional(SECTION_CACHE_PROTECTED,
default=False): And(str, is_bool, Use(to_bool)),
Optional(SECTION_REMOTE_USER): str,
Optional(SECTION_REMOTE_PORT): Use(int),
Optional(SECTION_REMOTE_KEY_FILE): str,
Optional(SECTION_REMOTE_TIMEOUT): Use(int),
Optional(SECTION_REMOTE_PASSWORD): str,
Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)),
}
SECTION_STATE = 'state'
SECTION_STATE_ROW_LIMIT = 'row_limit'
SECTION_STATE_ROW_CLEANUP_QUOTA = 'row_cleanup_quota'
SECTION_STATE_SCHEMA = {
Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole),
Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent),
}
SCHEMA = {
Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
# backward compatibility
Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA,
Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
}
def __init__(self, dvc_dir):
self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
self.config_file = os.path.join(dvc_dir, self.CONFIG)
self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL)
try:
self._config = configobj.ConfigObj(self.config_file)
local = configobj.ConfigObj(self.config_local_file)
# NOTE: schema doesn't support ConfigObj.Section validation, so we
# need to convert our config to dict before passing it to
self._config = self._lower(self._config)
local = self._lower(local)
self._config = self._merge(self._config, local)
self._config = Schema(self.SCHEMA).validate(self._config)
# NOTE: now converting back to ConfigObj
self._config = configobj.ConfigObj(self._config,
write_empty_values=True)
self._config.filename = self.config_file
except Exception as ex:
raise ConfigError(ex)
@staticmethod
def _merge(first, second):
res = {}
sections = list(first.keys()) + list(second.keys())
for section in sections:
f = first.get(section, {}).copy()
s = second.get(section, {}).copy()
f.update(s)
res[section] = f
return res
@staticmethod
def _lower(config):
new_config = {}
for s_key, s_value in config.items():
new_s = {}
for key, value in s_value.items():
new_s[key.lower()] = value
new_config[s_key.lower()] = new_s
return new_config
@staticmethod
def init(dvc_dir):
config_file = os.path.join(dvc_dir, Config.CONFIG)
open(config_file, 'w+').close()
return Config(dvc_dir)
db = connect(username=os.environ.get('MONGO_USER'),
password=os.environ.get('MONGO_PASS'),
host=os.environ.get('MONGO_HOST', '127.0.0.1'),
db=os.environ.get('MONGO_DB', 'threat_playbook'),
port=int(os.environ.get('MONGO_PORT', 27017)))
except Exception as e:
print(bold(red(e)))
exit(1)
return db
validation_dictionary = {
'create_user': {
'email':
Regex(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)",
error="Not a valid email"),
'password':
Regex(r'[A-Za-z0-9@#$%^&+=]{8,}',
error="Password is not compliant with requirements")
},
'login': {
'email':
Regex(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)",
error="Not a valid email"),
'password':
Regex(r'[A-Za-z0-9@#$%^&+=]{8,}',
error="Password is not compliant with requirements")
}
}
if value not in Position.POSITIONS:
raise SchemaError("Unknown position '{p}'".format(p=value))
class UniqueId:
def __init__(self):
self._ids = set()
def validate(self, value):
if value in self._ids:
raise SchemaError("ID {id} is not unique".format(id=value))
self._ids.add(value)
employee = Schema({
"name": And(str, len, Regex("[A-Z][a-z]+")),
"surname": And(str, len, Regex("[A-Z][a-z]+")),
"id": UniqueId(),
Optional("salary"): Salary(),
Optional("position"): And(str, Position())
})
verbose_mode = "-v" in argv
validate(
employee, {
"name": "Eda",
"surname": "Wasserfall",
"id": 1,
"salary": 15000.0,
"position": "QA"
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import os
from schema import Schema, And, Use, Optional, Regex, Or
common_schema = {
'authorName': str,
'experimentName': str,
Optional('description'): str,
'trialConcurrency': And(int, lambda n: 1 <=n <= 999999),
Optional('maxExecDuration'): Regex(r'^[1-9][0-9]*[s|m|h|d]$'),
Optional('maxTrialNum'): And(int, lambda x: 1 <= x <= 99999),
'trainingServicePlatform': And(str, lambda x: x in ['remote', 'local', 'pai']),
Optional('searchSpacePath'): os.path.exists,
Optional('multiPhase'): bool,
'useAnnotation': bool,
'tuner': Or({
'builtinTunerName': Or('TPE', 'Random', 'Anneal', 'Evolution', 'SMAC', 'BatchTuner'),
'classArgs': {
'optimize_mode': Or('maximize', 'minimize'),
Optional('speed'): int
},
Optional('gpuNum'): And(int, lambda x: 0 <= x <= 99999),
},{
'codeDir': os.path.exists,
'classFileName': str,
from dataclasses import dataclass
from pathlib import Path
import yaml
from schema import Regex, Schema
CONF_SCHEMA = {
'polling_interval': int,
'slow_threshold': int,
'db_path': str,
'host': str,
'port': int,
'log_file': str,
'log_level': Regex(r'DEBUG|INFO|WARNING|ERROR'),
}
@dataclass
class Conf:
polling_interval: int
slow_threshold: int
db_path: str
host: str
port: str
log_file: str
log_level: str
@classmethod
def load(cls, path=None):
if path is None:
path = Path(__file__).resolve().parent.parent.parent / 'config.yaml'
Schema(self.algo_schema).validate(data)
self.validate_extras(data, self.algo_type)
common_schema = {
'authorName':
setType('authorName', str),
'experimentName':
setType('experimentName', str),
Optional('description'):
setType('description', str),
'trialConcurrency':
setNumberRange('trialConcurrency', int, 1, 99999),
Optional('maxExecDuration'):
And(
Regex(r'^[1-9][0-9]*[s|m|h|d]$',
error='ERROR: maxExecDuration format is [digit]{s,m,h,d}')),
Optional('maxTrialNum'):
setNumberRange('maxTrialNum', int, 1, 99999),
'trainingServicePlatform':
setChoice('trainingServicePlatform', 'remote', 'local', 'pai', 'kubeflow',
'frameworkcontroller', 'paiYarn', 'dlts'),
Optional('searchSpacePath'):
And(os.path.exists, error=SCHEMA_PATH_ERROR % 'searchSpacePath'),
Optional('multiPhase'):
setType('multiPhase', bool),
Optional('multiThread'):
setType('multiThread', bool),
Optional('nniManagerIp'):
setType('nniManagerIp', str),
Optional('logDir'):
And(os.path.isdir, error=SCHEMA_PATH_ERROR % 'logDir'),
Optional("file_cell_parameters", default=None):
Or(str, None),
# Quality of the auxiliar basis cFIT
Optional("aux_fit", default="verygood"):
any_lambda(("low", "medium", "good", "verygood", "excellent")),
# executable name
# "sdbg" Serial single core testing and debugging
# "sopt" Serial general single core usage
# "ssmp" Parallel (only OpenMP), single node, multi core
# "pdbg" Parallel (only MPI) multi-node testing and debugging
# "popt" Parallel (only MPI) general usage, no threads
# "psmp" parallel (MPI + OpenMP) general usage, threading might improve scalability and memory usage
Optional("executable", default="cp2k.psmp"):
Regex(r'.*cp2k\.(?:popt|psmp|sdbg|sopt|ssmp|pdbg)',
flags=2) # flag 2 == IGNORECASE
})
#: Dictionary with the options common to all workflows
dict_general_options = {
# Number of occupied/virtual orbitals to use
Optional('active_space', default=[10, 10]):
And(list, lambda xs: len(xs) == 2),
# Index of the H**O
Optional("nHOMO"):
int,
# Index of the orbitals to compute the couplings
Optional("mo_index_range"):
