def sign_up(): schema = Schema({ "username": str, "email": Regex(MAIL_REGEX, error="Mail address is invalid"), "password": str }) validated = schema.validate(request.json) if User.objects(username=validated["username"]): return jsonify({"error": "Username not available"}), 409 if User.objects(email=validated["email"]): return jsonify( {"error": "There is already an account with your email address"}), 409 # Hash password with sha256 hashed_password = generate_password_hash(validated["password"]) user = User(username=validated["username"], email=validated["email"], password=hashed_password).save() token = jwt.encode( { "username": user.username, "email": user.email, "password": user.password, "created": str(user.created) }, app.config["SECRET_KEY"]) return jsonify({ "success": True, "user": { "username": user.username, "email": user.email, "password": user.password, "created": str(user.created) }, "token": token.decode("UTF-8") })
def validate_instance_parameters(params: dict): from spotty.providers.aws.config.instance_config import VOLUME_TYPE_EBS instance_parameters = { 'region': And(str, Regex(r'^[a-z0-9-]+$')), Optional('availabilityZone', default=''): And(str, Regex(r'^[a-z0-9-]+$')), Optional('subnetId', default=''): And(str, Regex(r'^subnet-[a-z0-9]+$')), 'instanceType': And(str, And(is_valid_instance_type, error='Invalid instance type.')), Optional('onDemandInstance', default=False): bool, Optional('amiName', default=None): And(str, len, Regex(r'^[\w\(\)\[\]\s\.\/\'@-]{3,128}$')), Optional('amiId', default=None): And(str, len, Regex(r'^ami-[a-z0-9]+$')), Optional('rootVolumeSize', default=0): And(Or(int, str), Use(str), Regex(r'^\d+$', error='Incorrect value for "rootVolumeSize".'), Use(int), And(lambda x: x > 0, error='"rootVolumeSize" should be greater than 0 or should ' 'not be specified.'), ), Optional('maxPrice', default=0): And(Or(float, int, str), Use(str), Regex(r'^\d+(\.\d{1,6})?$', error='Incorrect value for "maxPrice".'), Use(float), And(lambda x: x > 0, error='"maxPrice" should be greater than 0 or ' 'should not be specified.'), ), Optional('managedPolicyArns', default=[]): [str], } volumes_checks = [ And(lambda x: len(x) < 12, error='Maximum 11 volumes are supported at the moment.'), ] instance_checks = [ And(lambda x: not (x['onDemandInstance'] and x['maxPrice']), error='"maxPrice" cannot be specified for on-demand instances'), And(lambda x: not (x['amiName'] and x['amiId']), error='"amiName" and "amiId" parameters cannot be used together'), ] schema = get_instance_parameters_schema(instance_parameters, VOLUME_TYPE_EBS, instance_checks, volumes_checks) return validate_config(schema, params)
def validate_ami_config(data): schema = Schema( { 'instance': { 'region': And(str, len), Optional('availabilityZone', default=''): str, Optional('subnetId', default=''): str, 'instanceType': And(str, len), Optional('amiName', default=DEFAULT_AMI_NAME): And(str, len, Regex(AMI_NAME_REGEX)), Optional('keyName', default=''): str, Optional('onDemandInstance', default=False): bool, }, }, ignore_extra_keys=True) return _validate(schema, data)
def validate_ebs_volume_parameters(params: dict): from spotty.providers.aws.deployment.project_resources.ebs_volume import EbsVolume schema = Schema({ Optional('volumeName', default=''): And(str, Regex(r'^[\w-]{1,255}$')), Optional('mountDir', default=''): str, # all the checks happened in the base configuration Optional('size', default=0): And(int, lambda x: x > 0), # TODO: add the "iops" parameter to support the "io1" EBS volume type Optional('type', default='gp2'): lambda x: x in ['gp2', 'sc1', 'st1', 'standard'], Optional('deletionPolicy', default=EbsVolume.DP_CREATE_SNAPSHOT): And(str, lambda x: x in [ EbsVolume.DP_CREATE_SNAPSHOT, EbsVolume.DP_UPDATE_SNAPSHOT, EbsVolume.DP_RETAIN, EbsVolume.DP_DELETE ], error='Incorrect value for "deletionPolicy".'), }) return validate_config(schema, params)
def validate_ebs_volume_parameters(params: dict): from spotty.providers.aws.config.ebs_volume import EbsVolume old_deletion_policies_map = { 'create_snapshot': EbsVolume.DP_CREATE_SNAPSHOT, 'update_snapshot': EbsVolume.DP_UPDATE_SNAPSHOT, 'retain': EbsVolume.DP_RETAIN, 'delete': EbsVolume.DP_DELETE, } schema = Schema({ Optional('volumeName', default=''): And(str, Regex(r'^[\w-]{1,255}$')), Optional('mountDir', default=''): And( str, And(os.path.isabs, error='Use absolute paths in the "mountDir" parameters'), Use(lambda x: x.rstrip('/'))), Optional('size', default=0): And(int, lambda x: x > 0), # TODO: add the "iops" parameter to support the "io1" EBS volume type Optional('type', default='gp2'): lambda x: x in ['gp2', 'sc1', 'st1', 'standard'], Optional('deletionPolicy', default=EbsVolume.DP_RETAIN): And( str, lambda x: x in [ EbsVolume.DP_CREATE_SNAPSHOT, EbsVolume.DP_UPDATE_SNAPSHOT, EbsVolume.DP_RETAIN, EbsVolume.DP_DELETE ] + list(old_deletion_policies_map.keys()), Use(lambda x: old_deletion_policies_map.get(x, x)), error='Incorrect value for "deletionPolicy".', ), }) return validate_config(schema, params)
from schema import Optional as SchemaOptional from schema import Regex, Schema from corehq.motech.dhis2.const import ( DHIS2_EVENT_STATUSES, DHIS2_PROGRAM_STATUSES, ) id_schema = Regex(r"^[A-Za-z0-9]+$") # DHIS2 accepts date values, but returns datetime values for dates: date_schema = Regex(r"^\d{4}-\d{2}-\d{2}(:?T\d{2}:\d{2}:\d{2}.\d{3})?$") datetime_schema = Regex(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}$") enrollment_status_schema = Regex(f"^({'|'.join(DHIS2_PROGRAM_STATUSES)})$") event_status_schema = Regex(f"^({'|'.join(DHIS2_EVENT_STATUSES)})$") def get_attribute_schema() -> dict: return { "attribute": id_schema, SchemaOptional("code"): str, SchemaOptional("created"): datetime_schema, SchemaOptional("displayName"): str, SchemaOptional("lastUpdated"): datetime_schema, SchemaOptional("storedBy"): str, "value": object, SchemaOptional("valueType"): str, } def get_event_schema() -> dict: """
from schema import Schema, And, Or, Use, Optional, Regex DATE_ISO = "^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])(T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\.[0-9]+)?(Z|[+-](?:2[0-3]|[01][0-9]):[0-5][0-9])?$)*" URL_FORMAT = "^http(s)*://" CONTEXT = "https://schema.org(/)*" organization_schema = Schema({ "@context": Regex(CONTEXT), "@type": Or("Corporation", "Organization"), "name": str, Optional("alternateName"): str, Optional("url"): Regex(URL_FORMAT), Optional("logo"): Regex(URL_FORMAT), Optional("sameAs"): Or(str, [str]), Optional(str): object }) person_schema = Schema({ "@context": Regex(CONTEXT), "@type": "Person", "name": str, Optional("url"): Regex(URL_FORMAT), Optional("image"): Regex(URL_FORMAT), Optional("sameAs"): Or(Regex(URL_FORMAT), [Regex(URL_FORMAT)]), Optional("jobTitle"):
def image_schema(file): valid_arches = [ 'x86_64', 'ppc64le', ] valid_distgit_namespaces = [ 'apbs', 'containers', 'rpms', ] valid_streams = support.get_valid_streams_for(file) valid_member_references = support.get_valid_member_references_for(file) valid_modes = [ 'auto', 'disabled', 'wip', ] valid_odcs_modes = [ 'auto', 'manual', ] return Schema({ Optional('additional_tags'): [ And(str, len), ], Optional('arches'): [Or(*valid_arches)], Optional('base_only'): True, Optional('container_yaml'): { 'go': { 'modules': [ { 'module': And(str, len), Optional('path'): str, }, ], }, }, Optional('content'): { 'source': { Optional('alias'): And(str, len), Optional('dockerfile'): And(str, len), Optional('git'): { 'branch': { Optional('fallback'): And(str, len), Optional('stage'): And(str, len), 'target': And(str, len), }, 'url': And(str, len, Regex(GIT_SSH_URL_REGEX)), }, Optional('modifications'): [modification], Optional('path'): str, }, }, Optional('dependents'): [And(str, len)], Optional('distgit'): { Optional('namespace'): Or(*valid_distgit_namespaces), Optional('component'): And(str, len), Optional('branch'): And(str, len), }, Optional('enabled_repos'): [ And(str, len), ], Optional('non_shipping_repos'): [ And(str, len), ], Optional('non_shipping_rpms'): [ And(str, len), ], 'from': { Optional('builder'): [ { Optional('stream'): Or(*valid_streams), Optional('member'): Or(*valid_member_references), Optional('image'): And(str, len), }, ], Optional('image'): And(str, len), Optional('stream'): Or(*valid_streams), Optional('member'): Or(*valid_member_references), }, Optional('labels'): { Optional('License'): And(str, len), Optional('io.k8s.description'): And(str, len), Optional('io.k8s.display-name'): And(str, len), Optional('io.openshift.tags'): And(str, len), Optional('vendor'): And(str, len), }, Optional('image_build_method'): And(str, len), Optional('mode'): Or(*valid_modes), 'name': And(str, len), Optional('odcs'): { 'packages': { Optional('exclude'): [ And(str, len), ], Optional('list'): [ And(str, len), ], 'mode': Or(*valid_odcs_modes), }, }, Optional('no_oit_comments'): bool, Optional('owners'): [ And(str, len), ], Optional('push'): { Optional('repos'): [ And(str, len), ], Optional('additional_tags'): [ And(str, len), ], Optional('late'): bool, }, Optional('required'): bool, Optional('update-csv'): { 'manifests-dir': And(str, len), 'bundle-dir': And(str, len), 'registry': And(str, len), Optional('channel'): And(str, len), }, Optional('wait_for'): And(str, len), Optional('maintainer'): { Optional('product'): And(str, len), 'component': And(str, len), Optional('subcomponent'): And(str, len), }, Optional('for_payload'): bool, })
data = { 'username': '******', 'password': '******', 'email': '*****@*****.**', 'nick_name': '12121' } schema = { 'username': And(And(str, error='用户名参数字符类型错误'), And(lambda x: 4 <= len(x) <= 16, error='用户名字符长度错误')), 'password': And(And(str, error='密码参数字符类型错误'), And(lambda x: 4 <= len(x) <= 16, error='密码字符长度错误')), 'email': And( And(str, error='邮箱参数字符类型错误'), And( Regex(r'^\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$', error='邮箱正则匹配错误'))), Optional('nick_name'): And(And(str, error='昵称参数字符类型错误'), And(lambda x: len(x) <= 64, error='昵称字符长度错误')) } try: result = Schema(schema).validate(data) print(result) except SchemaError as e: print(e.code)
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import os from schema import Schema, And, Use, Optional, Regex, Or common_schema = { 'authorName': str, 'experimentName': str, Optional('description'): str, 'trialConcurrency': And(int, lambda n: 1 <= n <= 999999), Optional('maxExecDuration'): Regex(r'^[1-9][0-9]*[s|m|h|d]$'), Optional('maxTrialNum'): And(int, lambda x: 1 <= x <= 99999), 'trainingServicePlatform': And(str, lambda x: x in ['remote', 'local', 'pai', 'kubeflow']), Optional('searchSpacePath'): os.path.exists, Optional('multiPhase'): bool, Optional('multiThread'): bool, Optional('nniManagerIp'): str, 'useAnnotation': bool, Optional('advisor'):
Schema(self.algo_schema).validate(data) self.validate_extras(data, self.algo_type) common_schema = { 'authorName': setType('authorName', str), 'experimentName': setType('experimentName', str), Optional('description'): setType('description', str), 'trialConcurrency': setNumberRange('trialConcurrency', int, 1, 99999), Optional('maxExecDuration'): And( Regex(r'^[1-9][0-9]*[s|m|h|d]$', error='ERROR: maxExecDuration format is [digit]{s,m,h,d}')), Optional('maxTrialNum'): setNumberRange('maxTrialNum', int, 1, 99999), 'trainingServicePlatform': setChoice('trainingServicePlatform', 'remote', 'local', 'pai', 'kubeflow', 'frameworkcontroller', 'paiYarn', 'dlts', 'aml', 'adl', 'hybrid'), Optional('searchSpacePath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'searchSpacePath'), Optional('multiPhase'): setType('multiPhase', bool), Optional('multiThread'): setType('multiThread', bool), Optional('nniManagerIp'): setType('nniManagerIp', str), Optional('logDir'):
class Config(object): APPNAME = 'dvc' APPAUTHOR = 'iterative' # NOTE: used internally in RemoteLOCAL to know config # location, that url should resolved relative to. PRIVATE_CWD = '_cwd' CONFIG = 'config' CONFIG_LOCAL = 'config.local' SECTION_CORE = 'core' SECTION_CORE_LOGLEVEL = 'loglevel' SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel) SECTION_CORE_REMOTE = 'remote' SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool)) SECTION_CORE_INTERACTIVE = 'interactive' SECTION_CORE_ANALYTICS = 'analytics' SECTION_CORE_ANALYTICS_SCHEMA = And(str, is_bool, Use(to_bool)) SECTION_CACHE = 'cache' SECTION_CACHE_DIR = 'dir' SECTION_CACHE_TYPE = 'type' SECTION_CACHE_TYPE_SCHEMA = supported_cache_type SECTION_CACHE_PROTECTED = 'protected' SECTION_CACHE_LOCAL = 'local' SECTION_CACHE_S3 = 's3' SECTION_CACHE_GS = 'gs' SECTION_CACHE_SSH = 'ssh' SECTION_CACHE_HDFS = 'hdfs' SECTION_CACHE_AZURE = 'azure' SECTION_CACHE_SCHEMA = { Optional(SECTION_CACHE_LOCAL): str, Optional(SECTION_CACHE_S3): str, Optional(SECTION_CACHE_GS): str, Optional(SECTION_CACHE_HDFS): str, Optional(SECTION_CACHE_SSH): str, Optional(SECTION_CACHE_AZURE): str, Optional(SECTION_CACHE_DIR): str, Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA, Optional(SECTION_CACHE_PROTECTED, default=False): And(str, is_bool, Use(to_bool)), Optional(PRIVATE_CWD): str, } # backward compatibility SECTION_CORE_CLOUD = 'cloud' SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud) SECTION_CORE_STORAGEPATH = 'storagepath' SECTION_CORE_SCHEMA = { Optional(SECTION_CORE_LOGLEVEL, default='info'): And(str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA), Optional(SECTION_CORE_REMOTE, default=''): And(str, Use(str.lower)), Optional(SECTION_CORE_INTERACTIVE, default=False): SECTION_CORE_INTERACTIVE_SCHEMA, Optional(SECTION_CORE_ANALYTICS, default=True): SECTION_CORE_ANALYTICS_SCHEMA, # backward compatibility Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA, Optional(SECTION_CORE_STORAGEPATH, default=''): str, } # backward compatibility SECTION_AWS = 'aws' SECTION_AWS_STORAGEPATH = 'storagepath' SECTION_AWS_CREDENTIALPATH = 'credentialpath' SECTION_AWS_ENDPOINT_URL = 'endpointurl' SECTION_AWS_REGION = 'region' SECTION_AWS_PROFILE = 'profile' SECTION_AWS_USE_SSL = 'use_ssl' SECTION_AWS_SCHEMA = { SECTION_AWS_STORAGEPATH: str, Optional(SECTION_AWS_REGION): str, Optional(SECTION_AWS_PROFILE): str, Optional(SECTION_AWS_CREDENTIALPATH): str, Optional(SECTION_AWS_ENDPOINT_URL): str, Optional(SECTION_AWS_USE_SSL, default=True): And(str, is_bool, Use(to_bool)), } # backward compatibility SECTION_GCP = 'gcp' SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH SECTION_GCP_PROJECTNAME = 'projectname' SECTION_GCP_SCHEMA = { SECTION_GCP_STORAGEPATH: str, Optional(SECTION_GCP_PROJECTNAME): str, } # backward compatibility SECTION_LOCAL = 'local' SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH SECTION_LOCAL_SCHEMA = { SECTION_LOCAL_STORAGEPATH: str, } SECTION_AZURE_CONNECTION_STRING = 'connection_string' SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$' SECTION_REMOTE_FMT = 'remote "{}"' SECTION_REMOTE_URL = 'url' SECTION_REMOTE_USER = '******' SECTION_REMOTE_PORT = 'port' SECTION_REMOTE_KEY_FILE = 'keyfile' SECTION_REMOTE_TIMEOUT = 'timeout' SECTION_REMOTE_PASSWORD = '******' SECTION_REMOTE_ASK_PASSWORD = '******' SECTION_REMOTE_SCHEMA = { SECTION_REMOTE_URL: str, Optional(SECTION_AWS_REGION): str, Optional(SECTION_AWS_PROFILE): str, Optional(SECTION_AWS_CREDENTIALPATH): str, Optional(SECTION_AWS_ENDPOINT_URL): str, Optional(SECTION_AWS_USE_SSL, default=True): And(str, is_bool, Use(to_bool)), Optional(SECTION_GCP_PROJECTNAME): str, Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA, Optional(SECTION_CACHE_PROTECTED, default=False): And(str, is_bool, Use(to_bool)), Optional(SECTION_REMOTE_USER): str, Optional(SECTION_REMOTE_PORT): Use(int), Optional(SECTION_REMOTE_KEY_FILE): str, Optional(SECTION_REMOTE_TIMEOUT): Use(int), Optional(SECTION_REMOTE_PASSWORD): str, Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)), Optional(SECTION_AZURE_CONNECTION_STRING): str, Optional(PRIVATE_CWD): str, } SECTION_STATE = 'state' SECTION_STATE_ROW_LIMIT = 'row_limit' SECTION_STATE_ROW_CLEANUP_QUOTA = 'row_cleanup_quota' SECTION_STATE_SCHEMA = { Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole), Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent), } SCHEMA = { Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA, Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA, Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA, Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA, # backward compatibility Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA, Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA, Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA, } def __init__(self, dvc_dir=None, validate=True): self.system_config_file = os.path.join(self.get_system_config_dir(), self.CONFIG) self.global_config_file = os.path.join(self.get_global_config_dir(), self.CONFIG) if dvc_dir is not None: self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) self.config_file = os.path.join(dvc_dir, self.CONFIG) self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL) else: self.dvc_dir = None self.config_file = None self.config_local_file = None self.load(validate=validate) @staticmethod def get_global_config_dir(): from appdirs import user_config_dir return user_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) @staticmethod def get_system_config_dir(): from appdirs import site_config_dir return site_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) @staticmethod def init(dvc_dir): config_file = os.path.join(dvc_dir, Config.CONFIG) open(config_file, 'w+').close() return Config(dvc_dir) def _load(self): self._system_config = configobj.ConfigObj(self.system_config_file) self._global_config = configobj.ConfigObj(self.global_config_file) if self.config_file is not None: self._project_config = configobj.ConfigObj(self.config_file) else: self._project_config = configobj.ConfigObj() if self.config_local_file is not None: self._local_config = configobj.ConfigObj(self.config_local_file) else: self._local_config = configobj.ConfigObj() self._config = None def _load_config(self, path): config = configobj.ConfigObj(path) config = self._lower(config) self._resolve_paths(config, path) return config @staticmethod def _resolve_path(path, config_file): assert os.path.isabs(config_file) config_dir = os.path.dirname(config_file) return os.path.abspath(os.path.join(config_dir, path)) def _resolve_cache_path(self, config, fname): cache = config.get(self.SECTION_CACHE) if cache is None: return cache_dir = cache.get(self.SECTION_CACHE_DIR) if cache_dir is None: return cache[self.PRIVATE_CWD] = os.path.dirname(fname) def _resolve_paths(self, config, fname): self._resolve_cache_path(config, fname) for name, section in config.items(): if self.SECTION_REMOTE_URL not in section.keys(): continue section[self.PRIVATE_CWD] = os.path.dirname(fname) def load(self, validate=True): self._load() try: self._config = self._load_config(self.system_config_file) user = self._load_config(self.global_config_file) config = self._load_config(self.config_file) local = self._load_config(self.config_local_file) # NOTE: schema doesn't support ConfigObj.Section validation, so we # need to convert our config to dict before passing it to for c in [user, config, local]: self._config = self._merge(self._config, c) if validate: self._config = Schema(self.SCHEMA).validate(self._config) # NOTE: now converting back to ConfigObj self._config = configobj.ConfigObj(self._config, write_empty_values=True) self._config.filename = self.config_file self._resolve_paths(self._config, self.config_file) except Exception as ex: raise ConfigError(ex) def _get_key(self, d, name, add=False): for k in d.keys(): if k.lower() == name.lower(): return k if add: d[name] = {} return name return None def save(self, config=None): if config is not None: clist = [config] else: clist = [ self._system_config, self._global_config, self._project_config, self._local_config ] for conf in clist: if conf.filename is None: continue try: logger.debug("Writing '{}'.".format(conf.filename)) dname = os.path.dirname(os.path.abspath(conf.filename)) try: os.makedirs(dname) except OSError as exc: if exc.errno != errno.EEXIST: raise conf.write() except Exception as exc: msg = "failed to write config '{}'".format(conf.filename) raise ConfigError(msg, exc) def unset(self, config, section, opt=None): if section not in config.keys(): raise ConfigError("section '{}' doesn't exist".format(section)) if opt is None: del config[section] return if opt not in config[section].keys(): raise ConfigError("option '{}.{}' doesn't exist".format( section, opt)) del config[section][opt] if len(config[section]) == 0: del config[section] def set(self, config, section, opt, value): if section not in config.keys(): config[section] = {} config[section][opt] = value def show(self, config, section, opt): if section not in config.keys(): raise ConfigError("section '{}' doesn't exist".format(section)) if opt not in config[section].keys(): raise ConfigError("option '{}.{}' doesn't exist".format( section, opt)) logger.info(config[section][opt]) @staticmethod def _merge(first, second): res = {} sections = list(first.keys()) + list(second.keys()) for section in sections: f = first.get(section, {}).copy() s = second.get(section, {}).copy() f.update(s) res[section] = f return res @staticmethod def _lower(config): new_config = {} for s_key, s_value in config.items(): new_s = {} for key, value in s_value.items(): new_s[key.lower()] = value new_config[s_key.lower()] = new_s return new_config
def validate_instance_parameters(params: dict): from spotty.providers.aws.config.ebs_volume import EbsVolume instance_parameters = { 'region': And(str, Regex(r'^[a-z0-9-]+$')), Optional('availabilityZone', default=''): And(str, Regex(r'^[a-z0-9-]+$')), Optional('subnetId', default=''): And(str, Regex(r'^subnet-[a-z0-9]+$')), 'instanceType': str, Optional('spotInstance', default=False): bool, Optional('amiName', default=None): And(str, len, Regex(r'^[\w\(\)\[\]\s\.\/\'@-]{3,128}$')), Optional('amiId', default=None): And(str, len, Regex(r'^ami-[a-z0-9]+$')), Optional('rootVolumeSize', default=0): And( Or(int, str), Use(str), Regex(r'^\d+$', error='Incorrect value for "rootVolumeSize".'), Use(int), And(lambda x: x > 0, error='"rootVolumeSize" should be greater than 0 or should ' 'not be specified.'), ), Optional('ports', default=[]): [And(int, lambda x: 0 < x < 65536)], Optional('maxPrice', default=0): And( Or(float, int, str), Use(str), Regex(r'^\d+(\.\d{1,6})?$', error='Incorrect value for "maxPrice".'), Use(float), And(lambda x: x > 0, error='"maxPrice" should be greater than 0 or ' 'should not be specified.'), ), Optional('managedPolicyArns', default=[]): [str], Optional('instanceProfileArn', default=None): str, } volumes_checks = [ And(lambda x: len(x) < 12, error='Maximum 11 volumes are supported at the moment.'), And(lambda x: not has_prefix([(volume['parameters']['mountDir'] + '/') for volume in x if volume['parameters']. get('mountDir')]), error='Mount directories cannot be prefixes for each other.'), ] instance_checks = [ And(lambda x: not (x['maxPrice'] and not x['spotInstance']), error='"maxPrice" can be specified only for spot instances.'), And(lambda x: not (x['amiName'] and x['amiId']), error='"amiName" and "amiId" parameters cannot be used together.'), ] schema = get_instance_parameters_schema(instance_parameters, EbsVolume.TYPE_NAME, instance_checks, volumes_checks) return validate_config(schema, params)
#-*- coding: utf-8 -*- import os import re from pyquery import PyQuery as pq from pykl import gdom, pyhttp, pyutils, pyfile from schema import Schema, SchemaError, Regex, And, Use, Optional from flask import request from mrq.dashboard.utils import jsonify from functools import update_wrapper from urlparse import urlparse, parse_qs HttpUrlSchema = And( basestring, len, Regex( r'^(http|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?$' )) IPSchema = And( basestring, len, Regex( r'^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$' )) def TaskSchemaWrapper(*args, **kwgs): schema = Schema(*args, **kwgs) def _wrapper(func): func.params_schema = schema return func
def _validate_PO(self, **kw): """ validate required fields are present and validate types uses schema to validate and return validated kw """ # following vars are used for validation #ISO-8601 format for date and datetime. rdatetime = r'^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?(([+-]\d\d:\d\d)|Z)?$' toleranceList = ('AllowOverRun', 'AllowUnderrun', 'AllowOverrunOrUnderrun', 'ExactOnly') uomList = [ 'BX', 'CA', 'DZ', 'EA', 'KT', 'PR', 'PK', 'RL', 'ST', 'SL', 'TH' ] conact_schema = Schema({ Optional("attentionTo"): And(lambda s: var_check(s, 35), error='"attentionTo" should evaluate to varchar(35)'), Optional("companyName"): And(lambda s: var_check(s, 35), error='"companyName" should evaluate to varchar(35)'), Optional("address1"): And(lambda s: var_check(s, 35), error='"address1" should evaluate to varchar(35)'), Optional("address2"): And(lambda s: var_check(s, 35), error='"address2" should evaluate to varchar(35)'), Optional("address3"): And(lambda s: var_check(s, 35), error='"address3" should evaluate to varchar(35)'), Optional("city"): And(lambda s: var_check(s, 30), error='"city" should evaluate to varchar(30)'), Optional("region"): And(lambda s: var_check(s, 3), error='"region" should evaluate to varchar(3)'), Optional("postalCode"): And(lambda s: var_check(s, 10), error='"postalCode" should evaluate to varchar(10)'), Optional("country"): And(lambda s: var_check(s, 2), error='"country" should evaluate to varchar(2)'), Optional("email"): And(lambda s: var_check(s, 128), error='"email" should evaluate to varchar(128)'), Optional("phone"): And(lambda s: var_check(s, 32), error='"phone" should evaluate to varchar(32)'), Optional("comments"): str }) quantity_schema = Schema({ "uom": And(str, lambda s: s in uomList), "value": And(Use(to_decimal_4), validDecimal_12_4) }) third_party_schema = Schema({ "accountName": And(lambda s: var_check(s, 64), error='"accountName" should evaluate to varchar(64)'), "accountNumber": And(lambda s: var_check(s, 64), error='"accountNumber" should evaluate to varchar(64)'), "ContactDetails": conact_schema }) # schema used for validating PO version 1.0.0 v1_0_0_schema = Schema({ "wsVersion": And(str, len), "id": And(str, len), Optional("password"): str, "PO": { "orderType": And(str, lambda s: s in ('Blank', 'Sample', 'Simple', 'Configured')), "orderNumber": And(lambda s: var_check(s, 64), error='"orderNumber" should evaluate to varchar(64)'), "orderDate": And(Const(Use(fromisoformat)), Regex(r'{}'.format(rdatetime))), Optional("lastModified"): And(Const(Use(fromisoformat)), Regex(r'{}'.format(rdatetime))), "totalAmount": And(Use(to_decimal_4), validDecimal_12_4), Optional("paymentTerms"): str, "rush": Use(xml_bool), "currency": And(lambda s: var_check(s, 3), error='"currency" should evaluate to varchar(3)'), Optional("DigitalProof"): { "DigitalProofAddressArray": [{ "DigitalProofAddress": { "type": And(lambda s: var_check(s, 64), error='"type" should evaluate to varchar(64)'), "email": And(lambda s: var_check(s, 128), error='"email" should evaluate to varchar(128)' ), "lineItemGroupingId": int } }], "required": Use(xml_bool) }, Optional("OrderContactArray"): [{ "Contact": { Optional("accountName"): And(lambda s: var_check(s, 64), error='"accountName" should evaluate to varchar(64)'), Optional("accountNumber"): And(lambda s: var_check(s, 64), error='"accountNumber" should evaluate to varchar(64)'), "contactType": And(str, lambda s: s in [ 'Art', 'Bill', 'Expeditor', 'Order', 'Sales', 'Ship', 'Sold' ]), "ContactDetails": conact_schema } }], "ShipmentArray": [{ "Shipment": { Optional("shipReferences"): [lambda s: var_check(s, 64)], Optional("comments"): str, Optional("ThirdPartyAccount"): third_party_schema, "allowConsolidation": Use(xml_bool), "blindShip": Use(xml_bool), "packingListRequired": Use(xml_bool), "FreightDetails": { "carrier": And(lambda s: var_check(s, 64), error='"carrier" should evaluate to varchar(64)' ), "service": And(lambda s: var_check(s, 64), error='"service" should evaluate to varchar(64)' ) }, "ShipTo": { "customerPickup": Use(xml_bool), "ContactDetails": conact_schema, "shipmentId": int } } }], "LineItemArray": [{ "LineItem": { "lineNumber": int, Optional("lineReferenceId"): And(lambda s: var_check(s, 64), error='"lineReferenceId" should evaluate to varchar(64)'), "description": str, "lineType": And(str, lambda s: s in ['New', 'Repeat', 'Reference']), Optional("Quantity"): quantity_schema, Optional("fobid"): And(lambda s: var_check(s, 64), error='"fobid" should evaluate to varchar(64)'), "ToleranceDetails": { Optional('uom'): And(str, lambda s: s in uomList), Optional("value"): And(Use(to_decimal_4), validDecimal_12_4), "tolerance": And(str, lambda s: s in toleranceList) }, "allowPartialShipments": Use(xml_bool), Optional("unitPrice"): And(Use(to_decimal_4), validDecimal_12_4), "lineItemTotal": And(Use(to_decimal_4), validDecimal_12_4), Optional("requestedShipDate"): And(Const(Use(fromisoformat)), Regex(r'{}'.format(rdatetime))), Optional("requestedInHands"): And(Const(Use(fromisoformat)), Regex(r'{}'.format(rdatetime))), Optional("referenceSalesQuote"): And(lambda s: var_check(s, 64), error='"referenceSalesQuote" should evaluate to varchar(64)'), Optional("Program"): { Optional("id"): And(lambda s: var_check(s, 64), error='"id" should evaluate to varchar(64)'), Optional("name"): And(lambda s: var_check(s, 64), error='"name" should evaluate to varchar(64)') }, Optional("endCustomerSalesOrder"): And(lambda s: var_check(s, 64), error= '"endCustomerSalesOrder" should evaluate to varchar(64)'), Optional("productId"): And(lambda s: var_check(s, 64), error='"productId" should evaluate to varchar(64)'), Optional("customerProductId"): And(lambda s: var_check(s, 64), error='"customerProductId" should evaluate to varchar(64)'), Optional("lineItemGroupingId"): int, Optional("PartArray"): [{ "Part": { Optional("partGroup"): And(lambda s: var_check(s, 64), error='"partGroup" should evaluate to varchar(64)'), "partId": And(lambda s: var_check(s, 64), error='"partId" should evaluate to varchar(64)'), Optional("customerPartId"): And(lambda s: var_check(s, 64), error='"customerPartId" should evaluate to varchar(64)'), "customerSupplied": Use(xml_bool), Optional("description"): str, "Quantity": quantity_schema, Optional("locationLinkId"): [int], Optional("unitPrice"): And(Use(to_decimal_4), validDecimal_12_4), Optional("extendedPrice"): And(Use(to_decimal_4), validDecimal_12_4), Optional("ShipmentLinkArray"): [{ "ShipmentLink": { "Quantity": quantity_schema, "shipmentId": int } }] } }], Optional("Configuration"): { Optional("ChargeArray"): [{ "Charge": { Optional("chargeName"): And(lambda s: var_check(s, 128), error='"chargeName" should evaluate to varchar(128)'), Optional("description"): str, Optional("extendedPrice"): And(Use(to_decimal_4), validDecimal_12_4), Optional("unitprice"): And(Use(to_decimal_4), validDecimal_12_4), "chargeId": And(lambda s: var_check(s, 64), error='"chargeId" should evaluate to varchar(64)'), "chargeType": And(str, lambda s: s in ['Freight', 'Order', 'Run', 'Setup']), "Quantity": quantity_schema } }], Optional("LocationArray"): [{ "Location": { Optional("locationName"): And(lambda s: var_check(s, 128), error='"locationName" should evaluate to varchar(128)'), "DecorationArray": [{ "Decoration": { Optional("decorationName"): And(lambda s: var_check(s, 128), error='"decorationName" should evaluate to varchar(128)'), "Artwork": { Optional("instructions"): str, Optional("refArtworkId"): And(lambda s: var_check(s, 64), error='"refArtworkId" should evaluate to varchar(64)'), Optional("totalStitchCount"): int, Optional("ArtworkFileArray"): [{ "ArtworkFile": { "artworkType": And(str, lambda s: s in [ 'ProductionReady', 'VirtualProof', 'SupplierArtTemplate', 'NonProductionReady' ]), "fileLocation": And(lambda s: var_check(s, 1024), error='"fileLocation" should evaluate to varchar(1024)'), "fileName": And(lambda s: var_check(s, 256), error='"fileName" should evaluate to varchar(256)'), "transportMechanism": And(str, lambda s: s in [ 'Email', 'Url', 'Ftp', 'ArtworkToFollow' ]), } }], Optional("description"): str, Optional("Dimensions"): { Optional("diameter"): And(Use(to_decimal_4),validDecimal_12_4), Optional("height"): And(Use(to_decimal_4),validDecimal_12_4), Optional('uom'): And(str, lambda s: s in uomList), Optional("width"): And(Use(to_decimal_4),validDecimal_12_4), "useMaxLocationDimensions": Use(xml_bool), "geometry": And(str, lambda s: s in [ 'Circle', 'Other', 'Rectangle' ]) }, Optional("Layers"): { "colorSystem": And( str, lambda s: s in [ 'Cmyk', 'Other', 'Pms', 'Rgb', 'Thread' ]), "LayerOrStopArray": [{ "LayerOrStop":{ "color": And(lambda s: var_check(s, 64), error='Layer "color" should evaluate to varchar(64)'), "nameOrNumber": And(lambda s: var_check(s, 64), error='"nameOrNumber" should evaluate to varchar(64)'), "description": str }} ] }, Optional("TypesetArray"): [{ "Typeset":{ Optional("fontSize"): Use(Decimal), Optional("font"): And(lambda s: var_check(s, 64), error='"font" should evaluate to varchar(64)'), "sequenceNumber": int, "value": And(lambda s: var_check(s, 1024), error='Typset "value" should evaluate to varchar(1024)') } }] }, "decorationId": int } }], "locationLinkId": int, "locationId": int } }], Optional("referenceNumberType"): And( str, lambda s: s in [ 'PurchaseOrder', 'SalesOrder', 'JobOrWorkOrder' ]), Optional("referenceNumber"): And(lambda s: var_check(s, 64), error='"referenceNumber" should evaluate to varchar(64)'), "preProductionProof": Use(xml_bool), } } }], "termsAndConditions": str, Optional("salesChannel"): And(lambda s: var_check(s, 64), error='"salesChannel" should evaluate to varchar(64)'), Optional("promoCode"): And(lambda s: var_check(s, 64), error='"promoCode" should evaluate to varchar(64)'), Optional("TaxInformationArray"): [{ "TaxInformation": { "taxJurisdiction": And(lambda s: var_check(s, 64), error='"taxJurisdiction" should evaluate to varchar(64)'), "taxExempt": Use(xml_bool), "taxId": And(lambda s: var_check(s, 64), error='"taxId" should evaluate to varchar(64)'), "taxType": And(lambda s: var_check(s, 64), error='"taxType" should evaluate to varchar(64)'), Optional("taxAmount"): Use(Decimal) } }] } }) # run the validation v1_0_0_schema.validate(kw)
Optional("restart_execution_on_update"): bool, } AWS_ACCOUNT_ID_REGEX_STR = r"\A[0-9]{12}\Z" AWS_ACCOUNT_ID_SCHEMA = Schema( And( Or(int, str), Use(str), Regex( AWS_ACCOUNT_ID_REGEX_STR, error=( "The specified account id is incorrect. " "This typically happens when you specify the account id as a " "number, while the account id starts with a zero. If this is " "the case, please wrap the account id in quotes to make it a " "string. An AWS Account Id is a number of 12 digits, which " "should start with a zero if the Account Id has a zero at " "the start too. " "The number shown to not match the regular expression could " "be interpreted as an octal number due to the leading zero. " "Therefore, it might not match the account id as specified " "in the deployment map." ) ) ) ) # CodeCommit Source CODECOMMIT_SOURCE_PROPS = { "account_id": AWS_ACCOUNT_ID_SCHEMA, Optional("repository"): str, Optional("branch"): str,
class Config(object): # pylint: disable=too-many-instance-attributes """Class that manages configuration files for a dvc repo. Args: dvc_dir (str): optional path to `.dvc` directory, that is used to access repo-specific configs like .dvc/config and .dvc/config.local. validate (bool): optional flag to tell dvc if it should validate the config or just load it as is. 'True' by default. Raises: ConfigError: thrown when config has an invalid format. """ APPNAME = "dvc" APPAUTHOR = "iterative" # NOTE: used internally in RemoteLOCAL to know config # location, that url should resolved relative to. PRIVATE_CWD = "_cwd" CONFIG = "config" CONFIG_LOCAL = "config.local" BOOL_SCHEMA = And(str, is_bool, Use(to_bool)) SECTION_CORE = "core" SECTION_CORE_LOGLEVEL = "loglevel" SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel) SECTION_CORE_REMOTE = "remote" SECTION_CORE_INTERACTIVE_SCHEMA = BOOL_SCHEMA SECTION_CORE_INTERACTIVE = "interactive" SECTION_CORE_ANALYTICS = "analytics" SECTION_CORE_ANALYTICS_SCHEMA = BOOL_SCHEMA SECTION_CACHE = "cache" SECTION_CACHE_DIR = "dir" SECTION_CACHE_TYPE = "type" SECTION_CACHE_TYPE_SCHEMA = supported_cache_type SECTION_CACHE_PROTECTED = "protected" SECTION_CACHE_LOCAL = "local" SECTION_CACHE_S3 = "s3" SECTION_CACHE_GS = "gs" SECTION_CACHE_SSH = "ssh" SECTION_CACHE_HDFS = "hdfs" SECTION_CACHE_AZURE = "azure" SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning" SECTION_CACHE_SCHEMA = { Optional(SECTION_CACHE_LOCAL): str, Optional(SECTION_CACHE_S3): str, Optional(SECTION_CACHE_GS): str, Optional(SECTION_CACHE_HDFS): str, Optional(SECTION_CACHE_SSH): str, Optional(SECTION_CACHE_AZURE): str, Optional(SECTION_CACHE_DIR): str, Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA, Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA, Optional(PRIVATE_CWD): str, Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): BOOL_SCHEMA, } # backward compatibility SECTION_CORE_CLOUD = "cloud" SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud) SECTION_CORE_STORAGEPATH = "storagepath" SECTION_CORE_SCHEMA = { Optional(SECTION_CORE_LOGLEVEL): And(str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA), Optional(SECTION_CORE_REMOTE, default=""): And(str, Use(str.lower)), Optional(SECTION_CORE_INTERACTIVE, default=False): SECTION_CORE_INTERACTIVE_SCHEMA, Optional(SECTION_CORE_ANALYTICS, default=True): SECTION_CORE_ANALYTICS_SCHEMA, # backward compatibility Optional(SECTION_CORE_CLOUD, default=""): SECTION_CORE_CLOUD_SCHEMA, Optional(SECTION_CORE_STORAGEPATH, default=""): str, } # backward compatibility SECTION_AWS = "aws" SECTION_AWS_STORAGEPATH = "storagepath" SECTION_AWS_CREDENTIALPATH = "credentialpath" SECTION_AWS_ENDPOINT_URL = "endpointurl" SECTION_AWS_LIST_OBJECTS = "listobjects" SECTION_AWS_REGION = "region" SECTION_AWS_PROFILE = "profile" SECTION_AWS_USE_SSL = "use_ssl" SECTION_AWS_SCHEMA = { SECTION_AWS_STORAGEPATH: str, Optional(SECTION_AWS_REGION): str, Optional(SECTION_AWS_PROFILE): str, Optional(SECTION_AWS_CREDENTIALPATH): str, Optional(SECTION_AWS_ENDPOINT_URL): str, Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA, Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA, } # backward compatibility SECTION_GCP = "gcp" SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH SECTION_GCP_CREDENTIALPATH = SECTION_AWS_CREDENTIALPATH SECTION_GCP_PROJECTNAME = "projectname" SECTION_GCP_SCHEMA = { SECTION_GCP_STORAGEPATH: str, Optional(SECTION_GCP_PROJECTNAME): str, } # backward compatibility SECTION_LOCAL = "local" SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH SECTION_LOCAL_SCHEMA = {SECTION_LOCAL_STORAGEPATH: str} SECTION_AZURE_CONNECTION_STRING = "connection_string" # Alibabacloud oss options SECTION_OSS_ACCESS_KEY_ID = "oss_key_id" SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret" SECTION_OSS_ENDPOINT = "oss_endpoint" SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$' SECTION_REMOTE_FMT = 'remote "{}"' SECTION_REMOTE_URL = "url" SECTION_REMOTE_USER = "******" SECTION_REMOTE_PORT = "port" SECTION_REMOTE_KEY_FILE = "keyfile" SECTION_REMOTE_TIMEOUT = "timeout" SECTION_REMOTE_PASSWORD = "******" SECTION_REMOTE_ASK_PASSWORD = "******" SECTION_REMOTE_SCHEMA = { SECTION_REMOTE_URL: str, Optional(SECTION_AWS_REGION): str, Optional(SECTION_AWS_PROFILE): str, Optional(SECTION_AWS_CREDENTIALPATH): str, Optional(SECTION_AWS_ENDPOINT_URL): str, Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA, Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA, Optional(SECTION_GCP_PROJECTNAME): str, Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA, Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA, Optional(SECTION_REMOTE_USER): str, Optional(SECTION_REMOTE_PORT): Use(int), Optional(SECTION_REMOTE_KEY_FILE): str, Optional(SECTION_REMOTE_TIMEOUT): Use(int), Optional(SECTION_REMOTE_PASSWORD): str, Optional(SECTION_REMOTE_ASK_PASSWORD): BOOL_SCHEMA, Optional(SECTION_AZURE_CONNECTION_STRING): str, Optional(SECTION_OSS_ACCESS_KEY_ID): str, Optional(SECTION_OSS_ACCESS_KEY_SECRET): str, Optional(SECTION_OSS_ENDPOINT): str, Optional(PRIVATE_CWD): str, } SECTION_STATE = "state" SECTION_STATE_ROW_LIMIT = "row_limit" SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota" SECTION_STATE_SCHEMA = { Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole), Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent), } SCHEMA = { Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA, Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA, Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA, Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA, # backward compatibility Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA, Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA, Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA, } def __init__(self, dvc_dir=None, validate=True): self.system_config_file = os.path.join(self.get_system_config_dir(), self.CONFIG) self.global_config_file = os.path.join(self.get_global_config_dir(), self.CONFIG) if dvc_dir is not None: self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) self.config_file = os.path.join(dvc_dir, self.CONFIG) self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL) else: self.dvc_dir = None self.config_file = None self.config_local_file = None self._system_config = None self._global_config = None self._repo_config = None self._local_config = None self.config = None self.load(validate=validate) @staticmethod def get_global_config_dir(): """Returns global config location. E.g. ~/.config/dvc/config. Returns: str: path to the global config directory. """ from appdirs import user_config_dir return user_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) @staticmethod def get_system_config_dir(): """Returns system config location. E.g. /etc/dvc.conf. Returns: str: path to the system config directory. """ from appdirs import site_config_dir return site_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) @staticmethod def init(dvc_dir): """Initializes dvc config. Args: dvc_dir (str): path to .dvc directory. Returns: dvc.config.Config: config object. """ config_file = os.path.join(dvc_dir, Config.CONFIG) open(config_file, "w+").close() return Config(dvc_dir) def _load(self): self._system_config = configobj.ConfigObj(self.system_config_file) self._global_config = configobj.ConfigObj(self.global_config_file) if self.config_file is not None: self._repo_config = configobj.ConfigObj(self.config_file) else: self._repo_config = configobj.ConfigObj() if self.config_local_file is not None: self._local_config = configobj.ConfigObj(self.config_local_file) else: self._local_config = configobj.ConfigObj() self.config = None def _load_config(self, path): config = configobj.ConfigObj(path) config = self._lower(config) self._resolve_paths(config, path) return config @staticmethod def _resolve_path(path, config_file): assert os.path.isabs(config_file) config_dir = os.path.dirname(config_file) return os.path.abspath(os.path.join(config_dir, path)) def _resolve_cache_path(self, config, fname): cache = config.get(self.SECTION_CACHE) if cache is None: return cache_dir = cache.get(self.SECTION_CACHE_DIR) if cache_dir is None: return cache[self.PRIVATE_CWD] = os.path.dirname(fname) def _resolve_paths(self, config, fname): if fname is None: return self._resolve_cache_path(config, fname) for section in config.values(): if self.SECTION_REMOTE_URL not in section.keys(): continue section[self.PRIVATE_CWD] = os.path.dirname(fname) def load(self, validate=True): """Loads config from all the config files. Args: validate (bool): optional flag to tell dvc if it should validate the config or just load it as is. 'True' by default. Raises: dvc.config.ConfigError: thrown if config has invalid format. """ self._load() try: self.config = self._load_config(self.system_config_file) user = self._load_config(self.global_config_file) config = self._load_config(self.config_file) local = self._load_config(self.config_local_file) # NOTE: schema doesn't support ConfigObj.Section validation, so we # need to convert our config to dict before passing it to for conf in [user, config, local]: self.config = self._merge(self.config, conf) if validate: self.config = Schema(self.SCHEMA).validate(self.config) # NOTE: now converting back to ConfigObj self.config = configobj.ConfigObj(self.config, write_empty_values=True) self.config.filename = self.config_file self._resolve_paths(self.config, self.config_file) except Exception as ex: raise ConfigError(ex) @staticmethod def _get_key(conf, name, add=False): for k in conf.keys(): if k.lower() == name.lower(): return k if add: conf[name] = {} return name return None def save(self, config=None): """Saves config to config files. Args: config (configobj.ConfigObj): optional config object to save. Raises: dvc.config.ConfigError: thrown if failed to write config file. """ if config is not None: clist = [config] else: clist = [ self._system_config, self._global_config, self._repo_config, self._local_config, ] for conf in clist: if conf.filename is None: continue try: logger.debug("Writing '{}'.".format(conf.filename)) dname = os.path.dirname(os.path.abspath(conf.filename)) try: os.makedirs(dname) except OSError as exc: if exc.errno != errno.EEXIST: raise conf.write() except Exception as exc: msg = "failed to write config '{}'".format(conf.filename) raise ConfigError(msg, exc) def get_remote_settings(self, name): import posixpath """ Args: name (str): The name of the remote that we want to retrieve Returns: dict: The content beneath the given remote name. Example: >>> config = {'remote "server"': {'url': 'ssh://localhost/'}} >>> get_remote_settings("server") {'url': 'ssh://localhost/'} """ settings = self.config.get(self.SECTION_REMOTE_FMT.format( name.lower())) if settings is None: raise ConfigError( "unable to find remote section '{}'".format(name)) parsed = urlparse(settings["url"]) # Support for cross referenced remotes. # This will merge the settings, giving priority to the outer reference. # For example, having: # # dvc remote add server ssh://localhost # dvc remote modify server user root # dvc remote modify server ask_password true # # dvc remote add images remote://server/tmp/pictures # dvc remote modify images user alice # dvc remote modify images ask_password false # dvc remote modify images password asdf1234 # # Results on a config dictionary like: # # { # "url": "ssh://localhost/tmp/pictures", # "user": "******", # "password": "******", # "ask_password": False, # } # if parsed.scheme == "remote": reference = self.get_remote_settings(parsed.netloc) url = posixpath.join(reference["url"], parsed.path.lstrip("/")) merged = reference.copy() merged.update(settings) merged["url"] = url return merged return settings @staticmethod def unset(config, section, opt=None): """Unsets specified option and/or section in the config. Args: config (configobj.ConfigObj): config to work on. section (str): section name. opt (str): optional option name. """ if section not in config.keys(): raise ConfigError("section '{}' doesn't exist".format(section)) if opt is None: del config[section] return if opt not in config[section].keys(): raise ConfigError("option '{}.{}' doesn't exist".format( section, opt)) del config[section][opt] if not config[section]: del config[section] @staticmethod def set(config, section, opt, value): """Sets specified option in the config. Args: config (configobj.ConfigObj): config to work on. section (str): section name. opt (str): option name. value: value to set option to. """ if section not in config.keys(): config[section] = {} config[section][opt] = value @staticmethod def show(config, section, opt): """Prints option value from the config. Args: config (configobj.ConfigObj): config to work on. section (str): section name. opt (str): option name. """ if section not in config.keys(): raise ConfigError("section '{}' doesn't exist".format(section)) if opt not in config[section].keys(): raise ConfigError("option '{}.{}' doesn't exist".format( section, opt)) logger.info(config[section][opt]) @staticmethod def _merge(first, second): res = {} sections = list(first.keys()) + list(second.keys()) for section in sections: first_copy = first.get(section, {}).copy() second_copy = second.get(section, {}).copy() first_copy.update(second_copy) res[section] = first_copy return res @staticmethod def _lower(config): new_config = {} for s_key, s_value in config.items(): new_s = {} for key, value in s_value.items(): new_s[key.lower()] = str(value) new_config[s_key.lower()] = new_s return new_config
it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. """ from schema import And, Optional, Or, Regex, Schema NAME_ID_VALIDATION_REGEX = Regex(r"^[A-Za-z0-9_. ()-]+$") RESOURCE_TYPE_REGEX = Regex( r"^AWS\.(ACM\.Certificate|CloudFormation\.Stack|CloudTrail\.Meta|CloudTrail|CloudWatch" r"\.LogGroup|Config\.Recorder\.Meta|Config\.Recorder|DynamoDB\.Table|EC2\.AMI|EC2\.Instance" r"|EC2\.NetworkACL|EC2\.SecurityGroup|EC2\.Volume|EC2\.VPC|ECS\.Cluster|EKS\.Cluster|ELBV2" r"\.ApplicationLoadBalancer|GuardDuty\.Detector\.Meta|GuardDuty\.Detector|IAM\.Group|IAM" r"\.Policy|IAM\.Role|IAM\.RootUser|IAM\.User|KMS\.Key|Lambda\.Function|PasswordPolicy|RDS" r"\.Instance|Redshift\.Cluster|S3\.Bucket|WAF\.Regional\.WebACL|WAF\.WebACL)$" ) LOG_TYPE_REGEX = Regex( r"^(Apache\.AccessCombined|Apache\.AccessCommon|AWS\.ALB|AWS\.AuroraMySQLAudit|AWS" r"\.CloudTrail|AWS\.CloudTrailDigest|AWS\.CloudTrailInsight|AWS\.CloudWatchEvents|AWS" r"\.GuardDuty|AWS\.S3ServerAccess|AWS\.VPCDns|AWS\.VPCFlow|Box\.Event|CiscoUmbrella" r"\.CloudFirewall|CiscoUmbrella\.DNS|CiscoUmbrella\.IP|CiscoUmbrella\.Proxy|Cloudflare" r"\.Firewall|Cloudflare\.HttpRequest|Cloudflare\.Spectrum|Crowdstrike\.AIDMaster|Crowdstrike" r"\.DNSRequest|Crowdstrike\.GroupIdentity|Crowdstrike\.ManagedAssets|Crowdstrike"
# # THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import os from schema import Schema, And, Use, Optional, Regex, Or common_schema = { 'authorName': str, 'experimentName': str, Optional('description'): str, 'trialConcurrency': And(int, lambda n: 1 <=n <= 999999), Optional('maxExecDuration'): Regex(r'^[1-9][0-9]*[s|m|h|d]$'), Optional('maxTrialNum'): And(int, lambda x: 1 <= x <= 99999), 'trainingServicePlatform': And(str, lambda x: x in ['remote', 'local', 'pai', 'kubeflow', 'frameworkcontroller']), Optional('searchSpacePath'): os.path.exists, Optional('multiPhase'): bool, Optional('multiThread'): bool, Optional('nniManagerIp'): str, Optional('logDir'): os.path.isdir, Optional('logLevel'): Or('trace', 'debug', 'info', 'warning', 'error', 'fatal'), 'useAnnotation': bool, Optional('advisor'): Or({ 'builtinAdvisorName': Or('Hyperband'), 'classArgs': { 'optimize_mode': Or('maximize', 'minimize'), Optional('R'): int, Optional('eta'): int
rpm_schema = Schema({ 'content': { Optional('build'): { 'use_source_tito_config': bool, 'tito_target': And(str, len), 'push_release_commit': bool, }, Optional('source'): { Optional('alias'): And(str, len), Optional('git'): { 'branch': { Optional('fallback'): And(str, len), Optional('stage'): And(str, len), 'target': And(str, len), }, 'url': And(str, len, Regex(GIT_SSH_URL_REGEX)), }, 'specfile': Regex(r'.+\.spec$'), Optional('modifications'): [modification], }, }, Optional('distgit'): { Optional('branch'): And(str, len), }, Optional('enabled_repos'): [ And(str, len), ], Optional('mode'): Or(*valid_modes), 'name': And(str, len), 'owners': [And(str, len)], Optional('maintainer'): {
def validate_basic_config(data): container = And( { Optional('name', default=DEFAULT_CONTAINER_NAME): And(str, Regex(r'^[\w-]+$')), 'projectDir': And( str, And(os.path.isabs, error= 'Use an absolute path when specifying the project directory' ), Use(lambda x: x.rstrip('/'))), Optional('image', default=''): And(str, len), Optional('file', default=''): And( str, # TODO: a proper regex that the filename is valid Regex(r'^[\w\.\/@-]*$', error='Invalid name for a Dockerfile'), And(lambda x: not x.endswith('/'), error='Invalid name for a Dockerfile'), And(lambda x: not os.path.isabs(x), error='Path to the Dockerfile should be relative to the ' 'project\'s root directory.'), ), Optional('runAsHostUser', default=False): bool, Optional('volumeMounts', default=[]): (And( [{ 'name': And(Or(int, str), Use(str), Regex(r'^[\w-]+$')), 'mountPath': And( str, And(os.path.isabs, error= 'Use an absolute path when specifying a mount directory' ), Use(lambda x: x.rstrip('/')), ), }], And(lambda x: is_unique_value(x, 'name'), error='Each volume mount must have a unique name.'), And(lambda x: not has_prefix([(volume['mountPath'] + '/') for volume in x]), error= 'Volume mount paths cannot be prefixes for each other.'), )), Optional('workingDir', default=''): And( str, And(os.path.isabs, error='Use an absolute path when specifying a ' 'working directory'), Use(lambda x: x.rstrip('/'))), Optional('env', default={}): { And(str, Regex(r'^[a-zA-Z_]+[a-zA-Z0-9_]*$')): str, }, Optional('hostNetwork', default=False): bool, Optional('ports', default=[]): [{ 'containerPort': And(int, lambda x: 0 < x < 65536), Optional('hostPort', default=None): And(int, lambda x: 0 < x < 65536), }], Optional('commands', default=''): str, # TODO: allow to use only certain runtime parameters Optional('runtimeParameters', default=[]): And([str], Use(lambda x: [p.strip() for p in x])), }, And(lambda x: x['image'] or x['file'], error='Either "image" or "file" should be specified.'), And(lambda x: not (x['image'] and x['file']), error='"image" and "file" cannot be specified together.'), And(lambda x: not (x['hostNetwork'] and x['ports']), error= 'Published ports and the host network mode cannot be used together.' ), ) schema = Schema({ 'project': { 'name': And(str, Regex(r'^[a-zA-Z0-9][a-zA-Z0-9-]{,26}[a-zA-Z0-9]$')), Optional('syncFilters', default=[]): And( [ And( { Optional('exclude'): [ And( str, len, And(lambda x: '**' not in x, error= 'Use single asterisks ("*") in sync filters' )) ], Optional('include'): [ And( str, len, And(lambda x: '**' not in x, error= 'Use single asterisks ("*") in sync filters' )) ], }, And(lambda x: x, error= 'Either "exclude" or "include" filter should be specified.' ), And(lambda x: not ('exclude' in x and 'include' in x), error= '"exclude" and "include" filters should be specified as different list items.' ), ) ], error='"project.syncFilters" field must be a list.', ) }, WrongKey('container', error='Use "containers" field instead of "container".'): object, Optional('containers', default=[]): And( [container], And(lambda x: is_unique_value(x, 'name'), error='Each container must have a unique name.'), error='"containers" field must be a list.', ), WrongKey('instance', error='Use "instances" field instead of "instance".'): object, 'instances': And( [{ 'name': And(Or(int, str), Use(str), Regex(r'^[\w-]+$')), 'provider': str, Optional('parameters', default={}): { And(str, Regex(r'^[\w]+$')): object, } }], And(lambda x: len(x), error= 'At least one instance must be specified in the configuration file.' ), And(lambda x: is_unique_value(x, 'name'), error='Each instance must have a unique name.'), ), Optional('scripts', default={}): { And(str, Regex(r'^[\w-]+$')): And(str, len), }, }) return validate_config(schema, data)
class Config(object): # pylint: disable=too-many-instance-attributes """Class that manages configuration files for a dvc repo. Args: dvc_dir (str): optional path to `.dvc` directory, that is used to access repo-specific configs like .dvc/config and .dvc/config.local. validate (bool): optional flag to tell dvc if it should validate the config or just load it as is. 'True' by default. Raises: ConfigError: thrown when config has an invalid format. """ APPNAME = "dvc" APPAUTHOR = "iterative" # NOTE: used internally in RemoteLOCAL to know config # location, that url should resolved relative to. PRIVATE_CWD = "_cwd" CONFIG = "config" CONFIG_LOCAL = "config.local" LEVEL_LOCAL = 0 LEVEL_REPO = 1 LEVEL_GLOBAL = 2 LEVEL_SYSTEM = 3 BOOL_SCHEMA = And(str, is_bool, Use(to_bool)) SECTION_CORE = "core" SECTION_CORE_LOGLEVEL = "loglevel" SECTION_CORE_LOGLEVEL_SCHEMA = And( Use(str.lower), Choices("info", "debug", "warning", "error")) SECTION_CORE_REMOTE = "remote" SECTION_CORE_INTERACTIVE_SCHEMA = BOOL_SCHEMA SECTION_CORE_INTERACTIVE = "interactive" SECTION_CORE_ANALYTICS = "analytics" SECTION_CORE_ANALYTICS_SCHEMA = BOOL_SCHEMA SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs" SECTION_CORE_CHECKSUM_JOBS_SCHEMA = And(Use(int), lambda x: x > 0) SECTION_CACHE = "cache" SECTION_CACHE_DIR = "dir" SECTION_CACHE_TYPE = "type" SECTION_CACHE_TYPE_SCHEMA = supported_cache_type SECTION_CACHE_PROTECTED = "protected" SECTION_CACHE_SHARED = "shared" SECTION_CACHE_SHARED_SCHEMA = And(Use(str.lower), Choices("group")) SECTION_CACHE_LOCAL = "local" SECTION_CACHE_S3 = "s3" SECTION_CACHE_GS = "gs" SECTION_CACHE_SSH = "ssh" SECTION_CACHE_HDFS = "hdfs" SECTION_CACHE_AZURE = "azure" SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning" SECTION_CACHE_SCHEMA = { Optional(SECTION_CACHE_LOCAL): str, Optional(SECTION_CACHE_S3): str, Optional(SECTION_CACHE_GS): str, Optional(SECTION_CACHE_HDFS): str, Optional(SECTION_CACHE_SSH): str, Optional(SECTION_CACHE_AZURE): str, Optional(SECTION_CACHE_DIR): str, Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA, Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA, Optional(SECTION_CACHE_SHARED): SECTION_CACHE_SHARED_SCHEMA, Optional(PRIVATE_CWD): str, Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): BOOL_SCHEMA, } SECTION_CORE_SCHEMA = { Optional(SECTION_CORE_LOGLEVEL): And(str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA), Optional(SECTION_CORE_REMOTE, default=""): And(str, Use(str.lower)), Optional(SECTION_CORE_INTERACTIVE, default=False): SECTION_CORE_INTERACTIVE_SCHEMA, Optional(SECTION_CORE_ANALYTICS, default=True): SECTION_CORE_ANALYTICS_SCHEMA, Optional(SECTION_CORE_CHECKSUM_JOBS, default=None): SECTION_CORE_CHECKSUM_JOBS_SCHEMA, } # backward compatibility SECTION_AWS = "aws" SECTION_AWS_STORAGEPATH = "storagepath" SECTION_AWS_CREDENTIALPATH = "credentialpath" SECTION_AWS_ENDPOINT_URL = "endpointurl" SECTION_AWS_LIST_OBJECTS = "listobjects" SECTION_AWS_REGION = "region" SECTION_AWS_PROFILE = "profile" SECTION_AWS_USE_SSL = "use_ssl" SECTION_AWS_SSE = "sse" SECTION_AWS_ACL = "acl" SECTION_AWS_SCHEMA = { SECTION_AWS_STORAGEPATH: str, Optional(SECTION_AWS_REGION): str, Optional(SECTION_AWS_PROFILE): str, Optional(SECTION_AWS_CREDENTIALPATH): str, Optional(SECTION_AWS_ENDPOINT_URL): str, Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA, Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA, Optional(SECTION_AWS_SSE): str, Optional(SECTION_AWS_ACL): str, } # backward compatibility SECTION_GCP = "gcp" SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH SECTION_GCP_CREDENTIALPATH = SECTION_AWS_CREDENTIALPATH SECTION_GCP_PROJECTNAME = "projectname" SECTION_GCP_SCHEMA = { SECTION_GCP_STORAGEPATH: str, Optional(SECTION_GCP_PROJECTNAME): str, } # backward compatibility SECTION_LOCAL = "local" SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH SECTION_LOCAL_SCHEMA = {SECTION_LOCAL_STORAGEPATH: str} SECTION_AZURE_CONNECTION_STRING = "connection_string" # Alibabacloud oss options SECTION_OSS_ACCESS_KEY_ID = "oss_key_id" SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret" SECTION_OSS_ENDPOINT = "oss_endpoint" SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$' SECTION_REMOTE_FMT = 'remote "{}"' SECTION_REMOTE_URL = "url" SECTION_REMOTE_USER = "******" SECTION_REMOTE_PORT = "port" SECTION_REMOTE_KEY_FILE = "keyfile" SECTION_REMOTE_TIMEOUT = "timeout" SECTION_REMOTE_PASSWORD = "******" SECTION_REMOTE_ASK_PASSWORD = "******" SECTION_REMOTE_GSS_AUTH = "gss_auth" SECTION_REMOTE_NO_TRAVERSE = "no_traverse" SECTION_REMOTE_SCHEMA = { SECTION_REMOTE_URL: str, Optional(SECTION_AWS_REGION): str, Optional(SECTION_AWS_PROFILE): str, Optional(SECTION_AWS_CREDENTIALPATH): str, Optional(SECTION_AWS_ENDPOINT_URL): str, Optional(SECTION_AWS_LIST_OBJECTS, default=False): BOOL_SCHEMA, Optional(SECTION_AWS_USE_SSL, default=True): BOOL_SCHEMA, Optional(SECTION_AWS_SSE): str, Optional(SECTION_AWS_ACL): str, Optional(SECTION_GCP_PROJECTNAME): str, Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA, Optional(SECTION_CACHE_PROTECTED, default=False): BOOL_SCHEMA, Optional(SECTION_REMOTE_USER): str, Optional(SECTION_REMOTE_PORT): Use(int), Optional(SECTION_REMOTE_KEY_FILE): str, Optional(SECTION_REMOTE_TIMEOUT): Use(int), Optional(SECTION_REMOTE_PASSWORD): str, Optional(SECTION_REMOTE_ASK_PASSWORD): BOOL_SCHEMA, Optional(SECTION_REMOTE_GSS_AUTH): BOOL_SCHEMA, Optional(SECTION_AZURE_CONNECTION_STRING): str, Optional(SECTION_OSS_ACCESS_KEY_ID): str, Optional(SECTION_OSS_ACCESS_KEY_SECRET): str, Optional(SECTION_OSS_ENDPOINT): str, Optional(PRIVATE_CWD): str, Optional(SECTION_REMOTE_NO_TRAVERSE, default=True): BOOL_SCHEMA, } SECTION_STATE = "state" SECTION_STATE_ROW_LIMIT = "row_limit" SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota" SECTION_STATE_SCHEMA = { Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole), Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent), } SCHEMA = { Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA, Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA, Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA, Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA, # backward compatibility Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA, Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA, Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA, } def __init__(self, dvc_dir=None, validate=True): self.dvc_dir = dvc_dir self.validate = validate if not dvc_dir: try: from dvc.repo import Repo self.dvc_dir = os.path.join(Repo.find_dvc_dir()) except NotDvcRepoError: self.dvc_dir = None else: self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) self.load() @staticmethod def get_global_config_dir(): """Returns global config location. E.g. ~/.config/dvc/config. Returns: str: path to the global config directory. """ from appdirs import user_config_dir return user_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) @staticmethod def get_system_config_dir(): """Returns system config location. E.g. /etc/dvc.conf. Returns: str: path to the system config directory. """ from appdirs import site_config_dir return site_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) @staticmethod def init(dvc_dir): """Initializes dvc config. Args: dvc_dir (str): path to .dvc directory. Returns: dvc.config.Config: config object. """ config_file = os.path.join(dvc_dir, Config.CONFIG) open(config_file, "w+").close() return Config(dvc_dir) def _resolve_cache_path(self, config): cache = config.get(self.SECTION_CACHE) if cache is None: return cache_dir = cache.get(self.SECTION_CACHE_DIR) if cache_dir is None: return cache[self.PRIVATE_CWD] = os.path.dirname(config.filename) def _resolve_paths(self, config): if config.filename is None: return config ret = copy.deepcopy(config) self._resolve_cache_path(ret) for section in ret.values(): if self.SECTION_REMOTE_URL not in section.keys(): continue section[self.PRIVATE_CWD] = os.path.dirname(ret.filename) return ret def _load_configs(self): system_config_file = os.path.join(self.get_system_config_dir(), self.CONFIG) global_config_file = os.path.join(self.get_global_config_dir(), self.CONFIG) self._system_config = configobj.ConfigObj(system_config_file) self._global_config = configobj.ConfigObj(global_config_file) self._repo_config = configobj.ConfigObj() self._local_config = configobj.ConfigObj() if not self.dvc_dir: return config_file = os.path.join(self.dvc_dir, self.CONFIG) config_local_file = os.path.join(self.dvc_dir, self.CONFIG_LOCAL) self._repo_config = configobj.ConfigObj(config_file) self._local_config = configobj.ConfigObj(config_local_file) @property def config_local_file(self): return self._local_config.filename @property def config_file(self): return self._repo_config.filename def load(self): """Loads config from all the config files. Raises: dvc.config.ConfigError: thrown if config has invalid format. """ self._load_configs() self.config = configobj.ConfigObj() for c in [ self._system_config, self._global_config, self._repo_config, self._local_config, ]: c = self._resolve_paths(c) c = self._lower(c) self.config.merge(c) if not self.validate: return d = self.config.dict() try: d = Schema(self.SCHEMA).validate(d) except SchemaError as exc: raise ConfigError("config format error", cause=exc) self.config = configobj.ConfigObj(d, write_empty_values=True) def save(self, config=None): """Saves config to config files. Raises: dvc.config.ConfigError: thrown if failed to write config file. """ if config is not None: clist = [config] else: clist = [ self._system_config, self._global_config, self._repo_config, self._local_config, ] for conf in clist: self._save(conf) self.load() @staticmethod def _save(config): if config.filename is None: return logger.debug("Writing '{}'.".format(config.filename)) dname = os.path.dirname(os.path.abspath(config.filename)) try: os.makedirs(dname) except OSError as exc: if exc.errno != errno.EEXIST: raise config.write() def unset(self, section, opt=None, level=None, force=False): """Unsets specified option and/or section in the config. Args: section (str): section name. opt (str): optional option name. level (int): config level to use. force (bool): don't error-out even if section doesn't exist. False by default. Raises: dvc.config.ConfigError: thrown if section doesn't exist and `force != True`. """ config = self.get_configobj(level) if section not in config.keys(): if force: return raise ConfigError("section '{}' doesn't exist".format(section)) if opt: if opt not in config[section].keys(): if force: return raise ConfigError("option '{}.{}' doesn't exist".format( section, opt)) del config[section][opt] if not config[section]: del config[section] else: del config[section] self.save(config) def set(self, section, opt, value, level=None, force=True): """Sets specified option in the config. Args: section (str): section name. opt (str): option name. value: value to set option to. level (int): config level to use. force (bool): set option even if section already exists. True by default. Raises: dvc.config.ConfigError: thrown if section already exists and `force != True`. """ config = self.get_configobj(level) if section not in config.keys(): config[section] = {} elif not force: raise ConfigError( "Section '{}' already exists. Use `-f|--force` to overwrite " "section with new value.".format(section)) config[section][opt] = value self.save(config) def get(self, section, opt=None, level=None): """Return option value from the config. Args: section (str): section name. opt (str): option name. level (int): config level to use. Returns: value (str, int): option value. """ config = self.get_configobj(level) if section not in config.keys(): raise ConfigError("section '{}' doesn't exist".format(section)) if opt not in config[section].keys(): raise ConfigError("option '{}.{}' doesn't exist".format( section, opt)) return config[section][opt] @staticmethod def _lower(config): new_config = configobj.ConfigObj() for s_key, s_value in config.items(): new_s = {} for key, value in s_value.items(): new_s[key.lower()] = str(value) new_config[s_key.lower()] = new_s return new_config def get_configobj(self, level): configs = { self.LEVEL_LOCAL: self._local_config, self.LEVEL_REPO: self._repo_config, self.LEVEL_GLOBAL: self._global_config, self.LEVEL_SYSTEM: self._system_config, } return configs.get(level, self._repo_config) def list_options(self, section_regex, option, level=None): ret = {} config = self.get_configobj(level) for section in config.keys(): r = re.match(section_regex, section) if r: name = r.group("name") value = config[section].get(option, "") ret[name] = value return ret
class Validator(object): """ Validator for the Pythonic pipeline definition. - validates matrix, hooks, model and pipeline as main items in yaml. - validates parallel or ordered in matrix and tasks - support for Bash, Python, Docker, Packer and Ansible tasks """ SCHEMA = { # optional matrix part of the schema Optional(Regex(r'((matrix\(parallel\)|matrix\(ordered\)|matrix))')): And(len, [{ 'name': And(str, len), Optional('env'): And(len, {Regex(r'([a-zA-Z][_a-zA-Z]*)'): And(str, len)}), Optional('tags'): And(len, [And(str, len)]) }]), # optional model part of the schema # you may have any value (any yaml structure as value) Optional('model'): And(len, {Regex(r'([a-z][_a-z]*)'): object}), # optional hooks part of the schema Optional('hooks'): { 'cleanup': { 'script': And(str, len) } }, # mandatory pipeline part of the schema 'pipeline': And( len, [ Or( # optional environment variables in a pipeline { 'env': And(len, {Regex(r'([a-zA-Z][_a-zA-Z]*)'): And(str, len)}) }, { Regex(r'(stage\(.+\))'): And( len, [ Or( # optional environment variables in a stage { 'env': And( len, { Regex(r'([a-zA-Z][_a-zA-Z]*)'): And(str, len) }) }, # optional tasks { Regex(r'((tasks\(parallel\)|matrix\(tasks\)|tasks))'): And( len, [ # optional environment variables { Optional('env'): And( len, { Regex(r'([a-zA-Z][_a-zA-Z]*)'): And(str, len) }) }, # optional shell task { Optional('shell'): { 'script': And( Or( type(' '), type(u' ')), len), Optional('title'): And(str, len), Optional('tags'): And([And(str, len)], len), Optional('with'): And( len, Or( type(' '), type(u' '), [object])), # when set store output of shell task into a variable with this name # can be referenced in jinja templating with {{ variables.name }} Optional('variable'): And( Or( type(' '), type(u' ')), len, Regex( r'([a-zA-Z][_a-zA-Z]*)' )), # condition when to run the task Optional('when', default=''): And( str, Condition.is_valid) } }, # optional Docker container task { Optional('docker(container)'): { 'script': And(str, len), # docker image name and docker tag (version) Optional('image'): And(str, len), # the title is printed to the logs (when given) Optional('title'): And(str, len), # when set the mount will be -v $PWD:/mnt/host Optional('mount', default=False): bool, # when set using -d option (docker run) Optional('background', default=False): bool, # when set using --rm option (docker run) Optional('remove', default=True): bool, # when set using --network= option (docker run) Optional('network', default=''): And(str, len), # when set using --labels= option (docker run) Optional('labels'): And( len, { Regex(r'(UL[_A-Z]*)'): And(str, len) }), # when defined used for --tags option (spline --tags=) Optional('tags'): And([And(str, len)], len), Optional('with'): And(len, [object]), # when set store output of docker container task into a variable with this name # can be referenced in jinja templating with {{ variables.name }} Optional('variable'): And( Or( type(' '), type(u' ')), len, Regex( r'([a-zA-Z][_a-zA-Z]*)' )), # condition when to run the task Optional('when', default=''): And( str, Condition.is_valid) } }, # optional Docker image task { Optional('docker(image)'): { 'script': And(str, len), 'name': And(str, len), 'tag': And(str, len), Optional('unique', default=True): bool, Optional('tags'): And([And(str, len)], len), Optional('with'): And(len, [object]), # condition when to run the task Optional('when', default=''): And( str, Condition.is_valid) } }, # optional Python script { Optional('python'): { 'script': And(str, len), Optional('title'): And(str, len), Optional('tags'): And([And(str, len)], len), Optional('with'): And(len, [object]), # when set store output of python task into a variable with this name # can be referenced in jinja templating with {{ variables.name }} Optional('variable'): And( Or( type(' '), type(u' ')), len, Regex( r'([a-zA-Z][_a-zA-Z]*)' )), # condition when to run the task Optional('when', default=''): And( str, Condition.is_valid) } }, # optional Packer task { Optional('packer'): { 'script': And(str, len), Optional('tags'): And([And(str, len)], len), Optional('with'): And(len, [object]), # condition when to run the task Optional('when', default=''): And( str, Condition.is_valid) } }, # optional Ansible task { Optional('ansible(simple)'): { 'script': And(str, len), 'inventory': And(str, len), Optional('limit', default=''): And(str, len), Optional('tags'): And([And(str, len)], len), Optional('with'): And(len, [object]), # condition when to run the task Optional('when', default=''): And( str, Condition.is_valid) } } ]) } # end of tasks ) ]) }, # end of stage ) ]) # end of pipeline } # end of schema @staticmethod def validate(data): """ Validate data against the schema. Args: data(dict): data structure to validate. Returns: dict: data as provided and defaults where defined in schema. """ try: return Schema(Validator.SCHEMA).validate(data) except SchemaError as exception: logging.getLogger(__name__).error(exception) return None
class Config(object): CONFIG = 'config' CONFIG_LOCAL = 'config.local' SECTION_CORE = 'core' SECTION_CORE_LOGLEVEL = 'loglevel' SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel) SECTION_CORE_REMOTE = 'remote' SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool)) SECTION_CORE_INTERACTIVE = 'interactive' SECTION_CACHE = 'cache' SECTION_CACHE_DIR = 'dir' SECTION_CACHE_TYPE = 'type' SECTION_CACHE_TYPE_SCHEMA = supported_cache_type SECTION_CACHE_PROTECTED = 'protected' SECTION_CACHE_LOCAL = 'local' SECTION_CACHE_S3 = 's3' SECTION_CACHE_GS = 'gs' SECTION_CACHE_SSH = 'ssh' SECTION_CACHE_HDFS = 'hdfs' SECTION_CACHE_AZURE = 'azure' SECTION_CACHE_SCHEMA = { Optional(SECTION_CACHE_LOCAL): str, Optional(SECTION_CACHE_S3): str, Optional(SECTION_CACHE_GS): str, Optional(SECTION_CACHE_HDFS): str, Optional(SECTION_CACHE_SSH): str, Optional(SECTION_CACHE_AZURE): str, Optional(SECTION_CACHE_DIR, default='cache'): str, Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA, Optional(SECTION_CACHE_PROTECTED, default=False): And(str, is_bool, Use(to_bool)), } # backward compatibility SECTION_CORE_CLOUD = 'cloud' SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud) SECTION_CORE_STORAGEPATH = 'storagepath' SECTION_CORE_SCHEMA = { Optional(SECTION_CORE_LOGLEVEL, default='info'): And(str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA), Optional(SECTION_CORE_REMOTE, default=''): And(str, Use(str.lower)), Optional(SECTION_CORE_INTERACTIVE, default=False): SECTION_CORE_INTERACTIVE_SCHEMA, # backward compatibility Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA, Optional(SECTION_CORE_STORAGEPATH, default=''): str, } # backward compatibility SECTION_AWS = 'aws' SECTION_AWS_STORAGEPATH = 'storagepath' SECTION_AWS_CREDENTIALPATH = 'credentialpath' SECTION_AWS_ENDPOINT_URL = 'endpointurl' SECTION_AWS_REGION = 'region' SECTION_AWS_PROFILE = 'profile' SECTION_AWS_SCHEMA = { SECTION_AWS_STORAGEPATH: str, Optional(SECTION_AWS_REGION): str, Optional(SECTION_AWS_PROFILE): str, Optional(SECTION_AWS_CREDENTIALPATH): str, Optional(SECTION_AWS_ENDPOINT_URL): str, } # backward compatibility SECTION_GCP = 'gcp' SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH SECTION_GCP_PROJECTNAME = 'projectname' SECTION_GCP_SCHEMA = { SECTION_GCP_STORAGEPATH: str, Optional(SECTION_GCP_PROJECTNAME): str, } # backward compatibility SECTION_LOCAL = 'local' SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH SECTION_LOCAL_SCHEMA = { SECTION_LOCAL_STORAGEPATH: str, } SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$' SECTION_REMOTE_FMT = 'remote "{}"' SECTION_REMOTE_URL = 'url' SECTION_REMOTE_USER = '******' SECTION_REMOTE_PORT = 'port' SECTION_REMOTE_KEY_FILE = 'keyfile' SECTION_REMOTE_TIMEOUT = 'timeout' SECTION_REMOTE_PASSWORD = '******' SECTION_REMOTE_ASK_PASSWORD = '******' SECTION_REMOTE_SCHEMA = { SECTION_REMOTE_URL: And(supported_url, error="Unsupported URL"), Optional(SECTION_AWS_REGION): str, Optional(SECTION_AWS_PROFILE): str, Optional(SECTION_AWS_CREDENTIALPATH): str, Optional(SECTION_AWS_ENDPOINT_URL): str, Optional(SECTION_GCP_PROJECTNAME): str, Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA, Optional(SECTION_CACHE_PROTECTED, default=False): And(str, is_bool, Use(to_bool)), Optional(SECTION_REMOTE_USER): str, Optional(SECTION_REMOTE_PORT): Use(int), Optional(SECTION_REMOTE_KEY_FILE): str, Optional(SECTION_REMOTE_TIMEOUT): Use(int), Optional(SECTION_REMOTE_PASSWORD): str, Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)), } SECTION_STATE = 'state' SECTION_STATE_ROW_LIMIT = 'row_limit' SECTION_STATE_ROW_CLEANUP_QUOTA = 'row_cleanup_quota' SECTION_STATE_SCHEMA = { Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole), Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent), } SCHEMA = { Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA, Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA, Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA, Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA, # backward compatibility Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA, Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA, Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA, } def __init__(self, dvc_dir): self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) self.config_file = os.path.join(dvc_dir, self.CONFIG) self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL) try: self._config = configobj.ConfigObj(self.config_file) local = configobj.ConfigObj(self.config_local_file) # NOTE: schema doesn't support ConfigObj.Section validation, so we # need to convert our config to dict before passing it to self._config = self._lower(self._config) local = self._lower(local) self._config = self._merge(self._config, local) self._config = Schema(self.SCHEMA).validate(self._config) # NOTE: now converting back to ConfigObj self._config = configobj.ConfigObj(self._config, write_empty_values=True) self._config.filename = self.config_file except Exception as ex: raise ConfigError(ex) @staticmethod def _merge(first, second): res = {} sections = list(first.keys()) + list(second.keys()) for section in sections: f = first.get(section, {}).copy() s = second.get(section, {}).copy() f.update(s) res[section] = f return res @staticmethod def _lower(config): new_config = {} for s_key, s_value in config.items(): new_s = {} for key, value in s_value.items(): new_s[key.lower()] = value new_config[s_key.lower()] = new_s return new_config @staticmethod def init(dvc_dir): config_file = os.path.join(dvc_dir, Config.CONFIG) open(config_file, 'w+').close() return Config(dvc_dir)
db = connect(username=os.environ.get('MONGO_USER'), password=os.environ.get('MONGO_PASS'), host=os.environ.get('MONGO_HOST', '127.0.0.1'), db=os.environ.get('MONGO_DB', 'threat_playbook'), port=int(os.environ.get('MONGO_PORT', 27017))) except Exception as e: print(bold(red(e))) exit(1) return db validation_dictionary = { 'create_user': { 'email': Regex(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)", error="Not a valid email"), 'password': Regex(r'[A-Za-z0-9@#$%^&+=]{8,}', error="Password is not compliant with requirements") }, 'login': { 'email': Regex(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)", error="Not a valid email"), 'password': Regex(r'[A-Za-z0-9@#$%^&+=]{8,}', error="Password is not compliant with requirements") } }
if value not in Position.POSITIONS: raise SchemaError("Unknown position '{p}'".format(p=value)) class UniqueId: def __init__(self): self._ids = set() def validate(self, value): if value in self._ids: raise SchemaError("ID {id} is not unique".format(id=value)) self._ids.add(value) employee = Schema({ "name": And(str, len, Regex("[A-Z][a-z]+")), "surname": And(str, len, Regex("[A-Z][a-z]+")), "id": UniqueId(), Optional("salary"): Salary(), Optional("position"): And(str, Position()) }) verbose_mode = "-v" in argv validate( employee, { "name": "Eda", "surname": "Wasserfall", "id": 1, "salary": 15000.0, "position": "QA"
# # THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import os from schema import Schema, And, Use, Optional, Regex, Or common_schema = { 'authorName': str, 'experimentName': str, Optional('description'): str, 'trialConcurrency': And(int, lambda n: 1 <=n <= 999999), Optional('maxExecDuration'): Regex(r'^[1-9][0-9]*[s|m|h|d]$'), Optional('maxTrialNum'): And(int, lambda x: 1 <= x <= 99999), 'trainingServicePlatform': And(str, lambda x: x in ['remote', 'local', 'pai']), Optional('searchSpacePath'): os.path.exists, Optional('multiPhase'): bool, 'useAnnotation': bool, 'tuner': Or({ 'builtinTunerName': Or('TPE', 'Random', 'Anneal', 'Evolution', 'SMAC', 'BatchTuner'), 'classArgs': { 'optimize_mode': Or('maximize', 'minimize'), Optional('speed'): int }, Optional('gpuNum'): And(int, lambda x: 0 <= x <= 99999), },{ 'codeDir': os.path.exists, 'classFileName': str,
from dataclasses import dataclass from pathlib import Path import yaml from schema import Regex, Schema CONF_SCHEMA = { 'polling_interval': int, 'slow_threshold': int, 'db_path': str, 'host': str, 'port': int, 'log_file': str, 'log_level': Regex(r'DEBUG|INFO|WARNING|ERROR'), } @dataclass class Conf: polling_interval: int slow_threshold: int db_path: str host: str port: str log_file: str log_level: str @classmethod def load(cls, path=None): if path is None: path = Path(__file__).resolve().parent.parent.parent / 'config.yaml'
Schema(self.algo_schema).validate(data) self.validate_extras(data, self.algo_type) common_schema = { 'authorName': setType('authorName', str), 'experimentName': setType('experimentName', str), Optional('description'): setType('description', str), 'trialConcurrency': setNumberRange('trialConcurrency', int, 1, 99999), Optional('maxExecDuration'): And( Regex(r'^[1-9][0-9]*[s|m|h|d]$', error='ERROR: maxExecDuration format is [digit]{s,m,h,d}')), Optional('maxTrialNum'): setNumberRange('maxTrialNum', int, 1, 99999), 'trainingServicePlatform': setChoice('trainingServicePlatform', 'remote', 'local', 'pai', 'kubeflow', 'frameworkcontroller', 'paiYarn', 'dlts'), Optional('searchSpacePath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'searchSpacePath'), Optional('multiPhase'): setType('multiPhase', bool), Optional('multiThread'): setType('multiThread', bool), Optional('nniManagerIp'): setType('nniManagerIp', str), Optional('logDir'): And(os.path.isdir, error=SCHEMA_PATH_ERROR % 'logDir'),
Optional("file_cell_parameters", default=None): Or(str, None), # Quality of the auxiliar basis cFIT Optional("aux_fit", default="verygood"): any_lambda(("low", "medium", "good", "verygood", "excellent")), # executable name # "sdbg" Serial single core testing and debugging # "sopt" Serial general single core usage # "ssmp" Parallel (only OpenMP), single node, multi core # "pdbg" Parallel (only MPI) multi-node testing and debugging # "popt" Parallel (only MPI) general usage, no threads # "psmp" parallel (MPI + OpenMP) general usage, threading might improve scalability and memory usage Optional("executable", default="cp2k.psmp"): Regex(r'.*cp2k\.(?:popt|psmp|sdbg|sopt|ssmp|pdbg)', flags=2) # flag 2 == IGNORECASE }) #: Dictionary with the options common to all workflows dict_general_options = { # Number of occupied/virtual orbitals to use Optional('active_space', default=[10, 10]): And(list, lambda xs: len(xs) == 2), # Index of the H**O Optional("nHOMO"): int, # Index of the orbitals to compute the couplings Optional("mo_index_range"):