def save_comment(self, commenter_id: str, post_id: str,
commenter_search_id: str, comment_json: dict):
schema = save_comment.schema
try:
validate_schema(instance=comment_json, schema=schema)
except Exception as e:
raise e
comment_json.pop('cover_url', None)
comment_json.pop('video_icon', None)
comment_json.pop('story_open', None)
comment_json.pop('video_cover', None)
comment_json.pop('ad_cover_url', None)
comment_json.pop('ad_cover_url', None)
comment_json.pop('avatar_medium', None)
comment_json.pop('avatar_168x168', None)
comment_json.pop('avatar_300x300', None)
try:
comment_id = self.db.comments.insert_one(comment_json).inserted_id
except Exception as e:
raise e
return comment_id
def __init__(self, filehandle):
self._config = yaml.load(filehandle)
self.logger.debug('Config: %s', self._config)
if 'defaults' not in self._config:
self._config['defaults'] = {}
if 'config' not in self._config:
self._config['config'] = {}
if 'templates' not in self._config:
self._config['templates'] = {}
if 'plugins' not in self._config:
self._config['plugins'] = []
instances = {}
for instance in self._config.get('instances', []):
if type(instance) is str:
instances[instance] = {}
elif type(instance) is dict:
key, value = next(iter(instance.items()))
instances[key] = value
self._config['instances'] = instances
validate_schema(self._config, config_schema)
def save_post(self, shangol_profile_id: str, post_json: dict):
""" Insert Douyin post into DouyinDB.
Args:
profile_id: id of the Douyin user profile.
post_json: the JSON representation of a single post, as obtained from the Douyin API (must match schema defined below).
Return:
_id of the inserted object in the DB.
"""
schema = save_post.schema
try:
validate_schema(instance=post_json, schema=schema)
except Exception as e:
print("JSON is incorrect.")
raise e
post_json["_id"] = post_json.pop("item_id")
post_json["create_time"] = datetime.utcfromtimestamp(
post_json["create_time"])
post_json["shangol_profile_id"] = shangol_profile_id
post_json["statistics"]["dt"] = datetime.now(timezone.utc)
post_json["latest_stats"] = post_json["statistics"]
post_json["statistics"] = [post_json["statistics"]]
try:
post_id = self.db.posts.insert_one(post_json).inserted_id
except Exception as e:
raise e
return post_id
def save_normie_follows(self, shangol_profile_id: str, normie_id: str,
followee_normie_id: str, followee_json: dict):
relation = {
"shangol_profile_id": shangol_profile_id,
"normie_id": normie_id,
"followee_normie_id": followee_normie_id
}
relation_id = self.db.normie.follows.insert_one(relation).inserted_id
schema = save_normie_follows.schema
try:
validate_schema(instance=followee_json, schema=schema)
except Exception as e:
raise e
followee_json.pop('cover_url', None)
followee_json.pop('video_icon', None)
followee_json.pop('avatar_medium', None)
followee_json.pop('avatar_168x168', None)
followee_json.pop('avatar_300x300', None)
try:
normie_id = self.db.normies.insert_one(followee_json).inserted_id
except Exception as e:
raise e
return normie_id
def save_post_statistics(self, post_id: str, statistics_json: dict):
""" Save statistics data point about a post into the DB.
Args:
post_id: the ID of the post (as found from the official Douyin API).
statistics_json: the JSON representation of the statistics block in a post object,
as obtained from the Douyin API (must match schema defined below).
Returns:
None
"""
schema = save_post_statistics.schema
try:
validate_schema(instance=statistics_json, schema=schema)
except Exception as e:
raise e
statistics_json["dt"] = datetime.now(timezone.utc)
self.db.posts.update_one({"_id": post_id}, {
"$push": {
"statistics": statistics_json
},
"$set": {
"latest_stats": statistics_json,
"last_updated": datetime.now(timezone.utc)
}
})
def validate(json_data) -> (bool, str):
# Validate with the event schema (without considering the content field)
try:
validate_schema(instance=json_data, schema=event_schema)
except ValidationError as ve:
log.debug(str(type(ve)) + ', ' + str(ve))
return False, str(ve.message)
# Get the event type
try:
event_type = MeetingEventType((json_data["type"]))
except ValueError as ve:
log.debug(str(type(ve)) + ', ' + str(ve))
return False, "Unknown value in 'type'"
# Return if the content is supposed to be empty/ignored
if event_type not in content_schemas_files.keys():
return True, None
# Get the schema file for the content
file = content_schemas_files.get(event_type, None)
with open(file) as f:
event_content_schema = json.load(f)
# Validate with the right content schema
try:
validate_schema(instance=json_data.get("content", {}),
schema=event_content_schema)
except ValidationError as ve:
log.debug(str(type(ve)) + ', ' + str(ve))
return False, str(ve.message)
return True, None
def save_normie_profile(self, kol_profile_id: str, normie_search_id: str,
normie_unique_id: str, normie_uid: str,
normie_json: dict):
schema = save_normie_profile.schema
try:
validate_schema(instance=normie_json, schema=schema)
except Exception as e:
raise e
normie_json.pop('cover_url', None)
normie_json.pop('video_icon', None)
normie_json.pop('story_open', None)
normie_json.pop('video_cover', None)
normie_json.pop('ad_cover_url', None)
normie_json.pop('ad_cover_url', None)
normie_json.pop('avatar_medium', None)
normie_json.pop('avatar_168x168', None)
normie_json.pop('avatar_300x300', None)
normie_json["kol_profile_id"] = kol_profile_id
normie_json["search_id"] = normie_search_id
normie_json["unique_id"] = normie_unique_id
normie_json["_id"] = normie_uid
try:
normie_id = self.db.normies.insert_one(normie_json).inserted_id
except Exception as e:
raise e
return normie_id
def __init__(self, manifest_bytes, validate=True):
assert isinstance(manifest_bytes, Bytes)
self._layers = None
self._bytes = manifest_bytes
try:
self._parsed = json.loads(manifest_bytes.as_encoded_str())
except ValueError as ve:
raise MalformedSchema1Manifest("malformed manifest data: %s" % ve)
try:
validate_schema(self._parsed, DockerSchema1Manifest.METASCHEMA)
except ValidationError as ve:
raise MalformedSchema1Manifest(
"manifest data does not match schema: %s" % ve)
self._signatures = self._parsed.get(DOCKER_SCHEMA1_SIGNATURES_KEY)
self._architecture = self._parsed.get(DOCKER_SCHEMA1_ARCH_KEY)
self._tag = self._parsed[DOCKER_SCHEMA1_REPO_TAG_KEY]
repo_name = self._parsed[DOCKER_SCHEMA1_REPO_NAME_KEY]
repo_name_tuple = repo_name.split("/")
if len(repo_name_tuple) > 1:
self._namespace, self._repo_name = repo_name_tuple
elif len(repo_name_tuple) == 1:
self._namespace = ""
self._repo_name = repo_name_tuple[0]
else:
raise MalformedSchema1Manifest("malformed repository name: %s" %
repo_name)
if validate:
self._validate()
def validate(manifest):
"""
Validate an instance of the userapp.json schema.
manifest can be a dict object (returned as is) or a filename.
"""
if isinstance(manifest, basestring):
manifest = json.load(open(manifest))
validate_schema(manifest, read_schema('userapp.json'))
return manifest
def validate_task(task, tasks=None):
"""
Validate the jsonschema configuration of a task
"""
name = task["name"]
config = task.get("config", {})
schema = getattr(TaskRegistry.get(name)[0], "SCHEMA", {})
format_checker = getattr(TaskRegistry.get(name)[0], "FORMAT_CHECKER", None)
try:
validate_schema(config, schema, format_checker=format_checker)
except ValidationError as exc:
return TemplateError.format_details(exc, task)
def validate_task(task, tasks=None):
"""
Validate the jsonschema configuration of a task
"""
name = task['name']
config = task.get('config', {})
schema = getattr(TaskRegistry.get(name)[0], 'SCHEMA', {})
format_checker = getattr(TaskRegistry.get(name)[0], 'FORMAT_CHECKER', None)
try:
validate_schema(config, schema, format_checker=format_checker)
except ValidationError as exc:
return TemplateError.format_details(exc, task)
def __init__(self, config_bytes):
assert isinstance(config_bytes, Bytes)
self._config_bytes = config_bytes
try:
self._parsed = json.loads(config_bytes.as_unicode())
except ValueError as ve:
raise MalformedSchema2Config('malformed config data: %s' % ve)
try:
validate_schema(self._parsed, DockerSchema2Config.METASCHEMA)
except ValidationError as ve:
raise MalformedSchema2Config('config data does not match schema: %s' % ve)
def validate(vocabulary):
"""Validate vocabulary."""
if not isinstance(vocabulary, Vocabulary):
raise VocabularyError("{} is not a vocabulary".format(vocabulary))
# Validate vocabulary type
if vocabulary.type not in current_app.config["ILS_VOCABULARIES"]:
raise VocabularyError("Invalid vocabulary type: {}".format(
vocabulary.type))
# JSONSchema validation
schema = current_jsonschemas.get_schema(vocabulary._schema)
data = vocabulary.dumps()
validate_schema(data, schema)
def __init__(self, manifest_bytes):
assert isinstance(manifest_bytes, Bytes)
self._layers = None
self._manifest_bytes = manifest_bytes
try:
self._parsed = json.loads(manifest_bytes.as_unicode())
except ValueError as ve:
raise MalformedSchema2ManifestList("malformed manifest data: %s" % ve)
try:
validate_schema(self._parsed, DockerSchema2ManifestList.METASCHEMA)
except ValidationError as ve:
raise MalformedSchema2ManifestList("manifest data does not match schema: %s" % ve)
def __init__(self, manifest_bytes, validate=False):
assert isinstance(manifest_bytes, Bytes)
self._payload = manifest_bytes
self._filesystem_layers = None
self._cached_built_config = None
try:
self._parsed = json.loads(self._payload.as_unicode())
except ValueError as ve:
raise MalformedSchema2Manifest("malformed manifest data: %s" % ve)
try:
validate_schema(self._parsed, DockerSchema2Manifest.METASCHEMA)
except ValidationError as ve:
raise MalformedSchema2Manifest("manifest data does not match schema: %s" % ve)
for layer in self.filesystem_layers:
if layer.is_remote and not layer.urls:
raise MalformedSchema2Manifest("missing `urls` for remote layer")
def is_valid(self, remote=False):
schema = TaskDescriptor().get_schema()
validate_schema(json.loads(self.json()), schema)
if remote:
# Ignore output ports as gbdxtools will override value.
ports = [
port for port in self.input_ports if port.port_type == 'directory'
]
for port in ports:
# Will raise exception if the port is invalid.
port.is_valid_s3_url(port.value)
else:
all_ports = self.ports[0] + self.ports[1]
ports = [
port for port in all_ports if port.port_type == 'directory' and port.name != 'source_bundle'
]
for port in ports:
# Will raise exception if the port is invalid.
port.is_valid_filesys(port.value)
return True
def validate(self, obj):
if type(obj) in (int, str, bool, unicode):
return 0
validated = 0
# TODO: (or is_update...)
if is_mediachain_object(obj) and is_canonical(obj):
validate_schema(obj['meta'], self.get_schema())
validated = 1
objects = {k: v for k, v in obj.iteritems()
if isinstance(v, dict)}
lists = {k: v for k,v in obj.iteritems()
if isinstance(v, list)}
objects_from_lists = [item for sublist in lists.values() for item in sublist]
for o in objects.values() + objects_from_lists:
validated = validated + self.validate(o)
return validated
def validate_vfd(data):
"""
Check if the given data is a well-built VFD
Args:
data (dict): The data to check
Raises:
jsonschema.ValidationError: If the data is not a well-built VFD.
"""
if "type" not in data:
raise ValueError("No type in provided file")
if data["type"] == "plot":
validate_schema(data, schema_plot)
elif data["type"] == "multiplot":
validate_schema(data, schema_multiplot)
elif data["type"] == "colorplot":
validate_schema(data, schema_colorplot)
else:
raise ValueError("Unknown type: %s" % data["type"])
def validate_model(cls, s):
validate_schema(s, cls.schema)
async def get_handler_team_that_wins_the_least(request, name, season):
"""
Get the team least wins in league
:param
name : String - the league name
season : Number - the season year
:return
"""
logger.info(
f'Server/Get Team that win the least in league - start | name: {name}, season: {season}'
)
try:
parsed_request = {"name": name, "season": season}
logger.debug(
f'Server/Get Team that win the least in league - calling validate_schema | request: {parsed_request}, schema: {LeagueSchema}'
)
validate_schema(instance=parsed_request, schema=LeagueSchema)
logger.debug(
'Server/Get Team that score the least in league - input validation succeeded'
)
logger.debug(
f'Server/Get Team that win the least in league - calling MongoDbService/find_league | request: {parsed_request}'
)
_league = db.find_league(parsed_request)
logger.debug(
f'Server/Get Team that win the least in league - MongoDbService/find_league succeeded | league: {_league}'
)
logger.debug(
f'Server/Get Team that win the least in league - calling leagueProvider/parse_league_from_db | league: {_league}'
)
parsed_league = parse_league_from_db(_league)
logger.debug(
f'Server/Get Team that win the least in league - leagueProvider/parse_league_from_db succeeded | parsed league: {parsed_league}'
)
logger.debug(
f'Server/Get Team that win the least in league - calling MongoDbService/find_teams_from_league | league: {parsed_league}'
)
teams = db.find_teams_from_league(parsed_league)
logger.debug(
f'Server/Get Team that win the least in league - MongoDbService/find_teams_from_league succeeded | parsed league: {parsed_league}'
)
logger.debug(
f'Server/Get Team that win the least in league - calling teamProvider/find_team_least_wins | teams: {teams}'
)
team_win_least = find_team_least_wins(teams)
logger.debug(
f'Server/Get Team that win the least in league - teamProvider/find_team_least_wins succeeded | team win most: {team_win_least}'
)
except ValidationError as error:
logger.error(
f'Server/Get Team that win the least in league failed - validation error | error: {error}'
)
return rjson({
'status': 'Error',
'message': str(error.message)
},
status=400)
except Exception as error:
logger.error(
f'Server/Get Team that win the least in league failed - error: {error}'
)
return rjson({'status': 'Error', 'message': str(error)}, status=400)
else:
logger.info(
f'Server/Get Team that win the least in league succeeded - id: {parsed_league}'
)
return rjson(
{
'status': "success",
'message':
f'The team that win the least in {parsed_request["name"]}, Amount of wins: {team_win_least["number_of_wins"]}',
'team name': str(team_win_least['name'])
},
status=200)
finally:
logger.info(f'Server/Get Team that win the least in league - end')
async def post_handler_team(request):
"""
Create a new Team
:param request:
name : String - the team name
season : Number - the season year
:return
request example
{
"name": Real Madrid
"season": 2020
}
"""
logger.info(f'Server/Create Team - start | request: {request.json}')
try:
logger.debug(
f'Server/Create Team - calling validate_schema | request: {request.json}, schema: {TeamSchema}'
)
validate_schema(instance=request.json, schema=TeamSchema)
logger.debug('Server/Create Team - input validation succeeded')
logger.debug(
f'Server/Create Team - calling teamProvider/parse_team_from_request | request: {request.json}, schema: {TeamSchema}'
)
parsed_request = parse_team_from_request(request.json)
logger.debug(
f'Server/Create League - teamProvider/parse_team_from_request succeeded | parsed request: {parsed_request}'
)
logger.debug(
f'Server/Create Team - calling MongoDbService/create_team | request: {parsed_request}'
)
_id = db.create_team(parsed_request)
logger.debug(
f'Server/Create Team - MongoDbService/create_team succeeded | team id : {_id}'
)
except ValidationError as error:
logger.error(
f'Server/Create Team failed - validation error | error: {error}')
return rjson({
'status': 'Error',
'message': str(error.message)
},
status=400)
except Exception as error:
logger.error(f'Server/Create Team failed - error: {error}')
return rjson({'status': 'Error', 'message': str(error)}, status=400)
else:
logger.info(f'Server/Create Team succeeded - id: {_id}')
return rjson(
{
'status': "success",
'message': 'the team added',
'id': str(_id)
},
status=200)
finally:
logger.info(f'Server/Create Team - end')
async def post_handler_future_match(request):
"""
Create a new Match
:param request:
home_team : String - the home team name
away_team : String - the away team name
date : String - the match date in format: YYYY-MM-DD
:return
request example
{
"home_team": Real Madrid,
"away_team": Hapoel Jerusalem
"season": 20/03/2020
}
"""
logger.info(
f'Server/Create Future Match - start | request: {request.json}')
try:
logger.debug(
f'Server/Create Future Match - calling validate_schema | request: {request.json}, schema: {MatchSchema}'
)
validate_schema(instance=request.json, schema=MatchSchema)
date = request.get("date", None),
if date != datetime.strptime(date, "%Y-%m-%d").strftime('%Y-%m-%d'):
raise ValueError('Date must be in the format: YYYY-MM-DD')
logger.debug('Server/Create Future Match - input validation succeeded')
logger.debug(
f'Server/Create Future Match - calling matchProvider/parse_match_from_request | request: {request.json}, schema: {MatchSchema}'
)
parsed_request = parse_match_from_request(request.json)
logger.debug(
f'Server/Create Future Match - matchProvider/parse_match_from_request succeeded | parsed request: {parsed_request}'
)
logger.debug(
f'Server/Create Future Match - calling MongoDbService/create_match | request: {parsed_request}'
)
_id = db.create_match(parsed_request)
logger.debug(
f'Server/Create Future Match - MongoDbService/create_match succeeded | match id : {_id}'
)
except (ValidationError, ValueError) as error:
logger.error(
f'Server/Create Future Match failed - validation error | error: {error}'
)
return rjson({'status': 'Error', 'message': str(error)}, status=400)
except Exception as error:
logger.error(f'Server/Create Future Match failed - error: {error}')
return rjson({'status': 'Error', 'message': str(error)}, status=400)
else:
logger.info(f'Server/Create Future Match succeeded - id: {_id}')
return rjson(
{
'status': "success",
'message': 'the team added',
'id': str(_id)
},
status=200)
finally:
logger.info(f'Server/Create Future Match - end')
async def post_handler_match(request):
"""
Create a new Match
:param request:
home_team : String - the home team name
away_team : String - the away team name
score : String - the match score
date : String - the match date in format: YYYY-MM-DD
:return
request example
{
}
"""
logger.info(f'Server/Create Ended Match - start | request: {request.json}')
try:
logger.debug(
f'Server/Create Ended Match - calling validate_schema | request: {request.json}, schema: {MatchSchema}'
)
validate_schema(instance=request.json, schema=MatchSchema)
date = request.json.get("date", None)
if date != datetime.strptime(date, "%Y-%m-%d").strftime('%Y-%m-%d'):
raise ValueError('Date must be in the format: YYYY-MM-DD')
score = request.json.get("score", None).replace(' ', '')
is_valid_score = regex_match("(0|[1-9]\d*)-(0|[1-9]\d*)", score)
if not is_valid_score:
raise ValueError('Score must be in the format: Number-Number')
logger.debug('Server/Create Ended Match - input validation succeeded')
logger.debug(
f'Server/Create Ended Match - calling matchProvider/parse_match_from_request | request: {request.json}, schema: {MatchSchema}'
)
parsed_request = parse_ended_match_from_request(request.json)
logger.debug(
f'Server/Create Ended Match - matchProvider/parse_match_from_request succeeded | parsed request: {parsed_request}'
)
logger.debug(
f'Server/Create Ended Match - calling MongoDbService/create_match_with_score | request: {parsed_request}'
)
_id = db.create_match_with_score(parsed_request)
logger.debug(
f'Server/Create Ended Match - MongoDbService/create_match_with_score succeeded | match id : {_id}'
)
except (ValidationError, ValueError) as error:
logger.error(
f'Server/Create Ended Match failed - validation error | error: {error}'
)
return rjson({'status': 'Error', 'message': str(error)}, status=400)
except Exception as error:
logger.error(f'Server/Create Ended Match failed - error: {error}')
return rjson({'status': 'Error', 'message': str(error)}, status=400)
else:
logger.info(f'Server/Create Ended Match succeeded - id: {_id}')
return rjson(
{
'status': "success",
'message': 'the match added',
'id': str(_id)
},
status=200)
finally:
logger.info(f'Server/Create Ended Match - end')
async def get_handler_league_by_name_season(request, name, season):
"""
Get a League by name & season
:param
name : String - the league name
season : Number - the season year
:return
response example
{
"name": Spanish
"season": 2020,
"teams": ["real madrid", "hapoel jerusalem"],
"matches": [match_id_#1, match_id_#2, match_id_#3]
}
"""
logger.info(
f'Server/Get League by name & season - start | name: {name}, season: {season}'
)
try:
parsed_request = {"name": name, "season": season}
logger.debug(
f'Server/Get League by name & season - calling validate_schema | request: {parsed_request}, schema: {LeagueSchema}'
)
validate_schema(instance=parsed_request, schema=LeagueSchema)
logger.debug(
'Server/Get League by name & season - input validation succeeded')
logger.debug(
f'Server/Get League by name & season - calling MongoDbService/find_league | request: {parsed_request}'
)
_league = db.find_league(parsed_request)
logger.debug(
f'Server/Get League by name & season - MongoDbService/find_league succeeded | league: {_league}'
)
logger.debug(
f'Server/Get League by name & season - calling leagueProvider/parse_league_from_db | league: {_league}'
)
parsed_league = parse_league_from_db(_league)
logger.debug(
f'Server/Get League by name & season - leagueProvider/parse_league_from_db succeeded | parsed league: {parsed_league}'
)
except ValidationError as error:
logger.error(
f'Server/Get League by name & season failed - validation error | error: {error}'
)
return rjson({
'status': 'Error',
'message': str(error.message)
},
status=400)
except Exception as error:
logger.error(
f'Server/Get League by name & season failed - error: {error}')
return rjson({'status': 'Error', 'message': str(error)}, status=400)
else:
logger.info(
f'Server/Get League by name & season succeeded - id: {parsed_league}'
)
return rjson(
{
'status': "success",
'message': 'success',
'league': str(parsed_league)
},
status=200)
finally:
logger.info(f'Server/Get League by name & season - end')
async def get_handler_match_by_name_season(request, home_team, away_team,
date):
"""
Get a Team by name & season
:param
home_team : String - the home team name
away_team : String - the away team name
date : String - the match date in format: YYYY-MM-DD
:return
response example
{
"name": Spanish
"season": 2020,
"teams": ["real madrid", "hapoel jerusalem"],
"matches": [match_id_#1, match_id_#2, match_id_#3]
}
"""
logger.info(
f'Server/Get Match by home_team, away_team & date - start | home_team: {home_team}, away_team: {away_team}, date: {date}'
)
try:
parsed_request = {
"home_team": home_team,
"away_team": away_team,
"date": date
}
logger.debug(
f'Server/Get Match by home_team, away_team & date - calling validate_schema | request: {parsed_request}, schema: {MatchSchema}'
)
validate_schema(instance=parsed_request, schema=MatchSchema)
date = request.get("date", None),
if date != datetime.strptime(date, "%Y-%m-%d").strftime('%Y-%m-%d'):
raise ValueError('Date must be in the format: YYYY-MM-DD')
logger.debug(
'Server/Get Match by home_team, away_team & date - input validation succeeded'
)
logger.debug(
f'Server/Get Match by home_team, away_team & date - calling MongoDbService/find_match | request: {parsed_request}'
)
_match = db.find_match(parsed_request)
logger.debug(
f'Server/Get Match by home_team, away_team & date - MongoDbService/find_match succeeded | match: {_match}'
)
logger.debug(
f'Server/Get Match by home_team, away_team & date - calling matchProvider/parse_match_from_db | match: {_match}'
)
parsed_match = parse_match_from_db(_match)
logger.debug(
f'Server/Get Match by home_team, away_team & date - matchProvider/parse_match_from_db succeeded | parsed_match: {parsed_match}'
)
except ValidationError as error:
logger.error(
f'Server/Get Match by home_team, away_team & date failed - validation error | error: {error}'
)
return rjson({
'status': 'Error',
'message': str(error.message)
},
status=400)
except Exception as error:
logger.error(
f'Server/Get Match by home_team, away_team & date failed - error: {error}'
)
return rjson({'status': 'Error', 'message': str(error)}, status=400)
else:
logger.info(
f'Server/Get Match by home_team, away_team & date succeeded - match: {parsed_match}'
)
return rjson(
{
'status': "success",
'message': 'success',
'team': str(parsed_match)
},
status=200)
finally:
logger.info(f'Server/Get Match by home_team, away_team & date - end')
def validate_profile_schema(profile, schema):
validate_schema(profile, schema)
def validate(self):
"""Validates the base configuration file is correct against the `config_schema`."""
try:
validate_schema(self.data, config_schema)
except ValidationError as e:
raise MalformedConfig(e.message)
def validate_profile_schema(profile, schema):
validate_schema(profile, schema)
def validate(config_file):
"""
Validates the metamorph configuration file with the schema
Arguments:
config_file {string} -- Path to the metamorph configuration file
Raises:
NotADirectoryError -- Training folder given in config is not a directory
NotADirectoryError -- Dev folder given in config is not a directory
NotADirectoryError -- Test folder given in config is not a directory
FileNotFoundError -- Labels file is not found
NotADirectoryError -- Output folder given in config is not a directory
Returns:
boolean -- True if config file is valid else throws exceptions and
return False
"""
with open(config_file, "r") as json_file:
json_data = json_file.read()
data = json.loads(json_data)
schema = {
"type":
"object",
"required": [
"training mode", "pre-trained model", "dataset information",
"image augmentation", "hyperparameters",
"image generator information", "verbosity"
],
"properties": {
"training mode": {
"type": "string",
"pattern": "^(backend|classifier)$"
},
"pre-trained model": {
"type": "string"
},
"dataset information": {
"type":
"object",
"required": [
"train folder", "dev folder", "test folder",
"samples folder", "labels file", "output folder"
],
"properties": {
"train folder": {
"type": "string"
},
"dev folder": {
"type": "string"
},
"test folder": {
"type": "string"
},
"samples folder": {
"type": "string"
},
"labels file": {
"type": "string"
},
"output folder": {
"type": "string"
}
}
},
"image augmentation": {
"type":
"object",
"required": [
"size", "depth", "shift", "rotation",
"validation data augmentation factor"
],
"properties": {
"size": {
"type": "integer"
},
"depth": {
"type": "integer"
},
"shift": {
"type": "number"
},
"rotation": {
"type": "number"
},
"validation data augmentation factor": {
"type": "number"
}
}
},
"hyperparameters": {
"type":
"object",
"required": [
"epochs", "batch size", "learning rate",
"learning rate decay after x epoch", "decay rate",
"momentum"
],
"properties": {
"epochs": {
"type": "integer"
},
"batch size": {
"type": "integer"
},
"learning rate": {
"type": "number"
},
"learning rate decay after x epoch": {
"type": "integer"
},
"decay rate": {
"type": "number"
},
"momentum": {
"type": "number"
}
}
},
"image generator information": {
"type": "object",
"required": [
"num of training samples",
"num of test samples",
],
"properties": {
"num of training samples": {
"type": "integer"
},
"num of test samples": {
"type": "integer"
}
}
},
"verbosity": {
"type": "integer"
}
}
}
try:
validate_schema(data, schema)
pretrained_model = data["pre-trained model"]
if not pretrained_model == "":
if not path.isfile(pretrained_model) and not path.exists(
pretrained_model):
raise FileNotFoundError(pretrained_model)
train_folder = data["dataset information"]["train folder"]
if not path.isdir(train_folder) and not path.exists(train_folder):
raise NotADirectoryError(train_folder)
dev_folder = data["dataset information"]["dev folder"]
if not path.isdir(dev_folder) and not path.exists(dev_folder):
raise NotADirectoryError(dev_folder)
test_folder = data["dataset information"]["test folder"]
if not path.isdir(test_folder) and not path.exists(test_folder):
raise NotADirectoryError(test_folder)
samples_folder = data["dataset information"]["samples folder"]
if not path.isdir(samples_folder) and not path.exists(samples_folder):
raise NotADirectoryError(samples_folder)
label_file = data["dataset information"]["labels file"]
if not path.isfile(label_file) and not path.exists(label_file):
raise FileNotFoundError(label_file)
output_folder = data["dataset information"]["output folder"]
if not path.isdir(output_folder) and not path.exists(output_folder):
raise NotADirectoryError(output_folder)
except NotADirectoryError as error:
print("Either this path is not a directory or does not exist")
print(error)
except FileNotFoundError as error:
print(f"{error} file is missing")
except ValidationError as error:
print("Error in configuration file")
print("#" * 80)
print(error.message)
else:
return True
return False
async def post_handler_league(request):
"""
Create a new League
:param request:
name : String - the league name
season : Number - the season year
:return
request example
{
"name": Spanish
"season": 2020
}
"""
logger.info(f'Server/Create League - start | request: {request.json}')
try:
logger.debug(
f'Server/Create League - calling validate_schema | request: {request.json}, schema: {LeagueSchema}'
)
validate_schema(instance=request.json, schema=LeagueSchema)
logger.debug('Server/Create League - input validation succeeded')
logger.debug(
f'Server/Create League - calling leagueProvider/parse_league_from_request | request: {request.json}, schema: {LeagueSchema}'
)
parsed_request = parse_league_from_request(request.json)
logger.debug(
f'Server/Create League - leagueProvider/parse_league_from_request succeeded | parsed request: {parsed_request}'
)
logger.debug(
f'Server/Create League - calling MongoDbService/create_league | request: {parsed_request}'
)
_id = db.create_league(parsed_request)
logger.debug(
f'Server/Create League - MongoDbService/create_league succeeded | league id: {_id}'
)
except ValidationError as error:
logger.error(
f'Server/Create League failed - validation error | error: {error}')
return rjson({
'status': 'Error',
'message': str(error.message)
},
status=400)
except Exception as error:
logger.error(f'Server/Create League failed - error: {error}')
return rjson({'status': 'Error', 'message': str(error)}, status=400)
else:
logger.info(f'Server/Create League succeeded - id: {_id}')
return rjson(
{
'status': "success",
'message': 'the league added',
'id': str(_id)
},
status=200)
finally:
logger.info(f'Server/Create League - end')
def location_schema_validator(data):
try:
validate_schema(data, location_schema)
except Draft4ValidationError as e:
raise ValidationError(e.message)