def validate(data, schema=None):
if schema is None:
schema = generate()
JsonSchemaValidator.check_schema(schema)
validator = JsonSchemaValidator(schema)
errors = list(validator.iter_errors(data))
if not errors:
return check_unique(data) or []
try:
resp = policy_error_scope(specific_error(errors[0]), data)
name = isinstance(
errors[0].instance,
dict) and errors[0].instance.get(
'name',
'unknown') or 'unknown'
return [resp, name]
except Exception:
logging.exception(
"specific_error failed, traceback, followed by fallback")
return list(filter(None, [
errors[0],
best_match(validator.iter_errors(data)),
]))
def findErrors(self, schema, json_data, errorMessage):
v = DraftValidator(schema)
errors = sorted(v.iter_errors(json_data), key=lambda e: e.path)
for error in errors:
self.errorMessage.append(
errorMessage if errorMessage != None else error.message)
return len(errors) != 0
class LocalJSONSchemaValidator(ConfigurationValidator):
def __init__(self, conf_type: ConfigurationType):
self.conf_type = ConfigurationType(conf_type)
self.schema_path = Path(CONFIG_JSON_SCHEMA_PATH,
f"{self.conf_type.value}.jsonschema.json")
with self.schema_path.open("r") as f:
self.schema_dict = json.load(f)
self.validator = JsonSchemaValidator(self.schema_dict)
def validate_dict(self, _dict: dict) -> None:
errors = self.validator.iter_errors(_dict)
failed = False
for error in errors:
failed = True
print(error)
print('------')
if failed:
raise ValidationError(
"JSONSchema validation error, see console output for info")
def validate_json(self, _json: str) -> None:
self.validate_dict(json.loads(_json))
def validate(self, _object: t.Union[str, t.Dict[str, t.Any]]) -> None:
if isinstance(_object, str):
self.validate_json(_object)
elif isinstance(_object, dict):
self.validate_dict(_object)
else:
raise ValueError("Tried to validate incompatible object type.")
def check_json(validator: Draft7Validator,
json_obj: json,
logger=app_logger) -> bool:
is_valid = False
if validator.is_valid(json_obj):
logger.debug(f"JSON Schema check PASSED.")
is_valid = True
else:
for e in validator.iter_errors(json_obj):
logger.error(e)
raise Exception(f"JSON Schema check FAILED. Json string: {json_obj}")
return is_valid
class DataValidator(Validator):
def __init__(self, schema, only_error=True):
super(DataValidator, self).__init__(schema, name_field='name', only_error=only_error)
self._v = JsonSchemaValidator(self._schema)
def validate(self, data):
try:
errors = sorted(self._v.iter_errors(data), key=lambda e: e.path)
print(errors)
for error in errors:
self.errors.append(error.message)
# for suberror in sorted(error.context, key=lambda e: e.schema_path):
# print(list(suberror.schema_path), suberror.message, sep=", ")
except SchemaError as e:
raise
def validate_json_schemata(json_schemata):
print("Validate JSON schemata")
for el in json_schemata:
with open(os.path.join(el), 'r') as schema_file:
try:
schema = json.loads(schema_file.read())
except json.JSONDecodeError as ex:
print("[error] Decoding JSON has failed for " + el)
print(ex.msg + " at line " + str(ex.lineno))
else:
v = Draft7Validator(schema)
for error in Draft7Validator.iter_errors(schema):
print(error)
try:
Draft7Validator.check_schema(schema)
print(el.ljust(31) + "is valid")
except jsonschema.exceptions.SchemaError as error_ex:
print("Bad JSON schema " + el)
def get_validation_errors(validator: Draft7Validator, data: json) -> dict:
errors = {}
for error in sorted(validator.iter_errors(data), key=str):
key = error.message.split()[0]
errors.update({key.replace("'", ""): error.message.replace(key, "").strip()})
return errors
class TranscodingJob:
'''Store essential parameters of the transcoding job'''
ENCODERS = {
None: VorbisTranscoder,
'copy': VerbatimFileCopy,
'symlink': SymlinkCreator,
'vorbis': VorbisTranscoder,
'lame': LameTranscoder,
'mp3': LameTranscoder,
'aac': AACTranscoder,
'm4a': AACTranscoder,
'opus': OpusTranscoder,
}
def __init__(self, config_file):
'''Initialize transcoding job'''
self.stats = TranscodingStats()
self.finished = False
self.config_file = config_file
self._timestamp = None
with open(config_file, encoding=CONFIG_ENCODING) as f:
config = yaml.load(f, Loader=yaml.RoundTripLoader)
output = config.get('output', {})
extras = config.get('extras', {})
self.validate(config)
self.job_id = config.get('name', DEFAULT_CONFIG['name'])
self.inputs = config.get('input', [])
self.output_dir = output.get('directory')
self.output_pattern = output.get('pattern', DEFAULT_CONFIG['pattern'])
self.cover_size = extras.get('cover', DEFAULT_CONFIG['cover'])
if output.get('category_blacklist'):
self.select_mode = 'blacklist'
self.select = set(output.get('category_blacklist'))
elif output.get('category_whitelist'):
self.select_mode = 'whitelist'
self.select = set(output.get('category_whitelist'))
else:
self.select_mode = None
self.select = set()
lyrics_source = extras.get('lyrics', DEFAULT_CONFIG['lyrics'])
if not lyrics_source:
self.get_lyrics = None
elif os.path.isdir(lyrics_source):
self.get_lyrics = partial(read_lyrics, lyricsdir=lyrics_source)
elif os.path.isfile(lyrics_source):
database = LyricsStorage(lyrics_source)
self.get_lyrics = database.get
else:
self.get_lyrics = None
encoder = output.get('format', DEFAULT_CONFIG['format'])
quality = output.get('quality', DEFAULT_CONFIG['quality'])
self.transcoder = self.ENCODERS.get(encoder)(quality)
lossy_action = output.get('lossy_source',
DEFAULT_CONFIG['lossy_source'])
if lossy_action == 'allow_bad_transcodes'\
or encoder == 'symlink' \
or encoder == 'copy':
self.lossy_action = self.transcoder
elif lossy_action == 'copy':
self.lossy_action = VerbatimFileCopy()
elif lossy_action == 'skip':
skip_marker = self.transcoder.STATUS_SKIP
self.lossy_action = lambda infile, outfile: (infile, skip_marker)
self.lossy_action.STATUS_SKIP = skip_marker
os.makedirs(self.output_dir, exist_ok=True)
log.debug('Initialized {}'.format(self))
def __repr__(self):
return '{cls}({config!r})'.format(
cls=self.__class__.__name__,
config=self.config_file,
)
def transcode(self, task):
'''Execute a single transcoding task'''
log.debug('Started {task}'.format(task=task))
if (self.select_mode == 'blacklist' and self.select.intersection(task.categories)) \
or (self.select_mode == 'whitelist' and not self.select.intersection(task.categories)):
self.stats.record_skip()
log.debug('Skipped {task}'.format(task=task))
return
source_format = os.path.splitext(task.source)[1][1:].lower()
if source_format in LOSSLESS_EXTENSIONS:
worker = self.transcoder
else:
worker = self.lossy_action
if self._timestamp is None: # record the time of first transcoding task
self._timestamp = int(time.time())
# Step 1: Transcode
task.result, task.status = worker(
task.source, os.path.join(self.output_dir, task.target))
# Step 1a: Process extras (cover art, lyrics)
if self.cover_size:
Thread(target=copy_coverart,
kwargs=dict(task=task, size=self.cover_size)).start()
if self.get_lyrics:
Thread(
target=copy_lyrics,
kwargs=dict(task=task, lyrics_finder=self.get_lyrics),
).start()
# Handle skipped transcodes
if task.status is worker.STATUS_SKIP:
if os.path.getmtime(task.result) > self.timestamp:
raise RuntimeError('Target path collision for {}'.format(
task.result))
self.stats.record_skip()
log.debug('Skipped {task}'.format(task=task))
return
# Step 2: Copy music tags
if not task.status is worker.STATUS_SKIPTAGS:
result = mutagen.File(task.result, easy=True)
for key in task.tags.keys(
): # mutagen is inconsistent about `for k in t.tags`
if hasattr(result.tags, 'valid_keys') \
and key not in result.tags.valid_keys:
continue
result.tags[key] = task.tags[key]
result.save()
self.stats.record_done()
log.debug('Finished {task}'.format(task=task))
@property
def timestamp(self):
'''
Date and time of starting the first transcoding task in this job
(in Unix time format)
'''
return self._timestamp
def write_report(self):
'''Keep a journal of transcoder runs'''
log_entry = '{time}Z: {stats}\n'.format(
time=datetime.utcnow().replace(microsecond=0),
stats=self.stats.show().strip(),
)
with open(os.path.join(self.output_dir, 'transcoding.log'),
'a') as logfile:
logfile.write(log_entry)
def validate(self, config):
'''Validate transcoding job configuration'''
try:
self.validator
except AttributeError:
package = __name__.rsplit('.', 1)[0]
path = 'schema.json'
schema = json.loads(resource_string(package, path).decode())
self.validator = JSONSchemaValidator(schema)
error_messages = []
for error in self.validator.iter_errors(config):
error_messages.append(' - {}: {}'.format(
'.'.join(error.path) if error.path else '[config]',
error.message))
if error_messages:
raise ValueError('invalid configuration values:\n{}'.format(
'\n'.join(sorted(error_messages))))
