def validate(data, schema=None): if schema is None: schema = generate() JsonSchemaValidator.check_schema(schema) validator = JsonSchemaValidator(schema) errors = list(validator.iter_errors(data)) if not errors: return check_unique(data) or [] try: resp = policy_error_scope(specific_error(errors[0]), data) name = isinstance( errors[0].instance, dict) and errors[0].instance.get( 'name', 'unknown') or 'unknown' return [resp, name] except Exception: logging.exception( "specific_error failed, traceback, followed by fallback") return list(filter(None, [ errors[0], best_match(validator.iter_errors(data)), ]))
def findErrors(self, schema, json_data, errorMessage): v = DraftValidator(schema) errors = sorted(v.iter_errors(json_data), key=lambda e: e.path) for error in errors: self.errorMessage.append( errorMessage if errorMessage != None else error.message) return len(errors) != 0
class LocalJSONSchemaValidator(ConfigurationValidator): def __init__(self, conf_type: ConfigurationType): self.conf_type = ConfigurationType(conf_type) self.schema_path = Path(CONFIG_JSON_SCHEMA_PATH, f"{self.conf_type.value}.jsonschema.json") with self.schema_path.open("r") as f: self.schema_dict = json.load(f) self.validator = JsonSchemaValidator(self.schema_dict) def validate_dict(self, _dict: dict) -> None: errors = self.validator.iter_errors(_dict) failed = False for error in errors: failed = True print(error) print('------') if failed: raise ValidationError( "JSONSchema validation error, see console output for info") def validate_json(self, _json: str) -> None: self.validate_dict(json.loads(_json)) def validate(self, _object: t.Union[str, t.Dict[str, t.Any]]) -> None: if isinstance(_object, str): self.validate_json(_object) elif isinstance(_object, dict): self.validate_dict(_object) else: raise ValueError("Tried to validate incompatible object type.")
def check_json(validator: Draft7Validator, json_obj: json, logger=app_logger) -> bool: is_valid = False if validator.is_valid(json_obj): logger.debug(f"JSON Schema check PASSED.") is_valid = True else: for e in validator.iter_errors(json_obj): logger.error(e) raise Exception(f"JSON Schema check FAILED. Json string: {json_obj}") return is_valid
class DataValidator(Validator): def __init__(self, schema, only_error=True): super(DataValidator, self).__init__(schema, name_field='name', only_error=only_error) self._v = JsonSchemaValidator(self._schema) def validate(self, data): try: errors = sorted(self._v.iter_errors(data), key=lambda e: e.path) print(errors) for error in errors: self.errors.append(error.message) # for suberror in sorted(error.context, key=lambda e: e.schema_path): # print(list(suberror.schema_path), suberror.message, sep=", ") except SchemaError as e: raise
def validate_json_schemata(json_schemata): print("Validate JSON schemata") for el in json_schemata: with open(os.path.join(el), 'r') as schema_file: try: schema = json.loads(schema_file.read()) except json.JSONDecodeError as ex: print("[error] Decoding JSON has failed for " + el) print(ex.msg + " at line " + str(ex.lineno)) else: v = Draft7Validator(schema) for error in Draft7Validator.iter_errors(schema): print(error) try: Draft7Validator.check_schema(schema) print(el.ljust(31) + "is valid") except jsonschema.exceptions.SchemaError as error_ex: print("Bad JSON schema " + el)
def get_validation_errors(validator: Draft7Validator, data: json) -> dict: errors = {} for error in sorted(validator.iter_errors(data), key=str): key = error.message.split()[0] errors.update({key.replace("'", ""): error.message.replace(key, "").strip()}) return errors
class TranscodingJob: '''Store essential parameters of the transcoding job''' ENCODERS = { None: VorbisTranscoder, 'copy': VerbatimFileCopy, 'symlink': SymlinkCreator, 'vorbis': VorbisTranscoder, 'lame': LameTranscoder, 'mp3': LameTranscoder, 'aac': AACTranscoder, 'm4a': AACTranscoder, 'opus': OpusTranscoder, } def __init__(self, config_file): '''Initialize transcoding job''' self.stats = TranscodingStats() self.finished = False self.config_file = config_file self._timestamp = None with open(config_file, encoding=CONFIG_ENCODING) as f: config = yaml.load(f, Loader=yaml.RoundTripLoader) output = config.get('output', {}) extras = config.get('extras', {}) self.validate(config) self.job_id = config.get('name', DEFAULT_CONFIG['name']) self.inputs = config.get('input', []) self.output_dir = output.get('directory') self.output_pattern = output.get('pattern', DEFAULT_CONFIG['pattern']) self.cover_size = extras.get('cover', DEFAULT_CONFIG['cover']) if output.get('category_blacklist'): self.select_mode = 'blacklist' self.select = set(output.get('category_blacklist')) elif output.get('category_whitelist'): self.select_mode = 'whitelist' self.select = set(output.get('category_whitelist')) else: self.select_mode = None self.select = set() lyrics_source = extras.get('lyrics', DEFAULT_CONFIG['lyrics']) if not lyrics_source: self.get_lyrics = None elif os.path.isdir(lyrics_source): self.get_lyrics = partial(read_lyrics, lyricsdir=lyrics_source) elif os.path.isfile(lyrics_source): database = LyricsStorage(lyrics_source) self.get_lyrics = database.get else: self.get_lyrics = None encoder = output.get('format', DEFAULT_CONFIG['format']) quality = output.get('quality', DEFAULT_CONFIG['quality']) self.transcoder = self.ENCODERS.get(encoder)(quality) lossy_action = output.get('lossy_source', DEFAULT_CONFIG['lossy_source']) if lossy_action == 'allow_bad_transcodes'\ or encoder == 'symlink' \ or encoder == 'copy': self.lossy_action = self.transcoder elif lossy_action == 'copy': self.lossy_action = VerbatimFileCopy() elif lossy_action == 'skip': skip_marker = self.transcoder.STATUS_SKIP self.lossy_action = lambda infile, outfile: (infile, skip_marker) self.lossy_action.STATUS_SKIP = skip_marker os.makedirs(self.output_dir, exist_ok=True) log.debug('Initialized {}'.format(self)) def __repr__(self): return '{cls}({config!r})'.format( cls=self.__class__.__name__, config=self.config_file, ) def transcode(self, task): '''Execute a single transcoding task''' log.debug('Started {task}'.format(task=task)) if (self.select_mode == 'blacklist' and self.select.intersection(task.categories)) \ or (self.select_mode == 'whitelist' and not self.select.intersection(task.categories)): self.stats.record_skip() log.debug('Skipped {task}'.format(task=task)) return source_format = os.path.splitext(task.source)[1][1:].lower() if source_format in LOSSLESS_EXTENSIONS: worker = self.transcoder else: worker = self.lossy_action if self._timestamp is None: # record the time of first transcoding task self._timestamp = int(time.time()) # Step 1: Transcode task.result, task.status = worker( task.source, os.path.join(self.output_dir, task.target)) # Step 1a: Process extras (cover art, lyrics) if self.cover_size: Thread(target=copy_coverart, kwargs=dict(task=task, size=self.cover_size)).start() if self.get_lyrics: Thread( target=copy_lyrics, kwargs=dict(task=task, lyrics_finder=self.get_lyrics), ).start() # Handle skipped transcodes if task.status is worker.STATUS_SKIP: if os.path.getmtime(task.result) > self.timestamp: raise RuntimeError('Target path collision for {}'.format( task.result)) self.stats.record_skip() log.debug('Skipped {task}'.format(task=task)) return # Step 2: Copy music tags if not task.status is worker.STATUS_SKIPTAGS: result = mutagen.File(task.result, easy=True) for key in task.tags.keys( ): # mutagen is inconsistent about `for k in t.tags` if hasattr(result.tags, 'valid_keys') \ and key not in result.tags.valid_keys: continue result.tags[key] = task.tags[key] result.save() self.stats.record_done() log.debug('Finished {task}'.format(task=task)) @property def timestamp(self): ''' Date and time of starting the first transcoding task in this job (in Unix time format) ''' return self._timestamp def write_report(self): '''Keep a journal of transcoder runs''' log_entry = '{time}Z: {stats}\n'.format( time=datetime.utcnow().replace(microsecond=0), stats=self.stats.show().strip(), ) with open(os.path.join(self.output_dir, 'transcoding.log'), 'a') as logfile: logfile.write(log_entry) def validate(self, config): '''Validate transcoding job configuration''' try: self.validator except AttributeError: package = __name__.rsplit('.', 1)[0] path = 'schema.json' schema = json.loads(resource_string(package, path).decode()) self.validator = JSONSchemaValidator(schema) error_messages = [] for error in self.validator.iter_errors(config): error_messages.append(' - {}: {}'.format( '.'.join(error.path) if error.path else '[config]', error.message)) if error_messages: raise ValueError('invalid configuration values:\n{}'.format( '\n'.join(sorted(error_messages))))