def pretty(infile): stdout = click.get_text_stream("stdout") try: for entity in read_entities(infile): data = json.dumps(entity.to_dict(), indent=2) stdout.write(data + "\n") except BrokenPipeError: raise click.Abort()
def expand(infile, outfile, enricher): enricher = load_enricher(enricher) try: for entity in read_entities(infile): for expanded in enricher.expand_entity(entity): write_object(outfile, expanded) except BrokenPipeError: raise click.Abort()
def sign(infile, outfile, signature): ns = Namespace(signature) try: for entity in read_entities(infile): signed = ns.apply(entity) write_object(outfile, signed) except BrokenPipeError: raise click.Abort()
def enrich(infile, outfile, enricher): enricher = load_enricher(enricher) try: for entity in read_entities(infile): for match in enricher.enrich_entity_raw(entity): write_object(outfile, match) except BrokenPipeError: raise click.Abort()
def ingest(self, file_path, entity): entity.set("mimeType", "application/json+ftm") with open(file_path, "r", encoding="utf-8") as fh: for idx, proxy in enumerate(read_entities(fh, cleaned=False)): if proxy.id is None: continue proxy.add("proof", entity.id, quiet=True) self.manager.emit_entity(proxy, fragment=idx)
def sieve(infile, outfile, schema, property, type): try: for entity in read_entities(infile): entity = sieve_entity(entity, schema, property, type) if entity is not None: write_object(outfile, entity) except BrokenPipeError: raise click.Abort()
def validate(infile, outfile): try: for entity in read_entities(infile, cleaned=False): clean = model.make_entity(entity.schema) clean.id = entity.id for (prop, value) in entity.itervalues(): clean.add(prop, value) write_object(outfile, clean) except BrokenPipeError: raise click.Abort()
def link(infile, outfile, matches): try: linker = Linker(model) for match in Match.from_file(model, matches): linker.add(match) log.info("Linker: %s clusters.", len(linker.lookup)) for entity in read_entities(infile): entity = linker.apply(entity) write_object(outfile, entity) except BrokenPipeError: raise click.Abort()
def aggregate(infile, outfile): buffer = {} namespace = Namespace(None) try: for entity in read_entities(infile): entity = namespace.apply(entity) if entity.id in buffer: buffer[entity.id].merge(entity) else: buffer[entity.id] = entity for entity in buffer.values(): write_object(outfile, entity) except BrokenPipeError: raise click.Abort()
def import_vis(infile, outfile): for entity in read_entities(infile, cleaned=False): write_object(outfile, entity)