def process_entities(args): wikisite = args.wikisite if wikisite == "zhwiki": title_db, redir_db = 1, 3 elif wikisite == "enwiki": title_db, redir_db = 4, 5 title_idx = get_redis_title(title_db) redir_idx = get_redis_redir(redir_db) logging.info('collect titles for entities...') # thousands level title_bulk = set() for i, entity in enumerate(reader(args.entities)): try: title = entity['sitelinks'][wikisite]['title'] except: continue title = canonicalize(title, redir_idx) title_bulk.add(title) logging.info("got %d title out of %d entities" % (len(title_bulk), i)) logging.debug('====================') logging.debug('required: ' + u','.join(title_bulk).encode('utf-8')) title_text = bulk_query_wikipage(title_idx, title_bulk, redir_idx) logging.info("found page amount: %d" % len(title_text)) logging.debug('====================') logging.debug('got text: ' + u','.join(title_text.keys()).encode('utf-8')) output = open(args.output, 'w') for i, entity in enumerate(reader(args.entities)): try: title = entity['sitelinks'][wikisite]['title'] except: title = "" title = canonicalize(title, redir_idx) text = title_text.get(title) if text is None: if title: logging.warning("specified title %s not found" % title.encode('utf-8')) text = "" else: logging.debug("title %s found" % title.encode('utf-8')) entity[wikisite] = text output.write(json.dumps(entity) + '\n') if i % 500 == 0: logging.info("%d entities processed" % i) output.flush() output.close()
def process_entities(args): logging.info('building title to wiki filename mapping...') title_idx = load_idx_mapping(args.wiki_index) logging.info('building title redirect mapping...') redir_idx = load_redir_mapping(args.redir_file) logging.info('collect titles for entities...') # thousands level wikisite = args.wikisite title_bulk = set() for i, entity in enumerate(reader(args.entities)): try: title = entity['sitelinks'][wikisite]['title'] except: continue title = redirect(redir_idx, title) title = cc.convert(title) title_bulk.add(title) logging.info("got %d title out of %d entities" % (len(title_bulk), i)) title_text = bulk_query_wikipage(title_idx, title_bulk) logging.info("found page amount: %d" % len(title_text)) logging.debug('====================') logging.debug('got text: ' + u','.join(title_text.keys()).encode('utf-8')) logging.debug('--------------------') logging.debug('required: ' + u','.join(title_bulk).encode('utf-8')) output = open(args.output, 'w') for i, entity in enumerate(reader(args.entities)): try: title = entity['sitelinks'][wikisite]['title'] except: title = "" title = redirect(redir_idx, title) title = cc.convert(title) text = title_text.get(title) if text is None: if title: logging.warning("specified title %s not found" % title.encode('utf-8')) text = "" else: logging.debug("title %s found" % title.encode('utf-8')) entity[wikisite] = text output.write(json.dumps(entity) + '\n') if i % 500 == 0: logging.info("%d entities processed" % i) output.flush() output.close()
def process_entities(args): logging.info('building title to wiki filename mapping...') title_idx = load_idx_mapping(args.wiki_index) logging.info('building title redirect mapping...') redir_idx = load_redir_mapping(args.redir_file) logging.info('collect titles for entities...') # thousands level wikisite = args.wikisite title_bulk = set() for i, entity in enumerate(reader(args.entities)): try: title = entity['sitelinks'][wikisite]['title'] except: continue title = redirect(redir_idx, title) title = cc.convert(title) title_bulk.add(title) logging.info("got %d title out of %d entities" % (len(title_bulk), i)) title_text = bulk_query_wikipage(title_idx, title_bulk) logging.info("found page amount: %d" % len(title_text)) logging.debug('====================') logging.debug('got text: ' + u','.join(title_text.keys()).encode('utf-8')) logging.debug('--------------------') logging.debug('required: ' + u','.join(title_bulk).encode('utf-8')) output = open(args.output, 'w') for i, entity in enumerate(reader(args.entities)): try: title = entity['sitelinks'][wikisite]['title'] except: title = "" title = redirect(redir_idx, title) title = cc.convert(title) text = title_text.get(title) if text is None: if title: logging.warning("specified title %s not found" % title.encode('utf-8')) text = "" else: logging.debug("title %s found" % title.encode('utf-8')) entity[wikisite] = text output.write(json.dumps(entity) + '\n') if i % 500 == 0: logging.info("%d entities processed" % i) output.flush() output.close()
def find_neighbor(args): filelist = [l.rstrip() for l in args.filelist] qids = set(v for _, _, v in next_neighbor_id(reader(args.entities))) written_qids = set() output = open(args.output, 'w') for d in xrange(args.depth): logging.info('========> %d-hop neighbors amount: %d' % (d + 1, len(qids))) if args.debug: logging.debug(u','.join(qids).encode('utf-8')) break next_qids = set() for i, e in enumerate(next_required_entity_from_files(qids, filelist)): if e['id'] not in written_qids: output.write(json.dumps(e) + '\n') written_qids.add(e['id']) for v in next_neighbor_of_entity(e): next_qids.add(v) if i % 1000 == 0: logging.info("found %d neighbors so far by reading files, %d have been written" % (i, len(written_qids))) output.flush() qids = next_qids output.close()
def collect_categories(args): categories = set() for i, entity in enumerate(reader(args.input)): if i % 20000 == 0: logging.info( "%d data (first) processed: %s" % (i, datetime.datetime.now().strftime('%m%d-%H:%M:%S'))) if args.debug and i / 20000 == 1: break try: subclass_claims = entity['claims']['P279'] values = filter(None, (claim_value(claim) for claim in subclass_claims)) # An entity with a non-empty 'subclass_of' property is itself a class. # The parent entities indicated by the 'subclass_of' property are all classes. # If the class is an 'instance_of' another entity, the other one must also be a class. if values: categories.add(entity['id']) categories.update(values) instance_claims = entity['claims']['P31'] values = filter(None, (claim_value(claim) for claim in instance_claims)) categories.update(values) except: continue return categories
def collect_categories(args): categories = set() for i, entity in enumerate(reader(args.input)): if i % 20000 == 0: logging.info("%d data (first) processed: %s" %(i, datetime.datetime.now().strftime('%m%d-%H:%M:%S'))) if args.debug and i / 20000 == 1: break try: subclass_claims = entity['claims']['P279'] values = filter(None, (claim_value(claim) for claim in subclass_claims)) # An entity with a non-empty 'subclass_of' property is itself a class. # The parent entities indicated by the 'subclass_of' property are all classes. # If the class is an 'instance_of' another entity, the other one must also be a class. if values: categories.add(entity['id']) categories.update(values) instance_claims = entity['claims']['P31'] values = filter(None, (claim_value(claim) for claim in instance_claims)) categories.update(values) except: continue return categories
def stat(args): for i, entity in enumerate(reader(args.entity_file)): if i / 50000 == 1: break for claims in entity['claims'].itervalues(): for c in claims: # datatype, datavalue.type, datavalue.value x = extract_kv(c) print entity['id'].encode('utf-8') + '\t' + u'\t'.join(x).encode('utf-8')
def next_required_entity_from_files(qids, filelist): i = 0 for f in filelist: logging.info("reading file %s ..." % f) for e in reader(f): if i % 10000 == 0: logging.info('read %d input entities in file' % i) i += 1 qid = e['id'] if qid in qids: yield e
def extract(args): categories = collect_categories(args) logging.info('got all categories, count %d' % len(categories)) logging.debug('first 100 categories repr: ' + ','.join(repr(x) for x in list(categories)[:100])) output = open(args.output, 'w') for i, entity in enumerate(reader(args.input)): if i % 20000 == 0: logging.info( "%d data (first) processed: %s" % (i, datetime.datetime.now().strftime('%m%d-%H:%M:%S'))) if args.debug and i / 20000 == 1: break try: qid = entity['id'] except: continue if qid not in categories: continue if 'claims' not in entity: continue claims = entity['claims'] subclass_claims = claims['P279'] if 'P279' in claims else [ ] # subclass_of instance_claims = claims['P31'] if 'P31' in claims else [ ] # instance_of if len(subclass_claims) + len(instance_claims) == 0: continue subclass_values = filter(None, (claim_value(claim) for claim in subclass_claims)) instance_values = filter(None, (claim_value(claim) for claim in instance_claims)) new_entity = {} new_entity['id'] = entity['id'] try: new_entity['enlabel'] = entity['labels']['en']['value'] except: new_entity['enlabel'] = "" try: new_entity['zhlabel'] = entity['labels']['zh']['value'] except: new_entity['zhlabel'] = "" new_entity['pids'] = subclass_values + instance_values output.write(json.dumps(new_entity) + "\n")
def main(filename, dbname, colname): conn = pymongo.MongoClient() db = conn.get_database(dbname) col = db.get_collection(colname) logging.getLogger().setLevel(logging.INFO) for obj in reader(filename): try: col.insert_one(obj) logging.info('inserted id=%s' % str(obj['id'])) except pymongo.errors.DuplicateKeyError: logging.info('duplicated id=%s' % str(obj['id'])) continue except Exception: logging.warning('exception when writing obj %s' % str(obj['id']))
def bulk_query_wikipage(title_idx, titles, redir_idx={}): filelist = sorted(set(filter(None, (title_idx.get(t) for t in titles)))) logging.info('there are %d files to read, given these %d titles' % (len(filelist), len(titles))) texts = {} # scan all filelist for f in filelist: logging.info("reading file %s .." % f) for page in reader(f): try: title = page['title'] title = cc.convert(redirect(redir_idx, title)) if title in titles: texts[title] = page['text'] except: continue return texts
def bulk_query_wikipage(title_idx, titles, redir_idx={}): filelist = sorted(set(filter(None, (title_idx.get(t) for t in titles)))) logging.info('there are %d files to read, given these %d titles' % (len(filelist), len(titles))) texts = {} # scan all filelist for f in filelist: logging.info("reading file %s .." % f) for page in reader(f): try: title = page['title'] title = cc.convert(redirect(redir_idx, title)) if title in titles: texts[title] = page['text'] except: continue return texts
def extract(args): categories = collect_categories(args) logging.info('got all categories, count %d' % len(categories)) logging.debug('first 100 categories repr: ' + ','.join(repr(x) for x in list(categories)[:100])) output = open(args.output, 'w') for i, entity in enumerate(reader(args.input)): if i % 20000 == 0: logging.info("%d data (first) processed: %s" %(i, datetime.datetime.now().strftime('%m%d-%H:%M:%S'))) if args.debug and i / 20000 == 1: break try: qid = entity['id'] except: continue if qid not in categories: continue if 'claims' not in entity: continue claims = entity['claims'] subclass_claims = claims['P279'] if 'P279' in claims else [] # subclass_of instance_claims = claims['P31'] if 'P31' in claims else [] # instance_of if len(subclass_claims) + len(instance_claims) == 0: continue subclass_values = filter(None, (claim_value(claim) for claim in subclass_claims)) instance_values = filter(None, (claim_value(claim) for claim in instance_claims)) new_entity = {} new_entity['id'] = entity['id'] try: new_entity['enlabel'] = entity['labels']['en']['value'] except: new_entity['enlabel'] = "" try: new_entity['zhlabel'] = entity['labels']['zh']['value'] except: new_entity['zhlabel'] = "" new_entity['pids'] = subclass_values + instance_values output.write(json.dumps(new_entity) + "\n")
def bulk_query_wikidata(qids, wikidata_idx): filelist = sorted(set(mquery_redis_idx(qids, wikidata_idx))) logging.info('there are %d files to read for %d entities' % (len(filelist), len(qids))) entities = {} for f in filelist: logging.info("reading file %s ..." % f) for i, entity in enumerate(reader(f)): try: qid = entity['id'] except KeyError: continue if qid in qids and qid not in entities: entities[qid] = entity if i % 10000 == 0: logging.info("%d items processed, current %s" % (i, qid)) logging.info("%d items cumulated after reading file %s" % (len(entities), f)) return entities
def find_neighbor(args): wikidata_idx = get_redis_wikidata() es = list(x for x in reader(args.entities)) logging.info('read %d input entities' % len(es)) output = open(args.output, 'w') for l in xrange(args.depth): if len(es) == 0 and l < args.depth: logging.info("early breaking at layer %d because of empty entities" % l) break logging.info('find %d-hop neighbors for %d inputs' % (l, len(es))) qids = set(filter(lambda v: v is not None and re.match('^Q\d+$', v), (claim_value(c) for e in es for _, cs in e['claims'].iteritems() for c in cs))) logging.debug('qids: %s' % u','.join(qids).encode('utf-8')) logging.info('there\'re %d neighbors to read' % len(qids)) es_d = bulk_query_wikidata(qids, wikidata_idx) es = [x for x in es_d.itervalues()] for e in es: output.write(json.dumps(e) + '\n')
def find_entities_by_kinships(args, kinships): logging.info('now filtering entities by kinships...') # iterate over the outputs output = open(args.output, 'w') dataset = reader_for_list(args.input_filelist) if args.input_filelist else reader(args.input) for i, entity in enumerate(dataset): if 'claims' not in entity: continue claims = entity['claims'] subclass_claims = claims['P279'] if 'P279' in claims else [] # subclass_of instance_claims = claims['P31'] if 'P31' in claims else [] # instance_of if len(subclass_claims) + len(instance_claims) == 0: continue categories = filter(lambda x: x is not None, (claim_value(claim) for claim in subclass_claims)) classes = filter(lambda x: x is not None, (claim_value(claim) for claim in instance_claims)) if any(x in kinships for x in categories + classes): output.write(json.dumps(entity) + '\n') if i % 20000 == 0: logging.info('categories: %s, classes: %s' % (repr(categories), repr(classes))) logging.info('%d entities iterated over: %s' % (i, datetime.datetime.now().strftime('%Y%m%d%H%M%S')))
trainer_id = int(sys.argv[1]) # trainer id for each guest job_path = "fl_job_config" job = FLRunTimeJob() job.load_trainer_job(job_path, trainer_id) job._scheduler_ep = "127.0.0.1:9091" # Inform the scheduler IP to trainer # print(job._trainer_send_program) trainer = FLTrainerFactory().create_fl_trainer(job) use_cuda = False place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() trainer._current_ep = "127.0.0.1:8192" trainer.start(place=place) trainer._logger.setLevel(logging.DEBUG) g = reader() if trainer_id > 0: for i in range(trainer_id): next(g) data = next(g) print(data) output_folder = "fl_model" step_i = 0 while not trainer.stop(): step_i += 1 print("batch %d start train" % step_i) trainer.run(feed=data, fetch=[]) if trainer_id == 0: print("start saving model") trainer.save_inference_program(output_folder)
def dataextractor(filelist): for f in open(filelist): f = f.rstrip() for i, entity in enumerate(reader(f)): qid = entity['id'] yield qid, f, i + 1