else: wiki_obj = doc_factory.wiki('dummy') return wiki_obj for index, row in df.iterrows(): print(index) try: new_address = 'Italia, ' if row[region_index]: new_address += row[region_index] if new_address not in lst_address: lst_address.append(new_address) gmap = gmap_by_address(new_address) if gmap.get('code'): gmap_obj = doc_factory.gmaps(gmap.get('code')) gmap_obj.update(gmap) else: gmap_obj = doc_factory.gmaps('dummy') istat_code = hash().make(str(['Italia', row[region_index]])) istat_obj = doc_factory.istat(istat_code) wiki_obj = doc_factory.wiki('dummy') internal_obj = make_internal(row, istat_obj, {}, wiki_obj, gmap, gmap_obj) new_address += ', ' if row[provincia_index]: new_address += row[provincia_index] if new_address not in lst_address: lst_address.append(new_address) gmap = gmap_by_address(new_address) if gmap.get('code'):
print(i) dct = {} wiki_parsed = {} wiki = parser_wiki(line) gmap = parser_gmap(line) wiki_obj = hash_lib.make(wiki.get('url')) if gmap.get('code'): code = gmap.get('code') elif gmap: code = hash().make(str(gmap)) else: code = 'dummy' print(code) gmap_obj = doc_factory.gmaps(code) gmap.update(code=code) gmap_obj.update(gmap) wiki_code = wiki_obj.get_document().get('code') doc = internal_collection.find_one({'source.wiki': wiki_code}) if doc and doc.get('code'): doc_obj = doc_factory.internal(doc.get('code')) source = doc_obj.get_document().get('source') source.update(gmap=gmap_obj.get_document().get('code')) print(gmap_obj.get_document().get('code')) doc_obj.update({'source': source})
actual_doc = document.get_document() added_requests = [(tuple(x) if isinstance(x, list) else x) for x in actual_doc.get('requests', ())] added_requests.append(request) actual_doc.update(requests=list(set(added_requests))) document.update(actual_doc) try: if use_address: address_content = loader.by_address(address=address) objects = MapFactory.france(address_content) for object in objects: code = object.get_place_id() if object.get_place_id(): doc = document_factory.gmaps(code) if doc.is_new() or force_update: dic = object.as_dictionary() if dic.get('type'): doc.update(object.as_dictionary()) update_meta(request=address, document=doc) else: log.add('Wrong command', log.ERROR) print('use parameters like -a as address string') except: message = str(sys.exc_info()) log.add('Unexpected error: [{0}]'.format(message), log.ERROR) raise log.add('Finish', log.INFO)
gmap_doc = gmap_docs.find_one({'code': gmap_code}) wiki_doc = wiki_docs.find_one({'code': wiki_code}) type_doc = gmap_doc.get('type') if not type_doc: empty_doc = empty_doc + 1 print(wiki_doc.get('type')) #if gmap_doc.get('code') and len(gmap_doc.get('code')) < 32: # g_objects = gmap_loader.by_place_id(gmap_doc.get('code')) # map_objects = MapFactory.france(g_objects) # result = map_objects[0].as_dictionary() # result.update(language=language) #else: result = gmap_by_address(wiki_doc) if result: gmap_obj = doc_factory.gmaps(code=result.get('code')) doc = gmap_obj.get_document() gmap_obj.update(new_data=result) source = obj.get('source', {}) source.update(gmap=doc.get('code')) internal_docs.update_one({'code': obj.get('code')}, {'$set': { 'source': source }}) if type_doc in exlude_type: locality = locality + 1 if type_doc in replace: need_replace = need_replace + 1