def prepare():
for area in mongo_functions.mongo_get(collection='pre_metropolitan_area'):
business = mongo_functions.mongo_get(
collection='pre_business',
filter={'tile15': {
'$in': area['city_tiles15']
}},
fields={
'tile10': 1,
'tile15': 1,
'tile18': 1,
'norm_categories': 1
})
business_ids = [doc['_id'] for doc in business]
reviews = mongo_functions.mongo_get(
collection='review', filter={'business_id': {
'$in': business_ids
}})
business_dict = {doc['_id']: doc for doc in business}
for review in reviews:
review['tile10'] = business_dict[review['business_id']]['tile10']
review['tile15'] = business_dict[review['business_id']]['tile15']
review['tile18'] = business_dict[review['business_id']]['tile18']
review['norm_categories'] = business_dict[
review['business_id']]['norm_categories']
review['city_area'] = area['_id']
mongo_functions.batch_upsert(reviews,
collection='pre_review',
update="{'$set': item}")
def prepare():
businesses = mongo_functions.mongo_get(collection='business')
pre_businessess = []
for business in businesses:
try:
business['tile10'] = '_'.join([
str(item) for item in tiles.deg2num(business['latitude'],
business['longitude'], 10)
])
business['tile15'] = '_'.join([
str(item) for item in tiles.deg2num(business['latitude'],
business['longitude'], 15)
])
business['tile18'] = '_'.join([
str(item) for item in tiles.deg2num(business['latitude'],
business['longitude'], 18)
])
pre_businessess.append(business)
except TypeError:
pass
except Exception as e:
raise
mongo_functions.batch_upsert(pre_businessess,
collection='pre_business',
update='{"$set": item}')
def data_import(filename):
concept = filename[:-5]
id_field = CONCEPT_ID_FIELD[concept] if concept in CONCEPT_ID_FIELD else None
# if id_field is None:
# return
data = data_read(filename)
bulk = data_save_prepare(id_field, data)
mongo_functions.batch_upsert(bulk, collection=concept, update="{'$set': item}")
def prepare():
tiles = {doc['tile10'] for doc in mongo_functions.mongo_get(collection='pre_business', fields={'tile10': 1})}
tiles_sorted = sorted(tiles, key=lambda x: x.split('_')[0])
tiles_grouped = tiles_group(tiles_sorted)
metropolitan_areas = groups_filter(tiles_grouped)
for metropolitan_area in metropolitan_areas:
metropolitan_area_center(metropolitan_area)
metropolitan_area_name(metropolitan_area)
mongo_functions.batch_upsert(metropolitan_areas, collection='pre_metropolitan_area', update="{'$set': item}")
def main():
for area in mongo_functions.mongo_get(collection='pre_metropolitan_area',
fields={}):
reviews = mongo_functions.mongo_get(collection='pre_review',
filter={
'city_area': area['_id'],
'user_from': {
'$exists': True
}
},
fields={
'tile_id': 1,
'user_id': 1,
'tile15': 1,
'tile18': 1,
'user_from': 1
})
city_unique_tourist = city_unique_tourist_get(reviews, area)
city_unique_residents = city_unique_residents_get(reviews, area)
total_tourist = len(city_unique_tourist)
total_residents = len(city_unique_residents)
for zoom in [15, 18]:
tile_reviews_dict = tile_reviews_dict_group(reviews, zoom)
pre_tile_list = []
for tile, tile_reviews in tile_reviews_dict.items():
tourist_reviews = [
review['user_id'] for review in tile_reviews
if review['user_from'] != area['_id']
]
resident_reviews = [
review['user_id'] for review in tile_reviews
if review['user_from'] == area['_id']
]
pre_tile = {
'_id': '{}_{}'.format(tile, zoom),
'area_id': area['_id'],
'tourist_unique': list(set(tourist_reviews)),
'resident_unique': list(set(resident_reviews)),
'tourist_review_counter': len(tourist_reviews),
'resident_review_counter': len(resident_reviews)
}
pre_tile['tourist_unique_counter'] = len(
pre_tile['tourist_unique'])
pre_tile['resident_unique_counter'] = len(
pre_tile['resident_unique'])
pre_tile['ratio_signed'] = tile_dissimilarity_ratio(
pre_tile['tourist_unique_counter'],
pre_tile['resident_unique_counter'], total_tourist,
total_residents)
pre_tile['ratio'] = pre_tile['ratio_signed'] if pre_tile[
'ratio_signed'] > 0 else -1 * pre_tile['ratio_signed']
pre_tile_list.append(pre_tile)
mongo_functions.batch_upsert(pre_tile_list,
collection='pre_tile',
update='{"$set": item}')
def prepare():
user_reviews_dict = user_reviews_dict_create()
_users = mongo_functions.mongo_get(collection='user')
users = []
index = 0
users_len = len(_users)
print('\n')
while len(_users) > 0:
index += 1
sys.stdout.write('\rProcessing user {}/{}...'.format(index, users_len))
sys.stdout.flush()
_user = _users.pop()
if _user['_id'] in user_reviews_dict:
_user.update({'reviews': user_reviews_dict[_user['_id']]})
_user_reviews_prepare(_user)
users.append(_user)
mongo_functions.batch_upsert(users,
collection='pre_user',
update='{"$set": item}')