def post_all_ad_versions_handler():
db_results = util.db_find('bee_csv', {
"is_processed_ad_version": True,
"is_processed_address": True
}, {
"_id": False,
"town": True,
"count": True,
"deliver_time": True,
"deliver_date": True,
"save_time": True,
"geo": True,
"county": True,
"address": True,
"user_name": True,
"is_processed_ad_version": True,
"is_processed_address": True,
"csv_key": True,
"ad_versions": True,
"version_text": True,
"memo": True,
"deliver_status": True
})
for db_result in db_results:
csv_key = db_result.get('csv_key', '')
util.db_update('bee', {'csv_key': csv_key}, db_result)
def _create_session_key(user_id=None, offset_timestamp=0):
the_timestamp = util.get_timestamp()
the_timestamp += offset_timestamp
session_key = _serialize_session_key(the_timestamp, util.gen_random_string())
if user_id:
util.db_update('session_user_map', {"session_key": session_key}, {"user_id": user_id, "the_timestamp": the_timestamp})
return session_key
def post_google_geo_handler(params):
csv_key = params.get('csv_key', '')
if not csv_key:
return {"success": False, "error_msg": "no csv_key"}
params['is_processed_address'] = True
cfg.logger.debug('params: %s', params)
util.db_update('bee_csv', {"csv_key": csv_key}, params)
return {"success": True}
def post_google_geo_handler(params):
csv_key = params.get('csv_key', '')
if not csv_key:
return {"success": False, "error_msg": "no csv_key"}
params['is_processed_address'] = True
cfg.logger.debug('params: %s', params)
util.db_update('bee_csv', {"csv_key": csv_key}, params)
return {"success": True}
def save_session_user_map(session_struct, user_id):
session_key = session_struct.get('key', '')
if not session_key:
return
util.db_update('session_user_map', {"session_key": session_key}, {'the_timestamp': session_struct.get('the_timestamp', 0), 'user_id': user_id})
is_cron_remove_expire = cfg.config.get('is_cron_remove_expire', True)
if not is_cron_remove_expire:
expire_timestamp_session = cfg.config.get('expire_unix_timestamp_session', EXPIRE_UNIX_TIMESTAMP_SESSION) * 1000
util.db_remove('session_user_map', {"the_timestamp": expire_timestamp_session})
def _parse_csv(data):
f = StringIO(data)
df = pd.read_csv(f)
funnel_dict = {"error_code": S_OK, "error_msg": "", "fail": set()}
for each_column in df.columns:
df[each_column].fillna('', inplace=True)
df['csv_key'] = df.apply(lambda x: _parse_csv_key(dict(x), funnel_dict), axis=1)
csv_key_list = list(df['csv_key'])
db_csv_keys = util.db_find('bee', {'csv_key': {'$in': csv_key_list}}, {"_id": False, "csv_key": True})
db_csv_keys = [db_csv_key.get('csv_key', '') for db_csv_key in db_csv_keys]
db_csv_keys = [each_key for each_key in db_csv_keys if each_key]
#is_csv_key_not_in_db = df['csv_key'].isin(db_csv_keys) == False
#df = df[is_csv_key_not_in_db]
df['address'] = df.apply(lambda x: _parse_address(dict(x), funnel_dict), axis=1)
df['county_and_town'] = df.apply(lambda x: _parse_county_and_town(dict(x), funnel_dict), axis=1)
df['google_address'] = df.apply(lambda x: _parse_google_address(dict(x), funnel_dict), axis=1)
df['deliver_time'] = df.apply(lambda x: _parse_deliver_time(dict(x), funnel_dict), axis=1)
df['save_time'] = df.apply(lambda x: _parse_save_time(dict(x), funnel_dict), axis=1)
df['deliver_date'] = df.apply(lambda x: _parse_deliver_date(dict(x), funnel_dict), axis=1)
df['user_name'] = df.apply(lambda x: _parse_user_name(dict(x), funnel_dict), axis=1)
df['count'] = df.apply(lambda x: _parse_count(dict(x), funnel_dict), axis=1)
df['deliver_status'] = df.apply(lambda x: _parse_deliver_status(dict(x), funnel_dict), axis=1)
df['memo'] = df.apply(lambda x: _parse_memo(dict(x), funnel_dict), axis=1)
df['version_text'] = df.apply(lambda x: _parse_version_text(dict(x), funnel_dict), axis=1)
df['versions'] = df.apply(lambda x: _parse_versions(dict(x), funnel_dict), axis=1)
cfg.logger.debug('df_len: %s', len(df))
parsed_dict_list = [_parse_dict_row(row, funnel_dict) for (idx, row) in df.iterrows()]
df = pd.DataFrame(parsed_dict_list)
df = df[['csv_key', 'deliver_time', 'deliver_date', 'user_name', 'address', 'county_and_town', 'google_address', 'versions', 'version_text', 'count', 'save_time', 'deliver_status', 'memo']]
results = util.df_to_dict_list(df)
for each_result in results:
csv_key = each_result.get('csv_key', '')
versions = each_result.get('versions', [])
version_text = each_result.get('version_text', [])
cfg.logger.debug('to db_update: each_result: %s', each_result)
util.db_update('bee_csv', {'csv_key': csv_key}, each_result)
for each_version in versions:
util.db_update('bee_csv_versions', {'version': each_version}, {csv_key: version_text})
return (funnel_dict['error_code'], funnel_dict['error_msg'], len(results), results)
def _create_session_key(user_id=None, offset_timestamp=0):
the_timestamp = util.get_timestamp()
the_timestamp += offset_timestamp
session_key = _serialize_session_key(the_timestamp,
util.gen_random_string())
if user_id:
util.db_update('session_user_map', {"session_key": session_key}, {
"user_id": user_id,
"the_timestamp": the_timestamp
})
return session_key
def save_session_user_map(session_struct, user_id):
session_key = session_struct.get('key', '')
if not session_key:
return
util.db_update('session_user_map', {"session_key": session_key}, {
'the_timestamp': session_struct.get('the_timestamp', 0),
'user_id': user_id
})
is_cron_remove_expire = cfg.config.get('is_cron_remove_expire', True)
if not is_cron_remove_expire:
expire_timestamp_session = cfg.config.get(
'expire_unix_timestamp_session',
EXPIRE_UNIX_TIMESTAMP_SESSION) * 1000
util.db_remove('session_user_map',
{"the_timestamp": expire_timestamp_session})
def post_ad_version_handler(params):
csv_key = params.get('csv_key', '')
if not csv_key:
return {"success": False, "error_msg": "no csv_key"}
ad_versions = params.get('ad_versions', [])
cfg.logger.debug('to update ad_versions: csv_key: %s ad_versions: %s', csv_key, ad_versions)
util.db_update('bee_csv', {'csv_key': csv_key}, {'ad_versions': ad_versions, "is_processed_ad_version": True})
db_result = util.db_find_one('bee_csv', {"csv_key": csv_key}, {"_id": False, "town": True, "count": True, "deliver_time": True, "deliver_date": True, "save_time": True, "geo": True, "county": True, "address": True, "user_name": True, "is_processed_ad_version": True, "is_processed_address": True, "csv_key": True, "ad_versions": True, "version_text": True, "memo": True, "deliver_status": True})
the_id = str(db_result.get('save_time', 0)) + "_" + util.uuid()
db_result['the_id'] = the_id
if db_result.get('is_processed_ad_version', False) and db_result.get('is_processed_address', False):
util.db_update('bee', {'csv_key': csv_key}, db_result)
def p_img_info_handler(data):
cfg.logger.debug('data: %s', data)
if not data.get('the_id', ''):
return {"success": False}
result = util.db_update('bee_img', {'the_id': data.get('the_id', '')}, data, upsert=False)
if not result:
return {"success": False}
return {"success": True}
def p_img_info_handler(data):
cfg.logger.debug('data: %s', data)
if not data.get('the_id', ''):
return {"success": False}
result = util.db_update('bee_img', {'the_id': data.get('the_id', '')},
data,
upsert=False)
if not result:
return {"success": False}
return {"success": True}
def _process_each_data(user_id, each_data, server_timestamp):
offset_timestamp = util._int(each_data.get('offset_timestamp', 0))
lat = each_data.get('lat', 0)
lon = each_data.get('lon', 0)
yaw = each_data.get('yaw', 0)
pitch = each_data.get('pitch', 0)
roll = each_data.get('roll', 0)
x = each_data.get('x', 0)
y = each_data.get('y', 0)
z = each_data.get('z', 0)
the_timestamp = server_timestamp + offset_timestamp
key = {'user_id': user_id, 'the_timestamp': the_timestamp}
val = {'lat': lat, 'lon': lon, 'yaw': yaw, 'pitch': pitch, 'roll': roll, 'x': x, 'y': y, 'z': z}
cfg.logger.debug('to db_update reportDB: server_timestamp: %s offset_timestamp: %s the_timestamp: %s key: %s val: %s', server_timestamp, offset_timestamp, the_timestamp, key, val)
util.db_update('reportDB', key, val)
return {'success': True}
def _put_to_db(the_val):
the_key = {'the_id': the_val['the_id']}
util.db_update('roadDB', the_key, the_val)
def save_user(user_id, session_struct, session_struct2, user_info):
util.db_update('user_info', {"user_id": user_id}, {"user_info": user_info})
save_session_user_map(session_struct, user_id)
save_session_user_map(session_struct2, user_id)
def _put_to_db(data):
category = 'new_taipei_city_dig_point'
the_idx = data['IDpro']
the_id = category + '_' + the_idx
the_key = {'the_category': category, 'the_id': the_id, 'the_idx': the_idx}
util.db_update('roadDB', the_key, data)
def _put_to_db(the_val):
the_key = {'the_id': the_val['the_id']}
util.db_update('roadDB', the_key, the_val)
def post_all_ad_versions_handler():
db_results = util.db_find('bee_csv', {"is_processed_ad_version": True, "is_processed_address": True}, {"_id": False, "town": True, "count": True, "deliver_time": True, "deliver_date": True, "save_time": True, "geo": True, "county": True, "address": True, "user_name": True, "is_processed_ad_version": True, "is_processed_address": True, "csv_key": True, "ad_versions": True, "version_text": True, "memo": True, "deliver_status": True})
for db_result in db_results:
csv_key = db_result.get('csv_key', '')
util.db_update('bee', {'csv_key': csv_key}, db_result)
def save_user(user_id, session_struct, session_struct2, user_info):
util.db_update('user_info', {"user_id": user_id}, {"user_info": user_info})
save_session_user_map(session_struct, user_id)
save_session_user_map(session_struct2, user_id)