def _parse_http_result(dig_point, result):
data = util.json_loads(result)
cfg.logger.debug('after json_loads: result: %s data: %s', result, data)
if taipei_city.validate_http_result_json(data) != S_OK:
return {}
return _process_data(dig_point, data)
def _parse_http_result(road_case, result):
data = util.json_loads(result)
cfg.logger.debug('result: %s data: %s', result, data)
if taipei_city.validate_http_result_json(data) != S_OK:
return {}
return _process_data(road_case, data)
def _parse_http_result(dig_point, result):
data = util.json_loads(result)
cfg.logger.debug('after json_loads: result: %s data: %s', result, data)
if taipei_city.validate_http_result_json(data) != S_OK:
return {}
return _process_data(dig_point, data)
def post_add_handler(params):
json_struct = util.json_loads(params.get('json', '{}'))
(error_code, error_param) = util.check_valid_params(json_struct, ['county_name', 'the_category', 'the_idx', 'start_timestamp', 'end_timestamp', 'geo', 'the_data'])
if error_code != S_OK:
return (error_code, {"success": False, "error_msg": "no " + error_param})
cron_data.process_data(json_struct['county_name'], json_struct['the_category'], json_struct['the_idx'], json_struct['start_timestamp'], json_struct['end_timestamp'], json_struct['geo'], json_struct['the_data'])
return (S_OK, {"success": True})
def _process_data_text(the_category, the_idx, the_data_text, time_period_idx):
the_data = util.json_loads(the_data_text)
data_list = the_data if the_data.__class__.__name__ == 'list' else [the_data]
len_data_list = len(data_list)
if len_data_list == 1:
_process_each_data(the_category, the_idx, data_list[0], time_period_idx)
else:
[_process_each_data(the_category, the_idx + '.' + data_idx, each_data, time_period_idx) for (data_idx, each_data) in enumerate(data_list)]
def _get_params():
server = cfg.config.get('web_server', 'http://106.187.101.193:5346')
the_url = server + '/get/taipei_city_road_case_next_road_case'
http_result = util.http_multiget([the_url])
next_road_case = util._int(util.json_loads(http_result.get(the_url, ''), ''), START_TAIPEI_CITY_ROAD_CASE)
cfg.logger.debug('after http_multiget: http_result: %s next_road_case: %s', http_result, next_road_case)
return {'next_road_case': next_road_case}
def _post_register_google(content, state, session_struct, session_struct2, request, params):
the_struct = util.json_loads(content)
user_id = 'google_' + str(the_struct['id'])
# save
util_user.save_user(user_id, session_struct, session_struct2, {"google_id": the_struct['id'], 'name': the_struct['name'], 'given_name': the_struct['given_name'], 'family_name': the_struct['family_name'], 'extension': the_struct})
cfg.logger.debug('user_info: content: (%s, %s) the_struct: (%s, %s)', content, content.__class__.__name__, the_struct, the_struct.__class__.__name__)
# return
_redirect_register(state)
def _post_register_facebook(content, state, session_struct, session_struct2, request, params):
the_struct = util.json_loads(content)
user_id = 'facebook_' + str(the_struct['id'])
# save
cfg.logger.debug('user_info: content: (%s, %s) the_struct: (%s, %s)', content, content.__class__.__name__, the_struct, the_struct.__class__.__name__)
util_user.save_user(user_id, session_struct, session_struct2, {"facebook_id": the_struct['id'], 'name': the_struct.get('name', ''), 'given_name': the_struct['first_name'], 'family_name': the_struct['last_name'], 'extension': the_struct})
# return
_redirect_register(state)
def _get_params():
server = cfg.config.get('web_server', 'http://106.187.101.193:5346')
the_url = server + '/get/taipei_city_road_case_next_road_case'
http_result = util.http_multiget([the_url])
next_road_case = util._int(
util.json_loads(http_result.get(the_url, ''), ''),
START_TAIPEI_CITY_ROAD_CASE)
cfg.logger.debug('after http_multiget: http_result: %s next_road_case: %s',
http_result, next_road_case)
return {'next_road_case': next_road_case}
def _process_data_text(the_category, the_idx, the_data_text, time_period_idx):
the_data = util.json_loads(the_data_text)
data_list = the_data if the_data.__class__.__name__ == 'list' else [
the_data
]
len_data_list = len(data_list)
if len_data_list == 1:
_process_each_data(the_category, the_idx, data_list[0],
time_period_idx)
else:
[
_process_each_data(the_category, the_idx + '.' + data_idx,
each_data, time_period_idx)
for (data_idx, each_data) in enumerate(data_list)
]
def _crawl_road_case(first_road_case):
count_fail = 0
latest_road_case = first_road_case
offset_road_case = first_road_case
while True:
end_road_case = offset_road_case + N_ROAD_CASE
cfg.logger.debug('offset_road_case: %s end_road_case: %s', offset_road_case, end_road_case)
road_cases = range(offset_road_case, end_road_case)
offset_road_case += N_ROAD_CASE
the_urls = {idx: 'http://www.road.tcg.gov.tw/ROADRCIS/GetCaseGeo.ashx?CASE_ID=%04d' % (idx) for idx in road_cases}
results = util.http_multiget(the_urls.values())
cfg.logger.debug('road_case: after http_multiget: results: %s', results)
if not results:
results = {}
is_success = False
for idx in road_cases:
the_url = the_urls[idx]
the_val = results[the_url]
if not the_val:
continue
if the_val == '-1':
continue
if the_val == u'-1':
continue
is_success = True
the_val = util.json_loads(the_val)
cfg.logger.debug('with_data: the_url: %s the_val: %s', the_url, the_val)
latest_road_case = idx
_process_data(the_val, 'taipei_city_road_case', idx)
if not is_success:
count_fail += 1
if count_fail >= N_COUNT_FAIL_ROAD_CASE:
break
util.save_cache('cron_taipei_city_latest_road_case', latest_road_case)
return latest_road_case
def _crawl_dig_point(first_dig_point):
count_fail = 0
latest_dig_point = first_dig_point
offset_dig_point = first_dig_point
while True:
end_dig_point = offset_dig_point + N_DIG_POINT
cfg.logger.debug('offset_dig_point: %s end_dig_point: %s', offset_dig_point, end_dig_point)
dig_points = range(offset_dig_point, end_dig_point)
offset_dig_point += N_DIG_POINT
the_urls = {idx: 'http://www.road.tcg.gov.tw/ROADRCIS/GetDigPoint.ashx?AP_NO=%08d' % (idx) for idx in dig_points}
results = util.http_multiget(the_urls.values())
cfg.logger.debug('dig_point: after http_multiget: results: %s', results)
if not results:
results = {}
is_success = False
for idx in dig_points:
the_url = the_urls[idx]
the_val = results[the_url]
if not the_val:
continue
if the_val == '-1':
continue
if the_val == u'-1':
continue
is_success = True
the_val = util.json_loads(the_val)
cfg.logger.debug('with_data: the_url: %s the_val: %s', the_url, the_val)
latest_dig_point = idx
_process_data(the_val, 'taipei_city_dig_point', idx)
if not is_success:
count_fail += 1
if count_fail >= N_COUNT_FAIL_DIG_POINT:
break
util.save_cache('cron_taipei_city_latest_dig_point', latest_dig_point)
return latest_dig_point
def _validate_http_result(result):
if not result:
return S_ERR
result_json = util.json_loads(result)
if not result_json:
return S_ERR
if result == '-1':
return S_ERR
if result == u'-1':
return S_ERR
if result == -1:
return S_ERR
return S_OK
def _validate_http_result(result):
if not result:
return S_ERR
result_json = util.json_loads(result)
if not result_json:
return S_ERR
if result == '-1':
return S_ERR
if result == u'-1':
return S_ERR
if result == -1:
return S_ERR
return S_OK
def post_add_handler(params):
json_struct = util.json_loads(params.get('json', '{}'))
(error_code,
error_param) = util.check_valid_params(json_struct, [
'county_name', 'the_category', 'the_idx', 'start_timestamp',
'end_timestamp', 'geo', 'the_data'
])
if error_code != S_OK:
return (error_code, {
"success": False,
"error_msg": "no " + error_param
})
cron_data.process_data(json_struct['county_name'],
json_struct['the_category'], json_struct['the_idx'],
json_struct['start_timestamp'],
json_struct['end_timestamp'], json_struct['geo'],
json_struct['the_data'])
return (S_OK, {"success": True})
def _post_register_google(content, state, session_struct, session_struct2,
request, params):
the_struct = util.json_loads(content)
user_id = 'google_' + str(the_struct['id'])
# save
util_user.save_user(
user_id, session_struct, session_struct2, {
"google_id": the_struct['id'],
'name': the_struct['name'],
'given_name': the_struct['given_name'],
'family_name': the_struct['family_name'],
'extension': the_struct
})
cfg.logger.debug('user_info: content: (%s, %s) the_struct: (%s, %s)',
content, content.__class__.__name__, the_struct,
the_struct.__class__.__name__)
# return
_redirect_register(state)
def _post_register_facebook(content, state, session_struct, session_struct2,
request, params):
the_struct = util.json_loads(content)
user_id = 'facebook_' + str(the_struct['id'])
# save
cfg.logger.debug('user_info: content: (%s, %s) the_struct: (%s, %s)',
content, content.__class__.__name__, the_struct,
the_struct.__class__.__name__)
util_user.save_user(
user_id, session_struct, session_struct2, {
"facebook_id": the_struct['id'],
'name': the_struct.get('name', ''),
'given_name': the_struct['first_name'],
'family_name': the_struct['last_name'],
'extension': the_struct
})
# return
_redirect_register(state)
def _get_geo(county_name, each_road):
query_string = county_name.encode('utf-8') + '+' + each_road.encode('utf-8')
quote_qs = urllib.quote(query_string)
url = 'http://maps.googleapis.com/maps/api/geocode/json'
the_url = url + '?sensor=false&address=' + quote_qs
results = util.http_multiget([the_url])
cfg.logger.debug('county_name: %s each_road: %s results: %s', county_name, each_road, results)
result = util.json_loads(results.get(the_url, ''))
cfg.logger.debug('county_name: %s each_road: %s result: %s', county_name, each_road, result)
status = result.get('status', '')
if status != 'OK':
cfg.logger.error('unable to retrieve geo info now')
return (S_ERR, [])
def _get_params(params):
if not params:
server = cfg.config.get('web_server', 'http://106.187.101.193:5346')
the_url = server + '/get/taipei_city_dig_point_next_dig_point'
http_result = util.http_multiget([the_url])
next_dig_point = util._int(util.json_loads(http_result.get(the_url, ''), ''), START_TAIPEI_CITY_DIG_POINT)
cfg.logger.debug('after http_multiget: http_result: %s next_dig_point: %s', http_result, next_dig_point)
return (S_OK, {'next_dig_point': next_dig_point})
next_dig_point = params.get('next_dig_point', START_TAIPEI_CITY_DIG_POINT)
year = next_dig_point // MAX_TAIPEI_CITY_DIG_POINTS_BY_YEAR
the_timestamp = util.get_timestamp()
tw_year = util.timestamp_to_tw_year(the_timestamp)
if tw_year <= year:
return (S_ERR, None)
next_dig_point = (year + 1) * MAX_TAIPEI_CITY_DIG_POINTS_BY_YEAR
return (S_OK, {'next_dig_point': next_dig_point})
def _get_params(params):
if not params:
server = cfg.config.get('web_server', 'http://106.187.101.193:5346')
the_url = server + '/get/new_taipei_city_dig_point_next_year'
http_result = util.http_multiget([the_url])
next_year = util._int(util.json_loads(http_result.get(the_url, ''), ''), START_NEW_TAIPEI_CITY_DIG_POINT_YEAR)
this_year = _get_this_year()
next_year = min(next_year, this_year)
cfg.logger.debug('after http_multiget: http_result: %s next_year: %s', http_result, next_year)
return (S_OK, {'next_year': next_year})
next_year = params.get('next_year', START_NEW_TAIPEI_CITY_DIG_POINT_YEAR)
stop_year = _get_stop_year()
if next_year == stop_year:
return (S_ERR, {'next_year': next_year})
next_year += 1
return (S_OK, {'next_year': next_year})
def _process_json_params():
the_content = _process_body_request()
return util.json_loads(the_content)
def _process_json_request():
f = request.body
f.seek(0)
return util.json_loads(f.read())
def _process_json_request():
f = request.body
f.seek(0)
return util.json_loads(f.read())
def _process_json_params():
the_content = _process_body_request()
return util.json_loads(the_content)
def _process_json_request():
return util.json_loads(_process_body_request())
