def _parse_http_result(dig_point, result): data = util.json_loads(result) cfg.logger.debug('after json_loads: result: %s data: %s', result, data) if taipei_city.validate_http_result_json(data) != S_OK: return {} return _process_data(dig_point, data)
def _parse_http_result(road_case, result): data = util.json_loads(result) cfg.logger.debug('result: %s data: %s', result, data) if taipei_city.validate_http_result_json(data) != S_OK: return {} return _process_data(road_case, data)
def post_add_handler(params): json_struct = util.json_loads(params.get('json', '{}')) (error_code, error_param) = util.check_valid_params(json_struct, ['county_name', 'the_category', 'the_idx', 'start_timestamp', 'end_timestamp', 'geo', 'the_data']) if error_code != S_OK: return (error_code, {"success": False, "error_msg": "no " + error_param}) cron_data.process_data(json_struct['county_name'], json_struct['the_category'], json_struct['the_idx'], json_struct['start_timestamp'], json_struct['end_timestamp'], json_struct['geo'], json_struct['the_data']) return (S_OK, {"success": True})
def _process_data_text(the_category, the_idx, the_data_text, time_period_idx): the_data = util.json_loads(the_data_text) data_list = the_data if the_data.__class__.__name__ == 'list' else [the_data] len_data_list = len(data_list) if len_data_list == 1: _process_each_data(the_category, the_idx, data_list[0], time_period_idx) else: [_process_each_data(the_category, the_idx + '.' + data_idx, each_data, time_period_idx) for (data_idx, each_data) in enumerate(data_list)]
def _get_params(): server = cfg.config.get('web_server', 'http://106.187.101.193:5346') the_url = server + '/get/taipei_city_road_case_next_road_case' http_result = util.http_multiget([the_url]) next_road_case = util._int(util.json_loads(http_result.get(the_url, ''), ''), START_TAIPEI_CITY_ROAD_CASE) cfg.logger.debug('after http_multiget: http_result: %s next_road_case: %s', http_result, next_road_case) return {'next_road_case': next_road_case}
def _post_register_google(content, state, session_struct, session_struct2, request, params): the_struct = util.json_loads(content) user_id = 'google_' + str(the_struct['id']) # save util_user.save_user(user_id, session_struct, session_struct2, {"google_id": the_struct['id'], 'name': the_struct['name'], 'given_name': the_struct['given_name'], 'family_name': the_struct['family_name'], 'extension': the_struct}) cfg.logger.debug('user_info: content: (%s, %s) the_struct: (%s, %s)', content, content.__class__.__name__, the_struct, the_struct.__class__.__name__) # return _redirect_register(state)
def _post_register_facebook(content, state, session_struct, session_struct2, request, params): the_struct = util.json_loads(content) user_id = 'facebook_' + str(the_struct['id']) # save cfg.logger.debug('user_info: content: (%s, %s) the_struct: (%s, %s)', content, content.__class__.__name__, the_struct, the_struct.__class__.__name__) util_user.save_user(user_id, session_struct, session_struct2, {"facebook_id": the_struct['id'], 'name': the_struct.get('name', ''), 'given_name': the_struct['first_name'], 'family_name': the_struct['last_name'], 'extension': the_struct}) # return _redirect_register(state)
def _get_params(): server = cfg.config.get('web_server', 'http://106.187.101.193:5346') the_url = server + '/get/taipei_city_road_case_next_road_case' http_result = util.http_multiget([the_url]) next_road_case = util._int( util.json_loads(http_result.get(the_url, ''), ''), START_TAIPEI_CITY_ROAD_CASE) cfg.logger.debug('after http_multiget: http_result: %s next_road_case: %s', http_result, next_road_case) return {'next_road_case': next_road_case}
def _process_data_text(the_category, the_idx, the_data_text, time_period_idx): the_data = util.json_loads(the_data_text) data_list = the_data if the_data.__class__.__name__ == 'list' else [ the_data ] len_data_list = len(data_list) if len_data_list == 1: _process_each_data(the_category, the_idx, data_list[0], time_period_idx) else: [ _process_each_data(the_category, the_idx + '.' + data_idx, each_data, time_period_idx) for (data_idx, each_data) in enumerate(data_list) ]
def _crawl_road_case(first_road_case): count_fail = 0 latest_road_case = first_road_case offset_road_case = first_road_case while True: end_road_case = offset_road_case + N_ROAD_CASE cfg.logger.debug('offset_road_case: %s end_road_case: %s', offset_road_case, end_road_case) road_cases = range(offset_road_case, end_road_case) offset_road_case += N_ROAD_CASE the_urls = {idx: 'http://www.road.tcg.gov.tw/ROADRCIS/GetCaseGeo.ashx?CASE_ID=%04d' % (idx) for idx in road_cases} results = util.http_multiget(the_urls.values()) cfg.logger.debug('road_case: after http_multiget: results: %s', results) if not results: results = {} is_success = False for idx in road_cases: the_url = the_urls[idx] the_val = results[the_url] if not the_val: continue if the_val == '-1': continue if the_val == u'-1': continue is_success = True the_val = util.json_loads(the_val) cfg.logger.debug('with_data: the_url: %s the_val: %s', the_url, the_val) latest_road_case = idx _process_data(the_val, 'taipei_city_road_case', idx) if not is_success: count_fail += 1 if count_fail >= N_COUNT_FAIL_ROAD_CASE: break util.save_cache('cron_taipei_city_latest_road_case', latest_road_case) return latest_road_case
def _crawl_dig_point(first_dig_point): count_fail = 0 latest_dig_point = first_dig_point offset_dig_point = first_dig_point while True: end_dig_point = offset_dig_point + N_DIG_POINT cfg.logger.debug('offset_dig_point: %s end_dig_point: %s', offset_dig_point, end_dig_point) dig_points = range(offset_dig_point, end_dig_point) offset_dig_point += N_DIG_POINT the_urls = {idx: 'http://www.road.tcg.gov.tw/ROADRCIS/GetDigPoint.ashx?AP_NO=%08d' % (idx) for idx in dig_points} results = util.http_multiget(the_urls.values()) cfg.logger.debug('dig_point: after http_multiget: results: %s', results) if not results: results = {} is_success = False for idx in dig_points: the_url = the_urls[idx] the_val = results[the_url] if not the_val: continue if the_val == '-1': continue if the_val == u'-1': continue is_success = True the_val = util.json_loads(the_val) cfg.logger.debug('with_data: the_url: %s the_val: %s', the_url, the_val) latest_dig_point = idx _process_data(the_val, 'taipei_city_dig_point', idx) if not is_success: count_fail += 1 if count_fail >= N_COUNT_FAIL_DIG_POINT: break util.save_cache('cron_taipei_city_latest_dig_point', latest_dig_point) return latest_dig_point
def _validate_http_result(result): if not result: return S_ERR result_json = util.json_loads(result) if not result_json: return S_ERR if result == '-1': return S_ERR if result == u'-1': return S_ERR if result == -1: return S_ERR return S_OK
def post_add_handler(params): json_struct = util.json_loads(params.get('json', '{}')) (error_code, error_param) = util.check_valid_params(json_struct, [ 'county_name', 'the_category', 'the_idx', 'start_timestamp', 'end_timestamp', 'geo', 'the_data' ]) if error_code != S_OK: return (error_code, { "success": False, "error_msg": "no " + error_param }) cron_data.process_data(json_struct['county_name'], json_struct['the_category'], json_struct['the_idx'], json_struct['start_timestamp'], json_struct['end_timestamp'], json_struct['geo'], json_struct['the_data']) return (S_OK, {"success": True})
def _post_register_google(content, state, session_struct, session_struct2, request, params): the_struct = util.json_loads(content) user_id = 'google_' + str(the_struct['id']) # save util_user.save_user( user_id, session_struct, session_struct2, { "google_id": the_struct['id'], 'name': the_struct['name'], 'given_name': the_struct['given_name'], 'family_name': the_struct['family_name'], 'extension': the_struct }) cfg.logger.debug('user_info: content: (%s, %s) the_struct: (%s, %s)', content, content.__class__.__name__, the_struct, the_struct.__class__.__name__) # return _redirect_register(state)
def _post_register_facebook(content, state, session_struct, session_struct2, request, params): the_struct = util.json_loads(content) user_id = 'facebook_' + str(the_struct['id']) # save cfg.logger.debug('user_info: content: (%s, %s) the_struct: (%s, %s)', content, content.__class__.__name__, the_struct, the_struct.__class__.__name__) util_user.save_user( user_id, session_struct, session_struct2, { "facebook_id": the_struct['id'], 'name': the_struct.get('name', ''), 'given_name': the_struct['first_name'], 'family_name': the_struct['last_name'], 'extension': the_struct }) # return _redirect_register(state)
def _get_geo(county_name, each_road): query_string = county_name.encode('utf-8') + '+' + each_road.encode('utf-8') quote_qs = urllib.quote(query_string) url = 'http://maps.googleapis.com/maps/api/geocode/json' the_url = url + '?sensor=false&address=' + quote_qs results = util.http_multiget([the_url]) cfg.logger.debug('county_name: %s each_road: %s results: %s', county_name, each_road, results) result = util.json_loads(results.get(the_url, '')) cfg.logger.debug('county_name: %s each_road: %s result: %s', county_name, each_road, result) status = result.get('status', '') if status != 'OK': cfg.logger.error('unable to retrieve geo info now') return (S_ERR, [])
def _get_params(params): if not params: server = cfg.config.get('web_server', 'http://106.187.101.193:5346') the_url = server + '/get/taipei_city_dig_point_next_dig_point' http_result = util.http_multiget([the_url]) next_dig_point = util._int(util.json_loads(http_result.get(the_url, ''), ''), START_TAIPEI_CITY_DIG_POINT) cfg.logger.debug('after http_multiget: http_result: %s next_dig_point: %s', http_result, next_dig_point) return (S_OK, {'next_dig_point': next_dig_point}) next_dig_point = params.get('next_dig_point', START_TAIPEI_CITY_DIG_POINT) year = next_dig_point // MAX_TAIPEI_CITY_DIG_POINTS_BY_YEAR the_timestamp = util.get_timestamp() tw_year = util.timestamp_to_tw_year(the_timestamp) if tw_year <= year: return (S_ERR, None) next_dig_point = (year + 1) * MAX_TAIPEI_CITY_DIG_POINTS_BY_YEAR return (S_OK, {'next_dig_point': next_dig_point})
def _get_params(params): if not params: server = cfg.config.get('web_server', 'http://106.187.101.193:5346') the_url = server + '/get/new_taipei_city_dig_point_next_year' http_result = util.http_multiget([the_url]) next_year = util._int(util.json_loads(http_result.get(the_url, ''), ''), START_NEW_TAIPEI_CITY_DIG_POINT_YEAR) this_year = _get_this_year() next_year = min(next_year, this_year) cfg.logger.debug('after http_multiget: http_result: %s next_year: %s', http_result, next_year) return (S_OK, {'next_year': next_year}) next_year = params.get('next_year', START_NEW_TAIPEI_CITY_DIG_POINT_YEAR) stop_year = _get_stop_year() if next_year == stop_year: return (S_ERR, {'next_year': next_year}) next_year += 1 return (S_OK, {'next_year': next_year})
def _process_json_params(): the_content = _process_body_request() return util.json_loads(the_content)
def _process_json_request(): f = request.body f.seek(0) return util.json_loads(f.read())
def _process_json_request(): return util.json_loads(_process_body_request())