def create_weather_data_by_group(input_file_path, city_id):
read_lines = FileUtils.read_file_and_ret_lines(input_file_path)
if len(read_lines) == 0:
print 'The lines count is 0 in file: %s' % input_file_path
exit(1)
split_word = ': '
output_lines = []
for line in read_lines:
if line.find(split_word) < 0:
continue
json_data = HttpJsonUtils.json_parse(line.split(split_word)[1])
if json_data['errNum'] != 0:
continue
tmp_city_id = json_data['retData']['cityid']
if tmp_city_id == city_id:
# output_lines.append(line.decode(FileUtils.CHARSET_UTF8))
output_lines.append(line)
# end for
if len(output_lines) == 0:
logging.warn('The weather data for city %s is none.' % city_id)
exit(1)
output_file_name = 'baidu_weather_data_update_freq_%s.log' % city_id
output_file_path = os.path.join(g_log_dir, output_file_name)
# FileUtils.write_lines_to_file(output_file_path,output_lines,flag_override=True,charset=FileUtils.CHARSET_UTF8)
FileUtils.write_lines_to_file(output_file_path, output_lines)
return output_file_path
def create_weather_data_by_group(input_file_path, city_id):
read_lines = FileUtils.read_file_and_ret_lines(input_file_path)
if len(read_lines) == 0:
print 'The lines count is 0 in file: %s' % input_file_path
exit(1)
split_word = ': '
output_lines = []
for line in read_lines:
if line.find(split_word) < 0:
continue
json_data = HttpJsonUtils.json_parse(line.split(split_word)[1])
if json_data['errNum'] != 0:
continue
tmp_city_id = json_data['retData']['cityid']
if tmp_city_id == city_id:
# output_lines.append(line.decode(FileUtils.CHARSET_UTF8))
output_lines.append(line)
# end for
if len(output_lines) == 0:
logging.warn('The weather data for city %s is none.' % city_id)
exit(1)
output_file_name = 'baidu_weather_data_update_freq_%s.log' % city_id
output_file_path = os.path.join(g_log_dir, output_file_name)
# FileUtils.write_lines_to_file(output_file_path,output_lines,flag_override=True,charset=FileUtils.CHARSET_UTF8)
FileUtils.write_lines_to_file(output_file_path, output_lines)
return output_file_path
def run_instrument_tests_v2(cmd):
input_lines = WinSysUtils.run_sys_cmd_and_ret_lines(cmd)
if len(input_lines) == 0:
return
output_lines = []
for line in input_lines:
output_lines.append(line.rstrip('\r\n') + '\n')
FileUtils.append_lines_to_file(g_local_inst_run_log_file_path, (output_lines))
check_test_case_force_closed(output_lines)
check_test_case_failed(output_lines)
def parse_weather_data_update_freq(file_path):
changed_num = 0
input_lines = FileUtils.read_file_and_ret_lines(file_path)
lines_size = len(input_lines)
if lines_size == 0:
print 'The lines count is 0 in file %s' % file_path
return
output_lines = []
output_lines.append('\n\n--------- Weather data changes log ---------\n')
for i in xrange(0, (lines_size - 1)): # compare current record with next
tmp_line_1 = input_lines[i]
tmp_line_2 = input_lines[i + 1]
tmp_items_1 = tmp_line_1.split(': ')
tmp_items_2 = tmp_line_2.split(': ')
tmp_json_data_1 = HttpJsonUtils.json_parse(tmp_items_1[1])
tmp_json_data_2 = HttpJsonUtils.json_parse(tmp_items_2[1])
tmp_lines_today = diff_today_weather_data(tmp_json_data_1,
tmp_json_data_2)
tmp_lines_forecast = diff_forecast_weather_data(
tmp_json_data_1, tmp_json_data_2)
if len(tmp_lines_today) > 0 or len(tmp_lines_forecast) > 0:
changed_num += 1
tmp_line = '\nThe weather data changed, %s to %s\n' % (
tmp_items_1[0], tmp_items_2[0])
output_lines.append(tmp_line)
if len(tmp_lines_today) > 0:
output_lines.append('<Today> data changed: \n')
for line in tmp_lines_today:
output_lines.append(line)
if len(tmp_lines_forecast) > 0:
output_lines.append('<Forecast> data changed: \n')
for line in tmp_lines_forecast:
output_lines.append(line)
# end for
lines_summary = create_weather_data_summary(lines_size, changed_num)
for line in lines_summary:
output_lines.append(line)
FileUtils.append_lines_to_file(file_path, output_lines)
def diff_main():
global g_list_of_city_ids_not_match
global g_list_of_city_names_not_match
data_dir = os.path.join(os.getcwd(), 'data')
lines_src_local = FileUtils.read_file_and_ret_lines(
os.path.join(data_dir, 'weather_local.txt'))
lines_src_web = FileUtils.read_file_and_ret_lines(
os.path.join(data_dir, 'weather_web.txt'))
for line_local in lines_src_local:
fields_local = line_local.rstrip('\n').split(',')
city_id_local = fields_local[0]
city_name_local = fields_local[1]
logging.info('START ---> match city id %s, city name %s' %
(city_id_local, city_name_local))
flag_id_found = False
flag_name_found = False
for line_web in lines_src_web:
fields_web = line_web.rstrip('\n').split(',')
city_id_web = fields_web[0]
city_name_web = fields_web[1]
if city_id_local == city_id_web:
logging.info('match to web city id %s, city name %s' %
(city_id_web, city_name_web))
flag_id_found = True
if city_name_local == city_name_web:
flag_name_found = True
break
# end for
if not flag_id_found:
g_list_of_city_ids_not_match[city_id_local] = city_name_local
logging.error('The city id(%s) is NOT match!' % (city_name_local))
if not flag_name_found:
g_list_of_city_names_not_match[city_id_local] = city_name_local
logging.error('The city name(%s) is NOT match!' %
(city_name_local))
logging.info('END ---> match city id %s, city name %s\n' %
(city_name_local, city_name_local))
def read_city_list():
file_path = os.path.join(os.getcwd(), 'data', g_const_city_list_file_name)
output_lines = FileUtils.read_file_and_ret_lines(file_path)
if (len(output_lines) == 0):
logging.error('No lines read from city list file!')
exit(1)
global g_total_num_cities
g_total_num_cities = len(output_lines)
return output_lines
def parse_weather_data_update_freq(file_path):
changed_num = 0
input_lines = FileUtils.read_file_and_ret_lines(file_path)
lines_size = len(input_lines)
if lines_size == 0:
print 'The lines count is 0 in file %s' % file_path
return
output_lines = []
output_lines.append('\n\n--------- Weather data changes log ---------\n')
for i in xrange(0, (lines_size - 1)): # compare current record with next
tmp_line_1 = input_lines[i]
tmp_line_2 = input_lines[i + 1]
tmp_items_1 = tmp_line_1.split(': ')
tmp_items_2 = tmp_line_2.split(': ')
tmp_json_data_1 = HttpJsonUtils.json_parse(tmp_items_1[1])
tmp_json_data_2 = HttpJsonUtils.json_parse(tmp_items_2[1])
tmp_lines_today = diff_today_weather_data(tmp_json_data_1, tmp_json_data_2)
tmp_lines_forecast = diff_forecast_weather_data(tmp_json_data_1, tmp_json_data_2)
if len(tmp_lines_today) > 0 or len(tmp_lines_forecast) > 0:
changed_num += 1
tmp_line = '\nThe weather data changed, %s to %s\n' % (tmp_items_1[0], tmp_items_2[0])
output_lines.append(tmp_line)
if len(tmp_lines_today) > 0:
output_lines.append('<Today> data changed: \n')
for line in tmp_lines_today:
output_lines.append(line)
if len(tmp_lines_forecast) > 0:
output_lines.append('<Forecast> data changed: \n')
for line in tmp_lines_forecast:
output_lines.append(line)
# end for
lines_summary = create_weather_data_summary(lines_size, changed_num)
for line in lines_summary:
output_lines.append(line)
FileUtils.append_lines_to_file(file_path, output_lines)
def diff_main():
global g_list_of_city_ids_not_match
global g_list_of_city_names_not_match
data_dir = os.path.join(os.getcwd(), 'data')
lines_src_local = FileUtils.read_file_and_ret_lines(os.path.join(data_dir, 'weather_local.txt'))
lines_src_web = FileUtils.read_file_and_ret_lines(os.path.join(data_dir, 'weather_web.txt'))
for line_local in lines_src_local:
fields_local = line_local.rstrip('\n').split(',')
city_id_local = fields_local[0]
city_name_local = fields_local[1]
logging.info('START ---> match city id %s, city name %s' % (city_id_local, city_name_local))
flag_id_found = False
flag_name_found = False
for line_web in lines_src_web:
fields_web = line_web.rstrip('\n').split(',')
city_id_web = fields_web[0]
city_name_web = fields_web[1]
if city_id_local == city_id_web:
logging.info('match to web city id %s, city name %s' % (city_id_web, city_name_web))
flag_id_found = True
if city_name_local == city_name_web:
flag_name_found = True
break
# end for
if not flag_id_found:
g_list_of_city_ids_not_match[city_id_local] = city_name_local
logging.error('The city id(%s) is NOT match!' % city_name_local)
if not flag_name_found:
g_list_of_city_names_not_match[city_id_local] = city_name_local
logging.error('The city name(%s) is NOT match!' % city_name_local)
logging.info('END ---> match city id %s, city name %s\n' % (city_name_local, city_name_local))
def init_log():
FileUtils.create_dir(g_log_dir)
log_name = 'baidu_weather_data_update_freq_run_log_%s.log' % g_cur_date
log_file = os.path.join(g_log_dir, log_name)
LogUtils.init_log_config(logging.DEBUG, logging.INFO, log_file)
resp = None
try:
resp = HttpJsonUtils.send_get_request_with_header_and_return(
url, header_parms, {'cityid': id})
except Exception, e:
logging.error('Exception: %s' % e)
if resp is None:
return
line = '%s: %s\n' % (WinSysUtils.get_current_date_and_time(),
resp.decode('unicode_escape'))
output_lines.append(line)
# end for
output_lines.append('\n')
FileUtils.append_lines_to_file(file_path, output_lines,
FileUtils.CHARSET_UTF8)
def log_cur_run_time(run_time):
if run_time > 60:
logging.info('Current run time: %d minutes, %d seconds' %
(run_time / 60, run_time % 60))
else:
logging.info('Current run time: %d seconds' % (run_time % 60))
def run_loop(during, interval):
logging.info('----- START: test baidu weather data update frequency')
start_time = time.clock()
i = 1
def init_log():
FileUtils.create_dir(g_log_dir)
log_name = 'baidu_weather_data_update_freq_run_log_%s.log' % g_cur_date
log_file = os.path.join(g_log_dir, log_name)
LogUtils.init_log_config(logging.DEBUG, logging.INFO, log_file)
output_lines = []
for id in city_ids:
resp = None
try:
resp = HttpJsonUtils.send_get_request_with_header_and_return(url, header_parms, {'cityid':id})
except Exception, e:
logging.error('Exception: %s' % e)
if resp is None:
return
line = '%s: %s\n' % (WinSysUtils.get_current_date_and_time(), resp.decode('unicode_escape'))
output_lines.append(line)
# end for
output_lines.append('\n')
FileUtils.append_lines_to_file(file_path, output_lines, FileUtils.CHARSET_UTF8)
def log_cur_run_time(run_time):
if run_time > 60:
logging.info('Current run time: %d minutes, %d seconds' % (run_time / 60, run_time % 60))
else:
logging.info('Current run time: %d seconds' % (run_time % 60))
def run_loop(during, interval):
logging.info('----- START: test baidu weather data update frequency')
start_time = time.clock()
i = 1
# while True:
while 1:
logging.info('Send request %d times.' % i)