def get_raw_data(table_all, dir_name, table, table2, message):
is_yaml = 0
file_name = ""
file_names = os.listdir(dir_name) # get the file names in dir_name
# just ready for a file have one yaml file, if need more Please find all
for file_s in file_names:
if file_s[-5:] == '.yaml':
file_name = file_s
is_yaml = 1
break
if is_yaml == 0:
print("Can't find the yaml file!\n")
return
file_name = dir_name + '/' + file_name
print("Please wait to get the yaml data! name:\t %s" % file_name)
g_t1 = time.time()
f_value = open(file_name)
raw_values = yaml.load(f_value)
g_t2 = time.time()
'''
在这里需要加一个进行全局管理的函数将names 和 yaml 存入其中
同时 这两者的名字处理需要设定标准
'''
fm.storage_values_specie(table_all, table, table2, 'Likeyo', message,
file_name)
print("Load yaml over! Cost time: %.3f" % (g_t2 - g_t1))
raw_values = raw_values
g_all_length = len(raw_values)
# print g_all_length
print("Now read the data to move to database.")
for i in range(2, g_all_length):
value_temp = yl.deal_val_strategy(raw_values[i])
temp_id = test_db.insert_yaml(table2, message, value_temp)
print("The insert value's csv_file name: %s " %
value_temp[10]) # 0 1 2 3 4 5 6 7 8 9 10
if value_temp[10] in file_names:
s = dir_name + '/' + value_temp[10]
name = value_temp[10][:-4]
name1 = deal_name(name)
print(name1)
create_table(name1, message)
storage(name1, s, message)
storage_name(name1, table, message, temp_id)
print("Now make the single sort name table full!\n")
insert_index_names_16(table, message)
def test_unit1(table, message):
'''
dir_name1 = '/home/liziqiang/Desktop/SHOWyou/j_zs_WH_WOBV_DX_QG_GZ_20180102-81542_grid.yaml'
file_names = os.listdir(dir_name1)
yaml_file = ""
file_num = 0
is_yaml = 0
print "1 is ok"
for file_s in file_names:
if '.yaml' in file_s:
print file_s[-5:]
yaml_file = file_s
is_yaml = 1
break
# f_yaml = open(yaml_file)
print yaml_file
print file_names[0]
print len(file_names)
yaml_file = dir_name1+'/'+yaml_file
if is_yaml == 0:
print "can't find the yaml file"
return
'''
yaml_file = "5PP5LY1.yaml"
print "Please wait to get the yaml data! name:\t %s" % yaml_file
g_t1 = time.time()
f_value = open(yaml_file)
raw_values = yaml.load(f_value)
g_t2 = time.time()
print "Load yaml over! Cost time: %.3f" % (g_t2 - g_t1)
g_all_length = len(raw_values)
for i in range(2, g_all_length):
# g_t3 = time.time()
# value_temp = yl.deal_val_strategy(raw_values[i])
value_temp = yl.deal_val_strategy(raw_values[i])
if len(value_temp) < 11:
print "Wrong!"
return
temp_id = test_db.insert_yaml(table, message, value_temp)
def test_unit(table, message):
dir_name1 = '/home/liziqiang/Desktop/SHOWyou/j_zs_WH_WOBV_DX_QG_GZ_20180102-81542_grid.yaml'
file_names = os.listdir(dir_name1)
yaml_file = ""
file_num = 0
is_yaml = 0
print "1 is ok"
for file_s in file_names:
if '.yaml' in file_s:
print file_s[-5:]
yaml_file = file_s
is_yaml = 1
break
# f_yaml = open(yaml_file)
print yaml_file
print file_names[0]
print len(file_names)
yaml_file = dir_name1 + '/' + yaml_file
if is_yaml == 0:
print "can't find the yaml file"
return
# f_yaml = open(yaml_file)
print "Please wait to get the yaml data! name:\t %s" % yaml_file
g_t1 = time.time()
f_value = open(yaml_file)
raw_values = yaml.load(f_value)
g_t2 = time.time()
print "Load yaml over! Cost time: %.3f" % (g_t2 - g_t1)
g_all_length = len(raw_values)
print g_all_length - 2
for i in range(2, g_all_length):
value_temp = yl.deal_val_strategy(raw_values[i])
temp_id = test_db.insert_yaml(table1, message, value_temp)
if value_temp[10] in file_names:
file_num += 1
print "NUM: ", temp_id
print "The total file number: ", file_num
def test_unit3():
yaml_file = "5PP5LY1.yaml"
print "Please wait to get the yaml data! name:\t %s" % yaml_file
g_t1 = time.time()
f_value = open(yaml_file)
raw_values = yaml.load(f_value)
g_t2 = time.time()
print "Load yaml over! Cost time: %.3f" % (g_t2 - g_t1)
g_all_length = len(raw_values)
'''
val = yl.deal_val_strategy(raw_values[178])
for i in val:
print i
val1 = yl.deal_val_strategy(raw_values[179])
for i in val1:
print i
'''
val2 = yl.deal_val_strategy(raw_values[181])
test_db.insert_yaml(table1, message1, val2)
for i in val2:
print i
print raw_values[181][2]['Optimum']
def get_raw_data_selected_good(table_all, dir_name, message, file_name_vals):
is_yaml = 0
file_name = ""
file_names = os.listdir(dir_name) # get the file names in dir_name
# just ready for a file have one yaml file, if need more Please find all
for file_s in file_names:
if file_s[-5:] == '.yaml':
file_name = file_s
is_yaml = 1
break
if is_yaml == 0:
return
print(file_name)
print('success!')
'''
here need a name to make sure the single sort!
'''
file_temp_name = file_name
# make the table standard table = specie_table_name, table2 = specie_table_yaml
file_temp_name = get_name_yaml_table_name(file_temp_name)
specie_table_name = file_temp_name + '_name'
specie_table_yaml = file_temp_name + '_yaml'
table = specie_table_name
table2 = specie_table_yaml
file_name = dir_name + '/' + file_name
print("Create yaml_table: ")
test_db.create_yaml_table(table2, message) # 1
print("Create yaml_table success!")
print("Create name_table:")
create_names_table(table, table2, message) # 2
print("Create name_table success!")
print("Please wait to get the yaml data! name:\t %s" % file_name)
g_t1 = time.time()
f_value = open(file_name)
raw_values = yaml.load(f_value)
g_t2 = time.time()
print("Load yaml over! Cost time: %.3f" % (g_t2 - g_t1))
fm.storage_values_specie_new_test(table_all, table, table2, 'Likeyo',
message, file_name, raw_values[:2])
raw_values = raw_values
g_all_length = len(raw_values)
# print g_all_length
print("\nNow read the data to move to database.\n")
flag = 0
# in here create csv table and storage it, make name's can do it and yaml tables have it's information
for i in range(2, g_all_length):
value_temp = yl.deal_val_strategy(raw_values[i])
if value_temp[10] in file_name_vals:
print("The flag: ", flag)
temp_id = test_db.insert_yaml(table2, message, value_temp)
print("The insert value's csv_file name: %s " %
value_temp[10]) # 0 1 2 3 4 5 6 7 8 9 10
if value_temp[10] in file_names:
s = dir_name + '/' + value_temp[10]
name = value_temp[10][:-4]
name1 = deal_name(name)
print(name1)
create_table(name1, message)
storage(name1, s, message)
storage_name(name1, table, message, temp_id)
print
"\n\n"
insert_index_names_16(table, message)