if os.path.isdir(CSV_FILEPATH):
total_cost = 0
count = 0
while 1:
try:
csv_name = global_r0.lpop(GLOBAL_CSV_FIELS)
except Exception, e:
print e
continue
if not csv_name:
time.sleep(5)
continue
csv_input = load_items_from_csv(os.path.join(CSV_FILEPATH, csv_name))
load_origin_data_func = csv_input.__iter__
tmp_count, tmp_cost = send_all(load_origin_data_func, sender, pre_funcs=pre_funcs)
total_cost += tmp_cost
count += tmp_count
csv_input.close()
elif os.path.isfile(CSV_FILEPATH):
csv_input = load_items_from_csv(CSV_FILEPATH)
load_origin_data_func = csv_input.__iter__
count, total_cost = send_all(load_origin_data_func, sender, pre_funcs=pre_funcs)
csv_input.close()
# Send kill signal to workers
controller.send("KILL")
print 'send "KILL" to workers'
if from_bson:
pre_funcs = [bs_input_pre_func]
bs_input = load_items_from_bson()
load_origin_data_func = bs_input.reads
count, total_cost = send_all(load_origin_data_func, sender, pre_funcs=pre_funcs)
bs_input.close()
elif from_csv:
pre_funcs = [csv_input_pre_func]
from consts import CSV_FILEPATH
if os.path.isdir(CSV_FILEPATH):
total_cost = 0
count = 0
files = os.listdir(CSV_FILEPATH)
files = sorted([int(f[:-4]) for f in files])
for f in files:
csv_input = load_items_from_csv(os.path.join(CSV_FILEPATH, str(f) + '.csv'))
load_origin_data_func = csv_input.__iter__
tmp_count, tmp_cost = send_all(load_origin_data_func, sender, pre_funcs=pre_funcs)
total_cost += tmp_cost
count += tmp_count
csv_input.close()
elif os.path.isfile(CSV_FILEPATH):
csv_input = load_items_from_csv(CSV_FILEPATH)
load_origin_data_func = csv_input.__iter__
count, total_cost = send_all(load_origin_data_func, sender, pre_funcs=pre_funcs)
csv_input.close()
# Send kill signal to workers
controller.send("KILL")
print 'send "KILL" to workers'
load_origin_data_func = bs_input.reads
count, total_cost = send_all(load_origin_data_func,
sender,
pre_funcs=pre_funcs)
bs_input.close()
elif from_csv:
pre_funcs = [csv_input_pre_func]
from consts import CSV_FILEPATH
if os.path.isdir(CSV_FILEPATH):
total_cost = 0
count = 0
files = os.listdir(CSV_FILEPATH)
files = sorted([int(f[:-4]) for f in files])
for f in files:
csv_input = load_items_from_csv(
os.path.join(CSV_FILEPATH,
str(f) + '.csv'))
load_origin_data_func = csv_input.__iter__
tmp_count, tmp_cost = send_all(load_origin_data_func,
sender,
pre_funcs=pre_funcs)
total_cost += tmp_cost
count += tmp_count
csv_input.close()
elif os.path.isfile(CSV_FILEPATH):
csv_input = load_items_from_csv(CSV_FILEPATH)
load_origin_data_func = csv_input.__iter__
count, total_cost = send_all(load_origin_data_func,
sender,
pre_funcs=pre_funcs)
csv_input.close()
