if os.path.isdir(CSV_FILEPATH): total_cost = 0 count = 0 while 1: try: csv_name = global_r0.lpop(GLOBAL_CSV_FIELS) except Exception, e: print e continue if not csv_name: time.sleep(5) continue csv_input = load_items_from_csv(os.path.join(CSV_FILEPATH, csv_name)) load_origin_data_func = csv_input.__iter__ tmp_count, tmp_cost = send_all(load_origin_data_func, sender, pre_funcs=pre_funcs) total_cost += tmp_cost count += tmp_count csv_input.close() elif os.path.isfile(CSV_FILEPATH): csv_input = load_items_from_csv(CSV_FILEPATH) load_origin_data_func = csv_input.__iter__ count, total_cost = send_all(load_origin_data_func, sender, pre_funcs=pre_funcs) csv_input.close() # Send kill signal to workers controller.send("KILL") print 'send "KILL" to workers'
if from_bson: pre_funcs = [bs_input_pre_func] bs_input = load_items_from_bson() load_origin_data_func = bs_input.reads count, total_cost = send_all(load_origin_data_func, sender, pre_funcs=pre_funcs) bs_input.close() elif from_csv: pre_funcs = [csv_input_pre_func] from consts import CSV_FILEPATH if os.path.isdir(CSV_FILEPATH): total_cost = 0 count = 0 files = os.listdir(CSV_FILEPATH) files = sorted([int(f[:-4]) for f in files]) for f in files: csv_input = load_items_from_csv(os.path.join(CSV_FILEPATH, str(f) + '.csv')) load_origin_data_func = csv_input.__iter__ tmp_count, tmp_cost = send_all(load_origin_data_func, sender, pre_funcs=pre_funcs) total_cost += tmp_cost count += tmp_count csv_input.close() elif os.path.isfile(CSV_FILEPATH): csv_input = load_items_from_csv(CSV_FILEPATH) load_origin_data_func = csv_input.__iter__ count, total_cost = send_all(load_origin_data_func, sender, pre_funcs=pre_funcs) csv_input.close() # Send kill signal to workers controller.send("KILL") print 'send "KILL" to workers'
load_origin_data_func = bs_input.reads count, total_cost = send_all(load_origin_data_func, sender, pre_funcs=pre_funcs) bs_input.close() elif from_csv: pre_funcs = [csv_input_pre_func] from consts import CSV_FILEPATH if os.path.isdir(CSV_FILEPATH): total_cost = 0 count = 0 files = os.listdir(CSV_FILEPATH) files = sorted([int(f[:-4]) for f in files]) for f in files: csv_input = load_items_from_csv( os.path.join(CSV_FILEPATH, str(f) + '.csv')) load_origin_data_func = csv_input.__iter__ tmp_count, tmp_cost = send_all(load_origin_data_func, sender, pre_funcs=pre_funcs) total_cost += tmp_cost count += tmp_count csv_input.close() elif os.path.isfile(CSV_FILEPATH): csv_input = load_items_from_csv(CSV_FILEPATH) load_origin_data_func = csv_input.__iter__ count, total_cost = send_all(load_origin_data_func, sender, pre_funcs=pre_funcs) csv_input.close()