def run():
remove_creat_dir(shift_pro_dur_dir)
init_multiprocessor()
count_num_jobs = 0
for fn in get_all_files(shifts_dir, 'shift-hour-state-', '.csv'):
put_task(process_file, [fn])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(for_full_driver_dir)
init_multiprocessor()
count_num_jobs = 0
for fn in get_all_files(merged_trip_dir, 'trips', '.csv'):
# process_file(fn)
put_task(process_file, [fn])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(airport_trips_dir); remove_creat_dir(nightsafari_trips_dir)
csv_files = get_all_files(trips_dir, 'whole-trip-', '.csv')
init_multiprocessor()
count_num_jobs = 0
for fn in csv_files:
# process_file(fn)
put_task(process_file, [fn])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def process_files(ALPHA, GAMMA):
ALPHA_GAMMA_dir = for_learning_dir + '/ALPHA-%.2f-GAMMA-%.2f' % (ALPHA, GAMMA)
remove_creat_dir(ALPHA_GAMMA_dir)
#
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
process_file(ALPHA, GAMMA, ALPHA_GAMMA_dir, yymm)
def run():
remove_creat_dir(trips_dir)
csv_files = get_all_files(merged_trip_dir, 'trips', '.csv')
init_multiprocessor()
counter = 0
for fn in csv_files:
counter += 1
put_task(process_file, [fn])
end_multiprocessor(counter)
def process_files(ALPHA, GAMMA):
ALPHA_GAMMA_dir = for_learning_dir + '/ALPHA-%.2f-GAMMA-%.2f' % (ALPHA,
GAMMA)
remove_creat_dir(ALPHA_GAMMA_dir)
#
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
process_file(ALPHA, GAMMA, ALPHA_GAMMA_dir, yymm)
def run():
remove_creat_dir(hourly_summary)
csv_files = get_all_files(trips_dir, 'whole-trip-', '.csv')
#
init_multiprocessor()
count_num_jobs = 0
for fn in csv_files:
put_task(process_file, [fn])
# process_file(fn)
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(hourly_summary)
csv_files = get_all_files(trips_dir, 'whole-trip-', '.csv')
#
init_multiprocessor()
count_num_jobs = 0
for fn in csv_files:
put_task(process_file, [fn])
# process_file(fn)
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(for_learning_dir)
csv_files = get_all_files(merged_trip_dir, 'trips', '.csv')
#
init_multiprocessor()
count_num_jobs = 0
for fn in csv_files:
# process_file(fn)
put_task(process_file, [fn])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(airport_trips_dir)
remove_creat_dir(nightsafari_trips_dir)
csv_files = get_all_files(trips_dir, 'whole-trip-', '.csv')
init_multiprocessor()
count_num_jobs = 0
for fn in csv_files:
# process_file(fn)
put_task(process_file, [fn])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(full_shift_dir)
csv_files = get_all_files(shifts_dir, '', '.csv')
init_multiprocessor()
count_num_jobs = 0
for fn in csv_files:
try:
put_task(process_file, [fn])
except Exception as _:
logging_msg('Algorithm runtime exception (%s)\n' % (fn) + format_exc())
raise
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(general_dur_fare_dir)
# process_files('0901')
init_multiprocessor()
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
put_task(process_files, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(ap_dur_fare_q_time_dir)
init_multiprocessor()
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
# process_files('1007')
put_task(process_files, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(individual_detail_dir)
# process_files('1007')
init_multiprocessor()
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
try:
yymm = "%02d%02d" % (y, m)
if yymm in ["0912", "1010"]:
continue
put_task(process_files, [yymm])
except Exception as _:
logging_msg("Algorithm runtime exception (%s)\n" % (yymm) + format_exc())
raise
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(individual_detail_dir)
# process_files('1007')
init_multiprocessor()
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
try:
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
put_task(process_files, [yymm])
except Exception as _:
logging_msg('Algorithm runtime exception (%s)\n' % (yymm) +
format_exc())
raise
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(logs_dir)
csv_file_paths = []
for yd in os.listdir(server_prefix):
yd_path = '%s/%s' % (server_prefix, yd)
if not (os.path.isdir(yd_path) and yd.startswith('20')):
continue
if yd not in TARGET_YEARS:
continue
for md in os.listdir(yd_path):
md_path = '%s/%s' % (yd_path, md)
if not os.path.isdir(md_path):
continue
fn = 'logs-%s%s-normal.csv' % (yd[-2:], md)
csv_file_paths.append('%s/%s/%s' % (md_path, 'logs', fn))
init_multiprocessor()
count_num_jobs = 0
for path_to_csv_file in csv_file_paths:
# process_file(path_to_csv_file)
put_task(process_file, [path_to_csv_file])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(logs_dir)
csv_file_paths = []
for yd in os.listdir(server_prefix):
yd_path = '%s/%s' % (server_prefix, yd)
if not (os.path.isdir(yd_path) and yd.startswith('20')):
continue
if yd not in TARGET_YEARS:
continue
for md in os.listdir(yd_path):
md_path = '%s/%s' % (yd_path, md)
if not os.path.isdir(md_path):
continue
fn = 'logs-%s%s-normal.csv' % (yd[-2:], md)
csv_file_paths.append('%s/%s/%s' % (md_path, 'logs', fn))
init_multiprocessor()
count_num_jobs = 0
for path_to_csv_file in csv_file_paths:
# process_file(path_to_csv_file)
put_task(process_file, [path_to_csv_file])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
