def run():
remove_creat_dir(shift_pro_dur_dir)
init_multiprocessor()
count_num_jobs = 0
for fn in get_all_files(shifts_dir, 'shift-hour-state-', '.csv'):
put_task(process_file, [fn])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(for_full_driver_dir)
init_multiprocessor()
count_num_jobs = 0
for fn in get_all_files(merged_trip_dir, 'trips', '.csv'):
# process_file(fn)
put_task(process_file, [fn])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(airport_trips_dir); remove_creat_dir(nightsafari_trips_dir)
csv_files = get_all_files(trips_dir, 'whole-trip-', '.csv')
init_multiprocessor()
count_num_jobs = 0
for fn in csv_files:
# process_file(fn)
put_task(process_file, [fn])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(trips_dir)
csv_files = get_all_files(merged_trip_dir, 'trips', '.csv')
init_multiprocessor()
counter = 0
for fn in csv_files:
counter += 1
put_task(process_file, [fn])
end_multiprocessor(counter)
def run():
remove_creat_dir(for_learning_dir)
csv_files = get_all_files(merged_trip_dir, 'trips', '.csv')
#
init_multiprocessor()
count_num_jobs = 0
for fn in csv_files:
# process_file(fn)
put_task(process_file, [fn])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(hourly_summary)
csv_files = get_all_files(trips_dir, 'whole-trip-', '.csv')
#
init_multiprocessor()
count_num_jobs = 0
for fn in csv_files:
put_task(process_file, [fn])
# process_file(fn)
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(hourly_summary)
csv_files = get_all_files(trips_dir, 'whole-trip-', '.csv')
#
init_multiprocessor()
count_num_jobs = 0
for fn in csv_files:
put_task(process_file, [fn])
# process_file(fn)
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(airport_trips_dir)
remove_creat_dir(nightsafari_trips_dir)
csv_files = get_all_files(trips_dir, 'whole-trip-', '.csv')
init_multiprocessor()
count_num_jobs = 0
for fn in csv_files:
# process_file(fn)
put_task(process_file, [fn])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
# task_list = [(0, 0), (0, 1), (0, 2), (0, 3), (0, 4), (0, 5), (0, 6), (0, 7), (0, 8), (0, 9), (0, 10), (2, 0), (2, 1), (2, 2), (2, 3), (2, 4), (2, 5), (2, 6), (2, 7), (2, 8), (2, 9), (2, 10), (4, 0), (4, 1), (4, 2), (4, 3), (4, 4), (4, 5), (4, 6), (4, 7), (4, 8), (4, 9), (4, 10), (6, 0), (6, 1), (6, 2), (6, 3), (6, 4), (6, 5), (6, 6), (6, 7), (6, 8), (6, 9), (6, 10), (8, 0), (8, 1), (8, 2), (8, 3), (8, 4), (8, 5), (8, 6), (8, 7), (8, 8), (8, 9), (8, 10), (10, 0), (10, 1), (10, 2), (10, 3), (10, 4), (10, 5), (10, 6), (10, 7), (10, 8), (10, 9), (10, 10)]
init_multiprocessor()
counter = 0
# for i, j in task_list:
# ALPHA = i / 10
# GAMMA = j / 10
for ALPHA, GAMMA in [(0.3, 0.7),(0.3, 0.9),(0.5, 0.1),(0.5, 0.3),(0.5, 0.8),(0.5, 1.0)]:
put_task(process_files, [ALPHA, GAMMA])
counter += 1
end_multiprocessor(counter)
def run():
csv_files = get_all_files(airport_trips_dir, 'airport-trip-', '.csv')
init_multiprocessor()
count_num_jobs = 0
for fn in csv_files:
try:
put_task(process_file, [fn])
except Exception as _:
logging_msg('Algorithm runtime exception (%s)\n' % (fn) + format_exc())
raise
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(full_shift_dir)
csv_files = get_all_files(shifts_dir, '', '.csv')
init_multiprocessor()
count_num_jobs = 0
for fn in csv_files:
try:
put_task(process_file, [fn])
except Exception as _:
logging_msg('Algorithm runtime exception (%s)\n' % (fn) + format_exc())
raise
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
# task_list = [(0, 0), (0, 1), (0, 2), (0, 3), (0, 4), (0, 5), (0, 6), (0, 7), (0, 8), (0, 9), (0, 10), (2, 0), (2, 1), (2, 2), (2, 3), (2, 4), (2, 5), (2, 6), (2, 7), (2, 8), (2, 9), (2, 10), (4, 0), (4, 1), (4, 2), (4, 3), (4, 4), (4, 5), (4, 6), (4, 7), (4, 8), (4, 9), (4, 10), (6, 0), (6, 1), (6, 2), (6, 3), (6, 4), (6, 5), (6, 6), (6, 7), (6, 8), (6, 9), (6, 10), (8, 0), (8, 1), (8, 2), (8, 3), (8, 4), (8, 5), (8, 6), (8, 7), (8, 8), (8, 9), (8, 10), (10, 0), (10, 1), (10, 2), (10, 3), (10, 4), (10, 5), (10, 6), (10, 7), (10, 8), (10, 9), (10, 10)]
init_multiprocessor()
counter = 0
# for i, j in task_list:
# ALPHA = i / 10
# GAMMA = j / 10
for ALPHA, GAMMA in [(0.3, 0.7), (0.3, 0.9), (0.5, 0.1), (0.5, 0.3),
(0.5, 0.8), (0.5, 1.0)]:
put_task(process_files, [ALPHA, GAMMA])
counter += 1
end_multiprocessor(counter)
def run():
remove_creat_dir(general_dur_fare_dir)
# process_files('0901')
init_multiprocessor()
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
put_task(process_files, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(ap_dur_fare_q_time_dir)
init_multiprocessor()
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
# process_files('1007')
put_task(process_files, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
csv_files = get_all_files(airport_trips_dir, 'airport-trip-', '.csv')
init_multiprocessor()
count_num_jobs = 0
for fn in csv_files:
try:
put_task(process_file, [fn])
except Exception as _:
logging_msg('Algorithm runtime exception (%s)\n' % (fn) +
format_exc())
raise
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
candi_dirs = get_all_directories(for_learning_dir)
q_lerning_ended_dir = [dn for dn in candi_dirs if len(get_all_files(for_learning_dir + '/%s' % (dn), 'ALPHA-', '.pkl')) == 22]
init_multiprocessor()
counter = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
# process_files(yymm, q_lerning_ended_dir)
put_task(process_files, [yymm, q_lerning_ended_dir])
counter += 1
end_multiprocessor(counter)
def run():
remove_creat_dir(individual_detail_dir)
# process_files('1007')
init_multiprocessor()
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
try:
yymm = "%02d%02d" % (y, m)
if yymm in ["0912", "1010"]:
continue
put_task(process_files, [yymm])
except Exception as _:
logging_msg("Algorithm runtime exception (%s)\n" % (yymm) + format_exc())
raise
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(individual_detail_dir)
# process_files('1007')
init_multiprocessor()
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
try:
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
put_task(process_files, [yymm])
except Exception as _:
logging_msg('Algorithm runtime exception (%s)\n' % (yymm) +
format_exc())
raise
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
candi_dirs = get_all_directories(for_learning_dir)
q_lerning_ended_dir = [
dn for dn in candi_dirs
if len(get_all_files(for_learning_dir + '/%s' %
(dn), 'ALPHA-', '.pkl')) == 22
]
init_multiprocessor()
counter = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
# process_files(yymm, q_lerning_ended_dir)
put_task(process_files, [yymm, q_lerning_ended_dir])
counter += 1
end_multiprocessor(counter)
def run():
remove_creat_dir(logs_dir)
csv_file_paths = []
for yd in os.listdir(server_prefix):
yd_path = '%s/%s' % (server_prefix, yd)
if not (os.path.isdir(yd_path) and yd.startswith('20')):
continue
if yd not in TARGET_YEARS:
continue
for md in os.listdir(yd_path):
md_path = '%s/%s' % (yd_path, md)
if not os.path.isdir(md_path):
continue
fn = 'logs-%s%s-normal.csv' % (yd[-2:], md)
csv_file_paths.append('%s/%s/%s' % (md_path, 'logs', fn))
init_multiprocessor()
count_num_jobs = 0
for path_to_csv_file in csv_file_paths:
# process_file(path_to_csv_file)
put_task(process_file, [path_to_csv_file])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_creat_dir(logs_dir)
csv_file_paths = []
for yd in os.listdir(server_prefix):
yd_path = '%s/%s' % (server_prefix, yd)
if not (os.path.isdir(yd_path) and yd.startswith('20')):
continue
if yd not in TARGET_YEARS:
continue
for md in os.listdir(yd_path):
md_path = '%s/%s' % (yd_path, md)
if not os.path.isdir(md_path):
continue
fn = 'logs-%s%s-normal.csv' % (yd[-2:], md)
csv_file_paths.append('%s/%s/%s' % (md_path, 'logs', fn))
init_multiprocessor()
count_num_jobs = 0
for path_to_csv_file in csv_file_paths:
# process_file(path_to_csv_file)
put_task(process_file, [path_to_csv_file])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def combine_trip_data_with_multi():
'''
In the server, when two files are combined, a trip can totally be expressed
A trip data shows the below information;
trip-id,job-id,start-time,end-time,start-long,start-lat,end-long,end-lat,vehicle-id,distance,fare,duration,start-dow,start-day,start-hour,start-minute,end-dow,end-day,end-hour,end-minute,start-zone,end-zone,start-postal,end-postal,driver-id
ex.
6029L09010100005,001010000,1230739200,1230739680,103.95029,1.37226,103.93417,1.38599,8069,3.4,850,480,4,1,0,0,4,1,0,8,51,51,518457,519355,20108
Also, this function uses the multiprocess library in Python.
'''
logging_on_txt('Start combine!!', gen_txt=True)
init_multiprocessor()
print 'Start combine'
for y in YEARS:
y_two_digit = y[-2:]
for m in MONTHS:
normal_file = TAXI_HOME + '/%s/%s/trips/trips-%s%s-normal.csv' % (y, m, y_two_digit, m)
ext_file = TAXI_HOME + '/%s/%s/trips/trips-%s%s-normal-ext.csv' % (y, m, y_two_digit, m)
put_task(read_write_a_trip, [normal_file, ext_file])
end_multiprocessor(len(YEARS) * len(MONTHS))