def run():
init_multiprocessor(11)
count_num_jobs = 0
# for tm in ['spendingTime', 'roamingTime']:
for tm in ['spendingTime']:
for year in ['2009', '2010', '2011', '2012']:
gt_dpath = dpaths[tm, year, 'groupTrips']
gt_prefix = prefixs[tm, year, 'groupTrips']
check_dir_create(gt_dpath)
# gs_dpath = dpaths[tm, year, 'groupShifts']
# for dpath in [gt_dpath, gs_dpath]:
# check_dir_create(dpath)
#
gp_dpath = dpaths[tm, year, 'groupPartition']
gp_prefix = prefixs[tm, year, 'groupPartition']
gp_summary_fpath = '%s/%ssummary.csv' % (gp_dpath, gp_prefix)
#
gs_df = pd.read_csv(gp_summary_fpath)
for gn in gs_df['groupName'].values:
igG = ig.Graph.Read_Pickle('%s/%s%s.pkl' % (gp_dpath, gp_prefix, gn))
groupDrivers = set()
for e in igG.es:
did0, did1 = [igG.vs[nIndex]['name'] for nIndex in e.tuple]
groupDrivers.add(did0)
groupDrivers.add(did1)
# process_file(tm, year, gn, groupDrivers)
put_task(process_file, [tm, year, gn, groupDrivers])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
check_dir_create(ss_trips_dpath)
#
init_multiprocessor(11)
count_num_jobs = 0
y = 9
for m in range(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
put_task(process_month, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
check_dir_create(ss_trips_dpath)
#
init_multiprocessor(11)
count_num_jobs = 0
y = 9
for m in range(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
put_task(process_month, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
check_dir_create(shift_pro_dur_dir)
#
init_multiprocessor(11)
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
# process_file(yymm)
put_task(process_file, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
check_dir_create(productivity_dir)
#
init_multiprocessor(11)
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
# process_files('1007')
put_task(process_files, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
check_dir_create(trips_dpath)
#
init_multiprocessor(11)
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
# both years data are corrupted
continue
put_task(tripMode_prevTripTime, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_create_dir(ap_ep_dir); remove_create_dir(ns_ep_dir)
#
init_multiprocessor(3)
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
# process_files(yymm)
put_task(process_files, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
check_dir_create(logs_last_day_dir)
#
init_multiprocessor(8)
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
# both years data_20160826 are corrupted
continue
# process_file(yymm)
put_task(process_file, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
remove_create_dir(ap_ep_dir)
remove_create_dir(ns_ep_dir)
#
init_multiprocessor(3)
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
# process_files(yymm)
put_task(process_files, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
check_dir_create(ap_crossing_dir); check_dir_create(ns_crossing_dir)
#
init_multiprocessor(11)
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
# both years data are corrupted
continue
# process_file(yymm)
put_task(process_file, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
for dpath in [economicProfit_ap_dpath, economicProfit_ns_dpath]:
check_dir_create(dpath)
init_multiprocessor(6)
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
# process_files(yymm)
put_task(process_files, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
for dpath in [ssDriverTrip_dpath, ssDriverShiftProDur_dpath, ssDriverEP_ap_dpath, ssDriverEP_ns_dpath]:
check_dir_create(dpath)
#
init_multiprocessor(11)
count_num_jobs = 0
for y in xrange(10, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
# process_files(yymm)
put_task(process_files, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
check_dir_create(logs_dir)
#
init_multiprocessor(8)
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
# both years data_20160826 are corrupted
continue
# process_files(yymm)
put_task(process_file, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
for dpath in [queueingTime_ap_dpath, queueingTime_ns_dpath]:
check_dir_create(dpath)
#
init_multiprocessor(6)
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
# both years data are corrupted
continue
# process_file(yymm)
put_task(process_file, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
for dpath in [queueingTime_ap_dpath, queueingTime_ns_dpath]:
check_dir_create(dpath)
#
init_multiprocessor(6)
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
# both years data are corrupted
continue
# process_file(yymm)
put_task(process_file, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
for dpath in [log_dpath, log_last_day_dpath, crossingTime_ap_dpath, crossingTime_ns_dpath]:
check_dir_create(dpath)
#
init_multiprocessor(11)
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
# both years data_20160826 are corrupted
continue
put_task(log_location_labeling, [yymm])
# put_task(log_last_day, [yymm])
# put_task(process_file, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
for dn in [ftd_gen_stat_dir,
ftd_prev_in_ap_stat_dir, ftd_prev_out_ap_stat_dir,
ftd_prev_in_ns_stat_dir, ftd_prev_out_ns_stat_dir]:
remove_create_dir(dn)
#
init_multiprocessor()
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
# process_files(yymm)
put_task(process_files, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
init_multiprocessor(6)
count_num_jobs = 0
tm = 'spendingTime'
# for year in ['2009', '2010', '2011', '2012']:
for year in ['2009']:
gds_dpath = dpaths[tm, year, 'groupDriverStats']
check_dir_create(gds_dpath)
#
gm_dpath = dpaths[tm, year, 'groupMarginal']
gm_prefix = prefixs[tm, year, 'groupMarginal']
for fn in get_all_files(gm_dpath, '%s*.csv' % gm_prefix):
_, _, _, gn = fn[:-len('.csv')].split('-')
# process_file(tm, year, gn)
put_task(process_file, [tm, year, gn])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
for path in [ftd_trips_dir, ftd_shift_dir]:
remove_create_dir(path)
#
init_multiprocessor(3)
count_num_jobs = 0
for y in xrange(9, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
# process_files(yymm)
put_task(process_files, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
for dpath in [
ssDriverTrip_dpath, ssDriverShiftProDur_dpath, ssDriverEP_ap_dpath,
ssDriverEP_ns_dpath
]:
check_dir_create(dpath)
#
init_multiprocessor(11)
count_num_jobs = 0
for y in xrange(10, 11):
for m in xrange(1, 13):
yymm = '%02d%02d' % (y, m)
if yymm in ['0912', '1010']:
continue
# process_files(yymm)
put_task(process_files, [yymm])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
init_multiprocessor(6)
count_num_jobs = 0
for tm in ['spendingTime']:
# for year in ['2009', '2010', '2011', '2012']:
for year in ['2009']:
gm_dpath = dpaths[tm, year, 'groupMarginal']
check_dir_create(gm_dpath)
#
gp_dpath = dpaths[tm, year, 'groupPartition']
gp_prefix = prefixs[tm, year, 'groupPartition']
for fn in get_all_files(gp_dpath, '%s*.pkl' % gp_prefix):
_, _, _, gn = fn[:-len('.pkl')].split('-')
if gn == 'drivers' or gn == 'original':
continue
# process_file(tm, year, gn)
put_task(process_file, [tm, year, gn])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
check_dir_create(tfZ_TP_dpath)
numWorker = 6
init_multiprocessor(numWorker)
count_num_jobs = 0
numReducers = numWorker * 10
#
yyyy = '20%02d' % 12
logger.info('loading driversRelations %s' % yyyy)
driversRelations = load_pickle_file(driversRelations_fpaths[yyyy])
whole_drivers = driversRelations.keys()
driver_subsets = [[] for _ in range(numReducers)]
for i, did in enumerate(whole_drivers):
driver_subsets[i % numReducers].append(did)
for i, driver_subset in enumerate(driver_subsets):
# process_files(yyyy, i, driver_subset, driversRelations)
pickUp_drivers = set()
for did1 in driver_subset:
pickUp_drivers = pickUp_drivers.union(driversRelations[did1])
put_task(process_files, [yyyy, i, driver_subset, pickUp_drivers])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
check_dir_create(tfZ_TP_dpath)
numWorker = 6
init_multiprocessor(numWorker)
count_num_jobs = 0
numReducers = numWorker * 10
#
yyyy = '20%02d' % 12
logger.info('loading driversRelations %s' % yyyy)
driversRelations = load_pickle_file(driversRelations_fpaths[yyyy])
whole_drivers = driversRelations.keys()
driver_subsets = [[] for _ in range(numReducers)]
for i, did in enumerate(whole_drivers):
driver_subsets[i % numReducers].append(did)
for i, driver_subset in enumerate(driver_subsets):
# process_files(yyyy, i, driver_subset, driversRelations)
pickUp_drivers = set()
for did1 in driver_subset:
pickUp_drivers = pickUp_drivers.union(driversRelations[did1])
put_task(process_files, [yyyy, i, driver_subset, pickUp_drivers])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
init_multiprocessor(6)
count_num_jobs = 0
tm = 'baseline'
# for tm in ['spendingTime', 'roamingTime']:
# for year in ['2009', '2010', '2011', '2012']:
for year in ['2009']:
gz_dpath = dpaths[tm, year, 'groupZones']
check_dir_create(gz_dpath)
#
gt_dpath = dpaths[tm, year, 'groupTrips']
gt_prefix = prefixs[tm, year, 'groupTrips']
for fn in get_all_files(gt_dpath, '%s*' % gt_prefix):
if len(fn[:-len('.csv')].split('-')) != 4:
continue
_, _, _, gn = fn[:-len('.csv')].split('-')
if gn == 'X':
continue
gt_fpath = '%s/%s' % (gt_dpath, fn)
# process_file(tm, year, gt_fpath)
put_task(process_file, [tm, year, gt_fpath])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def run():
init_multiprocessor(6)
count_num_jobs = 0
tm = 'spendingTime'
# for tm in ['spendingTime', 'roamingTime']:
# for year in ['2009', '2010', '2011', '2012']:
for year in ['2009']:
gz_dpath = dpaths[tm, year, 'groupZones']
check_dir_create(gz_dpath)
#
gt_dpath = dpaths[tm, year, 'groupTrips']
gt_prefix = prefixs[tm, year, 'groupTrips']
for fn in get_all_files(gt_dpath, '%s*' % gt_prefix):
if len(fn[:-len('.csv')].split('-')) != 4:
continue
_, _, _, gn = fn[:-len('.csv')].split('-')
if gn == 'X':
continue
gt_fpath = '%s/%s' % (gt_dpath, fn)
# process_file(tm, year, gt_fpath)
put_task(process_file, [tm, year, gt_fpath])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)