def test():
extreme_drivers_policy = load_picle_file('extreme_drivers_policy.pkl')
q_learning_policy = load_picle_file('q_learning_policy.pkl')
for s1 in DAY_OF_WEEK:
for s2 in TIME_SLOTS:
k = (s1, s2, IN_AP)
if extreme_drivers_policy.has_key(k) and q_learning_policy.has_key(k):
print k, extreme_drivers_policy[k], q_learning_policy[k]
print ''
for s1 in DAY_OF_WEEK:
for s2 in TIME_SLOTS:
k = (s1, s2, OUT_AP)
if extreme_drivers_policy.has_key(k) and q_learning_policy.has_key(k):
print k, extreme_drivers_policy[k], q_learning_policy[k]
def process_files(yymm):
# Qsa_value, _ = load_picle_file('%s/q-value-fare-dur-%s.pkl' % (for_learning_dir, yymm))
Qsa_value, _ = load_picle_file(
'/Users/JerryHan88/taxi/for_learning/ALPHA-0.70-GAMMA-0.30/ALPHA-0.70-GAMMA-0.30-q-value-fare-dur-0904.pkl'
)
decision_inAP, decision_outAP = [], []
for s1 in DAY_OF_WEEK:
for s2 in TIME_SLOTS:
for s3 in [IN_AP, OUT_AP]:
# print (s1, s2, s3), Qsa_value[(s1, s2, s3, IN_AP)], Qsa_value[(s1, s2, s3, OUT_AP)]
max_a = IN_AP if Qsa_value[(s1, s2, s3, IN_AP)] >= Qsa_value[(
s1, s2, s3, OUT_AP)] else OUT_AP
c = 0 if max_a == IN_AP else 1
if s3 == IN_AP:
decision_inAP.append([s1, s2, c])
else:
decision_outAP.append([s1, s2, c])
for s1, s2, a in decision_inAP:
print 'inAP', s1, s2, a
for s1, s2, a in decision_outAP:
print 'outAP', s1, s2, a
_data = [[(s2, id_dow[s1], a) for s1, s2, a in decision_inAP],
[(s2, id_dow[s1], a) for s1, s2, a in decision_outAP]]
# titles = ['Decision in the airport', 'Decision outside of the airport']
# grid_charts(('24 Time slots', []), ('', DAY_OF_WEEK), ['In Airport', 'Out Airport'], titles, _data, '%s/q-value-fare-dur-%s.pdf' % (for_learning_dir, yymm))
one_grid_chart(('Time slot', []), ('', DAY_OF_WEEK),
['In Airport', 'Out Airport'], '',
[(s2, id_dow[s1], a)
for s1, s2, a in decision_inAP], 'decision_inAP_q_e')
def test():
extreme_drivers_policy = load_picle_file('extreme_drivers_policy.pkl')
q_learning_policy = load_picle_file('q_learning_policy.pkl')
for s1 in DAY_OF_WEEK:
for s2 in TIME_SLOTS:
k = (s1, s2, IN_AP)
if extreme_drivers_policy.has_key(k) and q_learning_policy.has_key(
k):
print k, extreme_drivers_policy[k], q_learning_policy[k]
print ''
for s1 in DAY_OF_WEEK:
for s2 in TIME_SLOTS:
k = (s1, s2, OUT_AP)
if extreme_drivers_policy.has_key(k) and q_learning_policy.has_key(
k):
print k, extreme_drivers_policy[k], q_learning_policy[k]
def process_files(yymm):
# Qsa_value, _ = load_picle_file('%s/q-value-fare-dur-%s.pkl' % (for_learning_dir, yymm))
Qsa_value, _ = load_picle_file('/Users/JerryHan88/taxi/for_learning/ALPHA-0.70-GAMMA-0.30/ALPHA-0.70-GAMMA-0.30-q-value-fare-dur-0904.pkl')
decision_inAP, decision_outAP = [], []
for s1 in DAY_OF_WEEK:
for s2 in TIME_SLOTS:
for s3 in [IN_AP, OUT_AP]:
# print (s1, s2, s3), Qsa_value[(s1, s2, s3, IN_AP)], Qsa_value[(s1, s2, s3, OUT_AP)]
max_a = IN_AP if Qsa_value[(s1, s2, s3, IN_AP)] >= Qsa_value[(s1, s2, s3, OUT_AP)] else OUT_AP
c = 0 if max_a == IN_AP else 1
if s3 == IN_AP:
decision_inAP.append([s1, s2, c])
else:
decision_outAP.append([s1, s2, c])
for s1, s2, a in decision_inAP:
print 'inAP', s1, s2, a
for s1, s2, a in decision_outAP:
print 'outAP', s1, s2, a
_data = [
[(s2, id_dow[s1], a) for s1, s2, a in decision_inAP],
[(s2, id_dow[s1], a) for s1, s2, a in decision_outAP]
]
# titles = ['Decision in the airport', 'Decision outside of the airport']
# grid_charts(('24 Time slots', []), ('', DAY_OF_WEEK), ['In Airport', 'Out Airport'], titles, _data, '%s/q-value-fare-dur-%s.pdf' % (for_learning_dir, yymm))
one_grid_chart(('Time slot', []), ('', DAY_OF_WEEK), ['In Airport', 'Out Airport'], '', [(s2, id_dow[s1], a) for s1, s2, a in decision_inAP], 'decision_inAP_q_e')
def process_file(fn):
_, _, yymm = fn[:-len('.csv')].split('-')
print 'handle the file; %s' % yymm
logging_msg('handle the file; %s' % yymm)
#
is_driver_vehicle = load_picle_file('%s/driver-vehicle-%s.pkl' % (shifts_dir, yymm))
full_drivers = set()
with open('%s/%s' % (shifts_dir, fn), 'rt') as r_csvfile:
reader = csv.reader(r_csvfile)
headers = reader.next()
id_yy, id_mm, id_dd, id_hh = headers.index('yy'), headers.index('mm'), headers.index('dd'), headers.index('hh')
id_vid, id_did = headers.index('vid'), headers.index('did')
id_pro_dur, id_x_pro_dur = headers.index('pro-dur'), headers.index('x-pro-dur')
with open('%s/shift-full-time-%s.csv' % (full_shift_dir, yymm), 'wt') as w_csvfile:
writer = csv.writer(w_csvfile)
new_headers = ['year', 'month', 'day', 'hour', 'vehicle-id', 'driver-id', 'productive-duration', 'x-productive-duration']
writer.writerow(new_headers)
for row in reader:
if len(is_driver_vehicle[row[id_vid]]) > 1:
continue
writer.writerow([row[id_yy], row[id_mm], row[id_dd], row[id_hh], row[id_vid], row[id_did], row[id_pro_dur], row[id_x_pro_dur]])
full_drivers.add(row[id_did])
save_pickle_file('%s/full-time-drivers-%s.pkl' % (full_shift_dir, yymm), full_drivers)
print 'end the file; %s' % yymm
logging_msg('end the file; %s' % yymm)
def run():
op_policies_q_learning = load_picle_file(
'/Users/JerryHan88/git/workspace_SMU/taxi_py/learning/q_learning_policy.pkl'
)
op_policies_ext_drivers = load_picle_file(
'/Users/JerryHan88/git/workspace_SMU/taxi_py/learning/extreme_drivers_policy.pkl'
)
decision_inAP_Q, decision_outAP_Q = [], []
decision_inAP_D, decision_outAP_D = [], []
for s1 in DAY_OF_WEEK:
for s2 in TIME_SLOTS:
for s3 in [IN_AP, OUT_AP]:
p_IN_AP, p_OUT_AP = op_policies_q_learning[(s1, s2, s3)]
max_a_Q = IN_AP if eval(p_IN_AP) >= eval(p_OUT_AP) else OUT_AP
c = 0 if max_a_Q == IN_AP else 1
if s3 == IN_AP:
decision_inAP_Q.append([s1, s2, c])
else:
decision_outAP_Q.append([s1, s2, c])
#
try:
p_IN_AP, p_OUT_AP = op_policies_ext_drivers[(s1, s2, s3)]
max_a_D = IN_AP if eval(p_IN_AP) >= eval(
p_OUT_AP) else OUT_AP
c = 0 if max_a_D == IN_AP else 1
except KeyError:
c = 2
print s1, s2, s3
if s3 == IN_AP:
decision_inAP_D.append([s1, s2, c])
else:
decision_outAP_D.append([s1, s2, c])
inAP_Q = [(s2, id_dow[s1], a) for s1, s2, a in decision_inAP_Q]
outAP_Q = [(s2, id_dow[s1], a) for s1, s2, a in decision_outAP_Q]
inAP_D = [(s2, id_dow[s1], a) for s1, s2, a in decision_inAP_D]
outAP_D = [(s2, id_dow[s1], a) for s1, s2, a in decision_outAP_D]
d = [inAP_Q, outAP_Q, inAP_D, outAP_D]
n = [
'decision_inAP_Q', 'decision_outAP_Q', 'decision_inAP_D',
'decision_outAP_D'
]
for i in xrange(4):
one_grid_chart(('Time slot', []), ('', DAY_OF_WEEK),
['Go to AP', 'Go to OA'], '', d[i], n[i])
def run():
dir_path = '/Users/JerryHan88/taxi/full_drivers_trips_q_comparision'
pickle_files = get_all_files(dir_path,
'comparision-', '.pkl')
for pkl_file in pickle_files:
yymm = pkl_file[:-len('.pkl')].split('-')[-1]
yy, mm = int(yymm[:2]), int(yymm[2:])
whole_rev, whole_count, sub_rev, sub_count = load_picle_file('%s/comparision-%s.pkl' % (dir_path, yymm))
with open(fn, 'a') as w_csvfile:
writer = csv.writer(w_csvfile)
writer.writerow([yy, mm, whole_rev / whole_count, sub_rev / sub_count])
def run():
dir_path = '/Users/JerryHan88/taxi/full_drivers_trips_q_comparision'
pickle_files = get_all_files(dir_path, 'comparision-', '.pkl')
for pkl_file in pickle_files:
yymm = pkl_file[:-len('.pkl')].split('-')[-1]
yy, mm = int(yymm[:2]), int(yymm[2:])
whole_rev, whole_count, sub_rev, sub_count = load_picle_file(
'%s/comparision-%s.pkl' % (dir_path, yymm))
with open(fn, 'a') as w_csvfile:
writer = csv.writer(w_csvfile)
writer.writerow(
[yy, mm, whole_rev / whole_count, sub_rev / sub_count])
def process_files(ALPHA, GAMMA):
ALPHA_GAMMA_dir = for_learning_dir + '/ALPHA-%.2f-GAMMA-%.2f' % (ALPHA, GAMMA)
if not os.path.exists(ALPHA_GAMMA_dir):
return None
pickle_files = get_all_files(ALPHA_GAMMA_dir, 'ALPHA-%.2f-GAMMA-%.2f' % (ALPHA, GAMMA), '.pkl')
for pkl_file in pickle_files:
yymm = pkl_file[:-len('.pkl')].split('-')[-1]
yy, mm = int(yymm[:2]), int(yymm[2:])
whole_rev, whole_count, sub_rev, sub_count = load_picle_file('%s/results-%s.pkl' % (ALPHA_GAMMA_dir, yymm))
with open(fn, 'a') as w_csvfile:
writer = csv.writer(w_csvfile)
writer.writerow([ALPHA, GAMMA, yy, mm, whole_rev / whole_count, sub_rev / sub_count])
def run():
op_policies_q_learning = load_picle_file('/Users/JerryHan88/git/workspace_SMU/taxi_py/learning/q_learning_policy.pkl')
op_policies_ext_drivers = load_picle_file('/Users/JerryHan88/git/workspace_SMU/taxi_py/learning/extreme_drivers_policy.pkl')
decision_inAP_Q, decision_outAP_Q = [], []
decision_inAP_D, decision_outAP_D = [], []
for s1 in DAY_OF_WEEK:
for s2 in TIME_SLOTS:
for s3 in [IN_AP, OUT_AP]:
p_IN_AP, p_OUT_AP = op_policies_q_learning[(s1, s2, s3)]
max_a_Q = IN_AP if eval(p_IN_AP) >= eval(p_OUT_AP) else OUT_AP
c = 0 if max_a_Q == IN_AP else 1
if s3 == IN_AP:
decision_inAP_Q.append([s1, s2, c])
else:
decision_outAP_Q.append([s1, s2, c])
#
try:
p_IN_AP, p_OUT_AP = op_policies_ext_drivers[(s1, s2, s3)]
max_a_D = IN_AP if eval(p_IN_AP) >= eval(p_OUT_AP) else OUT_AP
c = 0 if max_a_D == IN_AP else 1
except KeyError:
c = 2
print s1, s2, s3
if s3 == IN_AP:
decision_inAP_D.append([s1, s2, c])
else:
decision_outAP_D.append([s1, s2, c])
inAP_Q = [(s2, id_dow[s1], a) for s1, s2, a in decision_inAP_Q]
outAP_Q = [(s2, id_dow[s1], a) for s1, s2, a in decision_outAP_Q]
inAP_D = [(s2, id_dow[s1], a) for s1, s2, a in decision_inAP_D]
outAP_D = [(s2, id_dow[s1], a) for s1, s2, a in decision_outAP_D]
d = [inAP_Q, outAP_Q, inAP_D,
outAP_D]
n = ['decision_inAP_Q', 'decision_outAP_Q', 'decision_inAP_D',
'decision_outAP_D']
for i in xrange(4):
one_grid_chart(('Time slot', []), ('', DAY_OF_WEEK), ['Go to AP', 'Go to OA'], '', d[i], n[i])
def test():
def difference(data0, data1):
diff = {}
for k, v in data0.iteritems():
diff[k] = data1[k] - v
return diff
def ordering(dids_values):
order_v_did = []
for did, v in dids_values.iteritems():
order_v_did.append([v, did])
order_v_did.sort()
order_v_did.reverse()
return order_v_did
def find_extreme_range(order_v_did):
# more than mean's 50 percent
values = [v for v, _ in order_v_did]
mu, std = np.mean(values), np.std(values)
i = 0
while order_v_did[i][0] > mu + std * 2.0:
i += 1
return (0, i / len(order_v_did))
both_years_full_drivers, \
Y09_driver_genprod_hour, Y10_driver_genprod_hour, \
Y09_pin_driver_aprod_hour, Y10_pin_driver_aprod_hour, \
Y09_pout_driver_aprod_hour, Y10_pout_driver_aprod_hour, \
Y09_pin_driver_epro_month, Y10_pin_driver_epro_month, \
Y09_pout_driver_epro_month, Y10_pout_driver_epro_month = load_picle_file('%s/productivities_ext.pkl' % (individual_detail_dir))
#
diff_general_prod = difference(Y09_driver_genprod_hour, Y10_driver_genprod_hour)
diff_pin_prod = difference(Y09_pin_driver_aprod_hour, Y10_pin_driver_aprod_hour)
diff_pout_prod = difference(Y09_pout_driver_aprod_hour, Y10_pout_driver_aprod_hour)
diff_pin_eco = difference(Y09_pin_driver_epro_month, Y10_pin_driver_epro_month)
diff_pout_eco = difference(Y09_pout_driver_epro_month, Y10_pout_driver_epro_month)
order_v_did = ordering(diff_pin_eco)
print len(diff_pin_eco)
r1, r2 = find_extreme_range(order_v_did)
extreme_drivers = [int(did) for _, did in order_v_did[int(r1 * len(order_v_did)):int(r2 * len(order_v_did))]]
for fn in get_all_files(for_full_driver_dir, 'full-drivers-trips-', '.csv'):
_, _, _, yymm = fn[:-len('.csv')].split('-')
with open('%s/%s' % (for_full_driver_dir, fn), 'rb') as r_csvfile:
reader = csv.reader(r_csvfile)
headers = reader.next()
id_did = headers.index('did')
with open('%s/diff-pin-eco-extreme-drivers-trip-%s.csv' % (for_full_driver_dir, yymm), 'wt') as w_csvfile:
writer = csv.writer(w_csvfile)
writer.writerow(headers)
for row in reader:
did = int(row[id_did])
if did not in extreme_drivers:
continue
writer.writerow(row)
def process_files(yymm):
print "handle the file; %s" % yymm
logging_msg("handle the file; %s" % yymm)
#
init_csv_files(yymm)
#
full_dids = sorted(
[eval(x) for x in load_picle_file("%s/%s%s.pkl" % (full_shift_dir, monthly_full_did_prefix, yymm))]
)
s_df = pd.read_csv("%s/%s%s.csv" % (full_shift_dir, sh_full_prefix, yymm))
trip_df = pd.read_csv("%s/%s%s.csv" % (trips_dir, trip_prefix, yymm))
ap_trip_df = pd.read_csv("%s/%s%s.csv" % (airport_trips_dir, ap_trip_op_ep_prefix, yymm))
#
yy, mm = int(yymm[:2]), int(yymm[2:])
for did in full_dids:
# General
did_sh = s_df[(s_df["driver-id"] == did)]
pro_dur = sum(did_sh["productive-duration"]) * SEC
did_wt = trip_df[(trip_df["did"] == did)]
total_fare = sum(did_wt["fare"])
if pro_dur > 0 and total_fare != 0:
total_prod = total_fare / pro_dur
with open("%s/%s%s.csv" % (individual_detail_dir, general_prefix, yymm), "a") as w_csvfile:
writer = csv.writer(w_csvfile)
writer.writerow([yy, mm, did, pro_dur, total_fare, total_prod])
#
did_ap = ap_trip_df[(ap_trip_df["did"] == did)]
prev_in_ap_trip = did_ap[(did_ap["trip-mode"] == DInAP_PInAP)]
prev_out_ap_trip = did_ap[(did_ap["trip-mode"] == DOutAP_PInAP)]
#
if len(did_ap) != 0:
# prev in ap trip
ap_qu, ap_dur = sum(prev_in_ap_trip["queue-time"]), sum(prev_in_ap_trip["duration"])
ap_fare = sum(prev_in_ap_trip["fare"])
ap_op_cost, ap_eco_profit = sum(prev_in_ap_trip["op-cost"]), sum(prev_in_ap_trip["economic"])
if ap_qu + ap_dur > 0 and ap_fare != 0:
ap_prod = ap_fare / (ap_qu + ap_dur)
with open("%s/%s%s.csv" % (individual_detail_dir, prev_in_ap_prefix, yymm), "a") as w_csvfile:
writer = csv.writer(w_csvfile)
writer.writerow([yy, mm, did, ap_qu, ap_dur, ap_fare, ap_prod, ap_op_cost, ap_eco_profit])
#
# prev out ap trip
ap_qu, ap_dur = sum(prev_out_ap_trip["queue-time"]), sum(prev_out_ap_trip["duration"])
ap_fare = sum(prev_out_ap_trip["fare"])
ap_op_cost, ap_eco_profit = sum(prev_out_ap_trip["op-cost"]), sum(prev_out_ap_trip["economic"])
if ap_qu + ap_dur > 0 and ap_fare != 0:
ap_prod = ap_fare / (ap_qu + ap_dur)
with open("%s/%s%s.csv" % (individual_detail_dir, prev_out_ap_prefix, yymm), "a") as w_csvfile:
writer = csv.writer(w_csvfile)
writer.writerow([yy, mm, did, ap_qu, ap_dur, ap_fare, ap_prod, ap_op_cost, ap_eco_profit])
print "End the file; %s" % yymm
logging_msg("End the file; %s" % yymm)
def q_learning():
policies_dir = for_learning_dir + '/%s' % ('ALPHA-0.10-GAMMA-0.50')
policies = {}
for fn in get_all_files(policies_dir, 'ALPHA-', '.pkl'):
Qsa_value, _ = load_picle_file(policies_dir + '/%s' % fn)
for s1 in DAY_OF_WEEK:
for s2 in TIME_SLOTS:
for s3 in [IN_AP, OUT_AP]:
if not policies.has_key((s1, s2, s3)):
policies[(s1, s2, s3)] = [0, 0]
i = index_IN_OUT_AP[IN_AP] if Qsa_value[(s1, s2, s3, IN_AP)] >= Qsa_value[(s1, s2, s3, OUT_AP)] else index_IN_OUT_AP[OUT_AP]
policies[(s1, s2, s3)][i] += 1
op_policies = {}
for k, v in policies.iteritems():
op_policies[k] = ('%.2f' % (v[0] / (v[0] + v[1])), '%.2f' % (v[1] / (v[0] + v[1])))
save_pickle_file('q_learning_policy.pkl', op_policies)
def process_files(yymm):
Qsa_value, state_action_fare_dur = load_picle_file('%s/ALPHA-0.10-GAMMA-0.50/ALPHA-0.10-GAMMA-0.50-q-value-fare-dur-%s.pkl'%(for_learning_dir,yymm))
argmax_as = {}
for s1 in DAY_OF_WEEK:
for s2 in TIME_SLOTS:
for s3 in [IN_AP, OUT_AP]:
argmax_as[(s1, s2, s3)] = IN_AP if Qsa_value[(s1, s2, s3, IN_AP)] >= Qsa_value[(s1, s2, s3, OUT_AP)] else OUT_AP
#
whole_rev, sub_rev = 0, 0
whole_count, sub_count = 0,0
count = 0
with open('%s/diff-pin-eco-extreme-drivers-trip-%s.csv' % (for_full_driver_dir, yymm), 'rb') as r_csvfile:
reader = csv.reader(r_csvfile)
headers = reader.next()
id_prev_tetime, id_prev_teloc = headers.index('prev-trip-end-time'), headers.index('prev-trip-end-location')
id_stime, id_sloc = headers.index('start-time'), headers.index('start-location'),
id_dur, id_fare = headers.index('duration'), headers.index('fare')
for row in reader:
prev_tetime, stime = eval(row[id_prev_tetime]), eval(row[id_stime])
setup_time = stime - prev_tetime
#
prev_tetime_datetime = datetime.datetime.fromtimestamp(prev_tetime)
s1, s2 = prev_tetime_datetime.strftime("%a"), prev_tetime_datetime.hour
s3 = row[id_prev_teloc]
#
a = row[id_sloc]
#
dur, fare = eval(row[id_dur]), eval(row[id_fare])
alter_a = OUT_AP if a == IN_AP else IN_AP
if state_action_fare_dur[(s1, s2, s3, alter_a)][1] == 0:
op_cost = 0
else:
op_cost = (setup_time + dur) * state_action_fare_dur[(s1, s2, s3, alter_a)][0] / state_action_fare_dur[(s1, s2, s3, alter_a)][1]
economic_profit = fare - op_cost
#
whole_rev += economic_profit
whole_count += 1
if argmax_as[(s1, s2, s3)] == a:
sub_rev += economic_profit
sub_count += 1
count += 1
if count % MOD_STAN == 0:
print '%s, %d' % (yymm, count)
logging_msg('%s, %d' % (yymm, count))
save_pickle_file('%s/comparision-%s.pkl'%(for_full_driver_dir, yymm), [whole_rev, whole_count, sub_rev, sub_count])
def q_learning():
policies_dir = for_learning_dir + '/%s' % ('ALPHA-0.10-GAMMA-0.50')
policies = {}
for fn in get_all_files(policies_dir, 'ALPHA-', '.pkl'):
Qsa_value, _ = load_picle_file(policies_dir + '/%s' % fn)
for s1 in DAY_OF_WEEK:
for s2 in TIME_SLOTS:
for s3 in [IN_AP, OUT_AP]:
if not policies.has_key((s1, s2, s3)):
policies[(s1, s2, s3)] = [0, 0]
i = index_IN_OUT_AP[IN_AP] if Qsa_value[(
s1, s2, s3, IN_AP)] >= Qsa_value[(
s1, s2, s3, OUT_AP)] else index_IN_OUT_AP[OUT_AP]
policies[(s1, s2, s3)][i] += 1
op_policies = {}
for k, v in policies.iteritems():
op_policies[k] = ('%.2f' % (v[0] / (v[0] + v[1])),
'%.2f' % (v[1] / (v[0] + v[1])))
save_pickle_file('q_learning_policy.pkl', op_policies)
def process_files(ALPHA, GAMMA):
ALPHA_GAMMA_dir = for_learning_dir + '/ALPHA-%.2f-GAMMA-%.2f' % (ALPHA,
GAMMA)
if not os.path.exists(ALPHA_GAMMA_dir):
return None
pickle_files = get_all_files(ALPHA_GAMMA_dir,
'ALPHA-%.2f-GAMMA-%.2f' % (ALPHA, GAMMA),
'.pkl')
for pkl_file in pickle_files:
yymm = pkl_file[:-len('.pkl')].split('-')[-1]
yy, mm = int(yymm[:2]), int(yymm[2:])
whole_rev, whole_count, sub_rev, sub_count = load_picle_file(
'%s/results-%s.pkl' % (ALPHA_GAMMA_dir, yymm))
with open(fn, 'a') as w_csvfile:
writer = csv.writer(w_csvfile)
writer.writerow([
ALPHA, GAMMA, yy, mm, whole_rev / whole_count,
sub_rev / sub_count
])
def process_file(ALPHA, GAMMA, ALPHA_GAMMA_dir, yymm):
print 'handle the file; %s' % yymm
logging_msg('handle the file; %s' % yymm)
#
print yymm
if yymm == '0901':
prev_yymm = None
elif yymm == '1001':
prev_yymm = '0911'
elif yymm == '1011':
prev_yymm = '1009'
else:
yy, mm = int(yymm[:2]), int(yymm[2:])
prev_yymm = '%02d%02d' % (yy, mm - 1)
#
if not prev_yymm:
Qsa_value, state_action_fare_dur = {}, {}
locations = [IN_AP, OUT_AP]
actions = [IN_AP, OUT_AP]
for s1 in DAY_OF_WEEK:
for s2 in TIME_SLOTS:
for s3 in locations:
for a in actions:
Qsa_value[(s1, s2, s3, a)] = 0
state_action_fare_dur[(s1, s2, s3, a)] = [0, 0]
else:
Qsa_value, state_action_fare_dur = load_picle_file(
'%s/ALPHA-%.2f-GAMMA-%.2f-q-value-fare-dur-%s.pkl' %
(ALPHA_GAMMA_dir, ALPHA, GAMMA, prev_yymm))
#
with open('%s/whole-trip-%s.csv' % (for_learning_dir, yymm),
'rb') as r_csvfile:
reader = csv.reader(r_csvfile)
headers = reader.next()
id_prev_tetime, id_prev_teloc = headers.index(
'prev-trip-end-time'), headers.index('prev-trip-end-location')
id_stime, id_sloc = headers.index('start-time'), headers.index(
'start-location'),
id_etime, id_eloc = headers.index('end-time'), headers.index(
'end-location'),
id_dur, id_fare = headers.index('duration'), headers.index('fare')
#
count = 0
for row in reader:
prev_tetime, stime, etime = eval(row[id_prev_tetime]), eval(
row[id_stime]), eval(row[id_etime])
setup_time = stime - prev_tetime
#
if setup_time < 0 or HOUR * 2 < setup_time:
continue
#
prev_tetime_datetime = datetime.datetime.fromtimestamp(prev_tetime)
s1, s2 = prev_tetime_datetime.strftime(
"%a"), prev_tetime_datetime.hour
s3 = row[id_prev_teloc]
#
etime_datetime = datetime.datetime.fromtimestamp(etime)
new_s1, new_s2 = etime_datetime.strftime("%a"), etime_datetime.hour
new_s3 = row[id_eloc]
#
a = row[id_sloc]
dur, fare = eval(row[id_dur]), eval(row[id_fare])
#
state_action_fare_dur[(s1, s2, s3, a)][0] += fare
state_action_fare_dur[(s1, s2, s3, a)][1] += setup_time + dur
#
if Qsa_value[(new_s1, new_s2, new_s3, IN_AP)] > Qsa_value[(
new_s1, new_s2, new_s3, OUT_AP)]:
future_max_q_value = Qsa_value[(new_s1, new_s2, new_s3, IN_AP)]
else:
future_max_q_value = Qsa_value[(new_s1, new_s2, new_s3,
OUT_AP)]
#
alter_a = OUT_AP if a == IN_AP else IN_AP
if state_action_fare_dur[(s1, s2, s3, alter_a)][1] == 0:
op_cost = 0
else:
op_cost = (setup_time + dur) * state_action_fare_dur[
(s1, s2, s3, alter_a)][0] / state_action_fare_dur[
(s1, s2, s3, alter_a)][1]
qrs = fare - op_cost + GAMMA * future_max_q_value
Qsa_value[(s1, s2, s3, a)] = \
(1 - ALPHA) * Qsa_value[(s1, s2, s3, a)] + ALPHA * qrs
count += 1
if count % MOD_STAN == 0:
print '%s, %d' % (yymm, count)
logging_msg('%s, %d' % (yymm, count))
save_pickle_file(
'%s/ALPHA-%.2f-GAMMA-%.2f-q-value-fare-dur-%s.pkl' %
(ALPHA_GAMMA_dir, ALPHA, GAMMA, yymm),
[Qsa_value, state_action_fare_dur])
save_pickle_file(
'%s/ALPHA-%.2f-GAMMA-%.2f-q-value-fare-dur-%s.pkl' %
(ALPHA_GAMMA_dir, ALPHA, GAMMA, yymm),
[Qsa_value, state_action_fare_dur])
print 'end the file; %s' % yymm
logging_msg('end the file; %s' % yymm)
def process_file(ALPHA, GAMMA, ALPHA_GAMMA_dir, yymm):
print 'handle the file; %s' % yymm
logging_msg('handle the file; %s' % yymm)
#
print yymm
if yymm == '0901':
prev_yymm = None
elif yymm == '1001':
prev_yymm = '0911'
elif yymm == '1011':
prev_yymm = '1009'
else:
yy, mm = int(yymm[:2]), int(yymm[2:])
prev_yymm = '%02d%02d' % (yy, mm - 1)
#
if not prev_yymm:
Qsa_value, state_action_fare_dur = {}, {}
locations = [IN_AP, OUT_AP]
actions = [IN_AP, OUT_AP]
for s1 in DAY_OF_WEEK:
for s2 in TIME_SLOTS:
for s3 in locations:
for a in actions:
Qsa_value[(s1, s2, s3, a)] = 0
state_action_fare_dur[(s1, s2, s3, a)] = [0, 0]
else:
Qsa_value, state_action_fare_dur = load_picle_file('%s/ALPHA-%.2f-GAMMA-%.2f-q-value-fare-dur-%s.pkl' % (ALPHA_GAMMA_dir, ALPHA, GAMMA, prev_yymm))
#
with open('%s/whole-trip-%s.csv' % (for_learning_dir, yymm), 'rb') as r_csvfile:
reader = csv.reader(r_csvfile)
headers = reader.next()
id_prev_tetime, id_prev_teloc = headers.index('prev-trip-end-time'), headers.index('prev-trip-end-location')
id_stime, id_sloc = headers.index('start-time'), headers.index('start-location'),
id_etime, id_eloc = headers.index('end-time'), headers.index('end-location'),
id_dur, id_fare = headers.index('duration'), headers.index('fare')
#
count = 0
for row in reader:
prev_tetime, stime, etime = eval(row[id_prev_tetime]), eval(row[id_stime]), eval(row[id_etime])
setup_time = stime - prev_tetime
#
if setup_time < 0 or HOUR * 2 < setup_time:
continue
#
prev_tetime_datetime = datetime.datetime.fromtimestamp(prev_tetime)
s1, s2 = prev_tetime_datetime.strftime("%a"), prev_tetime_datetime.hour
s3 = row[id_prev_teloc]
#
etime_datetime = datetime.datetime.fromtimestamp(etime)
new_s1, new_s2 = etime_datetime.strftime("%a"), etime_datetime.hour
new_s3 = row[id_eloc]
#
a = row[id_sloc]
dur, fare = eval(row[id_dur]), eval(row[id_fare])
#
state_action_fare_dur[(s1, s2, s3, a)][0] += fare
state_action_fare_dur[(s1, s2, s3, a)][1] += setup_time + dur
#
if Qsa_value[(new_s1, new_s2, new_s3, IN_AP)] > Qsa_value[(new_s1, new_s2, new_s3, OUT_AP)] :
future_max_q_value = Qsa_value[(new_s1, new_s2, new_s3, IN_AP)]
else:
future_max_q_value = Qsa_value[(new_s1, new_s2, new_s3, OUT_AP)]
#
alter_a = OUT_AP if a == IN_AP else IN_AP
if state_action_fare_dur[(s1, s2, s3, alter_a)][1] == 0:
op_cost = 0
else:
op_cost = (setup_time + dur) * state_action_fare_dur[(s1, s2, s3, alter_a)][0] / state_action_fare_dur[(s1, s2, s3, alter_a)][1]
qrs = fare - op_cost + GAMMA * future_max_q_value
Qsa_value[(s1, s2, s3, a)] = \
(1 - ALPHA) * Qsa_value[(s1, s2, s3, a)] + ALPHA * qrs
count += 1
if count % MOD_STAN == 0:
print '%s, %d' % (yymm, count)
logging_msg('%s, %d' % (yymm, count))
save_pickle_file('%s/ALPHA-%.2f-GAMMA-%.2f-q-value-fare-dur-%s.pkl' % (ALPHA_GAMMA_dir, ALPHA, GAMMA, yymm), [Qsa_value, state_action_fare_dur])
save_pickle_file('%s/ALPHA-%.2f-GAMMA-%.2f-q-value-fare-dur-%s.pkl' % (ALPHA_GAMMA_dir, ALPHA, GAMMA, yymm), [Qsa_value, state_action_fare_dur])
print 'end the file; %s' % yymm
logging_msg('end the file; %s' % yymm)
def process_files(yymm, q_lerning_ended_dir):
candi_pkl_files = []
for dn in q_lerning_ended_dir:
if os.path.exists('%s/%s/results-%s.pkl' %
(for_learning_dir, dn, yymm)):
continue
candi_pkl_files.append('%s/%s/%s-q-value-fare-dur-%s.pkl' %
(for_learning_dir, dn, dn, yymm))
result_pkls = [
os.path.dirname(pkl_path) + '/results-%s.pkl' % yymm
for pkl_path in candi_pkl_files
]
#
list_argmax_as = []
state_action_fare_dur = None
for pkl_file_path in candi_pkl_files:
Qsa_value, state_action_fare_dur = load_picle_file(pkl_file_path)
argmax_as = {}
for s1 in DAY_OF_WEEK:
for s2 in TIME_SLOTS:
for s3 in [IN_AP, OUT_AP]:
argmax_as[(s1, s2, s3)] = IN_AP if Qsa_value[(
s1, s2, s3, IN_AP)] >= Qsa_value[(s1, s2, s3,
OUT_AP)] else OUT_AP
list_argmax_as.append(argmax_as)
#
whole_rev, whole_count = 0, 0
list_sub_rev, list_sub_count = [0 for _ in xrange(len(candi_pkl_files))], [
0 for _ in xrange(len(candi_pkl_files))
]
count = 0
with open('%s/whole-trip-%s.csv' % (for_learning_dir, yymm),
'rb') as r_csvfile:
reader = csv.reader(r_csvfile)
headers = reader.next()
id_prev_tetime, id_prev_teloc = headers.index(
'prev-trip-end-time'), headers.index('prev-trip-end-location')
id_stime, id_sloc = headers.index('start-time'), headers.index(
'start-location'),
id_dur, id_fare = headers.index('duration'), headers.index('fare')
for row in reader:
prev_tetime, stime = eval(row[id_prev_tetime]), eval(row[id_stime])
setup_time = stime - prev_tetime
#
prev_tetime_datetime = datetime.datetime.fromtimestamp(prev_tetime)
s1, s2 = prev_tetime_datetime.strftime(
"%a"), prev_tetime_datetime.hour
s3 = row[id_prev_teloc]
#
a = row[id_sloc]
#
dur, fare = eval(row[id_dur]), eval(row[id_fare])
alter_a = OUT_AP if a == IN_AP else IN_AP
if state_action_fare_dur[(s1, s2, s3, alter_a)][1] == 0:
op_cost = 0
else:
op_cost = (setup_time + dur) * state_action_fare_dur[
(s1, s2, s3, alter_a)][0] / state_action_fare_dur[
(s1, s2, s3, alter_a)][1]
economic_profit = fare - op_cost
#
whole_rev += economic_profit
whole_count += 1
for i, argmax_as in enumerate(list_argmax_as):
if argmax_as[(s1, s2, s3)] == a:
list_sub_rev[i] += economic_profit
list_sub_count[i] += 1
count += 1
if count % MOD_STAN == 0:
print '%s, %d' % (yymm, count)
logging_msg('%s, %d' % (yymm, count))
for i in xrange(len(result_pkls)):
result_fn = result_pkls[i]
save_pickle_file(result_fn, [
whole_rev, whole_count, list_sub_rev[i],
list_sub_count[i]
])
for i in xrange(len(result_pkls)):
result_fn = result_pkls[i]
save_pickle_file(
result_fn,
[whole_rev, whole_count, list_sub_rev[i], list_sub_count[i]])
from __future__ import division
#
import os, sys
sys.path.append(os.getcwd() + '/..')
#
from supports.handling_pkl import load_picle_file
from supports.etc_functions import get_all_files, remove_creat_dir
from supports._setting import merged_trip_dir, for_full_driver_dir, individual_detail_dir
from supports.logger import logging_msg
from supports.location_check import is_in_airport
from supports._setting import OUT_AP
from supports.multiprocess import init_multiprocessor, put_task, end_multiprocessor
#
import csv
#
full_time_drivers, _, _, _, _, _, _, _, _, _, _ = load_picle_file('%s/productivities_ext.pkl' % (individual_detail_dir))
driver_full_or_not = [False] * (max(full_time_drivers) + 1)
for did in full_time_drivers:
driver_full_or_not[int(did)] = True
check_progress = 10000000
#
def run():
remove_creat_dir(for_full_driver_dir)
init_multiprocessor()
count_num_jobs = 0
for fn in get_all_files(merged_trip_dir, 'trips', '.csv'):
# process_file(fn)
put_task(process_file, [fn])
count_num_jobs += 1
end_multiprocessor(count_num_jobs)
def process_files(yymm, q_lerning_ended_dir):
candi_pkl_files = []
for dn in q_lerning_ended_dir:
if os.path.exists('%s/%s/results-%s.pkl' % (for_learning_dir, dn, yymm)):
continue
candi_pkl_files.append('%s/%s/%s-q-value-fare-dur-%s.pkl' % (for_learning_dir, dn, dn, yymm))
result_pkls = [os.path.dirname(pkl_path) + '/results-%s.pkl'% yymm for pkl_path in candi_pkl_files]
#
list_argmax_as = []
state_action_fare_dur = None
for pkl_file_path in candi_pkl_files:
Qsa_value, state_action_fare_dur = load_picle_file(pkl_file_path)
argmax_as = {}
for s1 in DAY_OF_WEEK:
for s2 in TIME_SLOTS:
for s3 in [IN_AP, OUT_AP]:
argmax_as[(s1, s2, s3)] = IN_AP if Qsa_value[(s1, s2, s3, IN_AP)] >= Qsa_value[(s1, s2, s3, OUT_AP)] else OUT_AP
list_argmax_as.append(argmax_as)
#
whole_rev, whole_count = 0, 0
list_sub_rev, list_sub_count = [0 for _ in xrange(len(candi_pkl_files))], [0 for _ in xrange(len(candi_pkl_files))]
count = 0
with open('%s/whole-trip-%s.csv' % (for_learning_dir, yymm), 'rb') as r_csvfile:
reader = csv.reader(r_csvfile)
headers = reader.next()
id_prev_tetime, id_prev_teloc = headers.index('prev-trip-end-time'), headers.index('prev-trip-end-location')
id_stime, id_sloc = headers.index('start-time'), headers.index('start-location'),
id_dur, id_fare = headers.index('duration'), headers.index('fare')
for row in reader:
prev_tetime, stime = eval(row[id_prev_tetime]), eval(row[id_stime])
setup_time = stime - prev_tetime
#
prev_tetime_datetime = datetime.datetime.fromtimestamp(prev_tetime)
s1, s2 = prev_tetime_datetime.strftime("%a"), prev_tetime_datetime.hour
s3 = row[id_prev_teloc]
#
a = row[id_sloc]
#
dur, fare = eval(row[id_dur]), eval(row[id_fare])
alter_a = OUT_AP if a == IN_AP else IN_AP
if state_action_fare_dur[(s1, s2, s3, alter_a)][1] == 0:
op_cost = 0
else:
op_cost = (setup_time + dur) * state_action_fare_dur[(s1, s2, s3, alter_a)][0] / state_action_fare_dur[(s1, s2, s3, alter_a)][1]
economic_profit = fare - op_cost
#
whole_rev += economic_profit
whole_count += 1
for i, argmax_as in enumerate(list_argmax_as):
if argmax_as[(s1, s2, s3)] == a:
list_sub_rev[i] += economic_profit
list_sub_count[i] += 1
count += 1
if count % MOD_STAN == 0:
print '%s, %d' % (yymm, count)
logging_msg('%s, %d' % (yymm, count))
for i in xrange(len(result_pkls)):
result_fn = result_pkls[i]
save_pickle_file(result_fn, [whole_rev, whole_count, list_sub_rev[i], list_sub_count[i]])
for i in xrange(len(result_pkls)):
result_fn = result_pkls[i]
save_pickle_file(result_fn, [whole_rev, whole_count, list_sub_rev[i], list_sub_count[i]])
def process_file(fn):
_, _, yymm = fn[:-len('.csv')].split('-')
print 'handle the file; %s' % yymm
logging_msg('handle the file; %s' % yymm)
#
ap_pkl_files = get_all_files(logs_dir, 'ap-crossing-time-', '.pkl')
ap_pkl_file_path = None
for pkl_fn in ap_pkl_files:
_, _, _, pkl_yymm = pkl_fn[:-len('.pkl')].split('-')
if pkl_yymm == yymm:
ap_pkl_file_path = '%s/%s' % (logs_dir, pkl_fn)
break
else:
assert False, yymm
ap_crossing_times = load_picle_file(ap_pkl_file_path)
#
ns_pkl_files = get_all_files(logs_dir, 'ns-crossing-time-', '.pkl')
ns_pkl_file_path = None
for pkl_fn in ns_pkl_files:
_, _, _, pkl_yymm = pkl_fn[:-len('.pkl')].split('-')
if pkl_yymm == yymm:
ns_pkl_file_path = '%s/%s' % (logs_dir, pkl_fn)
break
else:
assert False, yymm
ns_crossing_times = load_picle_file(ns_pkl_file_path)
#
init_csv_files(yymm)
with open('%s/%s' % (trips_dir, fn), 'rb') as r_csvfile:
reader = csv.reader(r_csvfile)
headers = reader.next()
header_id = {h : i for i, h in enumerate(headers)}
for row in reader:
tid, did = row[header_id['tid']], row[header_id['did']]
et, duration = row[header_id['end-time']], row[header_id['duration']]
fare = row[header_id['fare']]
#
ap_tm, ns_tm = int(row[header_id['ap-trip-mode']]), int(row[header_id['ns-trip-mode']])
vid, st, prev_tet = row[header_id['vid']], eval(row[header_id['start-time']]), eval(row[header_id['prev-trip-end-time']])
#
is_ap_trip, is_ns_trip = False, False
#
if ap_tm == DInAP_PInAP:
is_ap_trip = True
ap_join_queue_time = prev_tet
elif ap_tm == DOutAP_PInAP:
is_ap_trip = True
try:
i = bisect(ap_crossing_times[vid], st)
except KeyError:
logging_msg('%s-tid-%s' % (yymm, row[header_id['tid']]))
continue
ap_join_queue_time = ap_crossing_times[vid][i - 1] if i != 0 else ap_crossing_times[vid][0]
if is_ap_trip:
with open('%s/airport-trip-%s.csv' % (airport_trips_dir, yymm), 'a') as w_csvfile:
writer = csv.writer(w_csvfile)
ap_queue_time = st - ap_join_queue_time
new_row = [tid, vid, did, st, et, duration, fare, prev_tet,
ap_tm, ap_join_queue_time, ap_queue_time]
writer.writerow(new_row)
#
if ns_tm == DInNS_PInNS:
is_ns_trip = True
ns_join_queue_time = prev_tet
elif ns_tm == DOutNS_PInNS:
is_ns_trip = True
try:
i = bisect(ns_crossing_times[vid], st)
except KeyError:
logging_msg('%s-tid-%s' % (yymm, row[header_id['tid']]))
continue
ns_join_queue_time = ns_crossing_times[vid][i - 1] if i != 0 else ns_crossing_times[vid][0]
if is_ns_trip:
with open('%s/nightsafari-trip-%s.csv' % (nightsafari_trips_dir, yymm), 'a') as w_csvfile:
writer = csv.writer(w_csvfile)
ns_queue_time = st - ns_join_queue_time
new_row = [tid, vid, did, st, et, duration, fare, prev_tet,
ns_tm, ns_join_queue_time, ns_queue_time]
writer.writerow(new_row)
print 'end the file; %s' % yymm
logging_msg('end the file; %s' % yymm)
def process_files(yymm):
print 'handle the file; %s' % yymm
logging_msg('handle the file; %s' % yymm)
#
init_csv_files(yymm)
#
full_dids = sorted([
eval(x) for x in load_picle_file('%s/%s%s.pkl' %
(full_shift_dir,
monthly_full_did_prefix, yymm))
])
s_df = pd.read_csv('%s/%s%s.csv' % (full_shift_dir, sh_full_prefix, yymm))
trip_df = pd.read_csv('%s/%s%s.csv' % (trips_dir, trip_prefix, yymm))
ap_trip_df = pd.read_csv('%s/%s%s.csv' %
(airport_trips_dir, ap_trip_op_ep_prefix, yymm))
#
yy, mm = int(yymm[:2]), int(yymm[2:])
for did in full_dids:
# General
did_sh = s_df[(s_df['driver-id'] == did)]
pro_dur = sum(did_sh['productive-duration']) * SEC
did_wt = trip_df[(trip_df['did'] == did)]
total_fare = sum(did_wt['fare'])
if pro_dur > 0 and total_fare != 0:
total_prod = total_fare / pro_dur
with open(
'%s/%s%s.csv' %
(individual_detail_dir, general_prefix, yymm),
'a') as w_csvfile:
writer = csv.writer(w_csvfile)
writer.writerow([yy, mm, did, pro_dur, total_fare, total_prod])
#
did_ap = ap_trip_df[(ap_trip_df['did'] == did)]
prev_in_ap_trip = did_ap[(did_ap['trip-mode'] == DInAP_PInAP)]
prev_out_ap_trip = did_ap[(did_ap['trip-mode'] == DOutAP_PInAP)]
#
if len(did_ap) != 0:
# prev in ap trip
ap_qu, ap_dur = sum(prev_in_ap_trip['queue-time']), sum(
prev_in_ap_trip['duration'])
ap_fare = sum(prev_in_ap_trip['fare'])
ap_op_cost, ap_eco_profit = sum(prev_in_ap_trip['op-cost']), sum(
prev_in_ap_trip['economic'])
if ap_qu + ap_dur > 0 and ap_fare != 0:
ap_prod = ap_fare / (ap_qu + ap_dur)
with open(
'%s/%s%s.csv' %
(individual_detail_dir, prev_in_ap_prefix, yymm),
'a') as w_csvfile:
writer = csv.writer(w_csvfile)
writer.writerow([
yy, mm, did, ap_qu, ap_dur, ap_fare, ap_prod,
ap_op_cost, ap_eco_profit
])
#
# prev out ap trip
ap_qu, ap_dur = sum(prev_out_ap_trip['queue-time']), sum(
prev_out_ap_trip['duration'])
ap_fare = sum(prev_out_ap_trip['fare'])
ap_op_cost, ap_eco_profit = sum(prev_out_ap_trip['op-cost']), sum(
prev_out_ap_trip['economic'])
if ap_qu + ap_dur > 0 and ap_fare != 0:
ap_prod = ap_fare / (ap_qu + ap_dur)
with open(
'%s/%s%s.csv' %
(individual_detail_dir, prev_out_ap_prefix, yymm),
'a') as w_csvfile:
writer = csv.writer(w_csvfile)
writer.writerow([
yy, mm, did, ap_qu, ap_dur, ap_fare, ap_prod,
ap_op_cost, ap_eco_profit
])
print 'End the file; %s' % yymm
logging_msg('End the file; %s' % yymm)
def process_file(fn):
_, _, yymm = fn[:-len('.csv')].split('-')
print 'handle the file; %s' % yymm
logging_msg('handle the file; %s' % yymm)
#
ap_pkl_files = get_all_files(logs_dir, 'ap-crossing-time-', '.pkl')
ap_pkl_file_path = None
for pkl_fn in ap_pkl_files:
_, _, _, pkl_yymm = pkl_fn[:-len('.pkl')].split('-')
if pkl_yymm == yymm:
ap_pkl_file_path = '%s/%s' % (logs_dir, pkl_fn)
break
else:
assert False, yymm
ap_crossing_times = load_picle_file(ap_pkl_file_path)
#
ns_pkl_files = get_all_files(logs_dir, 'ns-crossing-time-', '.pkl')
ns_pkl_file_path = None
for pkl_fn in ns_pkl_files:
_, _, _, pkl_yymm = pkl_fn[:-len('.pkl')].split('-')
if pkl_yymm == yymm:
ns_pkl_file_path = '%s/%s' % (logs_dir, pkl_fn)
break
else:
assert False, yymm
ns_crossing_times = load_picle_file(ns_pkl_file_path)
#
init_csv_files(yymm)
with open('%s/%s' % (trips_dir, fn), 'rb') as r_csvfile:
reader = csv.reader(r_csvfile)
headers = reader.next()
header_id = {h: i for i, h in enumerate(headers)}
for row in reader:
tid, did = row[header_id['tid']], row[header_id['did']]
et, duration = row[header_id['end-time']], row[
header_id['duration']]
fare = row[header_id['fare']]
#
ap_tm, ns_tm = int(row[header_id['ap-trip-mode']]), int(
row[header_id['ns-trip-mode']])
vid, st, prev_tet = row[header_id['vid']], eval(
row[header_id['start-time']]), eval(
row[header_id['prev-trip-end-time']])
#
is_ap_trip, is_ns_trip = False, False
#
if ap_tm == DInAP_PInAP:
is_ap_trip = True
ap_join_queue_time = prev_tet
elif ap_tm == DOutAP_PInAP:
is_ap_trip = True
try:
i = bisect(ap_crossing_times[vid], st)
except KeyError:
logging_msg('%s-tid-%s' % (yymm, row[header_id['tid']]))
continue
ap_join_queue_time = ap_crossing_times[vid][
i - 1] if i != 0 else ap_crossing_times[vid][0]
if is_ap_trip:
with open('%s/airport-trip-%s.csv' % (airport_trips_dir, yymm),
'a') as w_csvfile:
writer = csv.writer(w_csvfile)
ap_queue_time = st - ap_join_queue_time
new_row = [
tid, vid, did, st, et, duration, fare, prev_tet, ap_tm,
ap_join_queue_time, ap_queue_time
]
writer.writerow(new_row)
#
if ns_tm == DInNS_PInNS:
is_ns_trip = True
ns_join_queue_time = prev_tet
elif ns_tm == DOutNS_PInNS:
is_ns_trip = True
try:
i = bisect(ns_crossing_times[vid], st)
except KeyError:
logging_msg('%s-tid-%s' % (yymm, row[header_id['tid']]))
continue
ns_join_queue_time = ns_crossing_times[vid][
i - 1] if i != 0 else ns_crossing_times[vid][0]
if is_ns_trip:
with open(
'%s/nightsafari-trip-%s.csv' %
(nightsafari_trips_dir, yymm), 'a') as w_csvfile:
writer = csv.writer(w_csvfile)
ns_queue_time = st - ns_join_queue_time
new_row = [
tid, vid, did, st, et, duration, fare, prev_tet, ns_tm,
ns_join_queue_time, ns_queue_time
]
writer.writerow(new_row)
print 'end the file; %s' % yymm
logging_msg('end the file; %s' % yymm)
