def run():
    remove_creat_dir(shift_pro_dur_dir)
    init_multiprocessor()
    count_num_jobs = 0
    for fn in get_all_files(shifts_dir, 'shift-hour-state-', '.csv'):
        put_task(process_file, [fn])
        count_num_jobs += 1
    end_multiprocessor(count_num_jobs)
def run():
    remove_creat_dir(for_full_driver_dir)
    init_multiprocessor()
    count_num_jobs = 0
    for fn in get_all_files(merged_trip_dir, 'trips', '.csv'):
#         process_file(fn)
        put_task(process_file, [fn])
        count_num_jobs += 1
    end_multiprocessor(count_num_jobs)
def run():
    remove_creat_dir(airport_trips_dir); remove_creat_dir(nightsafari_trips_dir)
    csv_files = get_all_files(trips_dir, 'whole-trip-', '.csv')
    init_multiprocessor()
    count_num_jobs = 0
    for fn in csv_files:
#         process_file(fn)
        put_task(process_file, [fn])
        count_num_jobs += 1
    end_multiprocessor(count_num_jobs)
示例#4
0
def run():
    remove_creat_dir(trips_dir)
    csv_files = get_all_files(merged_trip_dir, 'trips', '.csv')
    
    init_multiprocessor()
    counter = 0
    for fn in csv_files:
        counter += 1
        put_task(process_file, [fn])
    end_multiprocessor(counter)
示例#5
0
def run():
    remove_creat_dir(for_learning_dir)
    csv_files = get_all_files(merged_trip_dir, 'trips', '.csv')
    #
    init_multiprocessor()
    count_num_jobs = 0
    for fn in csv_files:
#         process_file(fn)
        put_task(process_file, [fn])
        count_num_jobs += 1
    end_multiprocessor(count_num_jobs)    
示例#6
0
def run():
    remove_creat_dir(hourly_summary)
    csv_files = get_all_files(trips_dir, 'whole-trip-', '.csv')
    #
    init_multiprocessor()
    count_num_jobs = 0
    for fn in csv_files:
        put_task(process_file, [fn])
        #         process_file(fn)
        count_num_jobs += 1
    end_multiprocessor(count_num_jobs)
def run():
    remove_creat_dir(hourly_summary)
    csv_files = get_all_files(trips_dir, 'whole-trip-', '.csv')
    #
    init_multiprocessor()
    count_num_jobs = 0
    for fn in csv_files:
        put_task(process_file, [fn])
#         process_file(fn)
        count_num_jobs += 1
    end_multiprocessor(count_num_jobs)
示例#8
0
def run():
    remove_creat_dir(airport_trips_dir)
    remove_creat_dir(nightsafari_trips_dir)
    csv_files = get_all_files(trips_dir, 'whole-trip-', '.csv')
    init_multiprocessor()
    count_num_jobs = 0
    for fn in csv_files:
        #         process_file(fn)
        put_task(process_file, [fn])
        count_num_jobs += 1
    end_multiprocessor(count_num_jobs)
示例#9
0
def run():
#     task_list = [(0, 0), (0, 1), (0, 2), (0, 3), (0, 4), (0, 5), (0, 6), (0, 7), (0, 8), (0, 9), (0, 10), (2, 0), (2, 1), (2, 2), (2, 3), (2, 4), (2, 5), (2, 6), (2, 7), (2, 8), (2, 9), (2, 10), (4, 0), (4, 1), (4, 2), (4, 3), (4, 4), (4, 5), (4, 6), (4, 7), (4, 8), (4, 9), (4, 10), (6, 0), (6, 1), (6, 2), (6, 3), (6, 4), (6, 5), (6, 6), (6, 7), (6, 8), (6, 9), (6, 10), (8, 0), (8, 1), (8, 2), (8, 3), (8, 4), (8, 5), (8, 6), (8, 7), (8, 8), (8, 9), (8, 10), (10, 0), (10, 1), (10, 2), (10, 3), (10, 4), (10, 5), (10, 6), (10, 7), (10, 8), (10, 9), (10, 10)]
    init_multiprocessor()
    counter = 0
#     for i, j in task_list:
#         ALPHA = i / 10
#         GAMMA = j / 10
        
    for ALPHA, GAMMA in [(0.3, 0.7),(0.3, 0.9),(0.5, 0.1),(0.5, 0.3),(0.5, 0.8),(0.5, 1.0)]:
        put_task(process_files, [ALPHA, GAMMA])
        counter += 1
    end_multiprocessor(counter)     
def run():
    csv_files = get_all_files(airport_trips_dir, 'airport-trip-', '.csv')
    init_multiprocessor()
    count_num_jobs = 0
    for fn in csv_files:
        try:
            put_task(process_file, [fn])
        except Exception as _:
            logging_msg('Algorithm runtime exception (%s)\n' % (fn) + format_exc())
            raise
        count_num_jobs += 1
    end_multiprocessor(count_num_jobs)
示例#11
0
def run():
    remove_creat_dir(full_shift_dir)
    csv_files = get_all_files(shifts_dir, '', '.csv')
    init_multiprocessor()
    count_num_jobs = 0
    for fn in csv_files:
        try:
            put_task(process_file, [fn])
        except Exception as _:
            logging_msg('Algorithm runtime exception (%s)\n' % (fn) + format_exc())
            raise
        count_num_jobs += 1
    end_multiprocessor(count_num_jobs)
示例#12
0
def run():
    #     task_list = [(0, 0), (0, 1), (0, 2), (0, 3), (0, 4), (0, 5), (0, 6), (0, 7), (0, 8), (0, 9), (0, 10), (2, 0), (2, 1), (2, 2), (2, 3), (2, 4), (2, 5), (2, 6), (2, 7), (2, 8), (2, 9), (2, 10), (4, 0), (4, 1), (4, 2), (4, 3), (4, 4), (4, 5), (4, 6), (4, 7), (4, 8), (4, 9), (4, 10), (6, 0), (6, 1), (6, 2), (6, 3), (6, 4), (6, 5), (6, 6), (6, 7), (6, 8), (6, 9), (6, 10), (8, 0), (8, 1), (8, 2), (8, 3), (8, 4), (8, 5), (8, 6), (8, 7), (8, 8), (8, 9), (8, 10), (10, 0), (10, 1), (10, 2), (10, 3), (10, 4), (10, 5), (10, 6), (10, 7), (10, 8), (10, 9), (10, 10)]
    init_multiprocessor()
    counter = 0
    #     for i, j in task_list:
    #         ALPHA = i / 10
    #         GAMMA = j / 10

    for ALPHA, GAMMA in [(0.3, 0.7), (0.3, 0.9), (0.5, 0.1), (0.5, 0.3),
                         (0.5, 0.8), (0.5, 1.0)]:
        put_task(process_files, [ALPHA, GAMMA])
        counter += 1
    end_multiprocessor(counter)
示例#13
0
def run():
    remove_creat_dir(general_dur_fare_dir)
    #     process_files('0901')
    init_multiprocessor()
    count_num_jobs = 0
    for y in xrange(9, 11):
        for m in xrange(1, 13):
            yymm = '%02d%02d' % (y, m)
            if yymm in ['0912', '1010']:
                continue
            put_task(process_files, [yymm])
            count_num_jobs += 1
    end_multiprocessor(count_num_jobs)
def run():
    remove_creat_dir(ap_dur_fare_q_time_dir)
    init_multiprocessor()
    count_num_jobs = 0
    for y in xrange(9, 11):
        for m in xrange(1, 13):
            yymm = '%02d%02d' % (y, m) 
            if yymm in ['0912', '1010']:
                continue
#             process_files('1007')
            put_task(process_files, [yymm])
            count_num_jobs += 1
    end_multiprocessor(count_num_jobs)
示例#15
0
def run():
    csv_files = get_all_files(airport_trips_dir, 'airport-trip-', '.csv')
    init_multiprocessor()
    count_num_jobs = 0
    for fn in csv_files:
        try:
            put_task(process_file, [fn])
        except Exception as _:
            logging_msg('Algorithm runtime exception (%s)\n' % (fn) +
                        format_exc())
            raise
        count_num_jobs += 1
    end_multiprocessor(count_num_jobs)
def run():
    candi_dirs = get_all_directories(for_learning_dir)
    q_lerning_ended_dir = [dn for dn in candi_dirs if len(get_all_files(for_learning_dir + '/%s' % (dn), 'ALPHA-', '.pkl')) == 22]
    init_multiprocessor()
    counter = 0
    for y in xrange(9, 11):
        for m in xrange(1, 13):
            yymm = '%02d%02d' % (y, m) 
            if yymm in ['0912', '1010']:
                continue
#             process_files(yymm, q_lerning_ended_dir)
            put_task(process_files, [yymm, q_lerning_ended_dir])
            counter += 1
    end_multiprocessor(counter)
def run():
    remove_creat_dir(individual_detail_dir)
    #     process_files('1007')
    init_multiprocessor()
    count_num_jobs = 0
    for y in xrange(9, 11):
        for m in xrange(1, 13):
            try:
                yymm = "%02d%02d" % (y, m)
                if yymm in ["0912", "1010"]:
                    continue
                put_task(process_files, [yymm])
            except Exception as _:
                logging_msg("Algorithm runtime exception (%s)\n" % (yymm) + format_exc())
                raise
            count_num_jobs += 1
    end_multiprocessor(count_num_jobs)
示例#18
0
def run():
    remove_creat_dir(individual_detail_dir)
    #     process_files('1007')
    init_multiprocessor()
    count_num_jobs = 0
    for y in xrange(9, 11):
        for m in xrange(1, 13):
            try:
                yymm = '%02d%02d' % (y, m)
                if yymm in ['0912', '1010']:
                    continue
                put_task(process_files, [yymm])
            except Exception as _:
                logging_msg('Algorithm runtime exception (%s)\n' % (yymm) +
                            format_exc())
                raise
            count_num_jobs += 1
    end_multiprocessor(count_num_jobs)
def run():
    candi_dirs = get_all_directories(for_learning_dir)
    q_lerning_ended_dir = [
        dn for dn in candi_dirs
        if len(get_all_files(for_learning_dir + '/%s' %
                             (dn), 'ALPHA-', '.pkl')) == 22
    ]
    init_multiprocessor()
    counter = 0
    for y in xrange(9, 11):
        for m in xrange(1, 13):
            yymm = '%02d%02d' % (y, m)
            if yymm in ['0912', '1010']:
                continue


#             process_files(yymm, q_lerning_ended_dir)
            put_task(process_files, [yymm, q_lerning_ended_dir])
            counter += 1
    end_multiprocessor(counter)
def run():
    remove_creat_dir(logs_dir)
    csv_file_paths = []
    for yd in os.listdir(server_prefix):
        yd_path = '%s/%s' % (server_prefix, yd)
        if not (os.path.isdir(yd_path) and yd.startswith('20')):
            continue
        if yd not in TARGET_YEARS:
            continue
        for md in os.listdir(yd_path):
            md_path = '%s/%s' % (yd_path, md)
            if not os.path.isdir(md_path):
                continue
            fn = 'logs-%s%s-normal.csv' % (yd[-2:], md)
            csv_file_paths.append('%s/%s/%s' % (md_path, 'logs', fn))
    init_multiprocessor()
    count_num_jobs = 0
    for path_to_csv_file in csv_file_paths:
#         process_file(path_to_csv_file)
        put_task(process_file, [path_to_csv_file])
        count_num_jobs += 1
    end_multiprocessor(count_num_jobs)     
def run():
    remove_creat_dir(logs_dir)
    csv_file_paths = []
    for yd in os.listdir(server_prefix):
        yd_path = '%s/%s' % (server_prefix, yd)
        if not (os.path.isdir(yd_path) and yd.startswith('20')):
            continue
        if yd not in TARGET_YEARS:
            continue
        for md in os.listdir(yd_path):
            md_path = '%s/%s' % (yd_path, md)
            if not os.path.isdir(md_path):
                continue
            fn = 'logs-%s%s-normal.csv' % (yd[-2:], md)
            csv_file_paths.append('%s/%s/%s' % (md_path, 'logs', fn))
    init_multiprocessor()
    count_num_jobs = 0
    for path_to_csv_file in csv_file_paths:
        #         process_file(path_to_csv_file)
        put_task(process_file, [path_to_csv_file])
        count_num_jobs += 1
    end_multiprocessor(count_num_jobs)
示例#22
0
def combine_trip_data_with_multi():
    '''
    In the server, when two files are combined, a trip can totally be expressed
    A trip data shows the below information;
    
    trip-id,job-id,start-time,end-time,start-long,start-lat,end-long,end-lat,vehicle-id,distance,fare,duration,start-dow,start-day,start-hour,start-minute,end-dow,end-day,end-hour,end-minute,start-zone,end-zone,start-postal,end-postal,driver-id
    
    ex.
    6029L09010100005,001010000,1230739200,1230739680,103.95029,1.37226,103.93417,1.38599,8069,3.4,850,480,4,1,0,0,4,1,0,8,51,51,518457,519355,20108
    
    Also, this function uses the multiprocess library in Python.
    '''
    
    logging_on_txt('Start combine!!', gen_txt=True)
    init_multiprocessor()
    print 'Start combine'
    for y in YEARS:
        y_two_digit = y[-2:]
        for m in MONTHS:
            normal_file = TAXI_HOME + '/%s/%s/trips/trips-%s%s-normal.csv' % (y, m, y_two_digit, m)
            ext_file = TAXI_HOME + '/%s/%s/trips/trips-%s%s-normal-ext.csv' % (y, m, y_two_digit, m)
            put_task(read_write_a_trip, [normal_file, ext_file])
    end_multiprocessor(len(YEARS) * len(MONTHS))