def crossesThreshold(start_date, end_date, mahal_timeseries, threshold):
    for key in [
            keyFromDatetime(d)
            for d in dateRange(start_date, end_date, timedelta(hours=1))
    ]:
        if (key in mahal_timeseries and mahal_timeseries[key] > threshold):
            return True
def detectWindowedEvents(mahal_timeseries, zscore_timeseries, global_pace_timeseries, 
                          out_file, window_size=6, threshold_quant=.95):
                              
    logMsg("Detecting events at %d%% bound" % int(threshold_quant*100))
                              
    #Sort the keys of the timeseries chronologically    
    sorted_dates = sorted(mahal_timeseries)
    
    #Generate the list of values of R(t)
    mahal_list = [mahal_timeseries[d] for d in sorted_dates]

    
    #Use the quantile to determine the threshold
    sorted_mahal = sorted(mahal_list)
    threshold = getQuantile(sorted_mahal, threshold_quant)

    # Get the expected global pace    
    (expected_pace_timeseries, sd_pace_timeseries) = getExpectedPace(global_pace_timeseries)        
    
    
    
    start_date = datetime(2010,1,1)
    end_date = datetime(2014,1,1)
    shift = timedelta(hours=window_size)
    
    prev_above_threshold = False
    current_event_start = None
    current_event_end = None
    eventList = []
    for date in dateRange(start_date, end_date, shift):
        #print
        #print(str(date))
        #print(prev_above_threshold)
        if(crossesThreshold(date, date+shift, mahal_timeseries, threshold)):
            #print("CROSS")
            if(not prev_above_threshold):
                #print("RESET")
                current_event_start = date
                
            current_event_end = date+shift
            prev_above_threshold=True
        else:
            if(prev_above_threshold):
                #print("*************OUTPUTTING************")
                #print("%s -> %s" % (current_event_start, current_event_end))
                start_key = keyFromDatetime(current_event_start)
                end_key = keyFromDatetime(current_event_end)
                event = computeEventProperties(start_key, end_key, mahal_timeseries, 
                                           global_pace_timeseries, expected_pace_timeseries,
                                           zscore_timeseries, sorted_mahal=sorted_mahal,
                                           mahal_threshold=threshold)
                #Add to list            
                eventList.append(event)
                
            prev_above_threshold =False
    
    #Sort events by duration, in descending order
    eventList.sort(key = lambda x: x[5], reverse=True)
    
    #Write events to a CSV file
    w = csv.writer(open(out_file, "w"))
    w.writerow(["start_date", "end_date", "max_mahal", "mahal_quant", "duration", "hours_above_thresh", "max_pace_dev",
                "min_pace_dev", "worst_trip"])
                
    for event in eventList:
        [start_date, end_date, max_mahal, mahal_quant, duration, hours_above_thresh, max_pace_dev, min_pace_dev, worst_trip] = event
        formattedEvent = [start_date, end_date, "%.2f" % max_mahal, "%.3f" % mahal_quant, 
                          duration, hours_above_thresh, "%.2f" % max_pace_dev,
                          "%.2f" % min_pace_dev, worst_trip]
        w.writerow(formattedEvent)
    
    return eventList
def crossesThreshold(start_date, end_date, mahal_timeseries, threshold):
    for key in [keyFromDatetime(d) for d in dateRange(start_date, end_date, timedelta(hours=1))]:
        if(key in mahal_timeseries and mahal_timeseries[key] > threshold):
            return True
def detectWindowedEvents(mahal_timeseries,
                         zscore_timeseries,
                         global_pace_timeseries,
                         out_file,
                         window_size=6,
                         threshold_quant=.95):

    logMsg("Detecting events at %d%% bound" % int(threshold_quant * 100))

    #Sort the keys of the timeseries chronologically
    sorted_dates = sorted(mahal_timeseries)

    #Generate the list of values of R(t)
    mahal_list = [mahal_timeseries[d] for d in sorted_dates]

    #Use the quantile to determine the threshold
    sorted_mahal = sorted(mahal_list)
    threshold = getQuantile(sorted_mahal, threshold_quant)

    # Get the expected global pace
    (expected_pace_timeseries,
     sd_pace_timeseries) = getExpectedPace(global_pace_timeseries)

    start_date = datetime(2010, 1, 1)
    end_date = datetime(2014, 1, 1)
    shift = timedelta(hours=window_size)

    prev_above_threshold = False
    current_event_start = None
    current_event_end = None
    eventList = []
    for date in dateRange(start_date, end_date, shift):
        #print
        #print(str(date))
        #print(prev_above_threshold)
        if (crossesThreshold(date, date + shift, mahal_timeseries, threshold)):
            #print("CROSS")
            if (not prev_above_threshold):
                #print("RESET")
                current_event_start = date

            current_event_end = date + shift
            prev_above_threshold = True
        else:
            if (prev_above_threshold):
                #print("*************OUTPUTTING************")
                #print("%s -> %s" % (current_event_start, current_event_end))
                start_key = keyFromDatetime(current_event_start)
                end_key = keyFromDatetime(current_event_end)
                event = computeEventProperties(start_key,
                                               end_key,
                                               mahal_timeseries,
                                               global_pace_timeseries,
                                               expected_pace_timeseries,
                                               zscore_timeseries,
                                               sorted_mahal=sorted_mahal,
                                               mahal_threshold=threshold)
                #Add to list
                eventList.append(event)

            prev_above_threshold = False

    #Sort events by duration, in descending order
    eventList.sort(key=lambda x: x[5], reverse=True)

    #Write events to a CSV file
    w = csv.writer(open(out_file, "w"))
    w.writerow([
        "start_date", "end_date", "max_mahal", "mahal_quant", "duration",
        "hours_above_thresh", "max_pace_dev", "min_pace_dev", "worst_trip"
    ])

    for event in eventList:
        [
            start_date, end_date, max_mahal, mahal_quant, duration,
            hours_above_thresh, max_pace_dev, min_pace_dev, worst_trip
        ] = event
        formattedEvent = [
            start_date, end_date,
            "%.2f" % max_mahal,
            "%.3f" % mahal_quant, duration, hours_above_thresh,
            "%.2f" % max_pace_dev,
            "%.2f" % min_pace_dev, worst_trip
        ]
        w.writerow(formattedEvent)

    return eventList
def load_pace_data(perc_data_threshold, pool=DefaultPool()):
    weekday_names = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]

    # Connect to the database adn get hte available dates
    logMsg("Getting relevant dates.")
    db_main.connect("db_functions/database.conf")
    # dates = db_travel_times.get_available_dates()
    dates = list(dateRange(datetime(2014, 06, 01), datetime(2014, 07, 01)))

    """ Only Do Once for the whole dataset and store in link_counts_chicago table"""
    # logMsg ("Computing consistent link set")
    # compute_all_link_counts(dates, pool=pool)

    logMsg("Loading consistent link set")
    consistent_link_set = load_consistent_link_set(dates, perc_data_threshold)
    if len(consistent_link_set) == 0:
        logMsg("Find 0 consistent_links. Return.")
        return
    else:
        print ("len of consistent_link_set", len(consistent_link_set))
    db_main.close()

    logMsg("Generating vectors")

    # Initialize dictionaries
    pace_timeseries = {}
    pace_grouped = defaultdict(list)
    dates_grouped = defaultdict(list)
    weights_grouped = defaultdict(list)

    # Split the dates into several pieces and use parallel processing to load the
    # vectors for each of these dates.  We will use a partial function to hold the
    # consistent_link_set constant across all dates
    it = splitList(dates, pool._processes)
    load_pace_vectors_consistent = partial(load_pace_vectors, consistent_link_set=consistent_link_set)
    list_of_lists = pool.map(load_pace_vectors_consistent, it)

    logMsg("Merging outputs.")
    # Flatten the vectors into one big list
    vects = [vect for vect_lst, weight_lst in list_of_lists for vect in vect_lst]
    weights = [weight for vect_lst, weight_lst in list_of_lists for weight in weight_lst]

    # Loop through all dates - one vector will be created for each one
    for i in xrange(len(dates)):
        date = dates[i]
        vect = vects[i]
        weight = weights[i]

        # Extract the date, hour of day, and day of week
        just_date = str(date.date())
        hour = date.hour
        weekday = weekday_names[date.weekday()]

        # Save vector in the timeseries

        # save the vector into the group
        # pace_grouped[(weekday, hour)].append(vect)
        # weights_grouped[(weekday, hour)].append(weight)
        # dates_grouped[(weekday, hour)].append(just_date)

        # use constant as key for this moment
        # weekday = 0
        # hour = 0
        # print just_date
        pace_timeseries[(just_date, hour, weekday)] = vect
        # print "vect here =========", vect
        pace_grouped[(weekday, hour)].append(vect)
        weights_grouped[(weekday, hour)].append(weight)
        dates_grouped[(weekday, hour)].append(just_date)

    # print pace_timeseries.keys()
    print len(pace_grouped[(0, 0)]), len(pace_grouped[(0, 0)][0])
    # Assign trip names based on node ids
    trip_names = ["%d" % link_id for link_id in consistent_link_set]

    # print "    len", len(pace_grouped.values())
    return (pace_timeseries, pace_grouped, weights_grouped, dates_grouped, trip_names, consistent_link_set)