def get_df(cols, load_from_temp, temp_path):
    weather_df = load_weather_df(CONFIG.preprocessed_meteo_path_complete)
    if not load_from_temp:
        logger.info('Loading train Dataframe...')
        train_df = load_train_df(CONFIG.preprocessed_train_path_means)
        logger.info('Loading weather Dataframe...')

        logger.info('Creating features...')
        ff = FeatureFactory(train_df, weather_df)
        for col in cols:
            logger.info('Creating %s feature...' % col)
            ff(col)
        if 'ASS_ASSIGNMENT' not in cols:
            cols = ['ASS_ASSIGNMENT'] + cols
        if 'DATE' not in cols:
            cols = ['DATE'] + cols
        if 'CSPL_RECEIVED_CALLS' not in cols:
            cols += ['CSPL_RECEIVED_CALLS']

        logger.info('Selecting features...')
        ff.select_features(cols)
        train_df = ff.X
        if temp_path is not None:
            train_df.to_csv(temp_path)
    else:
        assert temp_path is not None
        logger.info('Loading train Dataframe...')
        train_df = pd.read_csv(temp_path, encoding='latin-1', index_col=0, parse_dates=['DATE'])
    weather_df.reset_index(inplace=True)
    return train_df, weather_df
def complete_meteo_with_zeros(in_path, out_path=None):
    logger.debug('Loading Dataframe...')
    weather_df = load_weather_df(in_path)
    means = weather_df.mean()
    weather_df.reset_index(inplace=True)

    logger.debug('Generating empty Dataframe...')
    d1 = date(2011, 1, 1)
    d2 = date(2012, 12, 31)
    dates = [d1]
    while d1 < d2:
        d1 += td(days=1)
        dates.append(d1)
    zero_df = pd.DataFrame({'DATE': dates}, dtype=weather_df.dtypes['DATE'])

    logger.debug('Completing Dataframe...')
    weather_df = zero_df.merge(weather_df.copy(), how='left', on='DATE')
    weather_df['NUMB_FROZEN_DEPT'].fillna(means['NUMB_FROZEN_DEPT'], inplace=True)
    weather_df['NUMB_WET_DEPT'].fillna(means['NUMB_WET_DEPT'], inplace=True)
    weather_df.set_index('DATE', inplace=True)

    logger.debug('Saving Dataframe...')
    if out_path is not None:
        weather_df.to_csv(out_path)
    return weather_df
def compare_calls(scale, out_path, assignments=None, datetime=None):
    """
    Plot the number of calls to compare them.

    Parameters
    ==========
    scale: 'DATETIME', 'DAY', 'WEEK' or 'YEAR', calls are averaged on all smaller scales,
        and plotted for larger scales.
    out_path: str, folder in which figures should be saved.
    assignments: str or list of str, assignments to take into account.
        None to take all columns into account.
    datetime: if 'DATETIME', the datetime to filter on

    Example
    =======
    Week comparison: For each day of the week, take the average number of calls, then compare for each week of the year.
    """
    assert scale in ['DATETIME', 'DAY', 'WEEK', 'YEAR']
    if assignments is not None:
        if isinstance(assignments, str):
            assignments = [assignments]
        assert not set(assignments).difference(CONFIG.submission_assignments)
    else:
        assignments = CONFIG.submission_assignments

    df = load_train_df(CONFIG.preprocessed_train_path)
    df = df[df["ASS_ASSIGNMENT"].isin(assignments)]
    # if remove_days_off:
    #     df = df[df["DAY_OFF"] == 0]
    #     df.drop("DAY_OFF", axis=1, inplace=True)
    ff = FeatureFactory(df)
    for column in ["WEEK_NUMBER", "WEEK_DAY", "TIME"]:
        ff(column)
    df = ff.X

    if scale == 'DATETIME':
        assert datetime is not None
        df = df[ff("WEEK_DAY") == datetime.isoweekday()]
        df = df[df['TIME'] == datetime.hour + float(datetime.minute)/60]
        for assignment in assignments:
            print(assignment)
            df_assignment = df[df['ASS_ASSIGNMENT'] == assignment].reset_index()
            plt.plot(df_assignment['CSPL_RECEIVED_CALLS'])
        weather_df = load_weather_df(CONFIG.preprocessed_meteo_path)
        good_days = [d for d in weather_df.index if d.isoweekday() == datetime.isoweekday()]
        weather_df = weather_df.loc[good_days, :].reset_index()
        plt.plot(weather_df['NUMB_FROZEN_DEPT'])
        plt.plot(weather_df['NUMB_WET_DEPT'])
        plt.savefig(os.path.join(out_path, scale+".jpg"))

    if scale == 'DAY':
        grouped = df.groupby(["ASS_ASSIGNMENT", "WEEK_NUMBER", "WEEK_DAY", "TIME"])
        df = grouped["CSPL_RECEIVED_CALLS"].sum().reset_index()
        for assignment in assignments:
            print(assignment)
            df_assignment = df[df['ASS_ASSIGNMENT'] == assignment]
            for day in range(366):
                df_day = df_assignment[df_assignment['WEEK_NUMBER'] == int(day/7 + 1)]
                df_day = df_day[df_day['WEEK_DAY'] == day % 7]
                plt.plot(df_day['TIME'], df_day["CSPL_RECEIVED_CALLS"])
            plt.savefig(os.path.join(out_path, scale+"_"+assignment+".jpg"))
            plt.clf()
    if scale == 'WEEK':
        grouped = df.groupby(["ASS_ASSIGNMENT", "WEEK_NUMBER", "WEEK_DAY"])
        df = grouped["CSPL_RECEIVED_CALLS"].mean().reset_index()
        for assignment in assignments:
            print(assignment)
            df_assignment = df[df['ASS_ASSIGNMENT'] == assignment]
            for week_number in range(53):
                df_week = df_assignment[df_assignment['WEEK_NUMBER'] == week_number]
                plt.plot(df_week['WEEK_DAY'], df_week["CSPL_RECEIVED_CALLS"])
            plt.savefig(os.path.join(out_path, scale+"_"+assignment+".jpg"))
            plt.clf()
    if scale == 'YEAR':
        grouped = df.groupby(["ASS_ASSIGNMENT", "WEEK_NUMBER"])
        df = grouped["CSPL_RECEIVED_CALLS"].mean().reset_index()
        for assignment in assignments:
            print(assignment)
            df_assignment = df[df['ASS_ASSIGNMENT'] == assignment]
            plt.plot(df_assignment['WEEK_DAY'], df_assignment["CSPL_RECEIVED_CALLS"])
        # plt.axis([0, 52, 0, 50])
        plt.savefig(os.path.join(out_path, scale+"_absolute_values.jpg"))
        plt.clf()