Exemple #1
0
def get_bus_dates_from_agg_method_and_contracts_back(**kwargs):

    ref_date = kwargs['ref_date']
    aggregation_method = kwargs['aggregation_method']
    contracts_back = kwargs['contracts_back']

    ref_datetime = cu.convert_doubledate_2datetime(ref_date)

    if aggregation_method == 12:
        cal_date_list = [ref_datetime - relativedelta(years=x) for x in range(1, contracts_back+1)]
    elif aggregation_method == 1:
        cal_date_list = [ref_datetime - relativedelta(months=x) for x in range(1, contracts_back+1)]

    bday_us = CustomBusinessDay(expcalendar=exp.get_calendar_4ticker_head(const.reference_tickerhead_4business_calendar))

    if 'shift_bus_days' in kwargs.keys():
        shift_bus_days = kwargs['shift_bus_days']
        if shift_bus_days >= 0:
            bus_date_list = [pd.date_range(x, periods=shift_bus_days+1, freq=bday_us)[shift_bus_days].to_datetime() for x in cal_date_list]
        elif shift_bus_days < 0:
            bus_date_list = [pd.date_range(start=x-relativedelta(days=(max(m.ceil(-shift_bus_days*7/5)+5, -shift_bus_days+5))), end=x, freq=bday_us)[shift_bus_days-1].to_datetime() for x in cal_date_list]
    else:
        bus_date_list = [pd.date_range(x, periods=1, freq=bday_us)[0].to_datetime() for x in cal_date_list]

    return bus_date_list
Exemple #2
0
def get_bus_dates_from_agg_method_and_contracts_back(**kwargs):

    ref_date = kwargs['ref_date']
    aggregation_method = kwargs['aggregation_method']
    contracts_back = kwargs['contracts_back']

    ref_datetime = cu.convert_doubledate_2datetime(ref_date)

    if aggregation_method == 12:
        cal_date_list = [
            ref_datetime - relativedelta(years=x)
            for x in range(1, contracts_back + 1)
        ]
    elif aggregation_method == 1:
        cal_date_list = [
            ref_datetime - relativedelta(months=x)
            for x in range(1, contracts_back + 1)
        ]

    bday_us = CustomBusinessDay(calendar=exp.get_calendar_4ticker_head(
        const.reference_tickerhead_4business_calendar))

    if 'shift_bus_days' in kwargs.keys():
        shift_bus_days = kwargs['shift_bus_days']
        if shift_bus_days >= 0:
            bus_date_list = [
                pd.date_range(x, periods=shift_bus_days + 1,
                              freq=bday_us)[shift_bus_days].to_pydatetime()
                for x in cal_date_list
            ]
        elif shift_bus_days < 0:
            bus_date_list = [
                pd.date_range(start=x - relativedelta(days=(max(
                    m.ceil(-shift_bus_days * 7 / 5) + 5, -shift_bus_days +
                    5))),
                              end=x,
                              freq=bday_us)[shift_bus_days -
                                            1].to_pydatetime()
                for x in cal_date_list
            ]
    else:
        bus_date_list = [
            pd.date_range(x, periods=1, freq=bday_us)[0].to_pydatetime()
            for x in cal_date_list
        ]

    return bus_date_list
def get_backtesting_dates(**kwargs):
    date_to = kwargs['date_to']
    years_back = kwargs['years_back']

    if 'day_of_week' in kwargs.keys():
        day_of_week = kwargs['day_of_week']
    else:
        day_of_week = 2

    date_from = cu.doubledate_shift(date_to, years_back*365)

    trading_calendar = exp.get_calendar_4ticker_head('CL')
    bday_us = CustomBusinessDay(calendar=trading_calendar)

    dts = pd.date_range(start=cu.convert_doubledate_2datetime(date_from),
                    end=cu.convert_doubledate_2datetime(date_to), freq=bday_us)

    dts = dts[dts.dayofweek==day_of_week]

    return {'date_time_dates': dts,
            'double_dates': [int(x.strftime('%Y%m%d')) for x in dts]}
Exemple #4
0
def get_backtesting_dates(**kwargs):
    date_to = kwargs['date_to']
    years_back = kwargs['years_back']

    if 'day_of_week' in kwargs.keys():
        day_of_week = kwargs['day_of_week']
    else:
        day_of_week = 2

    date_from = cu.doubledate_shift(date_to, years_back * 365)

    trading_calendar = exp.get_calendar_4ticker_head('CL')
    bday_us = CustomBusinessDay(calendar=trading_calendar)

    dts = pd.date_range(start=cu.convert_doubledate_2datetime(date_from),
                        end=cu.convert_doubledate_2datetime(date_to),
                        freq=bday_us)

    dts = dts[dts.dayofweek == day_of_week]

    return {
        'date_time_dates': dts,
        'double_dates': [int(x.strftime('%Y%m%d')) for x in dts]
    }
Exemple #5
0
def update_options_price_database_from_cme_files_4ticker(**kwargs):

    ticker = kwargs['ticker']
    contract_specs_output = cmi.get_contract_specs(ticker)
    ticker_head = contract_specs_output['ticker_head']
    ticker_month_num = contract_specs_output['ticker_month_num']
    ticker_year = contract_specs_output['ticker_year']

    if 'settle_date' in kwargs.keys():
        settle_date = kwargs['settle_date']
        kwargs['report_date'] = settle_date
    else:
        settle_date = int(time.strftime('%Y%m%d'))
        kwargs['settle_date'] = settle_date
        kwargs['report_date'] = settle_date

    if not exp.is_business_day(double_date=settle_date,
                               reference_tickerhead=ticker_head):
        return

    if 'expiration_date' in kwargs.keys():
        expiration_date = kwargs['expiration_date']
    else:
        expiration_date = exp.get_options_expiration(ticker)
        expiration_date = expiration_date.date()

    settle_datetime = cu.convert_doubledate_2datetime(settle_date)

    if 'cal_dte' in kwargs.keys():
        cal_dte = kwargs['cal_dte']
    else:
        cal_dte = (expiration_date - settle_datetime.date()).days

    if 'tr_dte' in kwargs.keys():
        tr_dte = kwargs['tr_dte']
    else:
        bday_us = CustomBusinessDay(
            calendar=exp.get_calendar_4ticker_head(ticker_head))
        dts = pd.date_range(start=settle_datetime,
                            end=expiration_date,
                            freq=bday_us)
        tr_dte = len(
            [x for x in dts if x.to_pydatetime().date() < expiration_date])

    data_vendor_id = 2
    now = dt.datetime.now()
    con = msu.get_my_sql_connection(**kwargs)

    process_output = pco.process_cme_options_4ticker(**kwargs)

    if process_output['success']:
        settle_frame = process_output['settle_frame']
    else:
        if 'con' not in kwargs.keys():
            con.close()
        return

    column_names = settle_frame.columns.tolist()

    option_type_indx = column_names.index('option_type')
    strike_indx = column_names.index('strike')
    settle_indx = column_names.index('settle')
    volume_indx = column_names.index('volume')
    interest_indx = column_names.index('interest')

    tuples = [
        tuple([
            data_vendor_id, ticker_head, ticker_month_num, ticker_year, ticker,
            x[option_type_indx], x[strike_indx],
            settle_datetime.date(), cal_dte, tr_dte, now, now,
            None if np.isnan(x[settle_indx]) else x[settle_indx],
            None if np.isnan(x[volume_indx]) else x[volume_indx],
            None if np.isnan(x[interest_indx]) else x[interest_indx]
        ]) for x in settle_frame.values
    ]

    column_str = "data_vendor_id, ticker_head, ticker_month, ticker_year, ticker, " \
                 " option_type, strike, price_date, cal_dte, tr_dte, " \
                 " created_date,last_updated_date, close_price, volume, open_interest"

    insert_str = ("%s, " * len(column_str.split(',')))[:-2]
    final_str = "REPLACE INTO daily_option_price (%s) VALUES (%s)" % (
        column_str, insert_str)
    msu.sql_execute_many_wrapper(final_str=final_str, tuples=tuples, con=con)

    if 'con' not in kwargs.keys():
        con.close()
def load_price_data_4ticker(load_price_data_input):

    ticker = load_price_data_input['ticker']
    expiration_date = load_price_data_input['expiration_date']
    data_vendor_id = load_price_data_input['data_vendor_id']
    symbol_id = load_price_data_input['symbol_id']

    quandl_input = {'ticker': ticker}

    if 'date_to' in load_price_data_input.keys():
        quandl_input['date_to'] = load_price_data_input['date_to']

    if 'date_from' in load_price_data_input.keys() and load_price_data_input['date_from'] is not None:
        quandl_input['date_from'] = load_price_data_input['date_from']

    quandl_out = gdq.get_daily_historic_data_quandl(**quandl_input)

    if not quandl_out['success']:
        return

    price_data = quandl_out['data_out']

    if price_data.empty:
        print('Empty Results For ' + ticker)
        return

    contract_specs_output = cmi.get_contract_specs(ticker)

    if contract_specs_output['ticker_head'] == 'JY':
        price_multiplier = 10
    else:
        price_multiplier = 1

    bday_us = CustomBusinessDay(calendar=exp.get_calendar_4ticker_head(contract_specs_output['ticker_head']))
    dts = pd.date_range(start=price_data.index[0], end=expiration_date, freq=bday_us)
    dts = [x.date() for x in dts]

    now = datetime.datetime.now()

    price_data['price_date'] = pd.Series(price_data.index, index=price_data.index)

    column_names = price_data.columns.tolist()

    open_indx = column_names.index('Open')
    high_indx = column_names.index('High')
    low_indx = column_names.index('Low')
    settle_indx = column_names.index('Settle')
    volume_indx = column_names.index('Volume')
    interest_indx = column_names.index('Open Interest')
    date_indx = column_names.index('price_date')

    tuples = [tuple([data_vendor_id, symbol_id,
                     contract_specs_output['ticker_head'],
                     contract_specs_output['ticker_month_num'],
                     x[date_indx].to_datetime().date(),
                     (expiration_date-x[date_indx].to_datetime().date()).days,
                     len([y for y in dts if y > x[date_indx].to_datetime().date()]),
                     now, now,
                     None if np.isnan(x[open_indx]) else price_multiplier*x[open_indx],
                     None if np.isnan(x[high_indx]) else price_multiplier*x[high_indx],
                     None if np.isnan(x[low_indx]) else price_multiplier*x[low_indx],
                     None if np.isnan(x[settle_indx]) else price_multiplier*x[settle_indx],
                     None if np.isnan(x[volume_indx]) else x[volume_indx],
                     None if np.isnan(x[interest_indx]) else x[interest_indx]]) for x in price_data.values]

    column_str = "data_vendor_id, symbol_id, ticker_head, ticker_month, price_date,cal_dte, tr_dte, created_date,last_updated_date, open_price, high_price, low_price, close_price, volume, open_interest"
    insert_str = ("%s, " * 15)[:-2]
    final_str = "REPLACE INTO daily_price (%s) VALUES (%s)" % (column_str, insert_str)

    con = msu.get_my_sql_connection(**load_price_data_input)

    msu.sql_execute_many_wrapper(final_str=final_str, tuples=tuples, con=con)

    if 'con' not in load_price_data_input.keys():
        con.close()
def update_futures_price_database_from_cme_file(**kwargs):

    ticker_head_list = cmi.cme_futures_tickerhead_list

    import time
    con = msu.get_my_sql_connection(**kwargs)

    if 'settle_date' in kwargs.keys():
        run_date = kwargs['settle_date']
    else:
        run_date = int(time.strftime('%Y%m%d'))

    #run_date = 20160225
    data_vendor_id = 2
    now = datetime.datetime.now()
    run_datetime = cu.convert_doubledate_2datetime(run_date)

    for ticker_head in ticker_head_list:
        #print(ticker_head)

        contract_list = []

        bday_us = CustomBusinessDay(calendar=exp.get_calendar_4ticker_head(ticker_head))

        if not exp.is_business_day(double_date=run_date, reference_tickerhead=ticker_head):
            continue

        cme_output = pcf.process_cme_futures_4tickerhead(ticker_head=ticker_head, report_date=run_date)
        settle_frame = cme_output['settle_frame']

        for ticker_month in cmi.futures_contract_months[ticker_head]:
            ticker_month_num = cmi.letter_month_string.find(ticker_month)+1
            max_cal_dte = cmi.get_max_cal_dte(ticker_head=ticker_head, ticker_month=ticker_month_num)

            contract_list.extend(cl.get_db_contract_list_filtered(expiration_date_from=run_date,
                                                            expiration_date_to=cu.doubledate_shift(run_date, -max_cal_dte),
                                                            ticker_head=ticker_head, ticker_month=ticker_month_num, con=con,
                                                                  instrument='futures'))

        contract_frame = pd.DataFrame(contract_list, columns=['symbol_id', 'ticker', 'expiration_date'])
        merged_frame = pd.merge(contract_frame,settle_frame, how='inner', on='ticker')
        merged_frame.sort('expiration_date', ascending=True, inplace=True)

        column_names = merged_frame.columns.tolist()

        symbol_id_indx = column_names.index('symbol_id')
        ticker_month_indx = column_names.index('ticker_month')
        open_indx = column_names.index('open')
        high_indx = column_names.index('high')
        low_indx = column_names.index('low')
        settle_indx = column_names.index('settle')
        volume_indx = column_names.index('volume')
        interest_indx = column_names.index('interest')
        expiration_indx = column_names.index('expiration_date')

        dts = pd.date_range(start=run_datetime, end=merged_frame['expiration_date'].iloc[-1], freq=bday_us)

        tuples = [tuple([data_vendor_id, x[symbol_id_indx],
                     ticker_head,
                     x[ticker_month_indx],
                     run_datetime.date(),
                    (x[expiration_indx]-run_datetime.date()).days,
                     len([y for y in dts if y.to_datetime().date() < x[expiration_indx]]),
                     now, now,
                     None if np.isnan(x[open_indx]) else x[open_indx],
                     None if np.isnan(x[high_indx]) else x[high_indx],
                     None if np.isnan(x[low_indx]) else x[low_indx],
                     None if np.isnan(x[settle_indx]) else x[settle_indx],
                     None if np.isnan(x[volume_indx]) else x[volume_indx],
                     None if np.isnan(x[interest_indx]) else x[interest_indx]]) for x in merged_frame.values]

        column_str = "data_vendor_id, symbol_id, ticker_head, ticker_month, price_date,cal_dte, tr_dte, created_date,last_updated_date, open_price, high_price, low_price, close_price, volume, open_interest"
        insert_str = ("%s, " * 15)[:-2]
        final_str = "REPLACE INTO daily_price (%s) VALUES (%s)" % (column_str, insert_str)
        msu.sql_execute_many_wrapper(final_str=final_str, tuples=tuples, con=con)

    if 'con' not in kwargs.keys():
        con.close()
def generate_and_update_futures_data_file_4tickerhead(**kwargs):

    ticker_head = kwargs['ticker_head']

    con = msu.get_my_sql_connection(**kwargs)

    if os.path.isfile(presaved_futures_data_folder + '/' + ticker_head +
                      '.pkl'):
        old_data = pd.read_pickle(presaved_futures_data_folder + '/' +
                                  ticker_head + '.pkl')
        last_available_date = int(
            old_data['settle_date'].max().to_pydatetime().strftime('%Y%m%d'))
        date_from = cu.doubledate_shift(last_available_date, 60)
        data4_tickerhead = gfp.get_futures_price_4ticker(
            ticker_head=ticker_head, date_from=date_from, con=con)
    else:
        data4_tickerhead = gfp.get_futures_price_4ticker(
            ticker_head=ticker_head, con=con)

    data4_tickerhead = pd.merge(data4_tickerhead,
                                dirty_data_points,
                                on=['settle_date', 'ticker'],
                                how='left')
    data4_tickerhead = data4_tickerhead[data4_tickerhead['discard'] != True]
    data4_tickerhead = data4_tickerhead.drop('discard', 1)

    data4_tickerhead['close_price'] = [
        float(x) if x is not None else float('NaN')
        for x in data4_tickerhead['close_price'].values
    ]
    data4_tickerhead['open_price'] = [
        float(x) if x is not None else float('NaN')
        for x in data4_tickerhead['open_price'].values
    ]
    data4_tickerhead['high_price'] = [
        float(x) if x is not None else float('NaN')
        for x in data4_tickerhead['high_price'].values
    ]
    data4_tickerhead['low_price'] = [
        float(x) if x is not None else float('NaN')
        for x in data4_tickerhead['low_price'].values
    ]

    data4_tickerhead['cont_indx'] = 100 * data4_tickerhead[
        'ticker_year'] + data4_tickerhead['ticker_month']
    unique_cont_indx_list = data4_tickerhead['cont_indx'].unique()
    num_contracts = len(unique_cont_indx_list)
    unique_cont_indx_list = np.sort(unique_cont_indx_list)
    merged_dataframe_list = [None] * num_contracts

    bday_us = CustomBusinessDay(calendar=exp.get_calendar_4ticker_head('CL'))
    full_dates = pd.date_range(start=data4_tickerhead['settle_date'].min(),
                               end=data4_tickerhead['settle_date'].max(),
                               freq=bday_us)

    for i in range(num_contracts):

        contract_data = data4_tickerhead[data4_tickerhead['cont_indx'] ==
                                         unique_cont_indx_list[i]]

        contract_full_dates = full_dates[
            (full_dates >= contract_data['settle_date'].min())
            & (full_dates <= contract_data['settle_date'].max())]
        full_date_frame = pd.DataFrame(contract_full_dates,
                                       columns=['settle_date'])
        merged_dataframe_list[i] = pd.merge(full_date_frame,
                                            contract_data,
                                            on='settle_date',
                                            how='left')

        merged_dataframe_list[i]['ticker'] = contract_data['ticker'][
            contract_data.index[0]]
        merged_dataframe_list[i]['ticker_head'] = contract_data['ticker_head'][
            contract_data.index[0]]
        merged_dataframe_list[i]['ticker_month'] = contract_data[
            'ticker_month'][contract_data.index[0]]
        merged_dataframe_list[i]['ticker_year'] = contract_data['ticker_year'][
            contract_data.index[0]]
        merged_dataframe_list[i]['cont_indx'] = contract_data['cont_indx'][
            contract_data.index[0]]

        merged_dataframe_list[i][
            'change1'] = merged_dataframe_list[i]['close_price'].shift(
                -2) - merged_dataframe_list[i]['close_price'].shift(-1)
        merged_dataframe_list[i]['change1_instant'] = merged_dataframe_list[i][
            'close_price'].shift(-1) - merged_dataframe_list[i]['close_price']
        merged_dataframe_list[i]['high1_instant'] = merged_dataframe_list[i][
            'high_price'].shift(-1)
        merged_dataframe_list[i]['low1_instant'] = merged_dataframe_list[i][
            'low_price'].shift(-1)
        merged_dataframe_list[i][
            'change2'] = merged_dataframe_list[i]['close_price'].shift(
                -3) - merged_dataframe_list[i]['close_price'].shift(-1)
        merged_dataframe_list[i][
            'change5'] = merged_dataframe_list[i]['close_price'].shift(
                -6) - merged_dataframe_list[i]['close_price'].shift(-1)
        merged_dataframe_list[i][
            'change10'] = merged_dataframe_list[i]['close_price'].shift(
                -11) - merged_dataframe_list[i]['close_price'].shift(-1)
        merged_dataframe_list[i][
            'change20'] = merged_dataframe_list[i]['close_price'].shift(
                -21) - merged_dataframe_list[i]['close_price'].shift(-1)
        merged_dataframe_list[i]['change_5'] = merged_dataframe_list[i][
            'close_price'] - merged_dataframe_list[i]['close_price'].shift(5)
        merged_dataframe_list[i]['change_1'] = merged_dataframe_list[i][
            'close_price'] - merged_dataframe_list[i]['close_price'].shift(1)

    data4_tickerhead = pd.concat(merged_dataframe_list)

    if os.path.isfile(presaved_futures_data_folder + '/' + ticker_head +
                      '.pkl'):
        data4_tickerhead['past_indx'] = [
            1 if np.isfinite(x) else 0
            for x in data4_tickerhead['change_5'].values
        ]
        clean_data = data4_tickerhead
        clean_data['frame_indx'] = 1

        data_columns = old_data.columns
        old_data['frame_indx'] = 0
        old_data['past_indx'] = [
            1 if np.isfinite(x) else 0 for x in old_data['change_5'].values
        ]
        merged_data = pd.concat([old_data, clean_data],
                                ignore_index=True,
                                sort=True)
        merged_data.sort_values(
            ['cont_indx', 'settle_date', 'past_indx', 'frame_indx'],
            ascending=[True, True, False, False],
            inplace=True)
        merged_data.drop_duplicates(subset=['settle_date', 'cont_indx'],
                                    keep='first',
                                    inplace=True)
        data4_tickerhead = merged_data.drop(['frame_indx', 'past_indx'],
                                            1,
                                            inplace=False)
        data4_tickerhead = data4_tickerhead[data_columns]

    data4_tickerhead.to_pickle(presaved_futures_data_folder + '/' +
                               ticker_head + '.pkl')

    if 'con' not in kwargs.keys():
        con.close()
def update_options_price_database_from_cme_files_4ticker(**kwargs):

    ticker = kwargs['ticker']

    contract_specs_output = cmi.get_contract_specs(ticker)
    ticker_head = contract_specs_output['ticker_head']
    ticker_month_num = contract_specs_output['ticker_month_num']
    ticker_year = contract_specs_output['ticker_year']

    if 'settle_date' in kwargs.keys():
        settle_date = kwargs['settle_date']
        kwargs['report_date'] = settle_date
    else:
        settle_date = int(time.strftime('%Y%m%d'))
        kwargs['settle_date'] = settle_date
        kwargs['report_date'] = settle_date

    if not exp.is_business_day(double_date=settle_date, reference_tickerhead=ticker_head):
        return

    if 'expiration_date' in kwargs.keys():
        expiration_date = kwargs['expiration_date']
    else:
        expiration_date = exp.get_options_expiration(ticker)
        expiration_date = expiration_date.date()

    settle_datetime = cu.convert_doubledate_2datetime(settle_date)

    if 'cal_dte' in kwargs.keys():
        cal_dte = kwargs['cal_dte']
    else:
        cal_dte = (expiration_date-settle_datetime.date()).days

    if 'tr_dte' in kwargs.keys():
        tr_dte = kwargs['tr_dte']
    else:
        bday_us = CustomBusinessDay(calendar=exp.get_calendar_4ticker_head(ticker_head))
        dts = pd.date_range(start=settle_datetime, end=expiration_date, freq=bday_us)
        tr_dte = len([x for x in dts if x.to_datetime().date() < expiration_date])

    data_vendor_id = 2
    now = dt.datetime.now()
    con = msu.get_my_sql_connection(**kwargs)

    process_output = pco.process_cme_options_4ticker(**kwargs)

    if process_output['success']:
        settle_frame = process_output['settle_frame']
    else:
        if 'con' not in kwargs.keys():
            con.close()
        return

    column_names = settle_frame.columns.tolist()

    option_type_indx = column_names.index('option_type')
    strike_indx = column_names.index('strike')
    settle_indx = column_names.index('settle')
    volume_indx = column_names.index('volume')
    interest_indx = column_names.index('interest')

    tuples = [tuple([data_vendor_id, ticker_head, ticker_month_num, ticker_year,
                     ticker, x[option_type_indx],x[strike_indx],settle_datetime.date(),
                     cal_dte, tr_dte, now, now,
                    None if np.isnan(x[settle_indx]) else x[settle_indx],
                    None if np.isnan(x[volume_indx]) else x[volume_indx],
                    None if np.isnan(x[interest_indx]) else x[interest_indx]]) for x in settle_frame.values]

    column_str = "data_vendor_id, ticker_head, ticker_month, ticker_year, ticker, " \
                 " option_type, strike, price_date, cal_dte, tr_dte, " \
                 " created_date,last_updated_date, close_price, volume, open_interest"

    insert_str = ("%s, " * len(column_str.split(',')))[:-2]
    final_str = "REPLACE INTO daily_option_price (%s) VALUES (%s)" % (column_str, insert_str)
    msu.sql_execute_many_wrapper(final_str=final_str, tuples=tuples, con=con)

    if 'con' not in kwargs.keys():
        con.close()
def generate_and_update_futures_data_file_4tickerhead(**kwargs):

    ticker_head = kwargs['ticker_head']

    con = msu.get_my_sql_connection(**kwargs)

    if os.path.isfile(presaved_futures_data_folder + '/' + ticker_head + '.pkl'):
        old_data = pd.read_pickle(presaved_futures_data_folder + '/' + ticker_head + '.pkl')
        last_available_date = int(old_data['settle_date'].max().to_datetime().strftime('%Y%m%d'))
        date_from = cu.doubledate_shift(last_available_date, 60)
        data4_tickerhead = gfp.get_futures_price_4ticker(ticker_head=ticker_head, date_from=date_from, con=con)
    else:
        data4_tickerhead = gfp.get_futures_price_4ticker(ticker_head=ticker_head, con=con)

    data4_tickerhead = pd.merge(data4_tickerhead, dirty_data_points, on=['settle_date', 'ticker'],how='left')
    data4_tickerhead = data4_tickerhead[data4_tickerhead['discard'] != True]
    data4_tickerhead = data4_tickerhead.drop('discard', 1)

    data4_tickerhead['close_price'] = [float(x) if x is not None else float('NaN') for x in data4_tickerhead['close_price'].values]
    data4_tickerhead['open_price'] = [float(x) if x is not None else float('NaN') for x in data4_tickerhead['open_price'].values]
    data4_tickerhead['high_price'] = [float(x) if x is not None else float('NaN') for x in data4_tickerhead['high_price'].values]
    data4_tickerhead['low_price'] = [float(x) if x is not None else float('NaN') for x in data4_tickerhead['low_price'].values]

    data4_tickerhead['cont_indx'] = 100*data4_tickerhead['ticker_year']+data4_tickerhead['ticker_month']
    unique_cont_indx_list = data4_tickerhead['cont_indx'].unique()
    num_contracts = len(unique_cont_indx_list)
    unique_cont_indx_list = np.sort(unique_cont_indx_list)
    merged_dataframe_list = [None]*num_contracts

    bday_us = CustomBusinessDay(calendar=exp.get_calendar_4ticker_head('CL'))
    full_dates = pd.date_range(start=data4_tickerhead['settle_date'].min(),end=data4_tickerhead['settle_date'].max(), freq=bday_us)

    for i in range(num_contracts):

        contract_data = data4_tickerhead[data4_tickerhead['cont_indx']==unique_cont_indx_list[i]]

        contract_full_dates = full_dates[(full_dates >= contract_data['settle_date'].min()) & (full_dates<=contract_data['settle_date'].max())]
        full_date_frame = pd.DataFrame(contract_full_dates, columns=['settle_date'])
        merged_dataframe_list[i] = pd.merge(full_date_frame,contract_data,on='settle_date',how='left')

        merged_dataframe_list[i]['ticker'] = contract_data['ticker'][contract_data.index[0]]
        merged_dataframe_list[i]['ticker_head'] = contract_data['ticker_head'][contract_data.index[0]]
        merged_dataframe_list[i]['ticker_month'] = contract_data['ticker_month'][contract_data.index[0]]
        merged_dataframe_list[i]['ticker_year'] = contract_data['ticker_year'][contract_data.index[0]]
        merged_dataframe_list[i]['cont_indx'] = contract_data['cont_indx'][contract_data.index[0]]

        merged_dataframe_list[i]['change1'] = merged_dataframe_list[i]['close_price'].shift(-2)-merged_dataframe_list[i]['close_price'].shift(-1)
        merged_dataframe_list[i]['change1_instant'] = merged_dataframe_list[i]['close_price'].shift(-1)-merged_dataframe_list[i]['close_price']
        merged_dataframe_list[i]['change2'] = merged_dataframe_list[i]['close_price'].shift(-3)-merged_dataframe_list[i]['close_price'].shift(-1)
        merged_dataframe_list[i]['change5'] = merged_dataframe_list[i]['close_price'].shift(-6)-merged_dataframe_list[i]['close_price'].shift(-1)
        merged_dataframe_list[i]['change10'] = merged_dataframe_list[i]['close_price'].shift(-11)-merged_dataframe_list[i]['close_price'].shift(-1)
        merged_dataframe_list[i]['change20'] = merged_dataframe_list[i]['close_price'].shift(-21)-merged_dataframe_list[i]['close_price'].shift(-1)
        merged_dataframe_list[i]['change_5'] = merged_dataframe_list[i]['close_price']-merged_dataframe_list[i]['close_price'].shift(5)
        merged_dataframe_list[i]['change_1'] = merged_dataframe_list[i]['close_price']-merged_dataframe_list[i]['close_price'].shift(1)

    data4_tickerhead = pd.concat(merged_dataframe_list)

    if os.path.isfile(presaved_futures_data_folder + '/' + ticker_head + '.pkl'):
        clean_data = data4_tickerhead[np.isfinite(data4_tickerhead['change_5'])]
        clean_data['frame_indx'] = 1
        old_data['frame_indx'] = 0
        merged_data = pd.concat([old_data,clean_data],ignore_index=True)
        merged_data.sort(['cont_indx', 'settle_date', 'frame_indx'], ascending=[True, True, False], inplace=True)
        merged_data.drop_duplicates(subset=['settle_date', 'cont_indx'], take_last=False, inplace=True)
        data4_tickerhead = merged_data.drop('frame_indx', 1, inplace=False)

    data4_tickerhead.to_pickle(presaved_futures_data_folder + '/' + ticker_head + '.pkl')

    if 'con' not in kwargs.keys():
        con.close()