def load_data(): START = datetime.datetime(2018, 1, 1, 0, 0) END = datetime.datetime(2018, 12, 31, 23, 59) Tele2 = ts.Telescope() Tele2.load_dfs([r"Z:\KaiData\Theo\2018\RTY.ESTheo.mpk"]) Tele2.choose_range(START, END) Tele2.parse_datetime() return Tele2
def __init__(self, x=6, y=6): self.axes = axes.Axes(x, y, self) self._telescope = telescope.Telescope(treepath, self.handle) self.stored = {} #self.load_themes() self.themes = themes.ThemesHandler() #self.theme_handler = themes.ThemesHandler() self.color_handler = colors.ColorsHandler()
def tele_adjust(Tele2): new_Tele = ts.Telescope() new_Tele.df = Tele2.df.resample('5S', fill_method = 'bfill') new_Tele.parse_datetime() tm.apply_all_metrics(new_Tele.df, sma_period = 600) new_Tele.clip_time(datetime.time(8,30,0), datetime.time(15,15,0)) return new_Tele
def __init__(self, log, dir, nrealize=1, nobs=1, clcDet=1, elv=None, pwv=None, specRes=1.e9, foregrounds=False): self.log = log self.dir = dir self.configDir = self.dir + '/config' self.name = self.dir.rstrip('/').split('/')[-1] #Store foreground parameters try: params, vals = np.loadtxt(self.configDir + '/foregrounds.txt', unpack=True, usecols=[0, 2], dtype=np.str, delimiter='|') fgndDict = { params[i].strip(): vals[i].strip() for i in range(len(params)) } if foregrounds: self.log.log( "Using foreground parameters in %s" % (self.configDir + '/foregrounds.txt'), 1) else: self.log.log( "Ignoring foreground parameters in %s" % (self.configDir + '/foregrounds.txt'), 1) except: fgndDict = None #Store telescope objects telescopeDirs = sorted(gb.glob(dir + '/*/')) telescopeDirs = [x for x in telescopeDirs if 'config' not in x] self.telescopes = [ tp.Telescope(self.log, dir, fgndDict=fgndDict, nrealize=nrealize, nobs=nobs, clcDet=clcDet, elv=elv, pwv=pwv, specRes=specRes, foregrounds=foregrounds) for dir in telescopeDirs ]
def __init__(self, experimentDir, atmFile=None): #***** Private Variables ***** self.__yrToSec = 365.25*24.*60.*60. #Retrieve experiment parameters params, vals = np.loadtxt(experimentDir+'/experiment.txt', unpack=True, usecols=[0,2], dtype=np.str) #Retrieve telescopes telescopeDirArr = sorted(gb.glob(experimentDir+'/*/')) #***** Public Variables ***** self.tobs = float(vals[params.tolist().index('ObservationTime')])*self.__yrToSec self.fsky = float(vals[params.tolist().index('SkyFraction')]) self.obsEff = float(vals[params.tolist().index('ObservationEfficiency')]) self.netMgn = float(vals[params.tolist().index('NETmargin')]) self.dir = experimentDir self.name = experimentDir.rstrip('/').split('/')[-1] self.telescopes = [] for telescopeDir in telescopeDirArr: self.telescopes.append(tp.Telescope(telescopeDir, atmFile))
def generate(self): #Store telescope objects in dictionary self.log.log("Generating telescopes for experiment %s" % (self.name), 1) telescopeDirs = sorted(gb.glob(self.dir + '/*/')) telescopeDirs = [ x for x in telescopeDirs if 'config' not in x and 'paramVary' not in x ] telescopeNames = [ telescopeDir.split('/')[-2] for telescopeDir in telescopeDirs ] self.telescopes = { telescopeNames[i]: tp.Telescope(self.log, telescopeDirs[i], fgndDict=self.fgndDict, nrealize=self.nrealize, nobs=self.nobs, clcDet=self.clcDet, specRes=self.specRes, foregrounds=self.fgnds) for i in range(len(telescopeNames)) }
@author: achen """ import pandas as pd import telescope as ts import telescope_metrics as tm import bounce_match_strategy as bms import datetime import numpy as np from tqdm import tqdm START = datetime.datetime(2018, 1, 1, 0, 0) END = datetime.datetime(2018, 12, 31, 23, 59) Tele2 = ts.Telescope() Tele2.load_dfs([r"Z:\KaiData\Theo\2019\NQ.ESTheo.mpk"]) Tele2.choose_range(START, END) Tele2.parse_datetime() criteria_df = Tele2.df.copy() criteria_df = ts.tele_resample_new(criteria_df, '1T') criteria_df = tm.theochange(criteria_df, period=10) criteria_df['date'] = criteria_df.index.date criteria_df['time'] = criteria_df.index.time criteria_df = criteria_df[criteria_df['time'] == datetime.time(8, 40, 0)] criteria_df.index = criteria_df['date'] criteria_df['Abs Diff'] = np.absolute(criteria_df['Theo Change']) criteria_df = criteria_df[criteria_df['Abs Diff'] >= 155]
def rolling_backtest(Tele2, end_datetime, trade_day_list): import telescope as ts # import telescope_metrics as tm import strategy_backtest2_loss as sbtest2l #dmb.dir_match_backtest(r"Z:\KaiData\Theo\2019\RTY.ESTheo.mpk", datetime.datetime(2019, 7, 16, 0, 0), datetime.datetime(2019, 7, 17, 23, 59)) START = end_datetime - datetime.timedelta(days=6) END = end_datetime print(START) print(END) # Tele2 = ts.Telescope() # Tele2.load_dfs([r"Z:\KaiData\Theo\2019\RTY.ESTheo.mpk"]) # Tele2.df = Tele2.df.resample('5S', fill_method = 'bfill') # tm.apply_all_metrics(Tele2.df, sma_period = 600) # Tele2.clip_time(datetime.time(8,30,0), datetime.time(12,0,0)) chosen_Tele = ts.Telescope(df=Tele2.choose_date(START, END)) # date_list = chosen_Tele.df['date'].unique() # # trade_day_list = [] # not_trade_day_list = [] # for date in date_list: # if ts.select_date_time(chosen_Tele.df, date, datetime.time(8,30,0))['TheoPrice'].empty: # not_trade_day_list.append(date) # else: # trade_day_list.append(date) test_day = trade_day_list[-1] trade_day_list = trade_day_list[:-1] # # Tele2.df = Tele2.df.resample('5S', fill_method = 'bfill') # # Tele2.parse_datetime() # tm.apply_all_metrics(Tele2.df, sma_period = 600) # # Tele2.clip_time(datetime.time(8,30,0), datetime.time(12,0,0)) base_time_start = datetime.time(8, 30, 0) base_time_end_list = [ datetime.time(8, 31, 0), datetime.time(8, 33, 0), datetime.time(8, 35, 0), datetime.time(8, 40, 0), datetime.time(8, 45, 0) ] test_length_list = [360] sma_list = [0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95] stop_list = [0] bounce_stop_list = [80, 120, 160, 200, 240, 280, 320] column_list = [[ 'Profit', 'Trade Number', ' Time Stop Count', 'Limite Stop Count', 'Bounce Stop Count', 'Base Start', 'Base End', 'Test Start', 'Test End', 'SMA Side Ratio Threshold', 'Stop Threshold', 'Bounce Stop Threshold', 'Trading Cost' ]] for base_time_end in base_time_end_list: test_time_start = base_time_end for test_length in test_length_list: test_time_end = (ts.cal_datetime(test_time_start) + datetime.timedelta(minutes=test_length)).time() for sma in sma_list: for stop in stop_list: for bounce_stop in bounce_stop_list: absolute_stop = bounce_stop - stop [ daily_profit_result, trade_count, limit_stop_count, bounce_stop_count, time_stop_count, loss_record ] = sbtest2l.strategy_backtest2_loss( chosen_Tele, trade_day_list, base_time_start, base_time_end, test_time_start, test_time_end, sma_threshold=sma, stop_threshold=absolute_stop, bounce_stop_threshold=bounce_stop, trade_cost=30) total_profit = np.sum( daily_profit_result[~np.isnan(daily_profit_result)] ) single_column = [ total_profit, trade_count, time_stop_count, limit_stop_count, bounce_stop_count, base_time_start, base_time_end, test_time_start, test_time_end, sma, stop, bounce_stop, 30 ] column_list.append(single_column) result_df = pd.DataFrame(column_list[1:], columns=column_list[0]) parameter_series = result_df.loc[result_df['Profit'].idxmax(), :] print(result_df['Profit'].max()) base_time_start = parameter_series['Base Start'] base_time_end = parameter_series['Base End'] test_time_start = parameter_series['Test Start'] test_time_end = parameter_series['Test End'] sma = parameter_series['SMA Side Ratio Threshold'] stop = parameter_series['Stop Threshold'] bounce_stop = parameter_series['Bounce Stop Threshold'] trading_cost = parameter_series['Trading Cost'] [ test_profit_result, trade_count, limit_stop_count, bounce_stop_count, time_stop_count, loss_record ] = sbtest2l.strategy_backtest2_loss(chosen_Tele, [test_day], base_time_start, base_time_end, test_time_start, test_time_end, sma_threshold=sma, stop_threshold=absolute_stop, bounce_stop_threshold=bounce_stop, trade_cost=trading_cost) return base_time_start, base_time_end, sma, bounce_stop, result_df[ 'Profit'].max(), test_profit_result[0]
def simulate(self): # read parameters loadparams = loadparameters.LoadParameters( sky_spreadsheet=self.sky_spreadsheet, sky_sheet=self.sky_sheet) obs_specification = loadparams.run() self.result['loadparameters'] = obs_specification # generate information on the FTS ftsd = fts.FTS(parameters=obs_specification) self.result['fts'] = ftsd.run() print ftsd # generate information on the flux collectors tel = telescope.Telescope(parameters=obs_specification) self.result['telescope'] = tel.run() print tel # generate UV map uvmapgen = uvmapgenerator.UVMapGenerator(parameters=obs_specification, previous_results=self.result) self.result['uvmapgenerator'] = uvmapgen.run() print uvmapgen # generate parameters of cube to be simulated cubeparams = cubeparameters.CubeParameters( previous_results=self.result) self.result['cubeparameters'] = cubeparams.run() print cubeparams # generate synthesis beams - dirty and clean synthbeamsgen = synthesisbeamsgenerator.SynthesisBeamsGenerator( previous_results=self.result, job_server=self.job_server) self.result['synthesisbeams'] = synthbeamsgen.run() print synthbeamsgen # generate primary beams primarybeamsgen = primarybeamsgenerator.PrimaryBeamsGenerator( previous_results=self.result, job_server=self.job_server) self.result['primarybeams'] = primarybeamsgen.run() print primarybeamsgen # construct sky skygen = skygenerator.SkyGenerator(parameters=obs_specification, previous_results=self.result) self.result['skygenerator'] = skygen.run() print skygen # generate observation framework timeline = timelinegenerator.TimeLineGenerator( previous_results=self.result) self.result['timeline'] = timeline.run() print timeline # calculate interferograms obs = observe.Observe(parameters=obs_specification, previous_results=self.result, job_server=self.job_server) self.result['observe'] = obs.run() print obs # recover spectra from interferograms reduceint = reduceinterferogram.ReduceInterferogram( previous_results=self.result, job_server=self.job_server) self.result['reduceinterferogram'] = reduceint.run() print reduceint # construct dirty image dirty = dirtyimage.DirtyImage(previous_results=self.result, job_server=self.job_server) self.result['dirtyimage'] = dirty.run() print dirty # construct clean image clean = cleanimage.CleanImage(previous_results=self.result, job_server=self.job_server) self.result['cleanimage'] = clean.run() print clean # construct html description of result self.render()
def rolling_pairwise_theo_diff_sum_profit(corr_df, data_mpk_file_path, start_date, end_date, start_time = datetime.time(8, 0, 0), end_time = datetime.time(15, 15, 0), time_frame_length = 5, rolling_step = '1T', corr_threshold = 3, up_threshold = 100, down_threshold = 100, trade_cost = 30, output_excel_path= r"C:\Users\achen\Desktop\Monocular\rolling_pairwise_theo_diff_corr.xlsx"): import numpy as np import pandas as pd import telescope as ts import telescope_metrics as tm import datetime from scipy.stats.stats import pearsonr from tqdm import tqdm #--------------Input Parameters--------------- CORR_DF = corr_df df_2019 = data_mpk_file_path START = start_date END = end_date STIME = start_time ETIME = end_time TIME_FRAME_LENGTH = time_frame_length ROLLING_STEP = rolling_step #(has to be a factor of TIME_FRAME_LENGTH) UP_THRESHOLD = up_threshold DOWN_THRESHOLD = down_threshold TRADE_COST = trade_cost CORR_THRESHOLD = corr_threshold #--------------------------------------------- #--------------Input Parameters--------------- #df_2019 = r"Z:\KaiData\Theo\2019\YM.ESTheo.mpk" #START = datetime.datetime(2019, 7, 8, 0, 0) #END = datetime.datetime(2019, 7, 12, 23, 59) #STIME = datetime.time(8, 0, 0) #ETIME = datetime.time(15, 15, 0) #TIME_FRAME_LENGTH = 5 #ROLLING_STEP = '1T' #(has to be a factor of TIME_FRAME_LENGTH) #--------------------------------------------- df_lst = [df_2019] Tele2 = ts.Telescope() Tele2.load_dfs(df_lst) Tele2.choose_range(START, END) Tele2.resample(ROLLING_STEP) tm.apply_all_metrics(Tele2.df, sma_period = 10) Tele2.parse_datetime() Tele2.specific_timeframe(STIME, ETIME) #print(Tele2.df) #print(Tele2.grouped) #print(Tele2.num_of_groups) #print(Tele2.group_names) date_list = Tele2.df['date'].unique() time_list = Tele2.df['time'].unique() ignore_col_num = int(TIME_FRAME_LENGTH/int(ROLLING_STEP[0])+1) # change_direction_match_ratio_list = [] # correlatation_list = [] print(time_list[:-ignore_col_num]) #print(Tele2.df['TheoPrice']) column_list = [time_list[:-ignore_col_num]] for first_time in tqdm(time_list[:-ignore_col_num]): if first_time < (ts.cal_datetime(ETIME) - datetime.timedelta(minutes = TIME_FRAME_LENGTH)).time(): first_start = first_time; first_end = (ts.cal_datetime(first_time) + datetime.timedelta(minutes = TIME_FRAME_LENGTH)).time(); print(first_start) first_temp_df = ts.select_timeframe(Tele2.df, first_start, first_end) single_column = [] for second_time in time_list[:-ignore_col_num]: if (second_time >= (ts.cal_datetime(first_time) + datetime.timedelta(minutes = TIME_FRAME_LENGTH)).time()) & (second_time < (ts.cal_datetime(ETIME) - datetime.timedelta(minutes = TIME_FRAME_LENGTH)).time()): second_start = second_time second_end = (ts.cal_datetime(second_start) + datetime.timedelta(minutes = TIME_FRAME_LENGTH)).time() second_temp_df = ts.select_timeframe(Tele2.df, second_start, second_end) first_time_move_list = [] second_time_move_list = [] for date in date_list: # print(first_temp_df[(first_temp_df['date'] == date) & (first_temp_df['time'] == first_end)]['TheoPrice']) # print(ts.select_date_time(first_temp_df, date, first_end)['TheoPrice'][0]) first_move_theo = ts.select_date_time(first_temp_df, date, first_end)['TheoPrice'][0] - ts.select_date_time(first_temp_df, date, first_start)['TheoPrice'][0] second_move_theo = ts.select_date_time(second_temp_df, date, second_end)['TheoPrice'][0] - ts.select_date_time(second_temp_df, date, second_start)['TheoPrice'][0] if ((not np.isnan(first_move_theo)) & (not np.isnan(second_move_theo))): first_time_move_list.append(first_move_theo) second_time_move_list.append(second_move_theo) # print(first_time_move_list) # print(second_time_move_list) # count = 0 # for i in range(len(first_time_move_list)): # if ((first_time_move_list[i] > 0) & (second_time_move_list[i] > 0)) or ((first_time_move_list[i] < 0) & (second_time_move_list[i] < 0)): # count += 1 # match_ratio = count/len(first_time_move_list) # change_direction_match_ratio_list.append(match_ratio) score = 0 correlation = corr_df.loc[first_start,second_start] for i in range(len(first_time_move_list)): if first_time_move_list[i] >= UP_THRESHOLD: if correlation >= CORR_THRESHOLD: print(correlation) score += second_time_move_list[i] score = score - TRADE_COST if correlation <= -CORR_THRESHOLD: score += -second_time_move_list[i] score = score - TRADE_COST if first_time_move_list[i] <= -DOWN_THRESHOLD: if correlation >= CORR_THRESHOLD: score += -second_time_move_list[i] score = score - TRADE_COST if correlation <= -CORR_THRESHOLD: score += second_time_move_list[i] score = score - TRADE_COST # corr = pearsonr(first_time_move_list,second_time_move_list)[0] single_column.append(score) else: single_column.append(np.nan) print(single_column) column_list.append(single_column) result = pd.DataFrame(column_list[1:], columns=column_list[0]) result['time'] = time_list[:-ignore_col_num] result.set_index('time', inplace=True) #output_df = pd.DataFrame(list(zip(time_list, correlatation_list, change_direction_match_ratio_list)), columns =['start time', 'correlation with next 5 min', 'change direction match ratio']) #output_df.to_csv(r"C:\Users\achen\Desktop\Monocular\5_min_rolling_correlation_direction_match.csv") result.to_excel(output_excel_path) return result
# constrains config obsconstr_file = conf.get('obsconstrains', 'data_file') obsconstr_conf = conf.get('obsconstrains', 'data_conf') obsconstrains = parsing_config_file(obsconstr_file, obsconstr_conf) filters = dict(conf.items('filters')) # images output path fits_path = conf.get('images', 'download_path') #Initialize telescope & camera camera = ccd.Camera(conf.get('camera', 'name'), conf.get('camera', 'address'), conf.get('camera', 'port'), conf.get('camera', 'timeout')) telescope = tel.Telescope(conf.get('telescope', 'name'), conf.get('telescope', 'address'), conf.get('telescope', 'port'), conf.get('telescope', 'timeout')) filterw = ccd.Filter(filters, conf.get('filterw', 'name'), conf.get('filterw', 'address'), conf.get('filterw', 'port'), conf.get('filterw', 'timeout')) #~ #Defining some targets: #~ vega = {'object': "Vega", #~ 'ra': '18:37:02.255', #~ 'dec': '38:48:03.64' } #~ mel20 = {'object': "Mel20", #~ 'ra':'03h25m34.2s', #~ 'dec':'+49d55m42s'}
""" import pandas as pd import telescope as ts import telescope_metrics as tm import daily_best_feature_record_test as dbfrt import datetime import numpy as np from tqdm import tqdm #dmb.dir_match_backtest(r"Z:\KaiData\Theo\2019\RTY.ESTheo.mpk", datetime.datetime(2019, 7, 16, 0, 0), datetime.datetime(2019, 7, 17, 23, 59)) START = datetime.datetime(2017, 1, 1, 0, 0) END = datetime.datetime(2017, 12, 31, 23, 59) ES_Tele = ts.Telescope() ES_Tele.load_all_dfs([r"Z:\KaiData\Stitched\2017\ESStitched.mpk"]) ES_Tele.df = ES_Tele.df[['TradePrice', 'SettlePrice', 'LeadSettle', 'WATR']] RTY_Tele = ts.Telescope() RTY_Tele.load_all_dfs([r"Z:\KaiData\Stitched\2017\RTYStitched.mpk"]) RTY_Tele.df = RTY_Tele.df[['TradePrice', 'SettlePrice', 'LeadSettle', 'WATR']] ES_Tele.choose_range(START, END) ES_Tele.parse_datetime() RTY_Tele.choose_range(START, END) RTY_Tele.parse_datetime() Tele2 = ts.Telescope() Tele2.load_dfs([r"Z:\KaiData\Theo\2017\RTY.ESTheo.mpk"])
Compute time (in seconds) since last alignment. :return: time measured in consecutive seconds """ current_time = datetime.now() return self.current_time_seconds(current_time) - self.alignment_time if __name__ == "__main__": from socket_client import SocketClient, SocketClientDebug app = QtWidgets.QApplication(sys.argv) c = configuration.Configuration() tel = telescope.Telescope(c) host = 'localhost' port = c.fire_capture_port_number if c.camera_debug: mysocket = SocketClientDebug(host, port, c.camera_debug_delay) else: try: mysocket = SocketClient(host, port) except: print( "Camera: Connection to FireCapture failed, expect exception.") exit() # date_time = '2015/05/18 15:20:30'
def dir_match_backtest(df_2019, start_date, end_date): import numpy as np import pandas as pd import telescope as ts import telescope_metrics as tm import datetime from tqdm import tqdm import rolling_pairwise_theo_diff_dir_match as rptddm import rolling_pairwise_theo_diff_sum_profit as rptdsp from datetime import timedelta #--------------Input Parameters--------------- df_2019 = r"Z:\KaiData\Theo\2019\NQ.ESTheo.mpk" START = start_date END = end_date df_lst = [df_2019] Tele2 = ts.Telescope() Tele2.load_dfs(df_lst) Tele2.choose_range(START, END) Tele2.parse_datetime() #print(Tele2.df) #print(Tele2.grouped) #print(Tele2.num_of_groups) #print(Tele2.group_names) date_list = Tele2.df['date'].unique() #print(Tele2.df['TheoPrice']) trade_day_list = [] for date in date_list: if ts.select_date_time(Tele2.df, date, datetime.time(12, 0, 0))['TheoPrice'].empty: continue else: trade_day_list.append(date) time_length = 60 #corr_threshold_list = [0.5,0.6,0.7,0.8,0.9] #up_threshold_list = [50, 75, 100] #down_threshold_list = [50, 75, 100] corr_threshold_list = [3] up_threshold_list = [60] down_threshold_list = [60] trading_cost = 30 count = 0 for corr_thre in corr_threshold_list: for up_thre in up_threshold_list: for date in tqdm(trade_day_list): print(date) down_thre = up_thre if date.weekday() == 4: corr_start_datetime = date - timedelta(days=4) else: corr_start_datetime = date - timedelta(days=6) corr_end_datetime = date - timedelta(minutes=1) test_start_datetime = date test_end_datetime = date + timedelta(days=1) - timedelta( minutes=1) output_file_path_corr = "C:\\Users\\achen\\Desktop\\Monocular\\Automatic Backtest\\Direction Match\\NQ_" + str( time_length) + "MIN_" + corr_start_datetime.strftime( "%Y_%m_%d") + "_" + corr_end_datetime.strftime( "%Y_%m_%d") + "_ROLL5_PAIR_DIRMATCH_auto.xlsx" output_file_path_backtest = "C:\\Users\\achen\Desktop\\Monocular\\Automatic Backtest\\Dirmatch PL\\NQ_" + str( time_length) + "MIN_" + test_start_datetime.strftime( "%Y_%m_%d") + "_ROLL5_PAIR_DIRMATCH_PROFIT_U" + str( up_thre) + "_D" + str(down_thre) + "_T" + str( trading_cost) + "_auto.xlsx" corr_df = rptddm.rolling_pairwise_theo_diff_dir_match( "Z:\\KaiData\\Theo\\2019\\NQ.ESTheo.mpk", corr_start_datetime, corr_end_datetime, datetime.time(8, 0, 0), datetime.time(15, 15, 0), time_frame_length=time_length, rolling_step='5T', output_excel_path=output_file_path_corr) if corr_df is None: continue else: count += 1 pl_df = rptdsp.rolling_pairwise_theo_diff_sum_profit( corr_df, "Z:\\KaiData\\Theo\\2019\\NQ.ESTheo.mpk", test_start_datetime, test_end_datetime, datetime.time(8, 0, 0), datetime.time(15, 15, 0), time_frame_length=time_length, rolling_step='5T', corr_threshold=corr_thre, up_threshold=up_thre, down_threshold=down_thre, trade_cost=trading_cost, output_excel_path=output_file_path_backtest) if count == 1: result_df = pl_df else: result_df = result_df.add(pl_df)