def GMM_Ineq_parall(Theta0, DATA_STRUCT, d_struct): Theta = { "comm_mu": Theta0[0], "priv_mu": Theta0[1], "epsilon_mu": Theta0[2], "comm_var": Theta0[3], "priv_var": Theta0[4], "epsilon_var": Theta0[5], } rng = np.random.RandomState(d_struct['rng_seed']) start = time.time() print('--------------------------------------------------------') print('current parameter set are :') print(Theta) ''' parallel programming with two levels data separating runing the estimation ''' data_n = len(DATA_STRUCT) work_pool = ThreadPool(nodes=data_n) cpu_num = multiprocessing.cpu_count() cpu_num_node = int((cpu_num - 1) / data_n) # change the submit to mpa so that we can run multi-part altogether results = work_pool.amap( partial(para_data_allo_1, Theta, cpu_num_node, rng, d_struct), iter(DATA_STRUCT)) work_pool.close() while not results.ready(): time.sleep(5) print(".") # work_pool.join() auction_result = np.nanmean(list(results.get())) end = time.time() print("object value : " + str(auction_result)) print("time spend in this loop: ") print(end - start) print('--------------------------------------------------------\n') ## save the parameters and objective value with open('para.txt', 'a+') as f: for item in Theta0: f.write("%f\t" % item) f.write("%f\t" % auction_result) f.write("%f\n" % (end - start) / 60) return auction_result
MRH = [] skip_rate = int(evalsubset_relations.shape[0]/BATCH_EVAL) for j in xrange(0, skip_rate): eval_batch_h = evalsubset_relations[j::skip_rate,0] eval_batch_r = evalsubset_relations[j::skip_rate,1] eval_batch_t = evalsubset_relations[j::skip_rate,2] assert eval_batch_h.shape[0]==BATCH_EVAL indexes_t = sess.run(indices_t, \ feed_dict = { pos_h:eval_batch_h, pos_r:eval_batch_r, eval_to_rank:xrange(VOCABULARY_SIZE), keep_prob:1, }) mrt = Evaluate_MR(eval_batch_t.tolist(), indexes_t.tolist(),P) MRT.extend(mrt) if not os.path.exists(LOG_DIR): os.makedirs(LOG_DIR) saver.save(sess, os.path.join(LOG_DIR, "model.ckpt"), epoch) with open(LOG_DIR+'/progress.txt','a+') as fp: fp.write('Epoch %i: Minibatch MRT: %f\n\n' % (epoch, \ np.mean(MRT))) NOW_DISPLAY = False step += 1 P.close()
def backtest_mt(params): global capital su = None saveIndicators(candleamount=candleamount) #fix later candleSplice = candleData.tail(candleamount) atrseries = pd.Series(dtype=np.uint16) keltner_signals = pd.Series(dtype=object) engulf_signals = pd.Series(dtype=object) signals = pd.DataFrame(columns=['S']) atrperiod = params['atrperiod'] #candleSplice = candleSplice.reset_index(drop=True) if (params['keltner'] == True) and (params['engulf'] == True): engulf_signals = pd.read_csv( 'IndicatorData//' + params['symbol'] + '//Engulfing//' + "SIGNALS_t" + str(params['engulfthreshold']) + '_ignoredoji' + str(params['ignoredoji']) + '.csv', sep=',') keltner_signals = pd.read_csv('IndicatorData//' + params['symbol'] + '//Keltner//' + "SIGNALS_kp" + str(params['kperiod']) + '_sma' + str(params['ksma']) + '.csv', sep=',') signals = pd.concat([engulf_signals, keltner_signals], axis=1) signals.columns = ["E", "K"] signals['S'] = np.where((signals['E'] == signals['K']), Signal(0), signals['E']) elif (params['keltner'] == True): keltner_signals = pd.read_csv('IndicatorData//' + params['symbol'] + '//Keltner//' + "SIGNALS_kp" + str(params['kperiod']) + '_sma' + str(params['ksma']) + '.csv', sep=',') signals['S'] = np.array(keltner_signals).reshape( 1, len(keltner_signals))[0] elif (params['engulf'] == True): engulf_signals = pd.read_csv( 'IndicatorData//' + params['symbol'] + '//Engulfing//' + "SIGNALS_t" + str(params['engulfthreshold']) + '_ignoredoji' + str(params['ignoredoji']) + '.csv', sep=',') signals['S'] = np.array(engulf_signals).reshape( 1, len(engulf_signals))[0] print(signals['S']) #signals.to_csv('BacktestData//Signals//' + currentTime + '.csv') atrseries = pd.read_csv('IndicatorData//' + params['symbol'] + "//ATR//" + "p" + str(atrperiod) + '.csv', sep=',') copyIndex = candleSplice.index candleSplice = candleSplice.reset_index(drop=True) #candleSplice.merge(atrseries, left_index=True) #candleSplice.merge(signals['S'], right_on='S', left_index=True) candleSplice = pd.DataFrame.join(candleSplice, atrseries) candleSplice = pd.DataFrame.join( candleSplice, signals['S']) #COMBINE SIGNALS AND CANDLE DATA candleSplice.index = copyIndex candleSplice['timestamp'] = pd.to_datetime(candleSplice.timestamp) finalCapitalData = None currentTime = datetime.now().strftime("%Y%m%d-%H%M") backtestDir = params['symbol'] + '//' + "len" + str( candleamount) + "_k" + str(params['keltner']) + "_e" + str( params['engulf'] ) + "_id" + str(params['ignoredoji']) + "_eThrs" + str( params['engulfthreshold'] ) + "_ATR" + str(params['atrperiod']) + "_kP" + str( params['kperiod']) + "_kSMA" + str(params['ksma']) + "_pm" + str( params['posmult']) + "_ST" + params['stoptype'] + "_sm" + str( params['stopmult']) + "_tm" + str( params['tmult']) + "_TR" + params['trade'] bt_profit = 0 if (percision != 1): isafe = [] candleSplit = [] initialLength = len(candleSplice) firstStart = candleSplice.index[0] lastDistanceSafe = None if params['symbol'] == 'XBTUSD': su = xbtusd_su elif params['symbol'] == 'ETHUSD': su = ethusd_su for i in range(percision - 1): #abs() is a temporary fix to running the backtest on short intervals isafe.append((i + 1) * ((abs(initialLength - percision * su)) / percision) + i * su) #candleSplit = list(np.array_split(candleSplice, percision)) #candleSplit = list(candleSplit) for i in isafe: ia = int(i) if isafe.index(i) != 0: candleSplit.append(candleSplice.iloc[int(isafe[isafe.index(i) - 1]):ia + 1]) lastDistanceSafe = ia #print("lds", lastDistanceSafe) # else: #candleSplit.append(candleSplice.iloc[:ia+1]) #print("lds", lastDistanceSafe) #if(len(isafe) > 1): candleSplit.append(candleSplice.iloc[lastDistanceSafe:]) #print(candleSplit) #time.sleep(100) #generate parameters for multithreading safe_length = len(candleSplit) safe_candleamount = np.repeat(candleamount, safe_length).tolist() safe_capital = np.repeat(capital, safe_length).tolist() safe_params = np.repeat(params, safe_length).tolist() withSafe = np.repeat(True, safe_length).tolist() print("safe thread amount:", safe_length) #create multithread pool start = time.time() #print(candleSplit) #time.sleep(1000) pool = ThreadPool(safe_length) #run initial chunks multithreaded to find safepoints safe_results = pool.uimap(backtest_strategy, safe_candleamount, safe_capital, safe_params, candleSplit, withSafe) pool.close() #Compute anything we need to while threads are running candleSafe = [] final_length = safe_length + 2 withoutSafe = np.repeat(False, final_length).tolist() final_candleamount = np.repeat(candleamount, final_length).tolist() final_capital = np.repeat(capital, final_length).tolist() final_params = np.repeat(params, final_length).tolist() static_capital = capital safePoints = list(safe_results) ###################################### #time.sleep(1000) pool.join() for i in safePoints: if i == -1: backtest_mt.q.put( 'Not all safe points found for given percision. Reduce percision, or increase timeframe' ) return safePoints = sorted(safePoints) if find_su: su = [] for i, point in enumerate(safePoints): su.append(point - candleSplit[i].index[0]) suAvg = mean(su) #only works on evenly spliced chunks chunkLength = len(candleSplit[0]) backtest_mt.q.put(["su average:", suAvg, ' / ', chunkLength]) return (su) print("safe points:", safePoints) idx = 0 for i in safePoints: ia = i - firstStart idx = safePoints.index(i) if safePoints.index(i) != 0: candleSafe.append(candleSplice.iloc[lastDistanceSafe - idx:ia + 1]) lastDistanceSafe = ia + 1 else: candleSafe.append(candleSplice.iloc[:ia + 1]) lastDistanceSafe = ia + 1 candleSafe.append(candleSplice.iloc[lastDistanceSafe - idx:]) print("final thread amount:", final_length) #print(candleSafe) #time.sleep(10000) fpool = ThreadPool(final_length) final_results = fpool.uimap(backtest_strategy, final_candleamount, final_capital, final_params, candleSafe, withoutSafe) fpool.close() final_result = list(final_results) fpool.join() ordered_result = sorted(final_result, key=lambda x: x[0]) for i in range(len(ordered_result)): #print(final_result.index) if i != 0: #for non-static position size: ##capital += capital*((i[1]-static_capital)/static_capital) ordered_result[i][1]['capital'] += bt_profit bt_profit = ordered_result[i][1].iloc[-1][ 'capital'] - static_capital finalCapitalData = pd.concat( [finalCapitalData, ordered_result[i][1]], ignore_index=True) else: bt_profit = ordered_result[i][1].iloc[-1][ 'capital'] - static_capital finalCapitalData = pd.DataFrame(ordered_result[i][1]) capital = finalCapitalData['capital'].iloc[-1] else: #run chunks spliced by safepoints multithreaded to retrieve fully accurate results final_results = backtest_strategy(candleamount, capital, params, candleSplice, False) final_result = list(final_results) capital = str(final_result[1]['capital'].iloc[-1]) finalCapitalData = final_result[1] print(finalCapitalData) #time.sleep(1000) visualize_trades(finalCapitalData, backtestDir) saveBacktest(capital, params, backtestDir) backtest_mt.q.put(capital) end = time.time() print("Thread time: ", end - start) return ('done')
def make_patches(data_root, patches_root, patch_size, outline_filled=None, remove_filled=False, min_widths=('def',), mirror=True, rotations=(0,), translations=((0, 0),), distinguishability_threshold=.5, num_workers=0, random_samples=None, leave_width_percentile=None): if num_workers != 0: from pathos.multiprocessing import cpu_count, ProcessingPool from pathos.threading import ThreadPool if num_workers == -1: optimal_workers = cpu_count() - 1 workers_pool = ProcessingPool(optimal_workers) else: workers_pool = ProcessingPool(num_workers) print(f'Workers pool: {workers_pool}') savers_pool = ThreadPool(1) saving_patches_in_bg = savers_pool.amap(lambda a: None, []) else: workers_pool = 0 path = lambda basename, origin, width='def', ori='def', rot=0, t=(0, 0): os.path.join(patches_root, basename, '{}x{}'.format(*patch_size), 'width_{}'.format(width), 'orientation_{}'.format(ori), 'rotated_deg_{}'.format(rot), 'translated_{}_{}'.format(*t), '{}_{}.svg'.format(*origin)) orientations = ['def'] if mirror: orientations.append('mir') if random_samples is not None: min_widths_all = deepcopy(min_widths) orientations_all = deepcopy(orientations) rotations_all = deepcopy(rotations) translations_all = deepcopy(translations) source_images = glob(os.path.join(data_root, '**', '*.svg'), recursive=True) for file in source_images: print('Processing file {}'.format(file)) basename = file[len(data_root) + 1:-4] # split data_root and extension vector_image = VectorImage.from_svg(file) if remove_filled: vector_image.remove_filled() if outline_filled is not None: vector_image.leave_only_contours(outline_filled) if leave_width_percentile is not None: vector_image.leave_width_percentile(leave_width_percentile) if random_samples is not None: min_widths = np.random.choice(min_widths_all, size=min(random_samples, len(min_widths_all)), replace=False) orientations = np.random.choice(orientations_all, size=min(random_samples, len(orientations_all)), replace=False) rotations = np.random.choice(rotations_all, size=min(random_samples, len(rotations_all)), replace=False) translations = translations_all[ np.random.choice(len(translations_all), size=min(random_samples, len(translations_all)), replace=False)] for width in min_widths: print('\twidth {}'.format(width)) if width == 'def': vector_image_scaled = vector_image else: vector_image_scaled = vector_image.copy() vector_image_scaled.scale_to_width('min', width) for orientation in orientations: print('\t\torientation {}'.format(orientation)) if orientation == 'def': vector_image_reoriented = vector_image_scaled else: vector_image_reoriented = vector_image_scaled.mirrored() for rotation in rotations: print('\t\t\trotation {}'.format(rotation)) vector_image_rotated = vector_image_reoriented.rotated(rotation, adjust_view=True) for translation in translations: print('\t\t\t\ttranslation {}'.format(translation)) vector_image_translated = vector_image_rotated.translated(translation, adjust_view=True) vector_patches = vector_image_translated.split_to_patches(patch_size, workers=workers_pool) if num_workers != 0: print('\t\t\t\t\twaiting for previous batch to be saved') saving_patches_in_bg.get() def simplify_and_save(vector_patch, basename=basename, width=width, orientation=orientation, rotation=rotation, translation=translation): vector_patch.simplify_segments(distinguishability_threshold=distinguishability_threshold) if len(vector_patch.paths) == 0: return save_path = path(basename, (int(vector_patch.x.as_pixels()), int(vector_patch.y.as_pixels())), width, orientation, rotation, translation) os.makedirs(os.path.dirname(save_path), exist_ok=True) vector_patch.save(save_path) if num_workers == 0: print('\t\t\t\t\tsaving patches') for vector_path in vector_patches.reshape(-1): simplify_and_save(vector_path) else: print('\t\t\t\t\tsaving patches') saving_patches_in_bg = savers_pool.amap(simplify_and_save, vector_patches.reshape(-1)) if num_workers != 0: workers_pool.close() workers_pool.join() workers_pool.clear() savers_pool.close() savers_pool.join() savers_pool.clear()