def add_candles_database(self, step2, q2):
"""
Function that stores process data in the select
:broker_instance Is the instance of the broker connection.
:q2 Queue 2 is the one that will transport processed data to be stored on the computer.
"""
# Initializes counter to verify if it is necessary to sanitize candles
size = 0
while bool(step2.value) or not q2.empty():
gc.collect()
time.sleep(1)
while q2.empty() is False:
list_candles = q2.get()
# Creates pd.DataFrame in case it is already not, it is here and not in step1 for performance reasons
if not isinstance(list_candles, pd.DataFrame):
list_candles = pd.DataFrame(list_candles,
columns=[
'datetime', 'fi', 'ts',
'open', 'high', 'low',
'close', 'volume'
])
list_candles.set_index('datetime', inplace=True)
if list_candles.shape[0] > 0:
self._broker_instance.store(list_candles)
size += list_candles.shape[0]
# Sanitizes candles (remove duplicates if any) in case it fetched more than 500 candles
if size > 500:
print('{}Sanitizing {} candles database.'.format(
dtfx.now(), self.asset))
self._broker_instance.sanitize(self.asset)
def generate_oanda_params(params, count):
"""
Generates params necessary as input to request candles from Oanda's server.
:entry_params is a dictionary comprised of:
start_date --> datetime,
financial_instrument --> str (Ex.: 'EUR_USD'),
ts --> ts in table's format (Ex.: 'g01', 'g05'...)
:return returns the parameters necessary to request candles from Oanda's
:rtype dictionary
"""
# Converts datetime to Oanda's datetime format (str)
if type(params['from']) == datetime.datetime:
params['from'] = params['from'].strftime('%Y-%m-%dT%H:%M:%SZ')
# 5000 is the maximum amount of candles allowed by request on Oanda
params.update({'count': count})
# Bid price is the standard we use for that analysis (There is also Mid and Ask quotes)
params.update({'price': 'B'})
# Outputs every param generation (one for each request to Oanda)
print('{}Requesting candles to {} for {} on {}'.format(dtfx.now(), 'Oanda', params['financial instrument'],
params['from']))
return params
def update_load_run_data(self):
"""
This method updates, then loads all the historic data and then run all the indicators through it.
It is great for generating historic databases of indicators output.
"""
print('{}Starting to update database for {}.'.format(
dtfx.now(), self.asset))
# Updates historic database (downloads new candles from remote server)
self.hdm.update_historic_database(q3=None)
print('{}Database updated. Starting simulations for {}.'.format(
dtfx.now(), self.asset))
# Execute indicators in downloaded database (can select initial date)
self.load_run_data()
print('{}Simulations completed for {}.'.format(dtfx.now(), self.asset))
def load_run_data(self):
"""
This method loads all the historic data and then run all the indicators through it.
"""
# TODO implement a select here (will work on HDF5, but might not work on other storage methods)
df = self.hdm.load_data()
# Guarantees that 'self.start_date' is instantiated.
if self.start_date is None:
self.start_date = df.index[0]
# TODO implement end_date slice as well
# Gets Usable DataFrame
df = df[self.start_date:]
# Executes DataFrame in batches to be able to save the current state of the IndicatorManager
for dt in self.feeder.exec_df(df[self.start_date:]):
print(
'{}Saving a batch into disk on {}. Last Candle analyzed: {}.'.
format(dtfx.now(), self.asset, dt))
self.start_date = dtfx.next_candle_datetime(
dt, 1) # Order is important here
self.save()
def save_state(self, dt):
print('{}Saving a batch into disk on {}. Last Candle analyzed: {}.'.
format(dtfx.now(), self.asset, dt))
self.start_date = dtfx.next_candle_datetime(
dt, 1) # Order is important here
self.pickle_state()