def initialization(universe):
# general setting, required for all algo's
universe.algo_name='SMA_RSI2'
universe.start=datetime(2012,1,1)
universe.end =datetime.today()
#universe.symbols = ['AAPL','VOO','SPY','TQQQ','CVTI']
universe.symbols = ['AMZN']
universe.save_data='True'
universe.data_source='web'
universe.account.starting_cash=len(universe.symbols)*10000
# algo dependent setting
# load data
if universe.data_source=='web':
universe.data=load_eod(universe.symbols, universe.start, universe.end,
universe.save_data)
elif universe.data_source=='csv':
universe.data=load_csv(universe.symbols)
if 'TQQQ' in universe.symbols:
universe.data['TQQQ'][:datetime(2014,1,23)]/=2
# compute indicators
indicator={sym:DataFrame(index=universe.data[sym].index) for sym in universe.symbols}
for sym in universe.symbols:
close=universe.data[sym].Close.values
indicator[sym]['SMA10'] =ta.SMA(close, timeperiod=10)
indicator[sym]['SMA200']=ta.SMA(close, timeperiod=200)
indicator[sym]['EMA10'] =ta.EMA(close, timeperiod=10)
indicator[sym]['RSI2'] =ta.RSI(close, timeperiod=2)
indicator[sym]['MOM1'] =ta.MOM(close, timeperiod=1)
universe.indicator=indicator
return universe
def backtest(algo_name=universe.algo_name):
global universe
# import trading algorithm
algo=import_module(algo_name)
# initialize universe
universe=algo.initialization(universe)
universe.account.portfolio_value=universe.account.starting_cash
universe.account.cash=universe.account.starting_cash
# load data if it is not done in algo.initialization
if universe.data=={}:
if universe.data_source=='web':
universe.data=load_eod(universe.symbols, universe.start,
universe.end, universe.save_data)
elif universe.data_source=='csv':
universe.data=load_csv(universe.symbols)
# backtest trading
universe._complete_data=universe.data
date_index=universe.data[universe.symbols[0]].index
date_index=date_index[date_index>=universe.start]
date_index=date_index[date_index<=universe.end]
for dtitr in date_index:
universe.now=dtitr
# use up-to-today data
universe.data={sym:universe._complete_data[sym][:dtitr] \
for sym in universe.symbols}
# apply trade algo
record=algo.trading(universe)
# save recorded data
universe.record=universe.record.append(DataFrame(record),
ignore_index=True)
# update account values with current day close proce
today_data ={sym:universe._complete_data[sym][:dtitr][dtitr:] \
for sym in universe.symbols}
today_close={sym:today_data[sym].Close[0] \
for sym in universe.symbols\
if len(today_data[sym])>0}
universe.account.refresh_portfolio(dtitr,today_close)
# finalize trading
algo.finalize_trading(universe)
print '\nbacktest finished'
def direct_load():
fields = loader.load_csv(
tmp_file.name,
resource_id=resource['id'],
mimetype=resource.get('format'),
logger=logger)
loader.calculate_record_count(
resource_id=resource['id'], logger=logger)
set_datastore_active(data, resource, api_key, ckan_url, logger)
job_dict['status'] = 'running_but_viewable'
callback_xloader_hook(result_url=input['result_url'],
api_key=api_key,
job_dict=job_dict)
logger.info('Data now available to users: {}'.format(resource_ckan_url))
loader.create_column_indexes(
fields=fields,
resource_id=resource['id'],
logger=logger)
file_hash = m.hexdigest()
tmp_file.seek(0)
# hash isn't actually stored, so this is a bit worthless at the moment
if (resource.get('hash') == file_hash and not data.get('ignore_hash')):
logger.info('Ignoring resource - the file hash hasn\'t changed: '
'{hash}.'.format(hash=file_hash))
return
logger.info('File hash: {}'.format(file_hash))
resource['hash'] = file_hash # TODO write this back to the actual resource
# Load it
logger.info('Loading CSV')
try:
fields = loader.load_csv(tmp_file.name,
resource_id=resource['id'],
mimetype=resource.get('format'),
logger=logger)
loader.calculate_record_count(resource_id=resource['id'],
logger=logger)
set_datastore_active(data, resource, api_key, ckan_url, logger)
job_dict['status'] = 'running_but_viewable'
callback_xloader_hook(result_url=input['result_url'],
api_key=input['api_key'],
job_dict=job_dict)
logger.info(
'Data now available to users: {}'.format(resource_ckan_url))
loader.create_column_indexes(fields=fields,
resource_id=resource['id'],
logger=logger)
except JobError as e:
logger.warning('Load using COPY failed: {}'.format(e))
def load(model_results, test):
for result in model_results:
y_pred = result[0]
name = result[1]
submission = loader.create_submission(y_pred, test)
loader.load_csv(submission, "cars_submission_" + name + ".csv")
resource['hash'] = file_hash
# Write it to a tempfile on disk
# TODO Can't we just pass the stringio object?
# TODO or stream it straight to disk and avoid it all being in memory?
url = resource.get('url')
filename = url.split('/')[-1].split('#')[0].split('?')[0]
with tempfile.NamedTemporaryFile(suffix=filename) as f_:
f_.write(f.read())
f_.seek(0)
# Load it
logger.info('Loading CSV')
try:
loader.load_csv(f_.name,
resource_id=resource['id'],
mimetype=resource.get('format'),
logger=logger)
except JobError as e:
logger.error('Error during load: {}'.format(e))
raise
logger.info('Finished loading CSV')
# try:
# table_set = messytables.any_tableset(f, mimetype=ct, extension=ct)
# except messytables.ReadError as e:
# # try again with format
# f.seek(0)
# try:
# format = resource.get('format')
# table_set = messytables.any_tableset(f, mimetype=format,
# extension=format)
from sklearn.preprocessing.label import LabelBinarizer
from tensorflow.python.keras.models import Model
import numpy as np
from tensorflow.keras.layers import Activation
from tensorflow.keras.layers import Conv2D
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Dropout
from tensorflow.keras.layers import Flatten
from tensorflow.keras.layers import MaxPooling2D
from tensorflow.keras.layers import concatenate
from tensorflow.keras.models import Sequential
# load data
samplesIMG, labels, namesIMG = loader.load_img("radio_img")
samplesCSV, labelsCSV, namesCSV = loader.load_csv("radio.csv")
# find pairs for samplesIMG in samplesCSV
samples_paired = []
for i in range(samplesIMG.shape[0]):
for j in range(samplesCSV.shape[0]):
if namesCSV[j] == namesIMG[i]:
samples_paired.append(samplesCSV[j])
samplesCSV = np.array(samples_paired)
samplesIMG = np.expand_dims(samplesIMG, axis=3)
print("Paired")
print("Samples IMG: {}".format(len(samplesIMG)))
print("Samples CSV: {}".format(len(samplesCSV)))