def calcpurchases(numtries,numpurchases):
buychoices = {}
# print ".. trying options for ",numpurchases," number of purchases...tries",numtries
# now running multiprocessor, only put out a status bar for the first job.
for i in xrange(numtries):
buyamounts = rebalance.constrained_sum_sample_pos(numpurchases,int(addedcash - (fee * numpurchases)))
buycode = []
intermedport = []
buypattern = []
asxcodestobuy = copy.copy(asxcodestochoosesorted)
# we have the sorted list of ASX codes, so randomly remove some from the list until we
# only have numpurchases left
while len(asxcodestobuy) > numpurchases:
asxcodestobuy.remove(random.choice(asxcodestobuy))
# print "for numpurchases",numpurchases,"have asxcodestobuy", asxcodestobuy
codestobuy = asxcodestochoosesorted
for purchase in xrange(numpurchases):
# since the intervals chosen are random anyway, we dont need to randomly choose an ASX code?
# so we can do it in order of largest to smallest shareprice
buycode.append(asxcodestobuy[purchase][1])
# lets grab the asx code in order of lowest to highest share price, deleting
# lets round the amount down to a multiple of share price, and add the remainder to the next purchase..
# proivided we got a share price (which we may have not if it is not a share already in the portfolio)
if starterport[buycode[purchase]][3] > 0.0:
remainder = buyamounts[purchase] % starterport[buycode[purchase]][3]
# print remainder,purchase,numpurchases,buyamounts
if purchase < numpurchases -1:
buyamounts[purchase] = buyamounts[purchase] - remainder
buyamounts[purchase + 1] = buyamounts[purchase + 1] + remainder
if purchase == 0:
intermedport.append(rebalance.rebalance(buyamounts[purchase],starterport,buycode[purchase]))
else:
intermedport.append(rebalance.rebalance(buyamounts[purchase],intermedport[purchase - 1],buycode[purchase]))
buypattern.append(buyamounts[purchase])
buypattern.append(buycode[purchase])
newport = intermedport[-1]
newrating = rebalance.isrebalancegood(newport)
buychoices[newrating] = buypattern
# just grab the top rated one, we dont have to pass the whole lot back
# this step a holdover from when the non-parallel code putting it all in one global dict.
sortratings = buychoices.keys()
sortratings.sort()
# print "ending worker with numtries",numtries,"numpurchases",numpurchases
return({ sortratings[0] : buychoices[sortratings[0]] })
def daily_rebalance(d, amount=1000000):
symbols = d.keys()
ndays = len(d['HLT'])
amount = amount
shares = {}
series1 = collections.defaultdict(list)
series2 = collections.defaultdict(list)
for day in range(ndays):
v = {symbol: d[symbol][day]['close'] for symbol in sorted(symbols)}
amount = amount + sum(v[symbol] * shares[symbol] for symbol in shares)
print day, d['HLT'][day]['date'], amount
shares = rebalance(v, amount)
amount = amount - sum(v[symbol] * shares[symbol] for symbol in shares)
for symbol in sorted(symbols):
series1[symbol].append((day, v[symbol]))
series2[symbol].append((day, shares[symbol]))
canvas1 = Canvas(title="Hotel Share Prices")
for symbol in series1:
canvas1.plot(series1[symbol], legend=symbol, color=color(symbol))
canvas1.save('prices.png')
canvas2 = Canvas(title="Hotel Portfolio Shares")
for symbol in series2:
canvas2.plot(series2[symbol], legend=symbol, color=color(symbol))
canvas2.save('shares.png')
def rebalance_data(matrix, dict_keys, rebal_name):
# Construct feature vectors and label vector
features = []
labels = []
keys = []
sizes = []
flag = True
for row in matrix:
feats = []
for key in dict_keys:
val = row[key]
if key == labelname:
labels.append(val)
elif key != "MG_ID":
if type(val) is list:
feats = feats + val
if flag:
keys.append(key)
sizes.append(len(val))
else:
feats.append(val)
if flag:
keys.append(key)
sizes.append(1)
features.append(feats)
if flag:
flag = False
new_feats, new_labels = rebalance.rebalance(features, labels, rebal_name)
new_matrix = []
for y in range(len(new_feats)):
row = new_feats[y]
label = labels[y]
index = 0
new_entry = {}
for x in range(len(sizes)):
key = keys[x]
size = sizes[x]
val = row[index:index+size]
index = index+size
new_entry[key] = val
new_entry[labelname] = label
new_matrix.append(new_entry)
return new_matrix
starterports = []
firstcsv = True
for csvfile in inputcsvfiles:
starterports.append(rebalance.read_cmc_pnl_to_portfdict(csvfile,desiredport))
if not firstcsv:
starterports.append(rebalance.add_portfdict(starterports[-2],starterports[-1]))
else:
firstcsv = False
portfolios = [starterports[-1]]
starterport = starterports[-1]
print "starter portfolio"
rebalance.printport(starterport)
starterrating = rebalance.isrebalancegood(starterport)
print "starter rating", starterrating
totalspend = 0
fees = 0
for buycode,buyamount in buysequence:
print "buy ",buyamount," of ",buycode
portfolios.append(rebalance.rebalance(float(buyamount),portfolios[-1],buycode))
totalspend = totalspend + float(buyamount)
fees = fees + fee
rebalance.printport(portfolios[-1])
print "rating of ",rebalance.isrebalancegood(portfolios[-1])
print "totalspend is: ",totalspend," (inc fees is: ",totalspend + fees," )"
print "fees are: ",fees," which is ",fees/(totalspend + fees) * 100," per cent"
except:
usage()
starterport = rebalance.read_cmc_pnl_to_portfdict(cmcpnlcsvfilename,desiredport)
buychoices = {}
print "starter portfolio, loaded from the file ",cmcpnlcsvfilename
rebalance.printport(starterport)
starterrating = rebalance.isrebalancegood(starterport)
print "starter balance rating (i.e. how close to the desired balance):", starterrating
print
# first gather all the single buy choices.
for asxcode in starterport.keys():
newport = rebalance.rebalance(addedcash - fee,starterport,asxcode)
newrating = rebalance.isrebalancegood(newport)
# print asxcode,newrating
# if newrating > starterrating or asxcode == 'TOTALS:':
if asxcode == rebalance.TOTALS:
# print "dont bother with", asxcode
pass
else:
buychoices[newrating] = [addedcash,asxcode]
# get list of ASXcodes in order of share price, highest to lowest, with TOTALS removed.
asxcodestochoose = starterport.keys()
asxcodestochoose.remove(rebalance.TOTALS)
asxcodestochoosesorted = []
for asxcode in asxcodestochoose:
asxcodestochoosesorted.append([starterport[asxcode][3],asxcode])
# Initialize our mutual pairs for Trading
polo = initializePoloniex()
currencies = getPairs.getPairs(base_a, base_b, polo)
# spread[[base,currency],[bid_a,ask_a,bid_b,ask_b],depth,[value, volume]]
spread = np.zeros([len(currencies) + 1, 4, depth, 2])
flow = np.zeros([len(currencies) - 1, 4])
print('Starting Trading... Live: ', live)
fund_value = float(polo.returnBalances()['BTC']) * exposure
print('Fund Value Exposed:', fund_value, ' BTC')
# Main Loop
while True:
if (live == True and rebalance == True):
fund_value = rebalance.rebalance(currencies[i], polo)
rebalance = False
if (fund_value < cutoff):
break
fees = calculate_fees(fund_value, rate)
# Get the Spread
spread = getSpread.getSpread(base_a, base_b, currencies, spread, depth,
polo) #polo.returnOrderBook()
# Calculate the Flow
flow = getFlow(spread, flow, fees)
print()
print(flow)
'''
def run(arg_model,
arg_modelname,
arg_train_feats,
arg_test_feats,
arg_result_file,
arg_prefix,
arg_labelname,
arg_n_feats=227,
arg_anova="chi2",
arg_nodes=192,
arg_activation='relu',
arg_dropout=0.5,
arg_rebalance=""):
total_start_time = time.time()
# Special handling for neural network models
is_nn = arg_model == "nn" or arg_model == "lstm" or arg_model == "rnn" or arg_model == "cnn"
# Params
num_feats = arg_n_feats
num_nodes = arg_nodes
global labelname
labelname = arg_labelname
trainids = [] # VA record id
trainlabels = [] # Correct ICD codes
X = [] # Feature vectors
Y = []
X2 = [] # Extra features for hybrid model
# Read in feature keys
print "reading feature keys..."
global keys
with open(arg_train_feats + ".keys", "r") as kfile:
keys = eval(kfile.read())
vec_keys = [] # vector/matrix features for CNN and RNN models
point_keys = [] # traditional features for other models
# If a mix of traditional and vector features is requested, use a hybrid nn model
vec_keys, point_keys = split_feats(keys)
hybrid = False
if len(vec_keys) > 0 and len(point_keys) > 2:
hybrid = True
print "hybrid features"
# Transform ICD codes and record types to numbers
global labelencoder, typeencoder
labelencoder = preprocessing.LabelEncoder()
typeencoder = preprocessing.LabelEncoder()
# Load the features
if hybrid:
Y = preprocess(arg_train_feats, trainids, trainlabels, X, Y, vec_keys,
True)
preprocess(arg_train_feats, [], [], X2, [], point_keys, True)
else:
test_labs = model_new.get_labels(arg_test_feats, arg_labelname)
Y = preprocess(arg_train_feats,
trainids,
trainlabels,
X,
Y,
keys,
True,
extra_labels=test_labs)
print "X: " + str(len(X)) + " Y: " + str(len(Y))
print "X2: " + str(len(X2))
# Rebalance
if arg_rebalance != "":
print "rebalance: " + arg_rebalance
X, Y = rebalance.rebalance(X, Y, arg_rebalance)
print "X: " + str(len(X)) + "\nY: " + str(len(Y))
# Train the model
print "training model..."
stime = time.time()
# Feature selection
global anova_filter
anova_function = f_classif
if arg_anova == "chi2":
anova_function = chi2
print "anova_function: " + arg_anova + ", num_feats: " + str(num_feats)
if not is_nn:
anova_filter, X = create_anova_filter(X, Y, anova_function, num_feats)
global model
model = None
# Neural network models
if is_nn:
modelfile = arg_prefix + "/" + arg_modelname + ".model"
if os.path.exists(modelfile):
print "using pre-existing model at " + modelfile
model = load_model(modelfile)
Y = to_categorical(Y)
X = numpy.asarray(X)
else:
print "creating a new neural network model"
embedding_dim = 200
if arg_model == "nn":
model, X, Y = create_nn_model(X, Y, anova_function, num_feats,
num_nodes, 'relu')
elif arg_model == "lstm":
num_nodes = 56
model, X, Y = create_lstm_model(X,
Y,
embedding_dim,
num_nodes,
arg_activation,
hybrid=hybrid,
X2=X2)
#score = model.evaluate(X_test, Y_test, batch_size=16
elif arg_model == "rnn":
model, X, Y = create_rnn_model(X, Y, embedding_dim, num_nodes,
arg_activation)
elif arg_model == "cnn":
model, X, Y = create_cnn_model(X,
Y,
embedding_dim,
hybrid=hybrid,
X2=X2)
# Save the model
print "saving the model..."
model.save(modelfile)
plotname = modelfile + ".png"
plot_model(model, to_file=plotname)
# Other models
else:
if arg_model == "svm":
print "svm model"
model = svm.SVC(kernel='linear',
decision_function_shape='ovr',
probability=True)
elif arg_model == "knn":
print "k-nearest neighbor model"
model = neighbors.KNeighborsClassifier(n_neighbors=1,
weights='distance',
n_jobs=-1)
elif arg_model == "nb":
print "naive bayes model"
model = MultinomialNB()
elif arg_model == "rf":
print "random forest model"
model = RandomForestClassifier(n_estimators=26,
max_features=0.0485,
min_samples_split=4,
class_weight='balanced',
n_jobs=-1)
model.fit(X, Y)
etime = time.time()
print "training took " + str(etime - stime) + " s"
# Test
testids, testlabels, predictedlabels = test(arg_model, model,
arg_test_feats, hybrid)
# Write results to a file
output = open(arg_result_file, 'w')
for i in range(len(testids)):
out = {}
out['MG_ID'] = testids[i]
out['Correct_ICD'] = testlabels[i]
out['Predicted_ICD'] = predictedlabels[i]
output.write(str(out) + "\n")
output.close()
total_time = (time.time() - total_start_time) / 60
print "total time: " + str(total_time) + " mins"
#Benchmark Index
bench_symbol = "SPY"
#RF Syntax: 6 MO, 2 YR etc.
rate = '1 YR'
#CUR, AVG
method = "AVG"
#For Quandl
api_key = ""
#Dirctory Input For Data and Reports
root_path = "C:/Users/fdupu/PycharmProjects/Portfolio Tracker"
#------------Run Program----------------------#
if __name__ == '__main__':
import rebalance
rebalance.rebalance(allocations=allocations)
# # 1.) Import the module
# import report
#
# # Select Functions
#end_date = dt.date.today()
# r = report.rep(fname=root_path + '/Reports/Daily Report ' + str(end_date) + '.pdf',fund_name="Valhalla Investments LLC",logo_path="C:/Users/fdupu/Desktop/logo.png")
# r.cover()
# r.perf()
# r.mets()
# r.diversification()
# r.savePDF()
def lambda_handler(event, context):
logger.info("Got Event: {}".format(event))
config = fetchFromTransitConfigTable(transitConfigTable)
if config:
response = getInUsePaGroups(config['TransitPaGroupInfo'],
int(config['PaGroupMaxVpc']))
if response:
if config['RebalanceInProgress'] == 'True':
if config['RebalanceStatus'] == 'Done':
apiKey = pan_vpn_generic.getApiKey(response[0]['N1Mgmt'],
config['UserName'],
config['Password'])
result = rebalance.rebalance(apiKey, response,
int(config['PaGroupMaxVpc']),
config)
if result:
# Get the VGW, Region, SubscriberSnsArn and SubscriberAssumeRoleArn from VpcTable
subscriberData = getSubscriberDataFromVpcTable(
config['TransitVpcTable'],
result['FromPaGroup']['PaGroupName'])
result['FromPaGroup']['VpcCount'] = str(
result['FromPaGroup']['VpcCount'])
result['ToPaGroup']['VpcCount'] = str(
result['ToPaGroup']['VpcCount'])
value = {
'FromPaGroupName':
result['FromPaGroup']['PaGroupName'],
'ToPaGroupName':
result['ToPaGroup']['PaGroupName'],
'VpcId':
subscriberData['VpcId'],
'VpcCidr':
subscriberData['VpcCidr'],
'Region':
subscriberData['Region'],
'SubscriberSnsArn':
subscriberData['SubscriberSnsArn'],
'SubscriberAssumeRoleArn':
subscriberData['SubscriberAssumeRoleArn'],
'CreateStatus':
'Pending',
'DeleteStatus':
'InProgress'
}
item = {'Property': 'RebalanceStatus', 'Value': value}
updateTransitConfig(transitConfigTable, item)
# Send DeleteOperatin first
deleteData = {
'Action': 'DeleteVpnConnection',
'VpcId': subscriberData['VpcId'],
'Region': subscriberData['Region'],
'Rebalance': 'True'
}
#Publish message to Transit SNS
publishToSns(subscriberData['SubscriberSnsArn'],
deleteData,
subscriberData['SubscriberAssumeRoleArn'])
logger.info(
"Published message to Subscriber SNS with data: {}"
.format(deleteData))
return
else:
previousTaskStatus = config['RebalanceStatus']
if previousTaskStatus['DeleteStatus'] == 'InProgress':
vpcStatus = checkVpcIdInVpcTable(
config['TransitVpcTable'],
previousTaskStatus['VpcId'])
logger.info("Got VPC Status: {}".format(vpcStatus))
if len(vpcStatus['Items']) > 0:
if vpcStatus['Items'][0][
'PaGroupName'] == previousTaskStatus[
'FromPaGroupName']:
logger.info(
"Previous Delete VPN Operation is still InProgress, hence exiting from the process"
)
return
else:
# Create FetchVpnServerDetails and send to Subscriber SNS
previousTaskStatus['CreateStatus'] = 'InProgress'
previousTaskStatus['DeleteStatus'] = 'Completed'
item = {
'Property': 'RebalanceStatus',
'Value': previousTaskStatus
}
updateTransitConfig(transitConfigTable, item)
data = {
'Action':
'FetchVpnServerDetails',
'Region':
previousTaskStatus['Region'],
'VpcId':
previousTaskStatus['VpcId'],
'SubscriberAssumeRoleArn':
previousTaskStatus['SubscriberAssumeRoleArn'],
'SubscriberSnsArn':
previousTaskStatus['SubscriberSnsArn'],
'VpcCidr':
previousTaskStatus['VpcCidr'],
'Rebalance':
'True'
}
#Publish message to Transit SNS
publishToSns(config['TransitSnsArn'], data)
logger.info(
"Published message to Transit SNS with data: {}"
.format(data))
return
elif previousTaskStatus['CreateStatus'] == 'InProgress':
logger.info(
"Previous task was CreateTask, now check whether it has completed or not"
)
vpcStatus = checkVpcIdInVpcTable(
config['TransitVpcTable'],
previousTaskStatus['VpcId'])
logger.info("Got VPC Status: {}".format(vpcStatus))
if not vpcStatus['Items']:
logger.info(
"Create Task is still in progress, hence exiting"
)
return
else:
if vpcStatus['Items'][0][
'PaGroupName'] == previousTaskStatus[
'ToPaGroupName']:
logger.info(
"Previous Rebalance task Completed successfully, updating the RebalanceStatus=Done"
)
item = {
'Property': 'RebalanceStatus',
'Value': 'Done'
}
updateTransitConfig(transitConfigTable, item)
return
else:
logger.error(
"Something terrible happened? Unknown status, Stop StateMachine and Exit"
)
#Something terrible happened? Unknown status, Stop StateMachine and Exit
return
else:
logger.info("No PaGroups for Rebalancing, PaGroups are Optimal")
else:
logger.error("Not Received any data from TransitConfig table")