def sequentialRunner():
# main parameters
sourceFileNumber = 1
performPCA = False # takes time, if false the code will use saved data from a previous run
utils.Constants(
).incrementalFunctionFit = True # linear sum of functions will fit functions incrementally if True, independantly if False
plotPCAResults = False # plots few charts using PCA results
utils.Constants().testType = utils.TEST_TYPE_RANDOM
utils.Constants().fractionFullSampleForTest = 0.5
utils.Constants().YVarianceBuckets = 14
# chose the file to process
sourceFile = getSourceFile(sourceFileNumber)
# derive the post PCA / regression file name
postLinearRegressionProcessingFile = getPCAProcessedFile(sourceFile)
# gets X values post PCA and prediction for all Ys
if performPCA:
# calls PCA / linear regression formula to prepare different sets of X0 to process and the prediction for each method
df = processRawFileWithLinearRegressions(
sourceFile, postLinearRegressionProcessingFile, steps=[1, 2, 3]
) # [1, 2, 3, 4]) step 4 takes time and proved not to work too well
else:
# recalls the last file saved post PCA
df = utils.parseFile(postLinearRegressionProcessingFile)
if plotPCAResults:
plotPCA(df)
classifiers.rateClassifiers(df)
def testFromFile(sourceFileTrain):
HEADERS = [
utils.HEADER_INDEX, utils.HEADER_MARKET, utils.HEADER_DAY,
utils.HEADER_STOCK
] + utils.HEADER_X + [utils.HEADER_Y, utils.HEADER_WEIGHT]
return (HEADERS, utils.parseFile(sourceFileTrain), utils.HEADER_X,
utils.HEADER_Y, utils.HEADER_WEIGHT)
def processRawFileWithLinearRegressions(sourceFile,
targetFile,
fractionFullSampleForTest=0.2,
YVarianceBucketsParam=10,
steps=None):
utils.Constants().fractionFullSampleForTest = fractionFullSampleForTest
utils.Constants().YVarianceBuckets = YVarianceBucketsParam
logging.basicConfig(
filename=
'C:\\Users\\LL\\Desktop\\Work\\Machine learning\\challenge forecast markets\\log\\PCA.log',
level=logging.DEBUG,
format='%(asctime)s %(levelname)s:%(message)s')
df = utils.parseFile(sourceFile)
if steps == None:
steps = [1, 2, 3, 4]
if 1 in steps:
# simple PCA on the whole serie
coeffDetermin, df = modStats.mktPredictor.addRegressionData(
modStats.TYPE_LINEAR, df, addRegressionVectors=True, addRest=False)
print(
'Linear predictor per market average coefficient of determination: '
+ str(coeffDetermin))
if 2 in steps:
# PCA on linear combination of functions on the whole serie
coeffDetermin, df = modStats.mktPredictor.addRegressionData(
modStats.TYPE_LSUM_OF_FUNCTIONS,
df,
addRegressionVectors=True,
addRest=False)
print(
'Linear sum of functions predictor per market average coefficient of determination: '
+ str(coeffDetermin))
if 3 in steps:
# simple PCA on each stock independantly
coeffDetermin, df = modStats.mktPredictor.addRegressionData(
modStats.TYPE_LINEAR_PER_STCK,
df,
addRegressionVectors=True,
addRest=True)
print(
'Linear predictor per stock average coefficient of determination: '
+ str(coeffDetermin))
if 4 in steps:
# PCA on linear combination of functions on each stock independantly
coeffDetermin, df = modStats.mktPredictor.addRegressionData(
modStats.TYPE_LSUM_OF_FUNCTIONS_PER_STCK,
df,
addRegressionVectors=True,
addRest=True)
print(
'Linear sum of functions predictor per stock average coefficient of determination: '
+ str(coeffDetermin))
# write the new file with predicted data
df.to_csv(targetFile)
return df
def importFile(conffile):
#if conffile[0] != "/":
# conffile = "../"+conffile
try:
document = utils.parseFile(conffile)
except Exception, e:
invalidError("Error parsing configuration file: " + str(e))
def importFile(conffile):
#if conffile[0] != "/":
# conffile = "../"+conffile
try:
document = utils.parseFile(conffile)
except Exception, e:
invalidError("Error parsing configuration file: " + str(e))
def calcRevision(self, svndir):
entriesFile = os.path.join(svndir, "entries")
doc = parseFile(entriesFile)
for child in doc.elements():
try:
num = int(child.getAttribute("committed-rev"))
self.version = max(num, self.version)
except TypeError:
pass
def calcRevision(self, svndir):
entriesFile = os.path.join(svndir, "entries")
doc = parseFile(entriesFile)
for child in doc.elements():
try:
num = int(child.getAttribute("committed-rev"))
self.version = max(num, self.version)
except TypeError:
pass
def main(input):
"""Console script for led_tester."""
''' takes two arguments from input as N and instructions, N is first number which I will use to make the
led, i.e will be the size of the led board squared. Instructions will then be each line which will be
passed to the reg ex expression to have infor extracted.
'''
N, instructions = parseFile(input)
print("The amount of LED's currently 'ON' is: ",
apply_led(N, instructions))
return 0
def getItem(self, jid, node, itemid): """ Loads the item from a node from disk and returns an element """ try: filename = self.dir(jid, node) + itemid + ".xml" if os.path.isfile(filename): LogEvent(INFO, msg="Getting item %r - %r" % (node, itemid)) document = utils.parseFile(filename) return document else: LogEvent(INFO, msg="Avatar not found %r" % (key)) except IOError, e: LogEvent(INFO, msg="IOError reading item %r - %r" % (node, itemid))
def getItem(self, jid, node, itemid):
""" Loads the item from a node from disk and returns an element """
try:
filename = self.dir(jid, node) + itemid + ".xml"
if os.path.isfile(filename):
LogEvent(INFO, msg="Getting item %r - %r" % (node, itemid))
document = utils.parseFile(filename)
return document
else:
LogEvent(INFO, msg="Avatar not found %r" % (key))
except IOError, e:
LogEvent(INFO, msg="IOError reading item %r - %r" % (node, itemid))
def main():
# Get cli arguments count
argc = len(sys.argv)
# Handle incorrect number of arguments.
if argc != 2:
print("Usage: python %s [file]" % sys.argv[0])
return
# Set the file name.
file_name = sys.argv[1]
# Open the file as an input stream and parse the file for the records.
with open(file_name, 'rb') as fin:
records = parseFile(fin)
# Once records are parsed, iterate through each record and process it.
user_id = 2456938384156277127
answers = Report()
for record in records:
# 0x00: Debit
if record.category == 0:
answers.debits += record.amount_in_dollars
if record.user_id == user_id:
print(user_id, record.amount_in_dollars)
answers.balance_of_user -= record.amount_in_dollars
# 0x01: Credit
if record.category == 1:
answers.credits += record.amount_in_dollars
if record.user_id == user_id:
print(user_id, record.amount_in_dollars)
answers.balance_of_user += record.amount_in_dollars
# 0x02: StartAutopay
if record.category == 2:
answers.autopays_started += 1
# 0x03: EndAutopay
if record.category == 3:
answers.autopays_ended += 1
# Print the answers.
print("The total amount in dollars of debit is $%.2f." % answers.debits)
print("The total amount in dollars of debit is $%.2f." % answers.credits)
print("%d autopays were started." % answers.autopays_started)
print("%d autopays were ended." % answers.autopays_ended)
print("The balance of user ID %d is $%.2f." %
(user_id, answers.balance_of_user))
def getFiles():
files = utils.getFile(DIR_DATA, ['txt'])
# validation
# ----------
# query params
initRotation = request.args.get('initRotation')
if (initRotation is None):
return bad_request('missing required query param \'initRotation\'')
if initRotation not in ROTATE_TRANSLATE_MAP:
return bad_request('invalid query param \'initRotation\', it must be one of ' +
str(list(ROTATE_TRANSLATE_MAP)) +
'. You can change this in \'/web-app/src/config.js\'')
# no .txt files
num_files = len(files)
if (num_files == 0):
return bad_request('no .txt files in ' + DIR_DATA)
# no more than one .txt file
if (num_files > 1):
return bad_request('only one .txt file should be in ' + DIR_DATA)
file = files[0]
file_data = utils.parseFile(files[0]['path'], ACCEPTED_CHARS)
# bad character in file
if ('error' in file_data):
return bad_request(file_data['error'])
# process data
# ------------
file_data_processed = processor.processCharData(file_data, initRotation, ROTATE_TRANSLATE_MAP, CHAR_ACTION)
# response
# --------
response = jsonify({
'fileName': file['name'],
'fileData': file_data_processed
})
return(response)
import os
import utils
import pdb
debug = 0
#-------------------------------------------------------------------------------
## Input Parsing
initObj = None
testObj = None
resFile = None
while( initObj is None ):
initFile = raw_input("Enter the name of the initialization file: ")
initObj = utils.parseFile(initFile, "weights")
if( initObj is None ):
print "Invalid weights file!\n"
while( testObj is None ):
testFile = raw_input("Enter the name of the testing data file: ")
testObj = utils.parseFile(testFile, "data")
if( testObj is None ):
print "Invalid data file!\n"
while( resFile is None ):
resFileInput = raw_input("Enter the name results output file: ")
# if( os.access(resFileInput, os.W_OK) ):
resFile = resFileInput
# else:
# print "Invalid output file path!"
def __getFile(self, file): file = utils.mangle(file) hash = file[0:2] document = utils.parseFile(self.name + X + hash + X + file + ".xml") return document
continue
for entry in listentries:
listentry = list.addElement("entry")
listentry["name"] = entry[0]
attrs = entry[1]
for attr in attrs:
entryattr = listentry.addElement(attr)
entryattr.addContent(attrs[attr])
f = open(args[1], "w")
f.write(out.toXml())
f.close()
elif args[0] == "restore":
import xdb
myxdb = xdb.XDB(config.jid)
input = utils.parseFile(args[1])
if input.getAttribute("transport") != transportname:
print "The dump file specified does not appear to be for this transport."
sys.exit(0)
for child in input.elements():
jid = child.getAttribute("jid")
print "Restoring " + jid + "..."
doesexist = myxdb.getRegistration(jid)
if doesexist:
myxdb.removeRegistration(jid)
username = child.getAttribute("username")
password = child.getAttribute("password")
myxdb.setRegistration(jid, username, password)
for child2 in child.elements():
if child2.name == "preferences":
for pref in child2.elements():
listentries = myxdb.getList(listtype, jid)
if not listentries: continue
for entry in listentries:
listentry = list.addElement("entry")
listentry["name"] = entry[0]
attrs = entry[1]
for attr in attrs:
entryattr = listentry.addElement(attr)
entryattr.addContent(attrs[attr])
f = open(args[1], "w")
f.write(out.toXml())
f.close()
elif args[0] == "restore":
import xdb
myxdb = xdb.XDB(config.jid)
input = utils.parseFile(args[1])
if input.getAttribute("transport") != transportname:
print "The dump file specified does not appear to be for this transport."
sys.exit(0)
for child in input.elements():
jid = child.getAttribute("jid")
print "Restoring "+jid+"..."
doesexist = myxdb.getRegistration(jid)
if doesexist:
myxdb.removeRegistration(jid)
username = child.getAttribute("username")
password = child.getAttribute("password")
myxdb.setRegistration(jid, username, password)
for child2 in child.elements():
if child2.name == "preferences":
for pref in child2.elements():
def __getFile(self, file): file = utils.mangle(file) document = utils.parseFile(self.name + X + file + ".xml") return document
from smart_open import open
from utils import sanitize_move, record_move, parseFile
import sys
limit = 1e5
db_name = "lichess_db_standard_rated_2020-07.pgn"
# python main.py lichess_db_standard_rated_2020-07.pgn 10000
# python main.py s3://gjgd-chess/lichess_db_standard_rated_2020-07.pgn 10000
if len(sys.argv) != 3:
print('Usage: python main.py <filename> <number-of-games-to-analyse>')
exit(1)
file_name = sys.argv[1]
limit = int(sys.argv[2])
# Read the file from an s3 bucket
if (file_name.startswith('s3://')):
session = boto3.Session(profile_name='chess')
with open(file_name, 'rb', transport_params={'session': session}) as f:
all_games = parseFile(f, limit)
# Or read the file from the local filesystem
else:
with open(db_name) as f:
all_games = parseFile(f, limit)
with open(
'./results/{}-{}.json'.format(str(datetime.datetime.now()),
int(limit)), 'w') as result_file:
pretty_json = json.dumps(all_games)
print(pretty_json, file=result_file)
def __getFile(self, file):
file = utils.mangle(file)
document = utils.parseFile(self.name + X + file + ".xml")
return document
def get_annotation(filename):
genre = os.path.split(U.parseFile(filename)[0])[-1]
return genre
def get_annotation(filename):
genre = os.path.split(U.parseFile(filename)[0])[-1]
return genre
def __getFile(self, file):
file = utils.mangle(file)
hash = file[0:2]
document = utils.parseFile(self.name + X + hash + X + file + ".xml")
return document
import pdb
debug = 0
#-------------------------------------------------------------------------------
## Input Parsing
trainObj = None
initObj = None
resFile = None
num_epochs = 0
learning_rate = 0
while (initObj is None):
initFile = raw_input("Enter the name of the initialization file: ")
initObj = utils.parseFile(initFile, "weights")
if (initObj is None):
print "Invalid weights file!\n"
while (trainObj is None):
trainFile = raw_input("Enter the name of the training data file: ")
trainObj = utils.parseFile(trainFile, "data")
if (trainObj is None):
print "Invalid data file!\n"
while (resFile is None):
resFileInput = raw_input(
"Enter the name of the trained weights output file: ")
# if( os.access(resFileInput, os.W_OK) ):
resFile = resFileInput
# else: