def RNStar(moviesTrain, movieEdges, moviesDict, maxItr, traditional,
threshold):
previousCount = dict()
for i in range(0, maxItr):
print("{0}/{1}".format(i, maxItr))
print(Helper.genreCount(moviesTrain))
if previousCount == Helper.genreCount(moviesTrain):
break
previousCount = Helper.genreCount(moviesTrain)
for index, movie in enumerate(moviesTrain):
if movie.markedAsUnknown and movie.id in movieEdges:
genres = dict()
totalNeighbours = 0
for neighbourEdge in movieEdges[movie.id]:
neighbourMovie = moviesDict[neighbourEdge]
if neighbourMovie.genres:
totalNeighbours += 1
for genre in neighbourMovie.genres:
if genre in genres:
genres[genre] += 1
else:
genres[genre] = 1
if traditional:
if bool(genres):
maxProb = max(genres.items(),
key=operator.itemgetter(1))[0]
movie.genres = [maxProb]
else:
movie.genres = []
for genre, number in genres.items():
if totalNeighbours != 0 and (
number / totalNeighbours) >= threshold:
movie.genres.append(genre)
def refinementV15(self, E=None):
global refinementCounter
refinementCounter += 1
# print ".",
C_star = copy.copy(self.net)
C_star.cs = copy.deepcopy(self.net.cs)
aClosed = Search(C_star).aClosureV2(E)
if aClosed is INCONSISTENT:
return INCONSISTENT
refineList = C_star.listOfNontractableConstraints()
if len(refineList) == 0: # all rel's have 1 base relation
if Search(C_star).aClosureV2() == CONSISTENT:
# print("Resulting QCSP:\n%s" % C_star)
return CONSISTENT
for c, i, j in refineList:
prevRel = C_star.cs[i][j]
# for baseRel in self.net.algebra.aTractableSet(prevRel):
for baseRel in Helper.bits(prevRel):
C_star.cs[i][j] = baseRel
if Search(C_star).refinementV15([[i, j]]) == CONSISTENT:
return CONSISTENT
C_star.cs[i][j] = prevRel
return INCONSISTENT
def createTraderWidgets(self, type):
temp = []
if type == 'signals':
for element in Helper.getAvailableSignals():
temp.append(urwid.Text(('neutral', element)))
elif type == 'indicators':
for element in Helper.getAvailableIndicators():
temp.append(urwid.Text(
('neutral', element + " : " + str(0.0))))
elif type == 'strategies':
for element in Helper.getAvailableStrategies():
temp.append(urwid.Text(
('neutral', element + " : " + str(0.0))))
else:
pass
return temp
def run(self, loop):
try:
while self.Scraper.running and self.running:
Files = Helper.getFiles(self.Unfiltered_dir + '/*')
if Files:
if not os.listdir(self.Sample_dir):
self.simpleAnalysis(Files, Helper.getText(self.Words_file))
else:
self.complexAnalysis(Files)
self.logger.info('Finished filtering files')
else:
self.logger.info('No files to filter')
sleep(self.Downloader_wait)
except Exception as e:
self.logger.error(str(e))
traceback.print_exc()
self.logger.debug('Fin.')
self.fin = True
def simpleAnalysis(self, files, keywords):
self.logger.info('Sorting files: Simple')
i = 0
while self.running and i < len(files):
f = files[i]
try:
text = self.getText(f)
count = 0
for word in text:
if text in keywords:
count += 1
if count > self.sThreshhold:
dest = self.Hit_dir + f.split('/')[-1]
else:
dest = self.Miss_dir + f.split('/')[-1]
except Helper.ParseError:
self.logger.error('Encountered Parse Error with %s', f)
dest = self.Error_dir + f.split('/')[:-1]
Helper.moveFile(f, dest)
i += 1
def getText(self, f):
try:
import textract
text = textract.process(f)
except Exception:
try:
from tika import parser
text = parser.from_file(f)['content']
except Exception:
raise Helper.ParseError('Whoops')
return text
def complexAnalysis(self, files):
# if os.path.isfile('./config/text_classifier'):
# model = self.loadDataset()
# else:
self.logger.info('Sorting files: Complex')
model = self.getDataset('./config/dataset')
i = 0
while self.running and i < len(files):
f = files[i]
try:
text = self.cleanText(self.getText(f))
if model.predict(text) > self.aThreshhold:
dest = self.fHit_dir + f.split('/')[-1]
else:
dest = self.Miss_dir + f.split('/')[-1]
Helper.moveFile(f, dest)
except Helper.ParseError:
self.logger.error('Encountered Parse Error with %s', f)
dest = self.Error_dir + f.split('/')[:-1]
Helper.moveFile(f, dest)
i += 1
def run(self, loop):
try:
Primary_words, Secondary_words = self.getWords()
FileTypes = Helper.readFile(self.Filetypes_file)
BaseQuery = str(" ".join(str(x) for x in Primary_words))
Queries = self.filterQueries(
self.generateQueries(BaseQuery, Secondary_words))
self.Scrape(Queries, FileTypes)
except Exception as e:
self.logger.error(str(e))
traceback.print_exc()
self.logger.debug('Fin.')
self.fin = True
self.running = False
def aClosureV1(self):
s = True
while s:
s = False
for i, j, k in Helper.tripleNested(self.net.nodeCount):
Cij = self.net.cs[i][j]
Cjk = self.net.cs[j][k]
Cik = self.net.cs[i][k]
# if Cij != self.net.algebra.Universal and\
# Cjk != self.net.algebra.Universal:
Cik_star = Cik & self.net.algebra.compose(Cij, Cjk)
if Cik != Cik_star:
self.net.cs[i][k] = Cik_star
s = True
if Cik_star == 0:
return INCONSISTENT # early exit
return CONSISTENT
def __init__(self, callback, db=DB.DB()):
self.streamer = Streamer.Streamer(callback)
self.db = db
self.ind = Indicator.Indicator(self.db)
self.sig = Signal.Signal(self.db)
self.strat = Strategy.Strategy(self.db)
self.runAsyncTasks = True
self.pw = PyWhale.PyWhale()
self.pw.verbose = False
for strategy in Helper.getAvailableStrategies():
self.strategies[strategy] = False
self.log = logging.getLogger(__name__)
self.fh = logging.FileHandler('log/trader.log')
self.fh.setLevel(logging.DEBUG)
self.sh = logging.StreamHandler(sys.stderr)
self.sh.setLevel(logging.DEBUG)
self.log.addHandler(self.sh)
self.log.addHandler(self.fh)
logging.basicConfig(level=logging.DEBUG, handlers=[self.fh, self.sh])
def verifyFiles():
files = [Words, Filetypes, ChromeDriver]
missing_files = [f for f in files if not Helper.checkFile(f)]
if missing_files:
logger.error('missing files: %s' % missing_files)
sys.exit(0)
for movieId in edgeList:
if movieId in moviesDict:
newList.append(movieId)
movieEdges[id] = newList
print("removed {0} movies".format(len(moviesPrev) - len(movies)))
moviesPrev = movies
movieEdgesPrev = movieEdges
print("removed total {0} movies".format(len(moviesRaw) - len(movies)))
actualMovies = 0
for movie in movies:
if movie.titleType == "movie":
actualMovies += 1
genreCountTotal = Helper.genreCount(movies)
print(Helper.genreCount(movies))
print(genreCountTotal.keys())
print(datetime.now() - time)
# print(edge)
print("nr of edges: {0}".format(len(movieEdges)))
print("nr of movies: {0}".format(len(movies)))
print("nr of actual movies: {0}".format(actualMovies))
print("nr of crew: {0}".format(len(crew)))
print("nr of crew members: {0}".format(len(crewMember)))
moviesTrain = copy.deepcopy(movies)
for movie in moviesTrain:
class Trader():
strategies = {key: False for key in Helper.getAvailableStrategies()}
def __init__(self, callback, db=DB.DB()):
self.streamer = Streamer.Streamer(callback)
self.db = db
self.ind = Indicator.Indicator(self.db)
self.sig = Signal.Signal(self.db)
self.strat = Strategy.Strategy(self.db)
self.runAsyncTasks = True
self.pw = PyWhale.PyWhale()
self.pw.verbose = False
for strategy in Helper.getAvailableStrategies():
self.strategies[strategy] = False
self.log = logging.getLogger(__name__)
self.fh = logging.FileHandler('log/trader.log')
self.fh.setLevel(logging.DEBUG)
self.sh = logging.StreamHandler(sys.stderr)
self.sh.setLevel(logging.DEBUG)
self.log.addHandler(self.sh)
self.log.addHandler(self.fh)
logging.basicConfig(level=logging.DEBUG, handlers=[self.fh, self.sh])
def print_fn(self, msg, color):
print(msg)
def post_order(self, direction='long', leverage=20, size=20000000):
price_data = self.pw.getPrice('BTC-USD')['BTC-USD']
bid_price = float(price_data['bid'])
ask_price = float(price_data['ask'])
if direction == 'long':
price = ask_price
else:
price = bid_price
tp = round((price * (20 / leverage)) / 100, 5)
sl = round((price * (10 / leverage)) / 100, 5)
if direction == 'long':
takeprofit = float(price) + tp
stoploss = float(price) - sl
else:
takeprofit = float(price) - tp
stoploss = float(price) + sl
result = self.pw.newPosition(direction,
'BTC-USD',
leverage,
size,
price,
stop_loss=stoploss,
take_profit=takeprofit)['id']
return result
def close_position(self, id, status):
if status == 'pending':
self.pw.cancelPosition(position_id=id)
else:
self.pw.closePosition(position_id=id)
self.db.setPositionState(id, 'closed')
def syncPositions(self):
#SYNCS POSITION STATES OF DB AND PYWHALE
PYPositions = self.pw.listPositions() + self.pw.listPositions(
position_state='pending')
closed_positions = self.pw.listPositions(position_state='closed')
#DBPositions = list(self.db.getAllPositions()['trade_id'])
#UPDATE POSITION STATE FROM PYWHALE TO DB
if len(PYPositions) != 0:
for PYPosition in PYPositions:
self.db.setPositionState(PYPosition.get('id'),
PYPosition.get('state'))
if len(closed_positions) != 0:
for clpos in closed_positions:
self.db.closePosition(clpos.get('id'),
str(clpos.get('profit')))
def enable_strategy(self, strategy=''):
if strategy in list(self.strategies.keys()):
self.strategies[strategy] = True
return True
else:
return False
def disable_strategy(self, strategy=''):
if strategy in list(self.strategies.keys()):
self.strategies[strategy] = False
return True
else:
return False
def evalStrategies(self):
self.syncPositions()
for strategy, enabled in self.strategies.items():
if enabled == True:
active_long = list(
self.db.getPositions(strategy, 'long',
'active')['trade_id'])
pending_long = list(
self.db.getPositions(strategy, 'long',
'pending')['trade_id'])
active_short = list(
self.db.getPositions(strategy, 'short',
'active')['trade_id'])
pending_short = list(
self.db.getPositions(strategy, 'short',
'pending')['trade_id'])
result = (getattr(self.strat, 'strat_' + strategy))()
if result == 1:
#STRATEGY SUGGESTS LONG
if len(active_short + pending_short) != 0:
self.print_fn(
"Strategy " + strategy +
" flipped. Canceling SHORT ORDERS", "green")
#WE HAVE A SHORT, CANCEL ALL SHORTS
for order in active_short:
self.close_position(order, 'active')
for order in pending_short:
self.close_position(order, 'pending')
if len(active_long + pending_long) != 0:
#WE ALREADY HAVE A LONG ORDER, KEEP IT
continue
#WE NEED TO CREATE A NEW LONG
id = self.post_order(direction='long')
self.db.savePosition(id, strategy, 'long', 'pending')
self.print_fn(
"Posting LONG order (id:" + str(id) +
") for Strategy " + strategy.upper(), "green")
elif result == -1:
#STRATEGY SUGGESTS SHORT
if len(active_long + pending_long) != 0:
#WE HAVE A LONG BUT NEED A SHORT, CANCEL ALL LONG
self.print_fn(
"Strategy " + strategy +
" flipped. CANCELLING LONG Orders", "green")
for order in active_long:
self.close_position(order, 'active')
for order in pending_long:
self.close_position(order, 'pending')
if len(active_short + pending_short) != 0:
#WE ALREADY HAVE A SHORT, KEEP IT RUNNING
continue
#WE NEED TO CREATE A NEW SHORT
id = self.post_order(direction='short')
self.db.savePosition(id, strategy, 'short', 'pending')
self.print_fn(
"Posted SHORT order (id:" + str(id) +
") for Strategy " + strategy.upper(), "green")
elif result == 0:
#STRATEGY IS INDIFFERENT, DO NOTHING
continue
else:
#CANT HAPPEN
continue
raise ValueError("Unknown parameter - start/stop/last")
if __name__ == '__main__':
#CALLBACK FOR TRADER AND STREAMER
#PASSED TO STRAMER VIA TRADER TO NOTIFY ABOUT NEW DATA
def handleData(self, data):
process(self, data)
trader = Trader(handleData)
commander = Commander("BTC Trader", cmd_cb=CommandHandler(trader))
trader.print_fn = commander.output
trader.runAsyncTasks = True
signals = Helper.getAvailableSignals()
indicators = Helper.getAvailableIndicators()
strategies = Helper.getAvailableStrategies()
trader.print_fn("\nREGISTERED SIGNALS: " + str(signals) + "\n", "blue")
trader.print_fn("REGISTERED INDICATORS: " + str(indicators) + "\n", "blue")
trader.print_fn("REGISTERED STRATEGIES: " + str(strategies) + "\n", "blue")
welcome = '''Welcome to the Trading Bot!
1. Start streaming market data from Kraken by calling "stream start"
2. Wait for the streamer to sync historical data
3. Wait another minute for the latest OHLC to arrive. Be patient.
Your Indicators, Signals and Strategies should light up now.
4. Enable trading strategies by calling "trader enable <strategyname>"
For further assistence call "help" or "help commandname" '''