from datetime import datetime, timedelta
import texttable
from bot.config import RSP_INVALID_PARAMETERS
from bot.my_log import get_logger
from bot.property import ATON_TAX
from bot.mongo import Portfolio as pf, mongo as db
LOG = get_logger('income_portfolio')
def for_portfolio(words):
if len(words) != 2:
LOG.error('Ivalid parameters %s' % str(words))
return RSP_INVALID_PARAMETERS
position = int(words[1])
is_max = True
if words[0] == 'min':
is_max = False
LOG.info('Start calculate for %d' % position)
profits = list()
stdevs = list()
table = texttable.Texttable(max_width=500)
table.add_row(['id', 'price', 'profit', 'stdev'])
for ordered in db.get_n_first_portfolios(position):
item = ordered.max_item if is_max else ordered.min_item
LOG.info('Predict for %s' % str(item))
profit, correct_summ = predict(porftolio=item, money=118000)
from bot.config import RSP_CAPITAL, RSP_NOF_FOUND_STOCK
import bot.loader_from_file as loader_from_file
import bot.parser_command.command as p
from bot.my_log import get_logger
LOG = get_logger('capital')
def capital(words):
name, privileged = p.name_and_priviledget(words)
LOG.info('Get capital for [%s, %s]' % (name, str(privileged)))
stock = loader_from_file.load_one_stock(name, privileged)
if stock is None:
return format(RSP_NOF_FOUND_STOCK % name)
else:
return format(
RSP_CAPITAL %
(stock.emitent_full_name, float(
stock.capitalisation), float(stock.volume_stock_on_market),
float(stock.capitalisation / stock.volume_stock_on_market)))
from bot.my_log import get_logger
from bot.loader_from_file import update_stock_from_file, load_stocks, load_all
import re
from bot.config import RSP_UPDATE_METAINFO, CMD_DOWNLOAD_FILES
from bot.parser_command.command import name_and_priviledget
LOG = get_logger('update')
def update(words):
num = None
download = False
company, privileged = name_and_priviledget(words)
count_words = words.__len__()
if count_words > 1:
if re.compile(r'[0-9]+').match(words[1]):
num = int(words[1])
elif words[count_words - 1] in CMD_DOWNLOAD_FILES:
company = ' '.join(words[1:count_words - 1])
download = True
else:
company = ' '.join(words[1:count_words])
LOG.info("Update %s files and %s download" % (str(num) if num is not None else company, str(download)))
if company is None or len(company) == 0:
load_stocks(num, download)
else:
update_stock_from_file(company, download, privileged)
import re
from datetime import datetime
from pip._vendor import requests
from pip._vendor.requests.exceptions import MissingSchema
from bot.my_log import get_logger
import os
from bot.extractor import get_id_and_ext_file
from bot.property import *
LOG = get_logger("resource.loader")
def download_file(url, file):
try:
headers = {'User-Agent': 'curl'}
u = requests.get(url, headers=headers)
with open(file=file, mode='wb') as f:
for chunk in u.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
f.flush()
f.close()
LOG.info(format('Load from %s file %s' % (url, file)))
return os.stat(file).st_size
except MissingSchema:
LOG.info(format('Not found URL %s' % url))
except requests.exceptions.SSLError:
LOG.info('Bad SLL of url: %s' % url)
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import pandas_datareader.data as web
from scipy.stats import mstats as ms
from bot.config import GA_NSGAII, RSP_INVALID_PARAMETERS, GA_SIMPLE, GA_NSGAIII
from bot.my_log import get_logger
from bot.analyse import nsga as simple, nsga_platypus as NSGAII
from bot.mongo import mongo as db
from bot.mongo.Portfolio import Portfolio, Item, ItemPortfolio
from bot.mongo.Stock import Stock
from bot.property import *
LOG = get_logger('solver')
def get_stock_from_portfolio(portfolios):
stocks = list()
sharpes = list()
for portfolio in portfolios:
portfolio_stock = list()
sharpes.append(portfolio.max_item.sharpe_ratio)
for stock in portfolio.max_item.stocks:
portfolio_stock.append(db.stock_by_trade_code(stock.trade_code))
stocks.append(portfolio_stock)
return stocks, sharpes
def parallel_solve(all_stocks, type_ga, curr, count):
import bot.mongo
import re
from bot.my_log import get_logger
LOG = get_logger('Parse_Portfolio')
def string_portfolios(path):
LOG.info('Start for path=%s', path)
str_file = ''
with open(path, mode='r') as file:
str_file = file.read()
file.close()
list_min_max = list()
for line in str_file.split("=========="):
list_min_max.append(line)
for pair in list_min_max:
pair_items = str(pair).split("----------")
if len(pair_items) != 2:
LOG.warn('Invalid parsed string: %s' % pair)
continue
max_item = pair_items[0]
min_item = pair_items[1]
portfolio = bot.mongo.Portfolio.Portfolio()
max_portfolio = parse(max_item)
min_portfolio = parse(min_item)
portfolio.max_item = max_portfolio
portfolio.min_item = min_portfolio
import re
from bot.my_log import get_logger
from bot.property import *
LOG = get_logger("extractor")
def get_value_capitalization(trade_code):
with open(CAPITALIZATION) as f:
next = 0
r = '0.0'
contain = False
for line in f:
if trade_code in line:
contain = True
if contain:
next += 1
if next == 7:
r = ''.join(filter(lambda x: x.isdigit() or x == ',', line))
if r == '':
return '0.0'
return (r.replace(',', '.'), '0.0')[r is None or type(r) is None]
def get_free_float(short_name):
with open(FREE_FLOAT) as f:
next = 0
r = '0.0'
contain = False
for line in f:
import time
from concurrent.futures import ThreadPoolExecutor
from csv import *
import traceback
from datetime import datetime
from os.path import isfile, join
from selenium import webdriver
from bot.extractor import get_free_float, get_value_capitalization
from bot.my_log import get_logger
import bot.property as property
from bot.mongo import Stock as s, mongo as db
from bot.resources.loader import download_file, load_files
LOG = get_logger('loader_from_file')
def load_all():
download_file(property.URL_DATA, property.DATA)
download_file(property.URL_DATA_DESCRIPTION, property.DATA_DESCRIPTION)
download_file(property.URL_CAPITALIZATION, property.CAPITALIZATION)
download_file(property.URL_FREE_FLOAT, property.FREE_FLOAT)
def read_to_list(file):
arr = list()
with open(file, newline='', encoding='cp1251') as csvfile:
moex_data = reader(csvfile, dialect='excel', delimiter=';')
for row in moex_data:
arr.append(row)
import yfinance as yf
from bot.my_log import get_logger
import bot.mongo.mongo as db
from bot.config import RSP_PRICE, RSP_NOF_FOUND_STOCK
from bot.parser_command.command import name_and_priviledget
LOG = get_logger('yahoo.price')
def price(words):
LOG.info('Get price for [%s]' % words)
name, privileged = name_and_priviledget(words)
stock = db.stock_by_emitet_name(name, is_privileged=privileged)
shape = yf.Ticker()
if shape.get_name() is None:
return format(RSP_NOF_FOUND_STOCK % name)
else:
return format(RSP_PRICE % (stock.emitent_full_name, stock.trade_code,
float(shape.get_price())))
from bot.my_log import get_logger
from bot.loader_from_file import get_stocks_contains
from bot.config import RSP_NOT_FOUND, RSP_FIND
LOG = get_logger("finder")
def find(words):
company = " ".join(words[1:])
LOG.info("Find companies contains '%s'" % company)
list_found_stocks = get_stocks_contains(company)
if list_found_stocks.__len__() == 0:
LOG.info(RSP_NOT_FOUND)
return RSP_NOT_FOUND
LOG.info("Found %s" % str(list_found_stocks.__len__()))
return format(RSP_FIND % "\n".join(list_found_stocks))
import numpy as np
import pandas as pd
from bot.my_log import get_logger
LOG = get_logger('NSGA')
def solve(stocks, iterations, mean_daily_returns, cov_matrix, days):
LOG.info('Start simple GA for %d' % iterations)
solver = SimpleSolver(stocks, mean_daily_returns, cov_matrix, days)
results = solver.run(iterations)
cols = ['ret', 'wgmean', 'stdev', 'sharpe']
for stock in stocks:
cols.append(stock.shape())
results_frame = pd.DataFrame(results.T, columns=cols)
return results_frame
class SimpleSolver:
def __init__(self, stocks, mean_daily_returns, cov_matrix, days) -> None:
self.stocks = stocks
self.mean_daily_returns = mean_daily_returns
self.cov_matrix = cov_matrix
self.days = days
def run(self, iterations):
cols = 4
results = np.zeros((cols + len(self.stocks), iterations))
for i in range(iterations):
import datetime
import os
import re
import zipfile
import py7zlib
import rarfile
from bot.config import RSP_FILES_NOT_FOUND, RSP_FILES
from bot.loader_from_file import load_one_stock
from bot.my_log import get_logger
from bot.property import ZIP, RAR, ARCHIVES, SEVEN_Z, TMP_EXTRACT, TYPE2_PATH
LOG = get_logger("sender_file")
def send_file(words):
num = words[1]
if re.compile(r'[0-9]+').match(num):
company = " ".join(words[2:])
else:
company = " ".join(words[1:])
stock = load_one_stock(company)
if stock.files_name.__len__ == 0:
return RSP_FILES_NOT_FOUND, list()
else:
list_file = list()
for count, file in enumerate(stock.files_name):
path_to_file = TYPE2_PATH + '/' + stock.trade_code + ARCHIVES + '/' + file
if num and num != '' and int(num) == count:
import threading
import os
import requests
from bot.my_log import get_logger
LOG = get_logger("Bot")
class Bot:
def __init__(self, TOKEN, BOT_ID):
self.TOKEN = TOKEN
self.READ_WEBSOCKET_DELAY = 1
self.BOT_ID = BOT_ID
def post_file(self, channels, filename, token=None):
f = {'file': (filename, open(filename, 'rb'), 'application/octet-stream', {'Expires': '0'})}
requests.post(url='https://slack.com/api/files.upload',
data={'token': token if token is not None else self.TOKEN, 'channels': channels, 'media': f},
headers={'Accept': 'application/json'}, files=f)
LOG.info("Send file %s to channel: %s" % (filename, channels))
rm = threading.Thread(os.remove(filename))
rm.start()
self.reset_delay()
def reset_delay(self):
self.READ_WEBSOCKET_DELAY = 1
def short_delay(self):
self.READ_WEBSOCKET_DELAY = 0.1
from datetime import datetime, timedelta
import mongoengine as me
from mongoengine.connection import disconnect as closeDb
from bot.property import DB_NOT_FOUNT_STOCK, NOT_PRIVILEGED, DB_CONTAINS_MORE_ONE, PRIVILEGED
import bot.mongo
from bot.my_log import get_logger
import re
import random
from os import environ as env
from bot.mongo.Portfolio import Portfolio
from bot.mongo.exception import *
LOG = get_logger('mongo')
def connect():
return me.connect(
env.get('DB_NAME_ENV'),
host=env.get('DB_HOST'),
port=int(env.get('DB_PORT')),
username=env.get('DB_USERNAME'),
password=env.get('DB_PASSWORD'),
)
def close():
return closeDb(env.get('DB_NAME_ENV'))
import re
from bot.my_log import get_logger
import bot.mongo.mongo as db
from bot.property import FINAM_PERIODS, FINAM_P_DAY, FINAM_P_WEEK, FINAM_P_HOUR, SELECTED_STOCKS, FINAM_P_MONTH
import numpy as np
import texttable as tt
LOG = get_logger("analyzer")
np.set_printoptions(formatter={'float': '{: 0.4f}'.format})
def analyse(words):
LOG.info("Start analyse portfolio [%s]" % " ".join(words[1:]))
companies = list()
count = None
size = words.__len__()
if size >= 3:
count, to = get_count(size, words)
companies = all_stock(words[1:to])
elif size == 2:
count, to = get_count(size, words)
if to != size:
companies = stocks_from_file()
else:
companies = all_stock(words[1:to])
else:
companies = stocks_from_file()
return response(companies, count)
def get_count(to, words):
import random
import numpy as np
import pandas as pd
import platypus as pt
from bot.my_log import get_logger
from bot.mongo import mongo as db
from bot.mongo.Stock import Stock
LOG = get_logger('platypus')
db.connect()
all_stocks = list()
for s in Stock.objects():
all_stocks.append(s)
db.close()
class ProblemPortfolio(pt.Problem):
def __init__(self, cov_matrix, mean_daily_returns, days):
super(ProblemPortfolio, self).__init__(nvars=15, nobjs=2, nconstrs=4)
self.cov_matrix = cov_matrix
self.mean_daily_returns = mean_daily_returns
self.days = days
self.types[:] = pt.Real(0.0, 1.0)
self.constraints.__setitem__(0, "==1")
self.constraints.__setitem__(1, "<=0.06")
self.constraints.__setitem__(2, ">=0.02")
self.constraints.__setitem__(3, "<=0.18")
#
# Distributed under terms of the MIT license.
"""
"""
import time
import matplotlib.pyplot as plt
import numpy as np
from bot.my_log import get_logger
from bot.analyse import solver, nsga as sm, nsga_platypus as nsagii
from bot.mongo import mongo as db
LOG = get_logger('compare-solver')
#iterations = [1000, 2000, 3000, 5000, 10000]#, 20000, 30000, 50000]
iterations = [5000]
# for ordered in db.get_portfolio_by_id('598dc26208ed83001321a939'):
for ordered in db.get_portfolio_by_id('60a1507fdb94780001237585'):
sharpes_sm = list()
sharpes_nsga = list()
sharpes_nsgaiii = list()
wgmean_sm = list()
wgmean_nsga = list()
wgmean_nsgaiii = list()
time_sm = list()
time_nsga = list()
time_nsgaiii = list()
from os import environ as env
from bot.capital import capital
import bot.config as config
from bot.finam.finam import loader
from bot.finder import find
from bot.my_log import get_logger
from bot.select_for_portfolio import get_list_selected, select
from bot.sender_file import send_file
from bot.updater import update_metainfo, update
from bot.yahoo.price import price
from bot.analyse import analyser
from bot.analyse import solver, income_portfolio as ip
from bot.bot_impl.bot_api import Bot
LOG = get_logger("main")
# starterbot's ID as an environment variable
BOT_ID = env.get("BOT_ID")
TOKEN = env.get('SLACK_BOT_TOKEN')
if len(sys.argv) == 3:
if TOKEN is None:
TOKEN = sys.argv[1]
if BOT_ID is None:
BOT_ID = sys.argv[2]
bot = Bot(TOKEN=TOKEN, BOT_ID=BOT_ID)
# constants
AT_BOT = "<@" + str(bot.BOT_ID) + ">"
# instantiate Slack & Twilio clients
import datetime as datetime
import re
import time
from dateutil import relativedelta
from bot.config import RSP_FINAM_CODE_ALL
import bot.mongo.Price as p
import bot.mongo.mongo as db
from bot.my_log import get_logger
import bot.property as property
from bot.mongo.mongo import connect, close
from bot.mongo import Stock as s
from bot.resources.loader import download_file
LOG = get_logger("Finam Loader")
def set_price(stock, price, period):
if period == property.FINAM_P_MONTH:
stock.month_history.append(price)
elif period == property.FINAM_P_WEEK:
stock.week_history.append(price)
elif period == property.FINAM_P_DAY:
stock.day_history.append(price)
elif period == property.FINAM_P_HOUR:
stock.hour_history.append(price)
def get_date_from(period):
if period == property.FINAM_P_WEEK:
from bot.my_log import get_logger
from bot.config import RSP_GET_LIST_SELECTED, RSP_SELECT_FOR_PORTFOLIO
from bot.property import SELECTED_STOCKS
from bot.loader_from_file import load_one_stock
from bot.parser_command.command import name_and_priviledget
LOG = get_logger("select_for_portfolio")
def save_stock(stock):
with open(file=SELECTED_STOCKS, mode='a+', encoding='UTF-8') as file:
string = ';'.join(get_parameters_stock(stock))
file.write(string + '\n')
file.flush()
file.close()
LOG.info("Company was added.")
def select(words):
company, privileged = name_and_priviledget(words)
LOG.info("Select: %s" % company)
stock = load_one_stock(company, privileged)
save_stock(stock)
return get_response(stock)
def get_response(stock):
return format(
RSP_SELECT_FOR_PORTFOLIO %
(stock.emitent_full_name, stock.trade_code, stock.last_price))
