def download_stock_list(self):
stock_list = ts.get_stock_basics()
stock_list[['esp']] = stock_list[['esp'
]].astype(float,
raise_on_error='ignore')
dbu.save_pd_data('sdtb_stock_list_tmp', stock_list)
dbu.call('prc_check_stock_list')
def export(self):
for year in range(self.start_year, self.end_year):
logger.info("Export data for stock code " + self.stock_code + " for year " + str(year))
current_dir = self.dir + os.sep + self.stock_code + os.sep + str(year)
first_day = str(year) + "-01-01"
last_day = str(year) + "-12-31"
'''EXPORT BASIC DATA'''
basic_data_filename = "BASIC_" + self.stock_code + "_" + str(year) + ".csv"
sql_basic_data = self.sql_basic_data + "and t.DATE BETWEEN '" + first_day + "' AND '" + last_day + "'"
basic_data = dbu.get_pd_data(sql_basic_data)
if basic_data.shape[0] > 0:
if not os.path.exists(current_dir):
logger.debug("Make dir because there is no existing dir")
os.makedirs(current_dir)
eu.export(current_dir, basic_data_filename, basic_data)
logger.info("Basic data exported")
'''EXPORT TECH DATA'''
tech_data_filename = "TECH_" + self.stock_code + "_" + str(year) + ".csv"
sql_tech_data = self.sql_tech_data + "and t.DATE BETWEEN '" + first_day + "' AND '" + last_day + "'"
tech_data = dbu.get_pd_data(sql_tech_data)
if tech_data.shape[0] > 0:
if not os.path.exists(current_dir):
logger.debug("Make dir because there is no existing dir")
os.makedirs(current_dir)
eu.export(current_dir, tech_data_filename, tech_data)
logger.info("Tech data exported")
def save_tech_data(self, stock_code, date, dict):
update_tech_sql = "update pdtb_stock_tech_data set "
for key in dict:
update_tech_sql = update_tech_sql + key+ "=" + str(dict[key]) + ", "
update_tech_sql = update_tech_sql[:-2] + " where code = '" + stock_code + "' and date = '" + str(date) + "'"
#print(update_tech_sql)
dbu.update(update_tech_sql)
def is_working_day(self, date):
sql = "SELECT fn_is_holiday('" + date + "')"
flag = dbu.get_data(sql)[0][0]
if flag == 'W':
return True
else:
return False
def save_data(self, basic_data, basic_data_fq=None):
try:
tech_data = pd.DataFrame(index=basic_data.index,
columns=['code'],
data=basic_data['code'])
dbu.save_pd_data('pdtb_stock_basic_data', basic_data)
dbu.save_pd_data('pdtb_stock_tech_data', tech_data)
if basic_data_fq is not None:
dbu.save_pd_data('pdtb_stock_basic_data_fq', basic_data_fq)
except Exception as e:
logger.error(e.message)
def update_end_date(self, stock_code, basic_data):
last_updated_date = basic_data[-1:].index.values[0]
sql = 'UPDATE pdtb_stock_list SET LASTUPDATEDDATE = \'' + last_updated_date + '\' WHERE CODE = \'' + stock_code + '\''
dbu.update(sql)
def get_stock_list(self, end_date):
sql = "SELECT CODE, LASTUPDATEDDATE FROM pdtb_stock_list where LASTUPDATEDDATE < '" + end_date + "' and TIMETOMARKET <> '0000-00-00' ORDER BY CODE"
return dbu.get_data(sql)
def clean(self):
logger.info("Delete data for stock code: " + self.stock_code)
dbu.update(self.sql_basic_data)
dbu.update(self.sql_tech_data)
logger.info("Basic and Tech data less than year " + self.year +
" are deleted")
class IA:
#Como tomar decisoes!
#Nome do produto => tem palavra do dicionario? Sim = 1, nao = 0
#Nome do author => e oficial? Sim = 1, nao = 0
#Preco => mais caro do que o original? (supondo que o original seja 1000) Sim = 1, nao = 0
#Descricao => tem palavra do dicionario? Sim = 1, nao = 0
key_words = ['aa', 'aaa', 'réplica', 'replica', 'primeira', 'linha']
sklearn = LinearSVC()
mysql = MySQLManager()
util_db = DatabaseUtil()
objectProducts = []
def trainning(self, productName):
connection = self.mysql.connectDefaultMySQL() #conecta com o banco
baseProducts = self.mysql.findProductBase(
connection=connection,
productName=productName) #obtem a base de dados
objectsBase = self.util_db.marshallingDataToObject(
data_products=baseProducts) #converte em objeto
self.util_db.closeConnection(connection=connection) #fecha a conexao
classification = self.classification(
objectsBase) #classifica os objetos
objectsBaseClassification = self.util_db.marshallingDataToObject(
data_products=baseProducts,
onlyPredict=True) #converte em objeto so a classificacao
resolvedPredict = self.resolverPredict(
objectsBaseClassification,
invert=True) #resolve os 'verdadeiro/falso' para '0/1'
print(classification)
self.sklearn.fit(classification, resolvedPredict) #treina
def resolverPredict(self, objects, invert=False):
resolvedPredict = []
if (invert != True):
for obj in objects:
print(obj)
if (obj == 1):
resolvedPredict.append('falso')
else:
resolvedPredict.append('verdadeiro')
else:
for obj in objects:
if ('falso' in obj.predict):
resolvedPredict.append(1)
else:
resolvedPredict.append(0)
return resolvedPredict
def predict(self, products):
self.objectProducts = self.util_db.marshallingDataToObject(
data_products=products,
isMongoDB=True) #converter produto pesquisado em objeto
arrayProducts = self.classification(
self.objectProducts) #classifica os objetos
print(arrayProducts)
arrayPredicts = []
arrayPercentual = []
for product in arrayProducts:
predict = self.sklearn.predict([product]) #predicao
arrayPredicts.append(self.resolverPredict(predict))
arrayPercentual.append(self.getPercentual(product))
i = 0
for obj in self.objectProducts:
obj.predict = arrayPredicts[i][0]
obj.percentual = arrayPercentual[i]
i = i + 1
def getPercentual(self, array):
sum = 0
for i in array:
sum = sum + i
return (100 / sum)
def countWords(self, text):
splitted = text.split(" ")
for i in splitted:
if (i.lower() in self.key_words):
return 1
return 0
def classification(self, products):
connection = self.mysql.connectDefaultMySQL()
print(len(products))
#minorPrice = self.mysql.findMinorPrice(connection=connection,productName=products[0].title)
minorPrice = 500
#print(minorPrice)
author = 'author'
classification = []
for product in products:
data = []
#titulo
data.append(self.countWords(product.title))
#preco
if (product.price < minorPrice):
data.append(1)
else:
data.append(0)
#descricao
data.append(self.countWords(product.description))
#autor
if (product.author is author):
data.append(0)
else:
data.append(1)
classification.append(data)
self.util_db.closeConnection(connection=connection)
return classification
import time
import sys
import machine
from mongo_manager import MongoManager
from mysql_manager import MySQLManager
from util import DatabaseUtil
import stomp
db_util = DatabaseUtil()
class MyListener(stomp.ConnectionListener):
def on_error(self, headers, message):
print('received an error "%s"' % message)
def on_message(self, headers, message):
print('Iniciou o processamento!')
ia = machine.IA()
ia.trainning(message)
print('Treinamento concluído!')
mongo = MongoManager()
connection_mongo = mongo.connectMongo(
mongo_address=mongo._mongo_address, database=mongo._database)
products = mongo.getProducts(mongoConnection=connection_mongo,
collection=mongo._collection)
print('Dados coletados do mongo')
ia.predict(products)
mongo.clearMongo(mongoConnection=connection_mongo,
collection=mongo._collection)
print('predições feitas')