def download_stock_list(self): stock_list = ts.get_stock_basics() stock_list[['esp']] = stock_list[['esp' ]].astype(float, raise_on_error='ignore') dbu.save_pd_data('sdtb_stock_list_tmp', stock_list) dbu.call('prc_check_stock_list')
def export(self): for year in range(self.start_year, self.end_year): logger.info("Export data for stock code " + self.stock_code + " for year " + str(year)) current_dir = self.dir + os.sep + self.stock_code + os.sep + str(year) first_day = str(year) + "-01-01" last_day = str(year) + "-12-31" '''EXPORT BASIC DATA''' basic_data_filename = "BASIC_" + self.stock_code + "_" + str(year) + ".csv" sql_basic_data = self.sql_basic_data + "and t.DATE BETWEEN '" + first_day + "' AND '" + last_day + "'" basic_data = dbu.get_pd_data(sql_basic_data) if basic_data.shape[0] > 0: if not os.path.exists(current_dir): logger.debug("Make dir because there is no existing dir") os.makedirs(current_dir) eu.export(current_dir, basic_data_filename, basic_data) logger.info("Basic data exported") '''EXPORT TECH DATA''' tech_data_filename = "TECH_" + self.stock_code + "_" + str(year) + ".csv" sql_tech_data = self.sql_tech_data + "and t.DATE BETWEEN '" + first_day + "' AND '" + last_day + "'" tech_data = dbu.get_pd_data(sql_tech_data) if tech_data.shape[0] > 0: if not os.path.exists(current_dir): logger.debug("Make dir because there is no existing dir") os.makedirs(current_dir) eu.export(current_dir, tech_data_filename, tech_data) logger.info("Tech data exported")
def save_tech_data(self, stock_code, date, dict): update_tech_sql = "update pdtb_stock_tech_data set " for key in dict: update_tech_sql = update_tech_sql + key+ "=" + str(dict[key]) + ", " update_tech_sql = update_tech_sql[:-2] + " where code = '" + stock_code + "' and date = '" + str(date) + "'" #print(update_tech_sql) dbu.update(update_tech_sql)
def is_working_day(self, date): sql = "SELECT fn_is_holiday('" + date + "')" flag = dbu.get_data(sql)[0][0] if flag == 'W': return True else: return False
def save_data(self, basic_data, basic_data_fq=None): try: tech_data = pd.DataFrame(index=basic_data.index, columns=['code'], data=basic_data['code']) dbu.save_pd_data('pdtb_stock_basic_data', basic_data) dbu.save_pd_data('pdtb_stock_tech_data', tech_data) if basic_data_fq is not None: dbu.save_pd_data('pdtb_stock_basic_data_fq', basic_data_fq) except Exception as e: logger.error(e.message)
def update_end_date(self, stock_code, basic_data): last_updated_date = basic_data[-1:].index.values[0] sql = 'UPDATE pdtb_stock_list SET LASTUPDATEDDATE = \'' + last_updated_date + '\' WHERE CODE = \'' + stock_code + '\'' dbu.update(sql)
def get_stock_list(self, end_date): sql = "SELECT CODE, LASTUPDATEDDATE FROM pdtb_stock_list where LASTUPDATEDDATE < '" + end_date + "' and TIMETOMARKET <> '0000-00-00' ORDER BY CODE" return dbu.get_data(sql)
def clean(self): logger.info("Delete data for stock code: " + self.stock_code) dbu.update(self.sql_basic_data) dbu.update(self.sql_tech_data) logger.info("Basic and Tech data less than year " + self.year + " are deleted")
class IA: #Como tomar decisoes! #Nome do produto => tem palavra do dicionario? Sim = 1, nao = 0 #Nome do author => e oficial? Sim = 1, nao = 0 #Preco => mais caro do que o original? (supondo que o original seja 1000) Sim = 1, nao = 0 #Descricao => tem palavra do dicionario? Sim = 1, nao = 0 key_words = ['aa', 'aaa', 'réplica', 'replica', 'primeira', 'linha'] sklearn = LinearSVC() mysql = MySQLManager() util_db = DatabaseUtil() objectProducts = [] def trainning(self, productName): connection = self.mysql.connectDefaultMySQL() #conecta com o banco baseProducts = self.mysql.findProductBase( connection=connection, productName=productName) #obtem a base de dados objectsBase = self.util_db.marshallingDataToObject( data_products=baseProducts) #converte em objeto self.util_db.closeConnection(connection=connection) #fecha a conexao classification = self.classification( objectsBase) #classifica os objetos objectsBaseClassification = self.util_db.marshallingDataToObject( data_products=baseProducts, onlyPredict=True) #converte em objeto so a classificacao resolvedPredict = self.resolverPredict( objectsBaseClassification, invert=True) #resolve os 'verdadeiro/falso' para '0/1' print(classification) self.sklearn.fit(classification, resolvedPredict) #treina def resolverPredict(self, objects, invert=False): resolvedPredict = [] if (invert != True): for obj in objects: print(obj) if (obj == 1): resolvedPredict.append('falso') else: resolvedPredict.append('verdadeiro') else: for obj in objects: if ('falso' in obj.predict): resolvedPredict.append(1) else: resolvedPredict.append(0) return resolvedPredict def predict(self, products): self.objectProducts = self.util_db.marshallingDataToObject( data_products=products, isMongoDB=True) #converter produto pesquisado em objeto arrayProducts = self.classification( self.objectProducts) #classifica os objetos print(arrayProducts) arrayPredicts = [] arrayPercentual = [] for product in arrayProducts: predict = self.sklearn.predict([product]) #predicao arrayPredicts.append(self.resolverPredict(predict)) arrayPercentual.append(self.getPercentual(product)) i = 0 for obj in self.objectProducts: obj.predict = arrayPredicts[i][0] obj.percentual = arrayPercentual[i] i = i + 1 def getPercentual(self, array): sum = 0 for i in array: sum = sum + i return (100 / sum) def countWords(self, text): splitted = text.split(" ") for i in splitted: if (i.lower() in self.key_words): return 1 return 0 def classification(self, products): connection = self.mysql.connectDefaultMySQL() print(len(products)) #minorPrice = self.mysql.findMinorPrice(connection=connection,productName=products[0].title) minorPrice = 500 #print(minorPrice) author = 'author' classification = [] for product in products: data = [] #titulo data.append(self.countWords(product.title)) #preco if (product.price < minorPrice): data.append(1) else: data.append(0) #descricao data.append(self.countWords(product.description)) #autor if (product.author is author): data.append(0) else: data.append(1) classification.append(data) self.util_db.closeConnection(connection=connection) return classification
import time import sys import machine from mongo_manager import MongoManager from mysql_manager import MySQLManager from util import DatabaseUtil import stomp db_util = DatabaseUtil() class MyListener(stomp.ConnectionListener): def on_error(self, headers, message): print('received an error "%s"' % message) def on_message(self, headers, message): print('Iniciou o processamento!') ia = machine.IA() ia.trainning(message) print('Treinamento concluído!') mongo = MongoManager() connection_mongo = mongo.connectMongo( mongo_address=mongo._mongo_address, database=mongo._database) products = mongo.getProducts(mongoConnection=connection_mongo, collection=mongo._collection) print('Dados coletados do mongo') ia.predict(products) mongo.clearMongo(mongoConnection=connection_mongo, collection=mongo._collection) print('predições feitas')