def scrapeEstabelecimentos(*argv):
"""Realiza scrape para estabelecimentos no banco de dados ou pelos params"""
if (argv):
"Dados de um usuario do banco"
codigo = argv[0]
result = DataAccess.getById(str(codigo))
if (not result):
logger.error("usuário não encontrado")
raise
user = result[0][0]
passw = result[0][1]
refoId = result[0][2]
logger.info('usuario %s', user)
scrapeLogIn(user, passw, argv[1], argv[2], refoId)
else:
estabs = DataAccess.getAll()
for estab in estabs:
user = estab[0]
passw = estab[1]
refoId = estab[2]
dataFrom = datetime.strftime(datetime.now() - timedelta(1),
'%d/%m/%Y')
logger.info('usuario %s - data %s', user, dataFrom)
scrapeLogIn(user, passw, dataFrom, dataFrom, refoId)
def scrapeTodosClientes(self):
co_id = DataAccess.getCoId('Moip')
estabs = DataAccess.getAllWithAuth(co_id[0][0])
for estab in estabs:
self.__refoEstabelecimento = estab[0]
self.__access_token = estab[1]
logger.info('estabelecimento %s - data %s',
self.__refoEstabelecimento, self.__begin_date)
self.scrapeLogIn()
def check_user(user, password):
db = DataAccess()
if len(user) == 0 or len(password) == 0:
return jsonify("Bad request"), 400
result = db.check_user(user, password)
if not result:
return jsonify("Authorization failed"), 401
elif result:
resp = make_response(jsonify(result["guid"]), 200)
return resp
def goToLogin(self):
firstname = self.lineEditFirstName.text().strip()
middlename = self.lineEditMiddleName.text().strip()
lastname = self.lineEditLastName.text().strip()
idnumber = self.lineEditIdNumber.text().strip()
gender = self.cbGender.currentIndex()
password = self.lineEditPassword.text().strip()
validGender = 1 <= gender <= 2
validInput = firstname and lastname and idnumber and password and validGender
if validInput:
result = DataAccess.insertUser(firstname, middlename, lastname,
idnumber, gender, password)
if result:
Dialog.information_message(self.Form,
'Registration successful')
goToLogin(self)
else:
Dialog.error_message(
self.Form,
'Data insertion error. Try again\nId numbers should be unique'
)
else:
Dialog.warning_message(
self.Form,
"A valid gender is required and All fields requires save middle name."
)
def api(company):
get_quarterly = False
indicator_list = ['PE', 'SR']
indicator_from_param = request.args.get('i')
period = request.args.get('p')
if period and period == 'Q':
get_quarterly = True
if indicator_from_param:
indicator_list = indicator_from_param.split(',')
print('Indicators : {}'.format(indicator_list))
start_time = time.time()
print('Received Request at {}'.format(str(start_time)))
data = DataAccess.get_data(company, indicator_list, get_quarterly=get_quarterly)
end_time = time.time()
print('Processed Request at {}' + str(end_time))
print('Time take to process request : {} seconds'.format(str(end_time - start_time)))
return jsonify(data)
def getNamePartyPercentage():
list = []
nameParty = DataAccess.getContestantNameParty()
percentages = getListOfAllPercentageVotes()
for i in range(0, 6):
nameParty[i]['Percentage'] = percentages[i]
list.append(nameParty[i])
return list
def getIndsutryData(company):
start_time = time.time()
print('Received Request at {}'.format(str(start_time)))
data = DataAccess.get_industry_data(company)
end_time = time.time()
print('Processed Request at {}' + str(end_time))
print('Time take to process request : {} seconds'.format(str(end_time - start_time)))
return jsonify(data)
def getMomentum(company):
start_time = time.time()
print('Received Request at {}'.format(str(start_time)))
data = DataAccess.getTechnicals(company)
end_time = time.time()
print('Processed Request at {}' + str(end_time))
print('Time take to process request : {} seconds'.format(str(end_time - start_time)))
return jsonify(data)
def goToMain(self, contestantId):
# contestant Id to set a vote
result = DataAccess.insertVote(self.userId, contestantId)
if result:
Dialog.information_message(self.Form, 'Vote Successful Registered')
goToMain(self, self.userId)
else:
Dialog.error_message(self.Form, 'Data access error. Try again')
def goToMain(self):
username = self.lineEditUserName.text()
password = self.lineEditPassword.text()
result = DataAccess.getLogin(username, password)
if result:
goToMain(self, result)
else:
Dialog.error_message(self.Form, "Please check login details")
def scrapeTodosClientes(self):
estabs = DataAccess.getAll()
for estab in estabs:
self.__client_id = estab[0]
self.__client_secret = estab[1]
self.__refoEstabelecimento = estab[2]
logger.info('usuario %s - data %s', self.__client_id,
self.__begin_date)
self.scrapeLogIn()
def execute(self, query):
try:
cursor = self.__db.query(query)
except:
raise DBCommError("stacktrace:\n%s" % traceBack())
cursor = DataAccess.DataAccessCursor(cursor, cursor.dictresult)
return cursor
def __init__(self):
self.name = os.path.splitext(os.path.basename(__file__))[0]
# 0 何もしない
# 1 購入
# 2 売却
self.Action = [0, 1, 2]
self.enable_actions = np.arange(3)
self.env = DA.DataAccess()
self.reset()
def __init__(self, listOfStocks):
self.listOfStocks= listOfStocks
self.DAY=86400
dataItemsList=list()
dataItemsList.append("alphaValue")
self.alphaData= DataAccess.DataAccess(False, "AAPLonlybollingerBandsAlphaVals.h5", "/alphaData", "alphaData", True, listOfStocks, None, None, None, dataItemsList)
print "Timestamps are: "
for ts in self.alphaData.timestamps:
print ts
def __init__(self, listOfStocks):
self.listOfStocks = listOfStocks
self.DAY = 86400
dataItemsList = list()
dataItemsList.append("alphaValue")
self.alphaData = DataAccess.DataAccess(
False, "curveFittingAlphaVals_Jan_85_to_2010.h5", "/alphaData",
"alphaData", True, listOfStocks, None, None, None, dataItemsList)
print("Timestamps are: ")
for ts in self.alphaData.timestamps:
print(ts)
def goToVote(self):
userVoted = DataAccess.getUserVoteStatus(self.userId)
if userVoted:
Dialog.information_message(self.Form, 'You have already voted')
return
result = BusinessLogic.getNamePartyPercentage()
if result:
goToVote(self, self.userId, result)
else:
Dialog.error_message(self.Form, 'Data access error. Try again')
def scrapePorCliente(self):
"""Realiza scrape para um cliente específico"""
result = DataAccess.getById(str(self.__codigo_cliente))
if (not result):
logger.error("usuário não encontrado")
raise
self.__client_id = result[0][0]
self.__client_secret = result[0][1]
self.__refoEstabelecimento = result[0][2]
logger.info('usuario %s', self.__client_id)
self.scrapeLogIn()
def goToAnnouncements(self):
userVoted = DataAccess.getUserVoteStatus(self.userId)
if not userVoted:
Dialog.information_message(self.Form,
'You have to vote first to view it')
return
result = BusinessLogic.getNamePartyPercentage()
if result:
goToAnnouncements(self, self.userId, result)
else:
Dialog.error_message(self.Form, 'Data access error. Try again')
def goToWinner(self):
userVoted = DataAccess.getUserVoteStatus(self.userId)
if not userVoted:
Dialog.information_message(self.Form,
'You have to vote first to view it')
return
result = BusinessLogic.winnerUI()
if result:
name = result.get('Name')
party = result.get('Party')
percentage = result.get('Percentage')
goToWinner(self, self.userId, name, party, percentage)
else:
Dialog.error_message(self.Form, 'Data access error. Try again')
def savetosqlite(folderPath, modifyList=[], logsql=False):
has_db = False
if not os.path.exists("steelray.db"):
has_db = False
else:
has_db = True
if has_db == False:
logsql = has_db
print logsql, has_db
fileList = []
if len(modifyList) <= 0 and has_db:
return
if len(modifyList) <= 0 or (not has_db):
fileList = glob.glob1(folderPath, "*.xls")
else:
for fileinfo in modifyList:
fileList.append(os.path.split(fileinfo["name"])[1])
sql_command_array = []
sql_count = 0
threads = []
t1 = threading.Thread(target=create_macro_file, args=(folderPath,))
threads.append(t1)
t1.start()
ticks = int(time.time())
for item in fileList:
# if not item.__contains__("action_speed_test"):
# continue
excel_data_dic = readexcel(item)
t2 = threading.Thread(target=create_csfile, args=(folderPath + os.sep + "Structs", excel_data_dic,))
threads.append(t2)
t2.start()
sql_command_array = DataAccess.SaveToSqlite("steelray.db", excel_data_dic, logsql)
# 防止第一次导入数据库时生成超大SQL文件
if logsql and len(sql_command_array) > 0:
create_command_file(has_db, ticks, sql_command_array)
sql_count += len(sql_command_array)
sql_command_array = []
print u"查询语句总条数:", sql_count
LogCtrl.write_log_file()
for item in threads:
item.join()
def InsBTC_VALUE(self):
res = requests.get('https://coincheck.com/api/ticker')
jsonData = res.json()
# last 直近売買
# bid 買値
# ask 売値
# high 24時間での最高取引価格
# low 24時間での最安取引価格
# volume 24時間での取引量
last = jsonData["last"]
bid = jsonData["bid"]
ask = jsonData["ask"]
high = jsonData["high"]
low = jsonData["low"]
volume = jsonData["volume"]
# 取得したデータを保存用プログラムに投げる
env = da.DataAccess()
env.InsertToBTC_VALUE(last, bid, ask, high, low, volume)
def main():
data = dal.getData()
BOT_TOKEN = data['botToken']
CHANNEL_ID = data['channelId']
twitterApi = getTwitterAPI(data)
lastId = data['lastId']
tweets = []
print('Begining the loop...' + '\n')
while True:
try:
print('Checks for tweets...' + '\n')
tweets = twitterApi.home_timeline(since_id=lastId)
if tweets:
print('Tweets found!' + '\n')
for tweet in tweets:
message = getMessage(tweet)
send_message(message, BOT_TOKEN, CHANNEL_ID)
print('telegramed!\n')
lastId = tweets[0].id
data['lastId'] = lastId
dal.setData(data)
else:
print('No new tweets!' + '\n')
except tweepy.RateLimitError as e:
lastId = tweets[0].id
data['lastId'] = lastId
dal.setData(data)
print('We hit rate limit...' + '\n')
print(str(e))
except Exception as e:
lastId = tweets[0].id
data['lastId'] = lastId
dal.setData(data)
print('We have a problem!' + '\n')
print(str(e))
print('Now I\'m sleeping for a minute...' + '\n')
time.sleep(60)
def preprocessArticles(articles) :
reEndOfSentence = re.compile('\\. ')
reNonAlphaNumeric = re.compile('\'s|/\n/|/\t/|[\W]+ | ')
processedArticles = {}
for i,article in articles.items() :
# collect contents
contents = [str(article['description'])+' ']+DataAccess.scrapeArticleUrl(article['url'])
# convert contents into words
words = []
for text in contents:
# covert to lower case
text = text.lower()
# replace full stops with ENDOFSEN
text = reEndOfSentence.sub(' endofsen ',text)
# split text into individual words
newWords = text.split(' ')
# remove remaining non-alphanumerics
newWords = [reNonAlphaNumeric.sub('',word) for word in newWords]
words = words + [word for word in newWords if word!='']
# combine into list of processed articles
processedArticles[i] = words
return processedArticles
def winnerUI():
winner = DataAccess.getWinner()
id = winner.get('ContestantId')
index = id - 1
winnerNamePartyPercentage = getNamePartyPercentage()[index]
return winnerNamePartyPercentage
def goToContestant(self, index):
result = DataAccess.getContestant(index)
if result:
goToContestant(self, self.userId, index, result)
else:
Dialog.error_message(self.Form, 'Data access error. Try again')
def watchlist():
return render_template('/watchlist.html',watchlist=DataAccess.get_watchlist(),canEditWatchlists=canEditWatchlists)
def delete_from_watchlist():
company = request.json['company']
DataAccess.remove_from_watchlist(company)
return jsonify({'success': True}), 201
def add_to_watchlist():
print(request.get_json())
company = request.json['company']
DataAccess.add_to_watchlist(company)
return jsonify({'success': True}), 201
def get_watchlist():
return jsonify({'watchlist': DataAccess.get_watchlist()})
def get_all_symbols():
data = {'companies':DataAccess.get_all_symbols()};
return jsonify(data)
def delete_location_type():
ensure_admin()
data = json.loads(request.stream.read())
db = DataAccess()
loc_info_handler = LocationHandler(db)
return jsonify(loc_info_handler.delete_location_type(data))
def add_location_type():
ensure_admin()
location_type = json.loads(request.stream.read())
loc_DB = DataAccess()
loc_handler = LocationHandler(loc_DB)
return jsonify(loc_handler.add_location_type(location_type))
def ckeck_token():
token = request.headers.get("Authorization")
db = DataAccess()
sessions = db.get_all_sessions()
g.isAdmin = token in sessions
def getPercentageVotesWon(contestantId):
totalVotes = DataAccess.getTotalVotes().get('TotalVotes')
votesReceived = DataAccess.getVotes(contestantId).get('VotesCount')
percentage = votesReceived * 100 / totalVotes
return ("%.2f%%" % percentage)
#readStocksFromFile done
#Main begins
noOfDaysToUse = 5
daysAhead = 5
folderList = list()
folderList.append("C:\\tempoutput\\")
listOfStocks = getStocks(folderList)
beginTS = 473490000 #2 Jan 1985 0500 hrs GMT
endTS = 1262235600 #31 DEc 2009 0500 hrs GMT
print "list of stocks is: " + str(listOfStocks)
dataAccess = DataAccess.DataAccess(True, folderList, "/StrategyData",
"StrategyData", True, listOfStocks, beginTS,
endTS)
timestamps = list(dataAccess.getTimestampArray())
print "Printing all timestamps: "
for ts in timestamps:
print ts
print "Printing ts done"
#alpha= alphaGenerator.AlphaDataModel.AlphaDataModelClass()
adm.openFile("curveFittingAlphaVals_Jan_85_to_2010.h5")
daysArray = numpy.zeros((noOfDaysToUse + 1), dtype=float)
ctr = 0
while (ctr <= noOfDaysToUse): #because we get back noOfDaysToUse+1 rows
daysArray[ctr] = ctr
def main():
'''
@summary: Calculates Bollinger bands
'''
folderList = list()
folderList.append("C:\\tempoutput\\")
listOfStocks = list()
# listOfStocks.append("ACY")
listOfStocks.append("AAPL")
#listOfStocks= getStocks(folderList)
dataAccess = DataAccess.DataAccess(True, folderList, "/StrategyData",
"StrategyData", True, listOfStocks)
timestamps = list(dataAccess.getTimestampArray())
adm.openFile("AAPLonlybollingerBandsAlphaVals.h5")
period = 10
stdMultiplier = 2
noOfDays = len(timestamps) #400
centerband = numpy.zeros(
noOfDays, dtype=float
) #len(timestamps)- period + 1 #Just to make it the same length as the adj_close to make it easier to plot
upperBand = numpy.zeros(noOfDays, dtype=float)
lowerBand = numpy.zeros(noOfDays, dtype=float)
x = numpy.zeros(noOfDays, dtype=float)
ctr = 0
while (ctr < noOfDays):
x[ctr] = ctr
ctr += 1
#end while
for stock in listOfStocks:
print "Processing: " + str(stock)
#adj_close= dataAccess.getStockDataList(str(stock), 'adj_close')
adj_close = dataAccess.getStockDataList(stock, 'adj_close',
timestamps[0],
timestamps[noOfDays - 1])
adj_close = removeNaNs(adj_close) #nan's removed, unless all are nans
#Now calculating bollinger bands
for ctr in range(period, noOfDays):
try:
centerband[ctr] = numpy.average(adj_close[ctr - period:ctr])
stdDev = numpy.std(adj_close[ctr - period:ctr])
upperBand[ctr] = centerband[ctr] + (stdMultiplier * stdDev)
lowerBand[ctr] = centerband[ctr] - (stdMultiplier * stdDev)
except IndexError:
print "ctr is: " + str(ctr)
#writing alpha values to file
for ctr in range(0, noOfDays):
if (upperBand[ctr] == lowerBand[ctr]) or (adj_close[ctr]
== centerband[ctr]):
adm.addRow(str(stock), "blah", 0.0, timestamps[ctr])
elif (adj_close[ctr] < centerband[ctr]):
alphaValue = lowerBand[ctr] / adj_close[ctr]
adm.addRow(str(stock), "blah", alphaValue, timestamps[ctr])
else:
alphaValue = -adj_close[ctr] / upperBand[ctr]
adm.addRow(str(stock), "blah", alphaValue, timestamps[ctr])
#done writing alpha values of this stock to file
#calculating bollinger bands done!
# fig = Figure()
# canvas = FigureCanvas(fig)
# ax = fig.add_subplot(111)
# ax.plot(centerband)
# ax.plot (lowerBand)
# ax.plot (upperBand)
# ax.plot (adj_close)
#
# ax.set_title(str(stock)+' Bollinger bands')
# ax.grid(True)
# ax.set_xlabel('time')
# ax.set_ylabel('')
# canvas.print_figure(str(listOfStocks.index(stock)))
#for stock in listOfStocks: done
adm.closeFile()
def getNewCursor(self):
tempcursor = self.__db.cursor()
return DataAccess.DataAccessCursor(tempcursor, tempcursor.fetchallDict)
import DataAccess as da db = da.get_db() da.add_country(db) print(da.get_country(db))
progress = None
if (count % 100000 == 0):
progress = count / 7301389 * 100
print('{} percent complete. {} records imported'.format(progress, count))
con.commit()
con.close()
# Clear the Redis caches
def flushRedisCache():
r = redis.StrictRedis(host='localhost', port=6379, db=1)
r.flushall()
print('Creating sqlite database..')
import_DEB()
print('Database created.')
print('Flushing Redis Cache..')
flushRedisCache()
print('Redis cache flushed.')
print('Creating Redis Annual database..')
DataAccess.create_redis_database()
print('Redis Annual database created.')
print('Creating Redis Quarterly database..')
DataAccess.create_redis_database(createQuarterly=True)
print('Redis Quarterly database created.')
