def cron_b3_memory():
"""
Funcao que prioriza o uso limitado da memoria.
"""
# Limpa a Base
Quotes.objects.filter(market='BOVESPA').delete()
download_hquotes(ibrx_tickers)
# Limpa a Base
PairStats.objects.filter(market='BOVESPA').delete()
bulk_list = []
for idx, pair in enumerate(gera_pares(ibrx_tickers)):
obj = b3_producer(idx, pair)
bulk_list.append(obj)
if len(bulk_list) > 1000:
# Grava dados no Banco
PairStats.objects.bulk_create(bulk_list)
# Libera a memória
del bulk_list
bulk_list = []
# Grava dados no Banco
PairStats.objects.bulk_create(bulk_list)
# Telegram
send_msg()
def cron_b3():
# Limpa a Base
Quotes.objects.filter(market='BOVESPA').delete()
download_hquotes(ibrx_tickers)
# Limpa a Base
PairStats.objects.filter(market='BOVESPA').delete()
with Pool(2) as p:
bulk_list = p.starmap(b3_producer, enumerate(gera_pares(ibrx_tickers)))
# Grava dados no Banco
PairStats.objects.bulk_create(bulk_list)
# Telegram
send_msg()
def main():
while 1:
try:
visit_site()
except Exception as e:
# if there is an exception/error happending, send message to admin
try:
exc_info_123 = sys.exc_info(
) # just a reference/func pointer, not a string
except Exception as e_:
print "exc_info show failed with", str(e_)
finally:
# Display the *original* exception
print "--tb--"
traceback_msg = traceback.print_exception(*exc_info_123)
print traceback_msg
print traceback_msg
#traceback_msg = "msg"+str(e)
#return_code = subprocess.call("tail nohup.out", shell=True)
return_code = os.popen("tail nohup.out").readlines()
return_code_clean = []
for line in return_code:
if "urllib3.connectionpool" in line:
pass
else:
return_code_clean.append(line)
time.sleep(0.3) #bug
traceback_msg = "".join(return_code_clean[-6:])
del exc_info_123
msg = " open.py - error: "
msg += "\n"
msg += str(traceback_msg)
print "-" * 30
print msg
print "-" * 30
bot.send_msg(msg, flag_notify_admin_only=1)
print "==" * 30
wait_before_next_request()
def cron_b3():
"""
Funcao bastante rapida, porem usa muita memoria do Heroku (640Mb / 512Mb)
"""
# Limpa a Base
Quotes.objects.filter(market='BOVESPA').delete()
download_hquotes(ibrx_tickers)
# Limpa a Base
PairStats.objects.filter(market='BOVESPA').delete()
with Pool(2) as p:
bulk_list = p.starmap(b3_producer, enumerate(gera_pares(ibrx_tickers)))
# Grava dados no Banco
PairStats.objects.bulk_create(bulk_list)
# Telegram
send_msg()
def send_notification():
"""
send notification to given telegram user
"""
username = request.args.get("username", "")
to_id = request.args.get("to", "")
text = request.args.get("msg", "")
msg_type = request.args.get("type", None) # "HTML" or "Markdown"
if not username:
msg = "required parameter username missing"
raise Error(msg)
if not to_id:
msg = "required parameter to missing"
raise Error(msg)
if not text:
msg = "message can not be blank"
raise Error(msg)
user_obj = User()
user = user_obj.get(username=username)
if not user:
msg = "you dont have permisson to send notification"
raise Error(msg)
to = user_obj.get(username=to_id)
if not to:
group_obj = Group()
to = group_obj.get(group_name=to_id)
if not to:
msg = "invalid to id"
raise Error(msg)
else:
send_msg(my_bot, user["uid"], -to["gid"], text, msg_type)
return jsonify({"ok": 1})
send_msg(my_bot, user["uid"], to["uid"], text, msg_type)
return jsonify({"ok": 1})
def visit_site():
# If plan_changes.pkl file exists load content to plan_changes_prev
try:
plan_changes_prev = pkl.read("plan_changes.pkl")
except IOError as e:
if e.errno == 2:
plan_changes_prev = None
else:
raise e
# read url of course table from file
lines = open("link.txt", "r").readlines()
# clean url
lines = [x.strip() for x in lines]
link = "".join(lines)
link = link.replace("__var_link_type_id__", conf.link_type_id)
print link[:50]
if 1:
# get content from link
r = requests.request(method='GET', url=link)
content = r.content
#{"vorlesungen":"
content = content.replace('{"vorlesungen":"', "").replace('"}', '"')
#content = content.replace('\\"','"')
content = content.decode('string_escape')
content = content.replace("\\/", "/")
# print first 50 chars of content, useful for debugging
print content[:50]
# for debugging purposes write content to temp.html
open("temp.html", "w").write(content)
else:
# for debugging purposes read data from temp.html, reduces requests to website
content = open("temp.html", "r").read()
if 1:
#######################################
# Parse content from response, store events, professors in "output" variable
#######################################
# create BeautifulSoup tree for parsing
tree = BeautifulSoup(content)
#attrs={'class': "twelve columns nop"}): #"b", { "class" : "lime" }
# load cnt from file
try:
cnt_pkl = open("cnt.pkl", "r").read()
cnt_pkl = int(cnt_pkl)
cnt_pkl += 1
except IOError as e:
if e.errno == 2:
cnt_pkl = 1
else:
raise e
# safe cnt incremented
open("cnt.pkl", "w").write(str(cnt_pkl))
# contains changes to the Stundenplan
plan_changes = []
for node in tree.findAll("div", {"class": "twelve columns nop"}):
# for debugging
if 0:
# print node.contents
print "---"
print cnt_pkl
print "---"
print str(node)[:50]
rows = node.find('table').findAll('tr')
# iterate over table cells, store them in output
for row in rows:
cells = row.findAll('td')
output = []
for i, cell in enumerate(cells):
if i == 0:
output.append(cell.text.strip())
if i > 0:
output.append(cell.text.strip())
#print "--",cell.text.strip()
"""
elif cell.find('img'):
output.append(cell.find('img')['title'])
elif cell.find('input'):
output.append(cell.find('input')['value'])
"""
#print output
plan_changes.append(output)
#tname = "table_previous.pkl"
#pkl.write( (plan_changes,cnt_pkl),tname)
#plan_changes_prev,cnt_pkl_prev = pkl.read(tname)
if len(plan_changes) > 0: #bug ??, not recognize entfall ?
pkl.write(plan_changes, "plan_changes.pkl")
#open("file_plan.html","w").write(str(plan_changes))
#open("file_plan_prev.html","w").write(str(plan_changes_prev))
# init this code block with intention to always send new updates,
# then later on check if there really is a change
# <-- that's reason for website_changed = 1
website_changed = 1
if plan_changes == plan_changes_prev:
website_changed = 0
else:
# If there are no plan_changes, dont resize hash ?
# if len(plan_changes) #bug ??, not recognize entfall ?
if len(plan_changes) > 0:
website_changed = 1
else:
website_changed = 0
print "website_changed", website_changed
# Formatting and Builder of bot message
line_pad = 20
line_pad_short = 14
line_pad_short_half = (line_pad_short) / 2
msg = u""
msg += "#stundenplan"
msg += "\n"
msg += "=" * line_pad
ch = "\n"
for change in plan_changes:
if 0:
print ".........."
print change
print len(change)
print ".........."
if len(change) > 0:
for n, e in enumerate(change):
# text from table cell for title
if n == 1:
try:
e = e[:e.index("(")]
except ValueError:
print " ( substring not found - ValueError"
# e is not changed
# replace titles that are too long, with a substitution
# [ "title_to_be_replace", "substitution_of_title" ]
names = [
["Formale Sprachen", "FS"],
["Praktikum Software Entwicklung", "PSE"],
["Sport und Gesundheit", "Sport"],
[
"Oracle 11g Release 2 - Administration Workshop",
"Oracle"
],
[
"Webtechnologie und Webmarketing in der Cloud",
"Cloudmarketing"
],
]
for name in names:
e = e.replace(name[0], name[1])
msg += "\n"
msg += e
msg += " "
# text from table cell for professor
if n == 2:
e = e.replace("Prof.", "").replace("Dr.", "")
e = e.strip()
msg += ""
msg += e
msg += "\n"
msg += "-" * line_pad_short_half
msg += "------------------"
msg += "-" * line_pad_short_half
# text from table cell for entfällt
if n == 3:
msg += "\n"
s = list(e)
e = "".join(s)
e = insert_char(e, " ", 10)
e = insert_char(e, " ", 10)
e = insert_char(e, " ", 10)
e = insert_char(e, " ", 22)
e = insert_char(e, "-", 22)
e = insert_char(e, ch, 22)
msg += e
msg += "\n"
msg += "-" * line_pad_short_half
msg += " Vertretung "
msg += "-" * line_pad_short_half
# text from table cell for vertreten
if n == 4:
#print e
if len(e.strip()) > 0:
msg += "\n"
s = list(e)
e = "".join(s)
e = insert_char(e, " ", 10)
e = insert_char(e, " ", 10)
e = insert_char(e, " ", 10)
e = insert_char(e, " ", 22)
e = insert_char(e, "-", 22)
e = insert_char(e, ch, 22)
msg += e
msg += "\n"
msg += "=" * line_pad
if website_changed:
endpoints = conf.endpoints
bot.send_msg(msg, endpoints=endpoints)
# coding:utf-8
import subprocess
import time
import bot
if __name__ == '__main__':
subprocess.call('systemctl stop scan.service', shell=True)
time.sleep(5)
subprocess.call('python3.6 forks.py', shell=True, cwd='/home/scan/')
time.sleep(5)
subprocess.call('python3.6 proxy_push.py', shell=True, cwd='/home/scan/')
time.sleep(5)
subprocess.call('systemctl restart scan.service', shell=True)
subprocess.call('systemctl restart bot.service', shell=True)
bot.send_msg('Сканер выполнил суточное обновление', for_admin=True)