def check_runtime():
try:
proc_list = utility.get_th_crossed_proc()
for proc in proc_list:
r_id = proc[0]
req_str = proc[1].strip()
diff = proc[2]
th = proc[3]
status = proc[4]
script = proc[5]
status = status + '..TH Crossed'
utility.update_status(status, r_id)
msgBody = "Request:" + req_str + ", " + "ID:" + str(r_id) + ", " + "Script:" + script + "\n"
msgBody += "Runtime TH: " + str(th) + " min" + "\n" + "\n"
msgBody += "Total time taken by the process upto now: " + str(diff) + " min" + "\n"
msgBody += "Runtime has crossed the TH value."
email_sender.send(r_id, req_str, msgBody)
except Exception as err:
log.log_debug(err, 1, -4)
log.log_debug("Error in check_runtime()", 1, -4)
def __init__(self, url, title_selector, price_selector):
self.url = url
self.title_selector = title_selector
self.price_selector = price_selector
try:
self.scrape()
except AttributeError:
email_sender.send("Something went wrong - Probably the scraped element no longer exists", self.url)
def scrape(self):
page = requests.get(self.url, headers = self.headers)
soup = BeautifulSoup(page.content, 'html.parser')
title = soup.find(self.title_selector[0], self.title_selector[1]).get_text()
price = re.sub("[^0-9]", "", soup.find(self.price_selector[0], self.price_selector[1]).get_text()) #used to use [:4] becuase of avigil.co.il
title = re.sub(r'[^\x00-\x7F]+|[/\\]',' ', "".join(filter(lambda c: not (ord(c) >= 1424 and ord(c) <= 1514), title))).strip() #remove all non ascii after removing all heb characters
read_value = input_output.read_file(title)
if read_value != price:
input_output.write_file(title, price)
email_sender.send([title, price, read_value], self.url)
# print(title + " is only " + price)
print(title)
def envoie():
s = sender.get()
r = recv.get()
email_receiver = r.strip(" ")
le_email_sender = s[:s.find(" ")].strip(" ")
password = s[s.find(" "):].strip(" ")
_mesg = message.get()
accuse = email_sender.send(le_email_sender, password, _mesg,
email_receiver)
msg.set(accuse)
def run(self):
driver = self.getDriverWithLoggin()
listAllLinks = []
listAllLinks = listAllLinks + self.getLinks(
driver, 'http://sc.olx.com.br/florianopolis-e-regiao/'
'outras-cidades/veiculos/caminhoes-onibus-e-vans'
'?sd=2609&sd=2597&sd=2567&sd=2579&sd=2613&sd=2616&sd=2610&sd=2569'
'&sd=2570&sd=2583&sd=2615&sd=2586&sd=2600&sd=2585&sd=2578&sd=2576'
'&sd=2603&sd=2608&sd=2591&sd=2588&sd=2611&sd=2595&sd=2575&sd=2598'
'&sd=2617&sd=2571&sd=2580&sd=2568&sd=2601&sd=2599&sd=2593&sd=2582'
'&sd=2605&sd=2577&sd=2572&sd=2584&sd=2573&sd=2592&sd=2607&sd=2581'
'&sd=2618&sd=2606&sd=2604&sd=2614&sd=2596')
listAllLinks = listAllLinks + self.getLinks(
driver,
'http://sc.olx.com.br/oeste-de-santa-catarina/regioes-de-curitibanos-e-c-dos-lages/veiculos/caminhoes-onibus-e-vans'
)
listAllLinks = listAllLinks + self.getLinks(
driver,
'http://sc.olx.com.br/norte-de-santa-catarina/veiculos/caminhoes-onibus-e-vans'
)
print('Foram encontrados no total %s' % str(len(listAllLinks)))
print('Iniciando envio de mensagens...')
listLinksError = self.sendMessagesAndReturnErrors(driver, listAllLinks)
print("%s links falharam" % str(len(listLinksError)))
print('Envio de mensagens terminado')
emailMsg = 'Olá, foi finalizado o envio dos chats, e foram enviadas %s novas mensagens!!' % str(
Scrapper.countSendMessage)
if config('LOCAL', default=False, cast=bool):
print(emailMsg)
else:
print(emailMsg)
send(config('EMAIL'), emailMsg)
setExecution(False)
#read search summary from mongodb and store it to spark dataframe
readUsers = mySpark.read\
.format("com.mongodb.spark.sql.DefaultSource")\
.option("uri","mongodb://127.0.0.1/{0}.{1}".format(
config['mongo']['db_name'],
config['mongo']['search_summary_collection']
)).load()
#create a view for the previously created dataframe. Store to new spark dataframe results
readUsers.createOrReplaceTempView("users")
sqlDF = mySpark.sql("SELECT * FROM users")
#Report creation procedure initialization. Convert spark dataframe to pandas, convert column date to datetime (format: YYYY-MM-DD)
pdf1 = sqlDF.toPandas()
pdf1['date'] = pd.to_datetime(pdf1['date'])
#select search volume for the last 3 months, plot resuts
pdf2 = pdf1.groupby(['date']).sum().last('3M')
line = pdf2.plot(colormap='winter_r', figsize=(7.5, 5), legend=False)
line.set_ylabel("Search Volume")
#Save plot as image
fig = line.get_figure()
fig.savefig(config['report']['output_image_name'])
# Send report via email. Plot image will in the attachment and in image body
send(email=config['report']['email_addr'],
pw=config['report']['pwd'],
subject=config['report']['subject'],
to_addr=config['report']['target_email_addr'],
output_image_name=config['report']['output_image_name'])
def feedback(self, name, email, text):
message = 'Feedback from: {0},\ne-mail address: {1}, \nfeedback body:\n\t{2}'.format(name, email,
text.lstrip().rstrip())
email_sender.send(message)
return "Success"
def hello1():
email_sender.send("test")
return "send"
import email_sender
import urls
import logging
import datetime
import traceback
from tornado.options import options
if __name__ == "__main__":
# Load settings
settings.load_settings()
application = tornado.web.Application(urls.urls, **options.as_dict())
application.listen(options.port)
server_instance = tornado.ioloop.IOLoop.instance()
# tornado.autoreload.add_reload_hook(database.release)
try:
server_instance.start()
except KeyboardInterrupt:
logging.error("Existing")
exit_error = u'Keyboard Exit'
except Exception, e:
logging.exception(e)
exit_error = traceback.format_exec()
finally:
server_instance.add_callback(server_instance.stop)
exit_error = str(datetime.datetime.now()) + '\n' + exit_error
email_sender.send(title='Server exit unexpectly', message=exit_error)
def start():
utility.connect()
try:
r_id=int(sys.argv[3])
hostname=sys.argv[4]
port=int(sys.argv[5])
username=sys.argv[6]
password=sys.argv[7]
req_str=sys.argv[8]
script=sys.argv[9]
args=sys.argv[10]
cmd_syntax=sys.argv[11]
log.log_debug("%s %s %s %s %s %s %s %s %s" % (str(r_id), hostname, str(port), username, password, req_str, script, args, cmd_syntax), 2, -2)
if script.lower().rfind(".c") > 0:
script = script[:script.lower().rfind(".c")]
else:
if script.lower().rfind(".cpp") > 0:
script = script[:script.lower().rfind(".cpp")]
#print hostname+", "+str(port)+", "+username+", "+password+", "+req_str+", "+script+", "+args+", "+cmd_syntax+", "+str(r_id)
if cmd_syntax == "NULL":
if args == "NULL":
cmd = script
else:
cmd = script + " " + args
else:
if args == "NULL":
cmd = cmd_syntax + " " + script
else:
cmd = cmd_syntax + " " + script + " " + args
cmd = cmd + ' ' + 'rid_' + str(r_id)
log.log_debug("%s" % (cmd), 2, -2)
client = ssh.SSHClient()
client.set_missing_host_key_policy(ssh.AutoAddPolicy())
client.connect(hostname, port=port, username=username, password=password)
utility.update_status('Running', r_id)
#utility.update_time('start', r_id, utility.get_current_time())
stdin, stdout, stderr = client.exec_command(cmd)
err = stderr.readlines()
errStr = ''
if len(err) > 0: # if failed to run script
log.log_debug(err, 1, -2)
log.log_debug("Failed to run script %s; Request:%s ID:%d" % (script, req_str, r_id), 1, -2)
errStr = script + "\n"
errStr += "Failed to run script. "+"\n"
errStr += '; '.join(err)
email_sender.send(r_id, req_str, errStr)
utility.update_status('Failed', r_id)
utility.update_time('end', r_id, utility.get_current_time())
else: # if process runs successfully
output = stdout.readlines()
# search for ERROR token
for line in output:
if line.lower().find("error")!=-1 or line.lower().find("fail")!=-1 or line.lower().find("failed")!=-1 or line.lower().find("exception")!=-1:
errStr += "> "
errStr += line + '\n'
if len(errStr) > 0: # if error found while running script
log.log_debug("Error found after running script; Request:%s, ID:%d" % (req_str, r_id), 2, -2)
errStr = script + '\n' + errStr
email_sender.send(r_id, req_str, errStr)
utility.update_status('Error', r_id)
utility.update_time('end', r_id, utility.get_current_time())
else: # no error found while running script
utility.update_status('Done', r_id)
utility.update_time('end', r_id, utility.get_current_time())
log.log_debug("No error found after running script; Request:%s, ID:%d" % (req_str, r_id), 2, -2)
client.close()
except Exception as err:
log.log_debug(err, 1, -2)
log.log_debug("Error in start(); Request:%s ID:%d" % (req_str, r_id), 1, -2)
errStr = script + '\n' + str(err)
email_sender.send(r_id, req_str, errStr)
finally:
utility.close_conn()
collection_list = mydb.list_collection_names()
if "faculty_coordinators" in collection_list:
print("Dropping previous collection 'faculty_coordinators'")
collection = mydb['faculty_coordinators']
collection.drop()
xl_object = xlrd.open_workbook(args.filename)
sheet = xl_object.sheet_by_index(0)
email_list = []
collection_records = []
for i in range(sheet.nrows):
email_list.append(sheet.cell_value(i, 3))
dictionary = {
'coordinator_id': sheet.cell_value(i, 0),
'name': sheet.cell_value(i, 1),
'type': sheet.cell_value(i, 2),
'email': sheet.cell_value(i, 3),
'api_token': str(abs(hash(sheet.cell_value(i, 3).strip())))
}
collection_records.append(dictionary)
mydb.faculty_coordinators.insert_many(collection_records)
if args.email == 'yes':
print('sending email')
email_sender.send(email_list)
