def update_all_stocks_sheets(
unsubscribe_url='Bad Developer... Always supply this.'):
"""
Get new stock prices, alert users, and update spreadsheets.
:param unsubscribe_url: Url that will be put in alert emails allowing users to unsubscribe.
"""
sheet_prices_dict, sheet_alerts_dict = db.update_stock_prices(alerts=True)
user_alerts_dict = db.get_user_alerts(sheet_alerts_dict)
l = emails.get_emails(user_alerts_dict, unsubscribe_url)
emails.send_emails(l)
for ssheet_id, prices in sheet_prices_dict.items():
credentials = db.get_credentials(ssheet_id=ssheet_id)
h = credentials.authorize(httplib2.Http())
# TODO(jalex): Implement using GAE's task queues (https://cloud.google.com/appengine/docs/python/taskqueue/)
if USE_THREADS:
t = threading.Thread(target=sheets.set_prices,
args=(ssheet_id, prices, h))
t.daemon = True
t.start()
else:
sheets.set_prices(ssheet_id, prices, h)
def vote():
emails = get_emails()
for item in emails:
print(item)
proxies = get_proxies()
proxy_index = random.randrange(0, 5)
proxy = proxies[proxy_index]
print(proxy)
full_name = item[0]
email = item[1]
print(full_name)
print(email)
proxy_config = Proxy()
proxy_config.proxy_type = ProxyType.MANUAL
proxy_config.http_proxy = proxy
proxy_config.ssl_proxy = proxy
capabilities = webdriver.DesiredCapabilities.CHROME
proxy_config.add_to_capabilities(capabilities)
chrome_options = Options()
chrome_options.headless = True
driver = webdriver.Chrome("./chromedriver",
desired_capabilities=capabilities,
options=chrome_options)
driver.get("") # FILL IN WITH WEBSITE
time.sleep(10) # give page time to load
form_full_name = driver.find_element_by_xpath('') #FILL IN WITH X PATH
form_full_name.send_keys(full_name)
form_radio_button = driver.find_element_by_xpath(
'') #FILL IN WITH X PATH
form_radio_button.click()
form_video_dropdown = driver.find_element_by_xpath(
'') #FILL IN WITH X PATH
form_video_dropdown.click()
form_video_choice = driver.find_element_by_xpath(
'') #FILL IN WITH X PATH
form_video_choice.click()
form_email_address = driver.find_element_by_xpath(
'') #FILL IN WITH X PATH
form_email_address.send_keys(email)
form_submit_button = driver.find_element_by_xpath(
'') #FILL IN WITH X PATH
form_submit_button.click()
time.sleep(10) # give time to submit
get_confirmation_text = driver.find_element_by_css_selector(
'') #FILL IN WITH X PATH
if ((get_confirmation_text.text
) == ""): #FILL IN WITH CONFIRMATION MESSAGE
print("Successfully voted!")
else:
print("Did not vote successfully.")
driver.quit()
time.sleep(300) # wait 5 mins
def compute_thresholds(out_results_dir, thresholds_dir, emails_list_file):
if not os.path.exists(thresholds_dir):
os.makedirs(thresholds_dir)
emails = get_emails(emails_list_file)
for mail in emails:
y_score, y = get_result(mail, out_results_dir)
th = find_threshold(y_score, y)
th_file = os.path.join(thresholds_dir, mail + ".txt")
print("Saving threshold:", th_file)
with open(th_file, "w+") as f:
f.write(str(th))
def load_and_run_all_models(config, emails_list_file, emails_data_base_dir):
emails = get_emails(emails_list_file)
total_err = 0.0
for mail in emails:
X_data, Y_data = dataset.load_data(mail, emails_data_base_dir)
model_path = os.path.join(config.save_model_dir, mail, "full")
model_object_path = os.path.join(model_path, "model.json")
model_weights_path = os.path.join(model_path, "weights.h5")
threshold = get_threshold(config.thresholds_dir, mail)
err = load_and_run_model(config, model_object_path, model_weights_path,
threshold, X_data, Y_data)
print("Error for %s with threshold %f: %f" % (mail, threshold, err))
total_err = (total_err + err) / 2
print("Total error:", total_err)
def update_all_stocks_sheets(unsubscribe_url='Bad Developer... Always supply this.'):
"""
Get new stock prices, alert users, and update spreadsheets.
:param unsubscribe_url: Url that will be put in alert emails allowing users to unsubscribe.
"""
sheet_prices_dict, sheet_alerts_dict = db.update_stock_prices(alerts=True)
user_alerts_dict = db.get_user_alerts(sheet_alerts_dict)
l = emails.get_emails(user_alerts_dict, unsubscribe_url)
emails.send_emails(l)
for ssheet_id, prices in sheet_prices_dict.items():
credentials = db.get_credentials(ssheet_id=ssheet_id)
h = credentials.authorize(httplib2.Http())
# TODO(jalex): Implement using GAE's task queues (https://cloud.google.com/appengine/docs/python/taskqueue/)
if USE_THREADS:
t = threading.Thread(target=sheets.set_prices, args=(ssheet_id, prices, h))
t.daemon = True
t.start()
else:
sheets.set_prices(ssheet_id, prices, h)
import os
import constant
import prep
import verifier
import emails
if __name__ == '__main__':
fnames = os.listdir(constant.RAW_DIR)
for fname in fnames:
people = prep.rearrange_info(fname)
for person in people.values():
print(person)
email_prob_list = emails.get_emails(person)
people[person['id']]['email_list_crawl'] = email_prob_list
print('start verify')
verifier.verify(fname, people, False)
print('end verify')
import os, sys
from emails import get_emails
from sklearn.metrics import roc_curve, auc
def get_result(mail, results_dir="training/keras/models/lstm2layer2dropout/results"):
filename = os.path.join(results_dir, mail + ".txt")
with open(filename) as f:
resultset = [d.split(',') for d in f.read().splitlines()]
y_score = resultset[0]
y = resultset[1]
y_score = [float(x) for x in y_score]
y = [int(x) for x in y]
return y_score, y
if __name__ == '__main__':
emails = get_emails()
for mail in emails:
if len(sys.argv) > 1:
y_score, y = get_result(mail, sys.argv[1])
else:
y_score, y = get_result(mail)
# Compute micro-average ROC curve and ROC area
fpr, tpr, _ = roc_curve(y, y_score, pos_label=1)
roc_auc = auc(fpr, tpr)
##############################################################################
# Plot of a ROC curve for a specific class
plt.figure()
plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % roc_auc)
def train_and_evaluate(config, emails_list_file, emails_data_base_dir):
emails = get_emails(emails_list_file)
totalFP = 0
totalFN = 0
totalTP = 0
totalTN = 0
with open(config.results_file, "w+") as f:
f.write("")
for mail in emails:
print('Loading data...')
X_data, Y_data = dataset.load_data(mail, emails_data_base_dir)
falsePositives = 0
falseNegatives = 0
truePositives = 0
trueNegatives = 0
predictions = []
shouldBe = []
for i in range(len(X_data)):
tmp = X_data[i]
X_data[i] = X_data[0]
X_data[0] = tmp
tmp = Y_data[i]
Y_data[i] = Y_data[0]
Y_data[0] = tmp
X_train = X_data[1:]
y_train = Y_data[1:]
X_test = X_data[0:1]
y_test = Y_data[0:1]
print(len(X_train), 'train sequences')
print("Pad sequences (samples x time)")
X_train = sequence.pad_sequences(X_train, maxlen=config.max_seq_len)
X_test = sequence.pad_sequences(X_test, maxlen=config.max_seq_len)
X_train, X_test = config.additional_data_transform(X_train, X_test)
max_value = max(X_train.max(), X_test.max()) + 1
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print('Build model...')
model = config.build_model(max_value)
model.compile(loss=config.loss_function,
optimizer=config.optimizer,
class_mode=config.class_mode)
print("Train...")
model.fit(X_train, y_train, batch_size=config.batch_size, nb_epoch=config.epochs,
validation_data=(X_test, y_test), show_accuracy=True)
score, acc = model.evaluate(X_test, y_test,
batch_size=config.batch_size,
show_accuracy=True)
prediction = model.predict(X_test)
predicted_class = round(abs(float(prediction)))
print("Model prediction:", prediction, "Should be:", y_test)
print('Test score:', score)
print('Test accuracy:', acc)
predictions.append(float(prediction))
shouldBe.append(y_test[0])
if(predicted_class == 0 and y_test[0] == 1):
falseNegatives += 1
elif(predicted_class == 1 and y_test[0] == 0):
falsePositives += 1
elif(predicted_class == 1 and y_test[0] == 1):
truePositives += 1
elif(predicted_class == 0 and y_test[0] == 0):
trueNegatives += 1
tmp = X_data[i]
X_data[i] = X_data[0]
X_data[0] = tmp
tmp = Y_data[i]
Y_data[i] = Y_data[0]
Y_data[0] = tmp
totalFP += falsePositives
totalFN += falseNegatives
totalTP += truePositives
totalTN += trueNegatives
model_path = os.path.join(config.save_model_dir, mail, str(i))
model_object_path = os.path.join(model_path, "model.json")
model_weights_path = os.path.join(model_path, "weights.h5")
save_model(model, model_object_path, model_weights_path)
train_on_full_data(config, mail, X_data, Y_data)
result_string = "\n".join(["For: %s FP: %d FN: %d TP: %d TN: %d",
"Predictions: %s", "Should be: %s"])
result_string = result_string % (mail, falsePositives, falseNegatives,
truePositives, trueNegatives, str(predictions),
str(shouldBe))
appendResults(result_string, config.results_file)
print(result_string)
result_string = "TotalFP: " + str(totalFP) + " TotalFN: " + str(totalFN) + \
" TotalTP: " + str(totalTP) + " TotalTN: " + str(totalTN)
appendResults(result_string, config.results_file)
print(result_string)
gather_results.from_file(config.results_file, config.out_results_dir)
print("Looking for FP-free threshold...")
compute_thresholds(config.out_results_dir, config.thresholds_dir, emails_list_file)