def request_retry_proxy(url):
try:
print('Try without proxy')
page = requests_retry_session().get(url)
if page.status_code == 200:
return page
else:
raise Exception()
except Exception as x:
print('It failed. Try with proxies ', x.__class__.__name__)
proxies = get_proxies()
print(proxies)
proxy_pool = cycle(proxies)
for proxy in proxy_pool:
print("Request", proxy)
try:
page = requests_retry_session().get(url,
proxies={
"http": proxy,
"https": proxy
})
print(page.status_code)
if page.status_code == 200:
return page
except Exception:
print("Skipping. Connnection error")
def jobs_response(response, link, logger):
if link["type"] == "greenhouse" or link["type"] == "jobscore":
return response.json()["jobs"]
elif link["type"] == "ultipro":
return response.json()["opportunities"]
elif link["type"] == "adp":
return response.json()["jobRequisitions"]
elif link["type"] == "lever":
return response.json()
elif link["type"] == "smartrecruiters":
result = response.json()["content"]
offset = 0
total = response.json()["totalFound"]
while total - offset > 100:
offset += 100
try:
partial = requests_retry_session().get("{}?offset={}".format(
link["url"], offset),
timeout=2)
result.extend(partial.json()["content"])
except Exception as x:
logger.error("{} : {}".format(
repr(x), "{}?offset={}".format(link["url"], offset)))
continue
return result
def process_price_alerts(mydb, cursor, logger, symbol, price_alert_symbol_map):
new_price_alerts = []
price_info_url = "{}api/get_spiels?channel={}".format(
os.environ['BASE_URL'], symbol)
try:
price_info_response = requests_retry_session().get(price_info_url,
timeout=10)
except Exception as x:
logger.error("{} : {}".format(repr(x), price_info_url))
return []
if price_info_response.status_code != 200:
logger.error("{} : {}".format(price_info_response.status_code,
price_info_url))
return []
price_info_response = price_info_response.json()
percent_change = float(price_info_response["quote"]["percent_change"])
percent_change_threshold = int(os.environ['PERCENT_CHANGE_THRESHOLD'])
date_last_price_alert = price_alert_symbol_map[symbol]
date_last_quote_update = datetime.utcfromtimestamp(
price_info_response["quote"]["timestamp"] / 1000)
if (percent_change < -percent_change_threshold
or percent_change > percent_change_threshold
) and date_last_price_alert.date() != date_last_quote_update.date():
created_at = date_last_quote_update.strftime("%Y-%m-%dT%H:%M:%SZ")
embed = {
"author": {
"name": price_info_response["stock_info"]["company"]
},
"title":
"Price alert: {}% ({} {} to {})".format(
percent_change, price_info_response["quote"]["direction"],
price_info_response["quote"]["price_change"],
price_info_response["quote"]["last_trade_price"]),
"description":
"Realtime price: {}{}".format(
"https://finance.yahoo.com/quote/",
price_info_response["stock_info"]["symbol"]),
"timestamp":
created_at
}
new_price_alerts.append(embed)
try:
statement = "UPDATE watchlist SET date_last_price_alert = current_timestamp() WHERE symbol = %s"
values = (symbol)
cursor.execute(statement, values)
mydb.commit()
except Exception as x:
logger.error("{} : {}".format(repr(x), price_info_url))
return []
return new_price_alerts
def process_news_releases(mydb, cursor, logger, symbol, sedar_id,
news_releases_list):
new_news_releases = []
news_releases_url = "{}api/articles/load_more?channel={}&type=news&before={}".format(
os.environ['BASE_URL'], symbol,
int(datetime.utcnow().timestamp() * 1e3))
try:
news_releases_response = requests_retry_session().get(
news_releases_url, timeout=10)
except Exception as x:
logger.error("{} : {}".format(repr(x), news_releases_url))
return
if news_releases_response.status_code != 200:
logger.error("{} : {}".format(news_releases_response.status_code,
news_releases_url))
return
news_releases_response = news_releases_response.json()[:10]
for news_release in news_releases_response:
if news_release["channel"] not in news_releases_list:
created_at = datetime.utcfromtimestamp(news_release["created_at"]/1000).\
strftime("%Y-%m-%dT%H:%M:%SZ")
url = os.environ['BASE_URL'] + news_release["channel"]
sedar_url = os.environ['SEDAR_URL'] + sedar_id if sedar_id else ""
embed = {
"author": {
"name": symbol
},
"title":
news_release["title"],
"url":
url,
"description":
"{}\n\n[See SEDAR filings]({})".format(url, sedar_url),
"timestamp":
created_at
}
new_news_releases.append(embed)
try:
statement = "INSERT INTO news_releases(`channel`, `symbol`, `title`, `url`, `created_at`) VALUES (%s, %s, %s, %s, %s)"
values = (news_release["channel"], symbol,
news_release["title"], url, created_at)
cursor.execute(statement, values)
mydb.commit()
except Exception as x:
logger.error("{} : {}".format(repr(x), news_releases_url))
continue
return new_news_releases
def process_transactions(mydb, cursor, logger, symbol, transaction_list):
new_transactions = []
transactions_url = "{}api/sedi/filings?symbol={}".format(os.environ['BASE_URL'], symbol)
try:
transactions_response = requests_retry_session().get(transactions_url, timeout=10)
except Exception as x:
logger.error("{} : {}".format(repr(x), transactions_url))
return
if transactions_response.status_code != 200:
logger.error("{} : {}".format(transactions_response.status_code, transactions_url))
return
transactions_response = transactions_response.json()[:10]
for transaction in transactions_response:
transaction_b = transaction["datab"]
if str(transaction_b["Transaction ID"]) not in transaction_list:
units = transaction_b["Number or value acquired or disposed of"].replace(',', '')
unit_price = transaction_b["Unit price or exercise price"]
try:
total = '${0:+,.2f}'.format(float(units or 0) * float(unit_price or 0))
total = re.sub(r"\.00$", "", total)
except Exception as x:
logger.debug("{} : {}".format(repr(x), transactions_url))
continue
price_statement = []
if units and unit_price:
price_statement.append({
"name": "Total",
"value": total,
"inline": True})
if units:
price_statement.append({
"name": "Volume",
"value": units,
"inline": True})
if unit_price:
price_statement.append({
"name": "Unit Price",
"value": unit_price,
"inline": True})
created_at = datetime.utcfromtimestamp(transaction["timestamp"]/1000).\
strftime("%Y-%m-%dT%H:%M:%SZ")
embed = {
"author": {
"name": transaction_b["Issuer Name"]
},
"title": transaction_b["Nature of transaction"],
"description": "{}\n{}\nDate of Transcation: {}\n[See more transactions]({})".format(
transaction_b["Insider Name"],
transaction_b["Insider's Relationship to Issuer"],
transaction_b["Date of transaction"],
os.environ['BASE_URL'] + "api/sedi?symbol={}".format(symbol)),
"fields": price_statement,
"timestamp": created_at
}
new_transactions.append(embed)
try:
statement = "INSERT INTO transactions(`transaction_id`, `symbol`, `transaction_date`, `insider_name`, `issuer_name`, `relationship_to_issuer`, `nature_of_transaction`, `price`, `units`) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)"
values = (transaction_b["Transaction ID"],
symbol,
transaction_b["Date of transaction"],
transaction_b["Insider Name"],
transaction_b["Issuer Name"],
transaction_b["Insider's Relationship to Issuer"],
transaction_b["Nature of transaction"],
transaction_b["Unit price or exercise price"],
transaction_b["Number or value acquired or disposed of"])
cursor.execute(statement, values)
mydb.commit()
except Exception as x:
logger.error("{} : {}".format(repr(x), transactions_url))
continue
return new_transactions
db=os.environ['MARIADB_DATABASE'])
cursor = mydb.cursor()
cursor.execute("call links()")
links_list = [{
'name': item[0],
'url': item[1],
'type': item[2]
} for item in cursor]
cursor.close()
email_list = []
for link in links_list:
try:
response = requests_retry_session().get(link["url"], timeout=10)
except Exception as x:
logger.error("{} : {}".format(repr(x), link["url"]))
continue
if response.status_code != 200:
logger.error("{} : {}".format(response.status_code, link["url"]))
continue
for job in jobs_response(response, link, logger):
job = create_job(job, link)
filter_check = job.title.lower() + job.location.lower()
if any(x in filter_check
for x in filter_words) and not any(x in filter_check
for x in blacklist):
email_list.append("{} - {} ({}): {}".format(
embeds += new_transactions
new_news_releases = process_news_releases(mydb, cursor, logger,
item["symbol"], item["sedarId"],
news_releases_list)
embeds += new_news_releases
new_price_alerts = process_price_alerts(mydb, cursor, logger,
item["symbol"],
price_alert_symbol_map)
embeds += new_price_alerts
time.sleep(1)
cursor.close()
if embeds:
data = {"embeds": embeds[::-1][:10]}
try:
discord_response = requests_retry_session().post(
os.environ['DISCORD_WEBHOOK'], json=data, timeout=10)
except Exception as x:
logger.error("{} : {}".format(repr(x), data))
if discord_response.status_code != 204:
logger.error("{} : {}".format(discord_response.status_code, data))
logger.info(discord_response.status_code)
logger.info(discord_response.headers)
logger.info(discord_response.text)
else:
logger.info("No new alerts for: {}".format(
datetime.now().strftime("%m/%d/%Y, %H:%M:%S")))