def get_slack_token(cls, team_id, user_id=None):
database = Database()
database.initialize()
try:
slack_token_object = cls.get_token_from_database(team_id=team_id,
user_id=user_id)
except TypeError:
print(
"Visit http://178.128.234.3:5000/eight_ball/begin_auth to authorize this app"
)
raise ValueError("Slack authorization failed")
if slack_token_object.token_expiry_time is None:
print(
"Visit http://178.128.234.3:5000/eight_ball/begin_auth to authorize this app"
)
raise ValueError("Slack authorization failed")
elif int(slack_token_object.token_expiry_time) < int(time.time()):
hook = cls.get_token_from_database(team_id=team_id,
user_id=user_id)
auth_response = hook.slack_token_request(hook.auth_code)
try:
hook.update_token(auth_response)
except TypeError:
hook.add_token(auth_response)
return SlackClient(str(auth_response['access_token']))
else:
return SlackClient(str(slack_token_object.access_token))
def find_entry(cls, text):
database = Database()
database.initialize()
entry = database.find_one("samples", {"text": text})
if entry is not None:
return cls(**entry)
else:
return None
def get_distributors(cls, type):
database = Database()
database.initialize()
distributors = database.find("distributors", {"type": type})
return [cls(**distributor) for distributor in distributors]
#Distributor.add_distributor(type="slack", slack_channel_id="C0JS385LP")
#Distributor.add_distributor(type="email", email_address="*****@*****.**")
def get_release_notes(version):
database = Database()
database.initialize()
jira_settings = database.find_one("jira_settings", query={})
auth_jira = JIRA(basic_auth=(jira_settings['email'], jira_settings['password']),
server=jira_settings['server'])
release_notes = "{} has been deployed to production! Release notes:\n".format(version)
issues = auth_jira.search_issues("""fixVersion in ("{}")""".format(version))
for issue in issues:
release_notes = release_notes + str(issue) + " - " + auth_jira.issue(issue).fields.summary + "\n"
return release_notes
def get_latest_release():
database = Database()
database.initialize()
entries = database.find(collection="version_record",
query={})
saved_entries = list()
times = list()
for i in entries:
times.append(i['time'])
saved_entries.append(i)
max_time = max(times)
for i in saved_entries:
if i['time'] == max_time:
version = i['version']
return version
def remove_fact(fact_text):
database = Database()
database.initialize()
if database.find_one("facts", {"fact_text": fact_text}) is None:
raise LookupError('Fact was not found in the database')
else:
database.remove("facts", {"fact_text": fact_text})
print("Fact was successfully removed")
def remove_distributor(email_address=None, slack_ids=None):
database = Database()
database.initialize()
if database.find_one("distributors", {"email_address": email_address,
"slack_ids": slack_ids}):
database.remove("distributors", {"email_address": email_address,
"slack_ids": slack_ids})
return True
class MatchProducts(object):
database = Database()
database.initialize()
def get_products_by_site(self, site_name):
products = Product.get_products(
database=self.database,
collection=site_name,
query={'site_name': '{0}'.format(site_name)})
return products
@staticmethod
def clean_names(text):
text = re.sub('[^a-zA-Z0-9]', '', text).lower().replace("\n", "")
return text
def check_matches(self, ulta, sephora):
ulta = self.clean_names(ulta['name'])
sephora = self.clean_names(sephora['name'])
result = jellyfish.jaro_distance(ulta, sephora)
return result
def format_names(self, site_name):
clean_products = list()
products = self.get_products_by_site(site_name=site_name)
for product in products:
brand = self.clean_names(product.brand)
brand = brand.replace(" cosmetics", "")
item_name = self.clean_names(product.item_name)
name = brand + " " + item_name
clean_products.append({
"name": name,
"image": product.image,
"link": product.link,
"price": product.price
})
return clean_products
def get_matches(self):
sephora = self.format_names("sephora")
ulta = self.format_names("ulta")
for u_product in ulta:
for s_product in sephora:
if self.check_matches(s_product, u_product) > 0.95:
print("Match! {0}, {1}".format(s_product, u_product))
self.database.insert("matches",
data=({
"sephora": s_product,
"ulta": u_product
}))
def add_fact(fact_type, fact_text):
if not any([(fact_type == "puppy_fact"),
(fact_type == "cat_fact"),
(fact_type == "horse_fact")]):
print("fact_type must be one of the following: puppy_fact, cat_fact, horse_fact")
else:
database = Database()
database.initialize()
fact = Facts(fact_type = fact_type, fact_text = fact_text)
database.insert("facts", fact.json())
print("Fact successfully added")
def log_staging_deploy(msg):
pattern = re.compile("finished deploying branch.* version.* to staging")
if pattern.findall(msg):
version = msg.split("branch")[1].split("version")[0].strip()
database = Database()
database.initialize()
database.insert(collection="version_record",
data={
"version": version,
"time": int(time.time())
})
def add_token(self, auth_response):
database = Database()
database.initialize()
self.access_token = auth_response['bot']['bot_access_token']
self.user_id = auth_response['user_id']
self.team_id = auth_response['team_id']
numbers = re.compile('\d+(?:\.\d+)?')
max_age = int(
numbers.findall(
auth_response['headers']['Strict-Transport-Security'])[0])
self.token_expiry_time = int(time.time()) + max_age
database.insert(collection="slack_tokens", data=self.json())
def migrate_facts():
dirpath = os.path.dirname(__file__)
fact_files = {'horse_fact': 'facts/horse_facts.txt',
'cat_fact': 'facts/cat_facts.txt',
'puppy_fact': 'facts/puppy_facts.txt'}
database = Database()
database.initialize()
for k, v in fact_files.items():
lines = open(os.path.join(dirpath, v)).read().splitlines()
for line in lines:
if line == "":
continue
else:
fact = Facts(fact_type=k, fact_text=line)
database.insert("facts", fact.json())
def get_token_from_database(cls, team_id, user_id=None):
database = Database()
database.initialize()
if user_id is not None:
credentials = database.find_one("slack_tokens",
query=({
"team_id": team_id,
"user_id": user_id
}))
else:
credentials = database.find_one("slack_tokens",
query=({
"team_id": team_id
}))
return cls(**credentials)
def add_distributor(type, email_address=None, slack_ids=None):
if "type" == "email":
if email_address is not None:
return "Please pass an email address"
if "type" == "slack":
if email_address is not None:
return "Please pass an slack_channel_id"
elif all([(type != "slack"), (type != "email")]):
return "Valid 'type's are 'slack' and 'email'"
else:
distributor = Distributor(type=type,
email_address=email_address,
slack_ids=slack_ids)
database = Database()
database.initialize()
database.insert("distributors", distributor.json())
def get_email_credentials(cls, returnless=False):
database = Database()
database.initialize()
emailer_credential_object = database.find_one("emailer_credentials",
({}))
if emailer_credential_object is None:
email_address = input(
"Please enter the gmail account address you would like to use to send emails:"
)
email_password = input("Please enter your gmail password:"******"Login failed. Please enter your GMAIL address:")
email_password = input("Please enter your GMAIL password:"******"emailer_credentials",
({
"email_address": email_address,
"email_password": email_password
}))
else:
email_address = emailer_credential_object['email_address']
email_password = emailer_credential_object['email_password']
if returnless is False:
return email_address, email_password
def add_entry(data):
database = Database()
database.initialize()
database.insert("logging", data)
class Sephora(object):
main_url = "https://www.sephora.com"
sitemap_url = "/sitemap/departments/"
database = Database()
database.initialize()
webfunctions = WebFunctions()
session = WebFunctions.get_session()
def get_department_links(self):
url = self.main_url + self.sitemap_url
soup = WebFunctions.make_soup(url=url, session=self.session)
raw_links = soup.find("div", {
"class": "Sitemap u-bt u-bw2"
}).findAll("h2")
clean_links = set()
for i in raw_links:
clean_links.add(i.find("a").get("href"))
# pprint(clean_links)
return clean_links
def get_category_links(self, page_url):
url = self.main_url + page_url
soup = WebFunctions.make_soup(url=url, session=self.session)
raw_category_links = soup.findAll("a",
{"data-at": "top_level_category"})
clean_category_links = set()
for i in raw_category_links:
clean_category_links.add(i.get("href"))
# print(i.get("href"))
return clean_category_links
def get_all_category_links(self):
departments = self.get_department_links()
all_category_links = set()
for department in departments:
category_links = self.get_category_links(page_url=department)
all_category_links = all_category_links | category_links
# pprint(all_category_links)
return all_category_links
def get_subcategory_links(self, page_url):
url = self.main_url + page_url
soup = WebFunctions.make_soup(url=url, session=self.session)
raw_subcategory_links = soup.findAll("a", {"data-at": "nth_level"})
if len(raw_subcategory_links) == 0:
raw_subcategory_links = soup.findAll("a",
{"class": "SideNav-link"})
clean_subcategory_links = set()
for i in raw_subcategory_links:
clean_subcategory_links.add(i.get("href"))
# print(i.get("href"))
return clean_subcategory_links
def get_all_subcategory_links(self):
category_links = self.get_all_category_links()
all_subcategory_links = set()
count = 0
for category in category_links:
count += 1
subcategory_links = self.get_subcategory_links(category)
all_subcategory_links = all_subcategory_links | subcategory_links
# print(subcategory_links)
# print(count, len(category_links))
print("Done {0}/{1}".format(count, len(category_links)))
return all_subcategory_links
@classmethod
def get_pages(cls, soup):
try:
num_str = soup.find("span", {"data-at": "number_of_products"}).text
num_of_products = WebFunctions.only_digits(num_str)
pages_num = int(math.ceil(num_of_products / 12))
pages_num += 1
pages = [int(i) for i in range(1, pages_num)]
return pages
except AttributeError:
return
def save_product(self, product):
link = product.find("a").get("href")
image = product.find("img").get("src")
site_product_id = product.find("button", {
"data-comp": "ProductQuicklook"
}).get("data-sku-number")
brand = product.find("span", {"data-at": "sku_item_brand"}).text
item_name = product.find("span", {"data-at": "sku_item_name"}).text
price = WebFunctions.only_digits(
product.find("span", {
"data-at": "sku_item_price_list"
}).text)
product = Product(site_name="sephora",
link=link,
image=image,
site_product_id=site_product_id,
brand=brand,
item_name=item_name,
price=price)
product.add_product(self.database)
def get_products(self, page_url):
main_url = self.main_url + page_url
print(page_url)
soup = WebFunctions.make_soup(url=main_url, session=self.session)
pages = self.get_pages(soup)
if pages is None:
return
for page in pages:
url = main_url + "?pageSize=12" + "¤tPage={0}".format(page)
print(url)
soup = WebFunctions.make_soup(url=url, session=self.session)
products = soup.findAll("div", {"class": "css-12egk0t"})
for product in products:
self.save_product(product)
def get_app_credentials():
database = Database()
database.initialize()
result = database.find_one("slack_credentials", query=({}))
return result
def get_token_from_database():
database = Database()
database.initialize()
result = database.find_one("giphy_tokens", query=({}))
return result['giphy_token']
def add_entry(self):
database = Database()
database.initialize()
database.insert("message_log", self.json())
def remove_user(self):
database = Database()
database.initialize()
database.remove("users", {"_id": self._id})
print("User successfully removed")
def get_one_user(cls, query=({})):
database = Database()
database.initialize()
user = database.find_one("users", query)
return cls(**user)
def get_users(cls, query=({})):
database = Database()
database.initialize()
users = database.find("users", query)
return [cls(**user) for user in users]
def onboard_user(self):
database = Database()
database.initialize()
database.insert("users", self.json())
print("User successfully added")
import re
import os
from commons.config import Config
from commons.validations import Validations
from commons.database import Database
from commons.dns import DNS
def error(message):
print message
sys.exit(-1)
if __name__ == "__main__":
config = Config()
dns = DNS(config.get("domain").get("dns"))
db = Database(config.get("database"))
validations = Validations(db, config.get("domain"))
parser = argparse.ArgumentParser(description="Manage RED redirections with command-line utility")
parser.add_argument("action", choices=['add', 'del', 'list', 'check'], help="Action")
parser.add_argument("domain", help="Domain name", nargs='?')
parser.add_argument("url", nargs='?', help="Destination URL")
args = parser.parse_args()
domain = args.domain
action = args.action
if action == 'list':
for red in db.get_redirections():
print "%(date_added)s %(domain)s -> %(url)s" % red
elif action == 'add':
def get_facts(cls, query=({})):
database = Database()
database.initialize()
facts = database.find("facts", query)
return [cls(**fact) for fact in facts]
def get_entry_by_ts(cls, ts):
database = Database()
database.initialize()
result = database.find_one("message_log", {"trigger_ts": ts})
if result is not None:
return cls(**result)
def add_to_send_count(self):
database = Database()
database.initialize()
self.send_count += 1
database.update("users", {"_id": self._id}, self.json())
def get_keyword_from_database():
database = Database()
database.initialize()
result = database.find_one("giphy_keyword", query=({}))
return result['giphy_keyword']
import re
import os
from commons.config import Config
from commons.validations import Validations
from commons.database import Database
from commons.dns import DNS
def error(message):
print message
sys.exit(-1)
if __name__ == "__main__":
config = Config()
dns = DNS(config.get("domain").get("dns"))
db = Database(config.get("database"))
validations = Validations(db, config.get("domain"))
parser = argparse.ArgumentParser(description="Manage RED redirections with command-line utility")
parser.add_argument("action", choices=['add', 'del', 'list', 'check'], help="Action")
parser.add_argument("domain", help="Domain name", nargs='?')
parser.add_argument("url", nargs='?', help="Destination URL")
args = parser.parse_args()
domain = args.domain
action = args.action
if action == 'list':
for red in db.get_redirections():
print "%(date_added)s %(domain)s -> %(url)s" % red
elif action == 'add':
def update_user(self, update):
database = Database()
database.initialize()
database.update("users", {"_id": self._id}, update)
print("User successfully updated")