def __init__(self) -> None:
if Mailer.__instance is not None:
raise Exception("This class is a singleton!")
Mailer.__instance = self
self.init_ok: bool = False
self.msg: EmailMessage = None
self.smtp_host: str = Config.get('MAILER', 'smtp_host')
self.smtp_port: str = Config.get('MAILER', 'smtp_port')
self.smtp_user: str = Config.get('MAILER', 'smtp_user')
self.smtp_password: str = Config.get('MAILER', 'smtp_password')
_sender = Config.get('MAILER', 'sender')
try:
_receivers = Config.get('MAILER', 'receivers', param_type="json")
except JSONDecodeError:
Logger.get(self.__class__.__name__).warn('Problems decoding email receivers from JSON in config!')
_receivers = None
if not _sender or not _receivers or not self.smtp_host or not self.smtp_port:
Logger.get(self.__class__.__name__).warn('Emails not being sent!')
return
self.msg = EmailMessage()
self.msg.set_charset('utf-8')
self.msg['From'] = _sender
self.msg['To'] = _receivers
self.init_ok = True
def __init__(self) -> None:
self.url = Config.get('URL_FETCHER', 'url')
try:
self.headers = Config.get('URL_FETCHER',
'headers',
param_type="json")
except JSONDecodeError:
Logger.get(self.__class__.__name__).warn(
"Problem decoding URL headers from JSON in config!")
self.headers = None
self.content: Optional[str] = None
def fetch(self) -> None:
# noinspection PyBroadException
try:
response = requests.get(self.url, headers=self.headers)
except Exception as e:
Logger.get(self.__class__.__name__).error(str(e))
self.content = None
return
if response.status_code >= 300:
Logger.get(self.__class__.__name__).error('{}: {}'.format(
response.status_code, response.reason))
self.content = None
else:
self.content = response.content.decode('utf-8')
def send(cls, content: AbstractMailTemplate, subject: str = None) -> None:
if Mailer.__instance is None:
Mailer()
if Mailer.__instance.init_ok:
smtp = smtplib.SMTP_SSL("{}:{}".format(Mailer.__instance.smtp_host, Mailer.__instance.smtp_port))
try:
smtp.login(Mailer.__instance.smtp_user, Mailer.__instance.smtp_password)
except smtplib.SMTPAuthenticationError as e:
Logger.get(cls.__name__).warn(str(e))
return
Mailer.__instance.msg['Subject'] = subject if subject is not None else Config.get('MAILER', 'Subject')
Mailer.__instance.msg.set_content(content.get())
try:
smtp.send_message(Mailer.__instance.msg)
except smtplib.SMTPDataError as e:
Logger.get(cls.__name__).warn(str(e))
smtp.quit()
def parse(self) -> None:
contents = self.html[self.html.index(Config.get('PARSER', 'start_tag')
) +
len(Config.get('PARSER', 'start_tag')):self.html.
index(Config.get('PARSER', 'end_tag'))].strip()
if Config.get('PARSER', 'empty_pattern') in contents:
Logger.get(self.__class__.__name__).info("No new contents.")
self.has_new_content = False
self.links = {}
try:
os.remove("{}/{}".format(Config.get('CRON', 'project_root'),
Config.get('PARSER', 'cache_file')))
except OSError as e:
if e.errno != errno.ENOENT:
raise
return
with open(Config.get('PARSER', 'cache_file'), 'a+') as f:
f.seek(0)
if f.read() == contents:
Logger.get(self.__class__.__name__).info("No new contents.")
self.has_new_content = False
self.links = {}
else:
Logger.get(
self.__class__.__name__).info("New content detected.")
self.has_new_content = True
self._extract_links(contents)
f.seek(0)
f.truncate()
f.write(contents)
def __init__(self, session):
self.session = session
self.logger = Logger.get(__name__)
class App:
DEBUG = True
if DEBUG:
CONFIG_PATH = "config_testing.json"
else:
CONFIG_PATH = "config.json"
with open(CONFIG_PATH) as f:
CONFIG = json.load(f)
logger = Logger.get(__name__)
def main(self):
if not database_exists(self.CONFIG["database_url"]):
self.logger.info("Creating new database")
Database.init_db(self.CONFIG["database_url"])
self.logger.info("Connecting to the database")
db = Database.from_url(self.CONFIG["database_url"])
users = self.update_users_preferences(db)
venues = db.get_distinctive_items("venue")
artists = db.get_distinctive_items("artist")
promoters = db.get_distinctive_items("promoter")
# go through venues
new_events = []
for venue in venues:
self.logger.info(f"Checking {venue['name']} venue...")
venue["type"] = "venue"
venue_events = self.get_events(
venue, self.CONFIG["venue_url_prefix"] + venue["tag"]
)
new_venue_events = self.add_to_database(db, venue_events)
new_events.extend(new_venue_events)
for artist in artists:
self.logger.info(f"Checking {artist['name']} artist...")
artist["type"] = "artist"
artist_events = self.get_events(
artist, self.CONFIG["artist_url_prefix"] + artist["tag"]
)
new_artist_events = self.add_to_database(db, artist_events)
new_events.extend(new_artist_events)
for promoter in promoters:
self.logger.info(f"Checking {promoter['name']} promoter...")
promoter["type"] = "promoter"
promoter_events = self.get_events(
promoter, self.CONFIG["promoter_url_prefix"] + promoter["tag"]
)
new_promoter_events = self.add_to_database(db, promoter_events)
new_events.extend(new_promoter_events)
db.commit()
for new_event in new_events:
self.add_event_notifications(new_event, users)
self.send_emails(users)
def update_users_preferences(self, db):
self.logger.info("Downloading users favourites")
users = self.get_users()
users = self.download_users_interests(users)
self.logger.info("Updating users database")
self.update_database(users, db)
return users
def get_users(self):
with open(self.CONFIG["users_path"]) as f:
users_json = json.load(f)
users = []
for user_data in users_json["users"]:
users.append(
User(
user_data["name"],
user_data["nickname"],
user_data["email"],
user_data["locations"],
)
)
return users
def download_users_interests(self, users):
with requests.Session() as s:
login_response = s.post(
self.CONFIG["login_url"],
data=self.CONFIG["payload"],
headers=self.CONFIG["headers"],
)
if not self.logged_in(login_response):
raise Exception("Could not login to Resident Advisor")
for user in users:
self.logger.info(f"Fetching user {user.nickname} preferences")
url = self.CONFIG["profile_url_prefix"] + user.nickname + "/favourites"
html = s.get(url, headers=self.CONFIG["headers"])
html.encoding = "utf-8"
soup = BeautifulSoup(html.text, "html.parser")
html_artists = soup.find_all("div", class_="fav")
if html_artists is not None:
for artist in html_artists:
info_tag = artist.find("div", class_="pb2").find("a")
artist_name = info_tag.get_text()
artist_tag = info_tag.get("href")[4:]
user.add_artist(artist_name, artist_tag)
else:
self.logger.warning(
f"User {user.nickname} does not follow any artists"
)
html_venues = soup.find("ul", class_="list venueListing")
if html_venues is not None:
for venue in html_venues.find_all("li", recursive=False):
info_tag = venue.find_all("a")[1]
venue_name = info_tag.get_text()
venue_tag = info_tag.get("href")[14:]
user.add_venue(venue_name, venue_tag)
else:
self.logger.warning(
f"User {user.nickname} does not follow any venues"
)
try:
try:
html_promoters = soup.find_all(
lambda tag: tag.name == "ul"
and tag.get("class") == ["list"]
)[1]
except Exception:
# if find_all doesn't succeed, it means the person does not
# follow any labels and only promoters are available
html_promoters = soup.find(
lambda tag: tag.name == "ul"
and tag.get("class") == ["list"]
)
for promoter in html_promoters.find_all("li", recursive=False):
info_tag = promoter.find_all("a")[1]
promoter_name = info_tag.get_text()
promoter_tag = info_tag.get("href")[18:]
user.add_promoter(promoter_name, promoter_tag)
except Exception:
self.logger.warning(
f"User {user.nickname} does not follow any promoters"
)
return users
def logged_in(self, login_response):
login_response.encoding = "utf-8"
soup = BeautifulSoup(login_response.text, "html.parser")
spans = soup.find_all("span")
for span in spans:
if span.get_text() == "Welcome":
return True
return False
def update_database(self, users, db):
for user in users:
db.update_user(user)
def get_events(self, entity, url):
html = requests.get(url)
html.encoding = "utf-8"
soup = BeautifulSoup(html.text, "html.parser")
events_html = soup.find_all("article", class_="event-item")
events = []
for event_html in events_html:
try:
if entity["type"] is "venue":
event = Event.from_venue_html(entity["name"], event_html)
elif entity["type"] is "artist":
event = Event.from_artist_html(entity["name"], event_html)
elif entity["type"] is "promoter":
event = Event.from_promoter_html(entity["name"], event_html)
except:
self.logger.warning(
f"Could not generate event from the following html: {event_html.get_text()}"
)
events.append(event)
return events
def add_to_database(self, db, events):
new_events = []
for event in events:
event_in_database = db.fetch_from_database(event.event_id, event.event_type)
if event_in_database is not None and event_in_database.tickets_available:
continue
event.tickets = self.get_tickets(event.event_url)
tickets_available = (True, False)[not event.tickets]
# If event is not in db, add
if event_in_database is None:
db.add_event(event.event_id, event.event_type, tickets_available)
self.logger.info(
f"NEW EVENT WITH URL {event.event_url} IS ADDED TO THE DATABASE. TICKETS: {tickets_available}"
)
new_events.append(event)
# The event was in database but didn't have tickets
else:
if tickets_available:
db.update_event(event.event_id, event.event_type, tickets_available)
self.logger.info(
f"EVENT WITH URL {event.event_url} WAS UPDATED WITH TICKETS"
)
new_events.append(event)
return new_events
def get_tickets(self, event_url):
html = requests.get(event_url)
html.encoding = "utf-8"
soup = BeautifulSoup(html.text, "html.parser")
tickets = []
try:
html_tickets = soup.find_all("li", class_="onsale but")
for html_ticket in html_tickets:
p = html_ticket.find("p")
tickets.append(
{
"name": p.find(text=True, recursive=False),
"price": p.find("span").get_text(),
}
)
except:
self.logger.warning(f"some problem getting tickets for {event_url}")
return tickets
def add_event_notifications(self, event, users):
for user in users:
user.add_to_email(event)
def send_emails(self, users):
service = discovery.build("gmail", "v1", credentials=self.make_credentials())
for user in users:
if user.number_of_new_events == 0:
continue
user.add_email_ending()
self.logger.info(f"Emailing {user.email}")
message = MIMEMultipart()
message["From"] = "me"
message["Subject"] = "New events on RA"
message["To"] = user.email
message.attach(MIMEText(user.email_body.get(), "html"))
raw_message = base64.urlsafe_b64encode(message.as_bytes()).decode()
body = {"raw": raw_message}
self.send_email_request(service, body)
return
def make_credentials(self):
with open(self.CONFIG["credentials_path"]) as f:
data = json.load(f)
return Credentials(
None,
refresh_token=data["refresh_token"],
client_id=data["installed"]["client_id"],
client_secret=data["installed"]["client_secret"],
token_uri=data["installed"]["token_uri"],
)
@backoff.on_exception(backoff.expo, HttpError, max_tries=4)
def send_email_request(self, service, body):
service.users().messages().send(userId="me", body=body).execute()
