def getBestEvent(self, filename='events.json'):
"""Returns the event most relevant to this user's interests."""
if filename not in [f for f in os.listdir('.') if os.path.isfile(f)]:
raise IOError('Error: events file ' + filename + ' not found.')
data = json.load(open(filename))
best_event = Event()
best_score = 0
for event in data['events']:
iter_event = Event()
# The declarations are split to keep the Item __init__ method for automatically calling the API to populate the model.
iter_event.description = event['description']
iter_event.name = event['name']
iter_event.model = ConceptModel(model=event['model']['concepts'], maturity=1)
iter_event.starttime = event['starttime']
iter_event.endtime = event['endtime']
iter_event.location = event['location']
iter_event.picture = event['picture']
iter_event.name = event['name']
iter_event.url = event['url']
score = best_event.compareTo(iter_event)
if score >= best_score:
# print(self.exceptions)
if iter_event.name not in self.exceptions:
best_event = iter_event
best_score = score
return best_event
def parse_event_list_from_html(html):
soup = BeautifulSoup(html, HTML_PARSER)
event_list_divs = soup.select("#list")[0].find_all('div', recursive=False)
event_list = list()
for div in event_list_divs:
name_tag = div.find("span", class_="eventname")
if not name_tag:
continue
host_tag = div.find("span", class_="event_by")
desc_tag = div.find("span", {"itemprop": "description"})
start_date_tag = div.find("time", {"itemprop": "startDate"})
end_date_tag = div.find("time", {"itemprop": "endDate"})
# No idea why it's just 'name'
location_tag = div.find("span", {"itemprop": "name"})
event = Event()
event.name = name_tag.string
event.host = host_tag.string[HOST_NAME_START:]
event.desc = desc_tag.string
event.start_date = datetime_string_to_obj(
start_date_tag['datetime'])
event.end_date = datetime_string_to_obj(end_date_tag['datetime'])
event.location = location_tag.string
event_list.append(event)
return event_list
def make_event_from_page(link):
event = Event()
event.link = link
try:
f = urlopen(link)
soup = BeautifulSoup(f)
main = soup.select(".sectionMain")[0]
event.name = main.select(".subHeading")[0].get_text().strip()
info = main.select(".secondaryContent")[0].get_text()
sstart = regex_start.search(info).group(1).strip()
event.start = datetime.strptime(sstart, date_format)
send = regex_end.search(info).group(1).strip()
event.end = datetime.strptime(send, date_format)
event.location = urllib.parse.unquote_plus(regex_loc.search(str(main)).group(1).strip())
if event.start is not None and event.start >= datetime.today() \
and event.location is not None:
event.coords = geocode(event.location)
except Exception:
pass
return event
def _add_event(self, event, events):
e = Event(event)
if e.start_time().date() == date.today():
if e.artists():
events.append(e)
else:
print "no artists for for event:", e.name(), "url:", e.url()
def GetSingleEvent(self, eventLandingPage):
eventLandingPage = self.urlify(eventLandingPage)
self.page = requests.get(eventLandingPage)
eventTree = html.fromstring(self.page.text)
new_event = Event()
new_event.source = eventLandingPage.encode('utf-8')
new_event.name = self.multipleXPaths(self.eventName_xpath, eventTree, 'name')
new_event.location += self.multipleXPaths(self.eventLocation_xpath, eventTree, 'location')
new_event.description = self.multipleXPaths(self.eventDescription_xpath, eventTree, 'description')
new_event.datetime = self.multipleXPaths(self.eventDateTime_xpath, eventTree, 'datetime')
new_event.price = self.multipleXPaths(self.eventPrice_xpath, eventTree, 'price')
new_event.imagePath = self.multipleXPaths(self.eventImage_xpath, eventTree, 'imagePath')
new_event.imagePath = self.urlify(new_event.imagePath)
new_event.dump = SiteText(eventLandingPage).gettext()
new_event.cleanContent()
# print (new_event)
# print (new_event.name), " name"
# print (new_event.location), " location"
# print (new_event.description), " description"
# print (new_event.datetime), " datetime"
# print (new_event.price), " price"
# print (new_event.imagePath), " imagePath"
return new_event
def open_file(self, file_path):
with open(file_path) as f:
data = f.read()
cal = Calendar.from_ical(data)
for component in cal.walk():
if component.name == 'VEVENT':
tz_offset = timedelta(seconds=time.timezone)
name = component.get('summary')
start = component.get('dtstart').dt - tz_offset
end = component.get('dtend').dt - tz_offset
location = component.get('location')
event = Event()
event.name = name
event.location = location
event.year = start.year
event.month = start.month
event.day = start.day
event.start_hour = int(start.strftime('%H')) + 1
event.start_minute = int(start.strftime('%M'))
event.end_hour = int(end.strftime('%H')) + 1
event.end_minute = int(end.strftime('%M'))
if self.config.get('google_sync'):
google = self.get_google_client()
google.set_calendar_id()
google.export_event(event)
event.save()
self.show_message('Successfully added event')
self.current_view.update_gui()
def google_format_to_event(src_event):
event = Event()
event.name = src_event.get('summary', '')
event.location = src_event.get('location', '')
event.desc = src_event.get('description', '')
event.start_date = dateutil.parser.parse(src_event['start']['dateTime'])
event.end_date = dateutil.parser.parse(src_event['end']['dateTime'])
event.host = src_event.get('extendedProperties', {}).get('shared', {}).get('host', '')
return event
def scrape_dance_cal(keys_from_spreadsheet):
"""Scrapes dancecal.com. and returns an event instance
that includes name, start date, end date, country, city, url,
dance styles, teachers, status, key, and an obsolete value"""
soup = get_soup(URL_DC)
for event_div in soup.findAll('div', {'class' : 'DCListEvent'}):
name = None
event = Event()
for span in event_div.findAll('span'):
if 'DCListName' in span['class']:
name = span.text.strip()
print(name)
if name == None:
continue
elif name.lower() in event_name_list:
# checks to see if the event name already exists in the instance list
# If it does, it skips it
continue
else:
# This means the event does not already exist in the instance list
# and will be added
if 'DCListName' in span['class']:
event.name = span.text.strip()
for a_tag in span.findAll('a', href=True):
event.url = a_tag['href']
if 'DCEventInfoDate' in span['class']:
event.start_date = parse(span.text)
# Now need to guess what the end_date will be since this site does not provide it
# I'm going to assume that events will tend to end on a Sunday
# For example, if an event starts on a friday, I will make it's end-date two days later.
weekday = event.start_date.weekday()
gap = datetime.timedelta(days = 6 - weekday)
event.end_date = event.start_date + gap
if 'DCEventInfoWhere' in span['class']:
location_list = span.text.replace(':',',').split(',')
if len(location_list) == 3:
event.country = location_list[2].strip()
event.city = location_list[1].strip()
if len(location_list) == 4:
event.country = location_list[3].strip()
event.state = location_list[2].strip()
event.city = location_list[1].strip()
if 'DCEventInfoDances' in span['class']:
event.dance_styles = span.text.split(': ')[1].lower().strip()
if 'DCEventInfoTeachers' in span['class']:
event.teachers = str(span).replace('<br/>', '$').replace(':', '$').replace('</i>', '$').replace('|', 'and').split('$')[1:-1]
if 'DCEventInfoDesc' in span['class']:
event.details = span.text.strip()
if 'DCEventInfoBands' in span['class']:
event.bands = span.text.split(':')[1].strip()
if event.name == None:
pass
else:
event.key = create_key(event)
event_list = append_to_event_list(event, event.key, keys_from_spreadsheet)
return event_list
def scrape_swing_planit(keys_from_spreadsheet):
"""Scrapes swingplanit.com. and returns an event instance
that includes name, start date, end date, country, city, url,
dance styles, teachers, status, key, and an obsolete value"""
soup = get_soup(URL_SP)
for event_list_item in soup.findAll('li', {'class' : 'color-shape'}):
for a_tag in event_list_item.findAll('a', href=True):
event_soup = get_soup(a_tag['href'])
event = Event()
event.name = event_soup.title.text
event_name_list.append(event.name.lower())
event.details = event_soup.findAll('p')[0].text
event.teachers = event_soup.findAll('p')[2].text
# event.teachers = event_soup.findAll('p')[2].text.split(', ')
li_tags = event_soup.findAll('li')
for li in li_tags:
li_text = (li.get_text())
for splitter in SPLITTERS:
if splitter in li_text:
print(event.name + li_text.split(splitter,1)[0] + ': ' +
li_text.split(splitter,1)[1])
if li_text.split(splitter,1)[0].lower() == 'when':
date_range = li_text.split(splitter,1)[1].strip()
date_range = date_range.split(' - ')
event.start_date = parse(date_range[0])
event.end_date = parse(date_range[1])
if li_text.split(splitter,1)[0].lower() == 'country':
event.country = li_text.split(splitter,1)[1].strip()
if li_text.split(splitter,1)[0].lower() == 'town':
event.city = li_text.split(splitter,1)[1].strip()
if li_text.split(splitter,1)[0].lower() == 'website':
event.url = li_text.split(splitter,1)[1].strip()
if li_text.split(splitter,1)[0].lower() == 'styles':
event.dance_styles = li_text.split(splitter,1)[1].lower().strip()
# event.dance_styles = li_text.split(splitter,1)[1].lower().strip().split(',')
event.key = create_key(event)
event_list = append_to_event_list(event, event.key, keys_from_spreadsheet)
# # pdb.set_trace()
# if str(event.key) not in keys_from_spreadsheet:
# event_list.append(event)
return event_list
def parseOneJson(self, obj):
count = 0;
if ((obj[self.eventResults] is not None) and (len(obj[self.eventResults])>0)):
for result in obj[self.eventResults]:
count += 1
sys.stdout.write("\rGetting " + str(count) + " of " + str(len(obj[self.eventResults])))
sys.stdout.flush()
event = Event()
event.name = self.checkIfList(self.eventName, result)
event.location = self.checkIfList(self.location, result)
event.description = self.checkIfList(self.description, result)
event.datetime = self.checkIfList(self.eventTime, result)
event.price = self.checkIfList(self.price, result)
event.imagePath = self.checkIfList(self.image, result)
event.dump = self.checkIfList(self.description, result)
event.forceStr()
self.allEvents.append(event)
print "\n"
return True
else:
return False
def create_event_list():
event_list = []
for row_number, row in enumerate(
utils.iter_worksheet(spreadsheet, 'Sheet1', header_row=1)):
if row['key'] != '' and row['obsolete'] != '1' and row[
'status'] != 'past':
event = Event()
event.key = row['key']
event.name = row['name']
event.start_date = parse(row['start date'])
event.end_date = parse(row['end date'])
event.city = row['city']
event.state = row['state']
event.country = row['country']
event.dance_styles = row['dance styles']
event.status = row['status']
event.url = row['url']
event.teachers = row['teachers']
event.bands = row['bands']
event.details = row['details']
event.obsolete = row['obsolete']
event.workshop_cost = get_cost(row, 'workshop cost')
event.party_pass_cost = get_cost(row, 'party pass cost')
event.distance = int(row['distance'])
event.flight_cost = int(row['flight cost'])
event.event_type = row['type']
if row['currency'] == '':
event.currency = 'USD'
else:
event.currency = row['currency']
if row['driving time'] == '':
event.driving_time = 99999
else:
event.driving_time = int(row['driving time'])
event_list.append(event)
return event_list
def create(self, aName, aDate, aVacancies):
newEvent = Event(parent=event.eventKey())
newEvent.name = aName
newEvent.date = aDate
newEvent.vacancies = aVacancies
newEvent.put()
