def add_presentation(url, category): print("Collecting from {}".format(url)) xpath = '//div[contains(@class,"presentation")]/h3/a' entries = html.fromstring(requests.get(url).text).xpath(xpath) ## Iterate through and extract the relevant content for a in entries: title = a.text if 'canceled' in title.lower(): continue root = html.fromstring( requests.get('https://us.pycon.org' + a.get('href')).text) speakers = root.xpath('//h4/a/text()') abstract = root.xpath('//div[@class="abstract"]')[0].text_content() try: level = root.xpath('//dl/dd/text()')[0] except ValueError: continue level = 'Beginner' if level == 'Novice' else level talk = Talk(category=category, conference_id=conference.id, title=title) data = db.TalkData(speakers, [], []) talk.abstract = abstract[:10000] talk.level = level db.add_talk(talk, **data._asdict())
def add_presentation_from_table(url, category): print("Collecting from {}".format(url)) xpath = '//td[contains(@class,"slot")]' entries = html.fromstring(requests.get(url).text).xpath(xpath) ## Iterate through and extract the relevant content for td in entries: a = td.find('./span[@class="title"]/a') if a is None: print('skipping...') continue title = a.text abstract = a.get('title') if 'canceled' not in title.lower(): if 'Jasmine Hsu' in title: speakers = title.split(',') title = 'Fire, bullets, and productivity' level = 'Beginner' else: speakers = td.findtext('./span[@class="speaker"]').strip() speakers = speakers.split( ',') if ',' in speakers else speakers.split('&') speakers = [s for s in speakers if s.strip() and not '?' in s] level = td.xpath('./comment()')[0].text.splitlines()[1].strip() level = 'Beginner' if level == 'Novice' else level talk = Talk(category=category, conference_id=conference.id, title=title) data = db.TalkData(speakers, [], []) talk.abstract = abstract[:10000] talk.level = level db.add_talk(talk, **data._asdict())
def add_new_talk(title, abstract, speaker, topic): if abstract and "__wbhack.init('https://web.archive.org/web');" in abstract: abstract = abstract.split( "__wbhack.init('https://web.archive.org/web');", 1)[-1] db.add_talk( Talk(category=Talk.TALK, conference_id=conference.id, title=title, abstract=abstract), **db.TalkData([speaker], [topic], [])._asdict()) print("adding new one not in list:", speaker, title, "\n***\n")
def add_presentation(url, category): print("Collecting from {}".format(url)) xpath = '//div[contains(@class,"box-content")]/*' entries = html.fromstring(requests.get(url).text).xpath(xpath) first = next(i for i, e in enumerate(entries) if e.tag == 'h2') ## Iterate through and extract the relevant content for i in range(int((len(entries) - first) / 3)): h2, p, div = entries[first + 3 * i:first + 3 * (1 + i)] title = h2.text_content() if 'canceled' in title.lower(): continue speakers = p.text_content().strip('\n ').split('\n', 1)[0].split(',') speakers = [s for s in speakers if s.strip() and not '?' in s] abstract = div.text_content().strip() talk = Talk(category=category, conference_id=conference.id, title=title) data = db.TalkData(speakers, [], []) talk.abstract = abstract[:10000] db.add_talk(talk, **data._asdict())
def add_presentation_from_table(url, category): print("Collecting from {}".format(url)) xpath = '//td[contains(@class,"slot")]' entries = html.fromstring(requests.get(url).text).xpath(xpath) ## Iterate through and extract the relevant content for td in entries: a = td.find('./span[@class="title"]/a') if a is None: continue title = a.text abstract = a.get('title') if 'canceled' not in title.lower(): speakers = td.findtext('./span[@class="speaker"]').strip() speakers = speakers.split( ',') if ',' in speakers else speakers.split('&') level = td.findtext('./span[@class="audience_level"]').strip() level = 'Beginner' if level == 'Novice' else level talk = Talk(category=category, conference_id=conference.id, title=title) data = db.TalkData(speakers, [], []) talk.abstract = abstract[:10000] talk.level = level db.add_talk(talk, **data._asdict())
volunteer.volunteering.append(conference) db.session.commit() ## Talks ## ~~~~~~ ## ## Keynotes keynotes = ( #(['Kelsey Hightower'], 'Kubernetes for Pythonistas', ['Google'], 'http://pyvideo.org/pycon-us-2017/keynote-kubernetes-for-pythonistas.html', ['voice', 'kubernetes', 'containers']), ) for speaker_names, title, org, url, topics in keynotes: talk = Talk(category=Talk.KEYNOTE, conference_id=conference.id) talk.title = title talk.video_url = url data = db.TalkData(speaker_names, topics, org) db.add_talk(talk, **data._asdict()) ## Tutorials, talks, and posters def add_presentation(url, category): print("Collecting from {}".format(url)) xpath = '//div[contains(@class,"box-content")]/*' entries = html.fromstring(requests.get(url).text).xpath(xpath) first = next(i for i, e in enumerate(entries) if e.tag == 'h2') ## Iterate through and extract the relevant content for i in range(int((len(entries) - first) / 3)): h2, p, div = entries[first + 3 * i:first + 3 * (1 + i)] title = h2.text_content() if 'canceled' in title.lower(): continue
db.session.commit() ## Talks keynotes = ( (['Guido van Rossum'], 'Update on the state of Python', None), (['Steve Huffman', 'Alexis Ohanian'], 'Reddit', "Reddit's origin and the switch to Python") ) for speaker_names, title, abstract in keynotes: talk = Talk(category=Talk.KEYNOTE, conference_id=conference.id) talk.title = title if title == 'Reddit': data.organization_names.append('Reddit') if abstract: talk.abstract = abstract data = db.TalkData(speaker_names, [], []) db.add_talk(talk, **data._asdict()) ## Tutorials ## ==> Ignore these...the links are broken and only the presenters' ## last names are given, so it is hard to create an entry. ## #wayback = 'https://web.archive.org/web/20090518174359/' #url = wayback + 'http://us.pycon.org:80/2009/tutorials/schedule' #xpath = '//div[@id="tutorials"]//li' #entries = html.fromstring(requests.get(url).text).xpath(xpath) ## Iterate through and extract the relevant content #for e in entries: # tmp = e.text_content() # if 'cancel' in tmp.lower():
## Talks ## ~~~~~~ ## ## Keynotes keynotes = ( (['Eben Upton'], 'The Raspberry Pi: providing children around the world the opportunity to learn programming', ['Raspberry Pi Foundation'], 'http://pyvideo.org/pycon-us-2013/keynote-2.html', ['education']), (['Raymond Hettinger'], 'What makes Python Awesome', [], 'http://pyvideo.org/pycon-us-2013/keynote-3.html', ['core']), #(['Jessica McKellar'], 'How the Internet works', [], 'http://pyvideo.org/pycon-us-2013/how-the-internet-works.html', ['web', 'twisted', 'scapy']), (['Guido van Rossum'], 'Announcing asyncio for the standard library (PEP 3156)', [], 'http://pyvideo.org/pycon-us-2013/keynote-1.html', ['concurrency', 'standard library']) ) for speaker_names, title, org, url, topics in keynotes: talk = Talk(category=Talk.KEYNOTE, conference_id=conference.id) talk.title = title talk.video_url = url data = db.TalkData(speaker_names, topics, org) db.add_talk(talk, **data._asdict()) ## Tutorials ## ==> Ignore these...the links are broken and only the presenters' ## last names are given, so it is hard to create an entry. ## url = 'https://us.pycon.org/2013/schedule/tutorials/list/' xpath = '//div[contains(@class,"presentation")]' entries = html.fromstring(requests.get(url).text).xpath(xpath) print('tutorials') print(url) ## Iterate through and extract the relevant content for e in entries:
print(url) for volunteer_name in html.fromstring(requests.get(url).text).xpath(xpath): volunteer_name = volunteer_name.text_content().strip() if len(volunteer_name) == 0: continue # There can be multiple comma-separated names. for name in volunteer_name.split(','): volunteer = db.fetch_or_add(Human(name=name)) if conference not in volunteer.volunteering: volunteer.volunteering.append(conference) db.session.commit() ## Talks talk = Talk(category=Talk.KEYNOTE, conference_id=conference.id) data = db.TalkData([], [], []) wayback = 'https://web.archive.org/web/20070207091801/' url = wayback + 'http://us.pycon.org:80/TX2007/Keynotes' xpath = '//div[@id="wikitext"]/*' entries = html.fromstring(requests.get(url).text).xpath(xpath) first_talk = next(i for i,e in enumerate(entries) if e.tag == 'h2') entries = entries[first_talk:-1] print('talks') print(url) for e in entries: if e.tag == 'h2': if talk.title is not None: # Finished one. db.add_talk(talk, **data._asdict()) data = db.TalkData([], [], []) talk = Talk(category=Talk.KEYNOTE, conference_id=conference.id)
keynotes = (( ['Paul Graham'], 'Frighteningly Ambitious Startup Ideas', ['YCombinator'], 'http://pyvideo.org/pycon-us-2012/keynote-paul-graham-ycombinator.html' ), (['Stormy Peters'], 'Growing the web community and the Python community', [ 'Mozilla' ], 'http://pyvideo.org/pycon-us-2012/keynote-stormy-peters-mozilla-corporation.html' ), (['David Beazley'], 'Tinkering with PyPy', [], 'http://pyvideo.org/pycon-us-2012/keynote-david-beazley.html'), (['Guido van Rossum' ], 'Addressing common questions about Python', [], 'http://pyvideo.org/pycon-us-2012/keynote-guido-van-rossum.html')) for speaker_names, title, org, url in keynotes: talk = Talk(category=Talk.KEYNOTE, conference_id=conference.id) talk.title = title talk.video_url = url data = db.TalkData(speaker_names, [], []) db.add_talk(talk, **data._asdict()) ## Startup Series url = 'https://us.pycon.org/2012/community/startuprow/' xpath = '//div[@class="page"]/*' entries = html.fromstring(requests.get(url).text).xpath(xpath) first_talk = next(i for i, e in enumerate(entries) if e.tag == 'h2') i = first_talk + 1 print('startup row') print(url) while i < len(entries): e = entries[i] i += 1 if e.tag == 'p': title = e.findtext('./a')
volunteer = db.fetch_or_add(Human(name=new_name)) if conference not in volunteer.volunteering: volunteer.volunteering.append(conference) db.session.commit() ## Talks ## ~~~~~~ ## ## Keynotes keynotes = ((['Hilary Mason'], 'Hello, PyCon'), (['Guido van Rossum'], 'A Fireside Chat with Guido van Rossum')) for speaker_names, title in keynotes: talk = Talk(category=Talk.KEYNOTE, conference_id=conference.id) talk.title = title data = db.TalkData(speaker_names, [], []) db.add_talk(talk, **data._asdict()) ## Startup Series talk = Talk(category=Talk.PLENARY, conference_id=conference.id) data = db.TalkData([], ['startup'], []) wayback = 'https://web.archive.org/web/20110316093256/' url = wayback + 'http://us.pycon.org:80/2011/home/keynotes/' xpath = '//div[@class="page"]/*' entries = html.fromstring(requests.get(url).text).xpath(xpath) first_talk = next(i for i, e in enumerate(entries) if e.tag == 'h1' and e.text.startswith('Startup')) entries = entries[first_talk + 1:] first_talk = next(i for i, e in enumerate(entries) if e.tag == 'h2') i = first_talk print('startup row')
print(url) for name in entries: volunteer = db.fetch_or_add(Human(name=name)) if conference not in volunteer.volunteering: volunteer.volunteering.append(conference) ## Talks # - Keynotes entries = (('The virtues of Open Source', 'Mitch Kapor', 'Open Source Applications Foundation (OSAF)'), ('Python State of the Union', 'Guido van Rossum', 'Zope corporation'), ('How to argue about typing', 'Bruce Eckel', None)) for title, speaker, org in entries: talk = Talk(category=Talk.KEYNOTE, conference_id=conference.id) data = db.TalkData([], [], []) talk.title = title data.speaker_names.append(speaker) if org is not None: data.organization_names.append(org) db.add_talk(talk, **data._asdict()) # - Tutorials # None? -- True. (PostMortem recommends tutorials for 2005) # - Regular talks #wayback = 'https://web.archive.org/web/20050206212138/' #url = wayback + 'http://www.python.org:80/pycon/dc2004/schedule.html url = 'https://wiki.python.org/moin/PyConDC2004/TalksByCategory' #xpath = '//td[@class="body"]/table/tbody/tr' xpath = '//div[@id="content"]/*'