def generate_wiki_schedules(wiki_url):
global wiki_schedule, workshop_schedule
data = Wiki(wiki_url)
print("Processing...")
wiki_schedule = Schedule.from_XC3_template('Wiki', congress_nr, 27, 4)
wiki_schedule.add_rooms(rooms)
# workshops are all events from the wiki, which are in workshop rooms – starting from day 0 (the 26.)
workshop_schedule = Schedule.from_XC3_template('Workshops', congress_nr, 26, 5)
workshop_schedule.add_rooms(rooms)
print("Combining data...")
global sessions_complete
sessions_complete = OrderedDict()
# process_wiki_events() fills global variables: out, wiki_schedule, workshop_schedule
process_wiki_events(data, wiki_schedule, workshop_schedule)
# write imported data from wiki to one merged file
with open("sessions_complete.json", "w") as fp:
json.dump(sessions_complete, fp, indent=2)
wiki_schedule.export("wiki")
# write all sessions in workshop rooms to an additional schedule.json/xml
workshop_schedule.export("workshops")
print('done')
return wiki_schedule
def export_schedule(filename, schedule):
with open("{}.schedule.json".format(filename), "w") as fp:
json.dump(schedule, fp, indent=4)
with open('{}.schedule.xml'.format(filename), 'w') as fp:
fp.write(Schedule(json=schedule).xml())
# validate xml
os.system('{} {}.schedule.xml'.format(validator, filename))
def generate_wiki_schedule(wiki_url: str, full_schedule: Schedule):
data = Wiki(wiki_url)
write('Wiki: Processing...')
wiki_schedule = Schedule.empty_copy_of(full_schedule, 'Wiki')
wiki_schedule.add_rooms(rooms)
load_sos_ids()
# process_wiki_events() fills global variables: out, wiki_schedule, workshop_schedule
process_wiki_events(data, wiki_schedule, timestamp_offset=-7200, options=options)
store_sos_ids()
# remove rooms from wiki schedule we already got via schedule.xml – looking at you THMS!
wiki_schedule.remove_room('Village:Three Headed Monkey')
write('Exporting... ')
wiki_schedule.export('wiki')
print('Wiki: done \n')
return wiki_schedule
def generate_wiki_schedule(wiki_url: str, full_schedule: Schedule):
data = Wiki(wiki_url)
write('Wiki: Processing...')
wiki_schedule = Schedule.empty_copy_of(full_schedule, 'Wiki', start_hour=9)
wiki_schedule.add_rooms(rooms)
load_sos_ids()
# process_wiki_events() fills global variables: out, wiki_schedule, workshop_schedule
process_wiki_events(data,
wiki_schedule,
timestamp_offset=-3600,
options=options)
store_sos_ids()
write('Exporting... ')
wiki_schedule.export('wiki')
print('Wiki: done \n')
return wiki_schedule
def main():
#main_schedule = get_schedule('main_rooms', main_schedule_url)
full_schedule = Schedule.from_url(main_schedule_url)
print(' version: ' + full_schedule.version())
# add addional rooms from this local config now, so they are in the correct order
full_schedule.add_rooms(rooms)
# add events from additional_schedule's to full_schedule
for entry in additional_schedule_urls:
try:
#other_schedule = get_schedule(entry['name'], entry['url'])
other_schedule = Schedule.from_url(entry['url'])
if 'version' in other_schedule.schedule():
full_schedule._schedule['schedule']['version'] += "; {}".format(entry['name'])
print(' version: ' + other_schedule.version())
else:
print(' WARNING: schedule "{}" does not have a version number'.format(entry['name']))
if full_schedule.add_events_from(other_schedule, id_offset=entry.get('id_offset'), options=entry.get('options')):
print(' success')
except:
print(' UNEXPECTED ERROR:' + str(sys.exc_info()[1]))
# write all events from the three big stages to a own schedule.json/xml
write('\nExporting three main stages... ')
full_schedule.export('stages')
print('\Building wiki schedule...')
# wiki
wiki_schedule = generate_wiki_schedule(wiki_url, full_schedule)
full_schedule._schedule['schedule']['version'] += "; wiki"
full_schedule.add_events_from(wiki_schedule)
# remove lighthing talk slot to fill with individual small events per lighthing talk
full_schedule.remove_event(id=10380)
# write all events to one big schedule.json/xml
write('\nExporting... ')
full_schedule.export('everything')
# write seperate file for each event, to get better git diffs
#full_schedule.foreach_event(lambda event: event.export('events/'))
def export_event(event):
with open("events/{}.json".format(event['guid']), "w") as fp:
json.dump(event, fp, indent=2, cls=ScheduleEncoder)
full_schedule.foreach_event(export_event)
print('\nDone')
print(' version: ' + full_schedule.version())
print('\n rooms of day 1: ')
for room in full_schedule.day(1)['rooms']:
print(' - ' + room)
if not local or options.git:
content_did_not_change = os.system('/usr/bin/env git diff -U0 --no-prefix | grep -e "^[+-] " | grep -v version > /dev/null')
def git(args):
os.system('/usr/bin/env git {}'.format(args))
if content_did_not_change:
print('nothing relevant changed, reverting to previous state')
git('reset --hard')
else:
git('add *.json *.xml events/*.json')
git('commit -m "version {}"'.format(full_schedule.version()))
git('push')
def fetch_schedule(wiki_url):
global template, days
# TODO refactor schedule class, to allow more generic templates
out = template
tz = pytz.timezone(out['schedule']['conference']['time_zone_name'])
conference_start_date = tz.localize(
dateutil.parser.parse(out['schedule']['conference']['start'] +
"T00:00:00"))
for i in range(out['schedule']['conference']['daysCount']):
date = conference_start_date + timedelta(days=i)
start = date + timedelta(hours=9) # conference day starts at 10:00
end = start + timedelta(hours=20) # conference day lasts 17 hours
days.append(
OrderedDict([
('index', i),
('date', date),
('start', start),
('end', end),
]))
out['schedule']['conference']['days'].append(
OrderedDict([('index', i + 1), ('date', date.strftime('%Y-%m-%d')),
('day_start', start.isoformat()),
('day_end', end.isoformat()),
('rooms', OrderedDict())]))
print("Requesting wiki events")
soup = BeautifulSoup(requests.get(wiki_url).text, 'html5lib')
#soup = BeautifulSoup(open("divoc-sessions.xhtml"), 'lxml')
#sections = soup.find_all('h3')
elements = soup.select('h3, table.inline')
print('Processing sections')
section_title = None
for element in elements:
if element.name == 'h3':
section_title = element
continue
# ignore some sections
if element.name == 'table':
if section_title.attrs['id'] in [
'durchgehende_treffpunkte_und_assemblies',
'wochentag_datum'
]:
continue
day = section_title.text.split(',')[1].strip() + "{}".format(year)
day_dt = tz.localize(datetime.strptime(day, '%d.%m.%Y'))
# ignore sections which are not in target time span
if day_dt < conference_start_date:
continue
rows = element.find_all('tr')
# skip header row
rows_iter = iter(rows)
next(rows_iter)
for row in rows_iter:
data = {}
for td in row.find_all('td'):
#if type(td) != NoneType:
key = td.attrs['class'][0]
data[key] = re.compile(r'\s*\n\s*').split(
td.get_text().strip())
try:
[start, end] = [
tz.localize(datetime.strptime(day + x, '%d.%m.%Y%H:%M'))
for x in re.compile(r'\s*(?:-|–)\s*').split(data['col0']
[0])
]
title = data['col1'][0]
abstract = "\n".join(data['col1'][1:])
persons = data['col2'][0]
links = data['col2'][1:]
guid = voc.tools.gen_uuid('{}-{}'.format(start, links[0]))
local_id = voc.tools.get_id(guid)
duration = (end - start).total_seconds() / 60
room = 'Kidspace' if 'Kidspace' in persons else 'Self-organized'
event_n = OrderedDict([
('id', local_id),
('guid', guid),
# ('logo', None),
('date', start.isoformat()),
('start', start.strftime('%H:%M')),
('duration', '%d:%02d' % divmod(duration, 60)),
('room', room),
('slug', '{slug}-{id}-{name}'.format(
slug=out['schedule']['conference']['acronym'].lower(),
id=local_id,
name=voc.tools.normalise_string(title.lower()))),
('url', wiki_url.split('?')[0]),
('title', title),
('subtitle', ''),
('track', 'Workshop'),
('type', 'Workshop'),
('language', 'de'),
('abstract', abstract or ''),
('description', ''),
(
'persons',
[
OrderedDict([
('id', 0),
('public_name', p.strip()),
#('#text', p),
]) for p in persons.split(',')
]),
('links', [{
'url': url,
'title': url
} for url in links])
])
day_rooms = out['schedule']['conference']['days'][get_day(
start)]['rooms']
if room not in day_rooms:
day_rooms[room] = []
day_rooms[room].append(event_n)
sys.stdout.write('.')
except Exception as e:
print(e)
print(data)
print(json.dumps(event_n, indent=2))
print()
#print(json.dumps(out, indent=2))
print()
print()
schedule = Schedule(json=out)
return schedule
def main():
try:
full_schedule = Schedule.from_url(main_schedule_url)
print(' version: ' + full_schedule.version())
#print(' contains {events_count} events, with local ids from {min_id} to {max_id}'.format(**full_schedule.stats.__dict__))
except:
full_schedule = Schedule.from_XC3_template(None, 37, 27, 4)
conference = full_schedule.conference()
conference['acronym'] = 'rC3'
conference['title'] = 'Remote Chaos Experience'
loaded_schedules = {main_schedule_url: True, 'https://frab.cccv.de/': True}
# add addional rooms from this local config now, so they are in the correct order
for key in rooms:
full_schedule.add_rooms(rooms[key])
previous_max_id = 0
# add events from additional_schedule's to full_schedule
for entry in additional_schedule_urls:
try:
print('\n== Channel ' + entry['name'])
url = entry['url'].replace('schedule.xml', 'schedule.json')
if entry.get('room_guid'):
full_schedule._room_ids[entry['schedule_room']
or entry['name']] = entry['room_guid']
if not url:
print(' has no schedule_url yet – ignoring')
continue
if url in loaded_schedules:
print(' schedule ' + url + ' was already loaded – ignoring')
continue
other_schedule = Schedule.from_url(url)
loaded_schedules[url] = True
if 'version' in other_schedule.schedule():
full_schedule._schedule['schedule'][
'version'] += "; {}".format(entry['name'])
print(' version: ' + other_schedule.version())
else:
print(
' WARNING: schedule "{}" does not have a version number'.
format(entry['name']))
id_offset = entry.get('id_offset') or id_offsets.get(
entry['name']) or 0
'''
print(' contains {events_count} events, with local ids from {min_id} to {max_id}'.format(**other_schedule.stats.__dict__))
min_id = other_schedule.stats.min_id + id_offset
max_id = other_schedule.stats.max_id + id_offset
print(' after adding the offset, ids reach from {} to {}'.format(min_id, max_id))
# TODO improve error message and check actual intervals
if previous_max_id >= min_id:
print(' WARNING: schedule "{}" might have ID overlap with other schedules'.format(entry['name']))
previous_max_id = max_id
'''
if full_schedule.add_events_from(other_schedule,
id_offset=id_offset,
options=entry.get('options')):
print(' success')
except KeyboardInterrupt:
exit()
except Exception as e:
print(' UNEXPECTED ERROR:' + str(sys.exc_info()[1]))
if options.exit_when_exception_occours:
raise e
# remove breaks from lightning talk schedule import
# full_schedule.remove_event(guid='bca1ec84-e62d-528a-b254-68401ece6c7c')
# write all events from the channels to a own schedule.json/xml
write('\nExporting channels... ')
channels = full_schedule.copy('Channels')
for day in channels._schedule['schedule']['conference']['days']:
i = 0
room_keys = list(day['rooms'].keys())
for room_key in room_keys:
if ('Workshop' in room_key or 'Meetup' in room_key) and \
not(i < 4 or room_key in rooms['channels']):
del day['rooms'][room_key]
i += 1
print('\n channels: ')
for room in channels.rooms():
print(' - ' + room)
channels.export('channels')
del channels
# remove talks starting before 9 am
def remove_too_early_events(room):
for event in room:
start_time = Event(event).start
if start_time.hour > 4 and start_time.hour < 9:
print(
'removing {} from full schedule, as it takes place at {} which is too early in the morning'
.format(event['title'], start_time.strftime('%H:%M')))
room.remove(event)
else:
break
full_schedule.foreach_day_room(remove_too_early_events)
# write all events to one big schedule.json/xml
write('\nExporting... ')
full_schedule.export('everything')
# write seperate file for each event, to get better git diffs
def export_event(event):
with open("events/{}.json".format(event['guid']), "w") as fp:
json.dump(
{
**event, 'room_guid':
full_schedule._room_ids.get(event['room'], None)
},
fp,
indent=2,
cls=ScheduleEncoder)
full_schedule.foreach_event(export_event)
print('\nDone')
print(' version: ' + full_schedule.version())
print('\n rooms: ')
for room in full_schedule.rooms():
print(' - ' + room)
if not local or options.git:
content_did_not_change = os.system(
'/usr/bin/env git diff -U0 --no-prefix | grep -e "^[+-] " | grep -v version > /dev/null'
)
def git(args):
os.system('/usr/bin/env git {}'.format(args))
if content_did_not_change:
print('nothing relevant changed, reverting to previous state')
git('reset --hard')
else:
git('add *.json *.xml events/*.json')
git('commit -m "version {}"'.format(full_schedule.version()))
git('push')
def main():
global local, options
full_schedule = Schedule.from_url(main_schedule_url)
print(' version: ' + full_schedule.version())
print(
' contains {events_count} events, with local ids from {min_id} to {max_id}'
.format(**full_schedule.stats.__dict__))
# add addional rooms from this local config now, so they are in the correct order
for key in rooms:
full_schedule.add_rooms(rooms[key])
previous_max_id = 0
# add events from additional_schedule's to full_schedule
for entry in additional_schedule_urls:
try:
#other_schedule = get_schedule(entry['name'], entry['url'])
other_schedule = Schedule.from_url(entry['url'])
if 'version' in other_schedule.schedule():
full_schedule._schedule['schedule'][
'version'] += "; {}".format(entry['name'])
print(' version: ' + other_schedule.version())
else:
print(
' WARNING: schedule "{}" does not have a version number'.
format(entry['name']))
print(
' contains {events_count} events, with local ids from {min_id} to {max_id}'
.format(**other_schedule.stats.__dict__))
id_offset = entry.get('id_offset')
if not id_offset:
id_offset = 0
min_id = other_schedule.stats.min_id + id_offset
max_id = other_schedule.stats.max_id + id_offset
print(
' after adding the offset, ids reach from {} to {}'.format(
min_id, max_id))
if previous_max_id >= min_id:
print(
' WARNING: schedule "{}" has ID overlap with previous schedule'
.format(entry['name']))
previous_max_id = max_id
if full_schedule.add_events_from(other_schedule,
id_offset=id_offset,
options=entry.get('options')):
print(' success')
except KeyboardInterrupt:
exit()
except Exception as e:
print(' UNEXPECTED ERROR:' + str(sys.exc_info()[1]))
if options.exit_when_exception_occours:
raise e
print('\nBuilding wiki schedule...')
# wiki
wiki_schedule = generate_wiki_schedule(wiki_url, full_schedule)
full_schedule._schedule['schedule']['version'] += "; wiki"
full_schedule.add_events_from(wiki_schedule)
# write all events to one big schedule.json/xml
write('\nExporting... ')
full_schedule.export('everything')
# write seperate file for each event, to get better git diffs
#full_schedule.foreach_event(lambda event: event.export('events/'))
#def export_event(event):
# with open("events/{}.json".format(event['guid']), "w") as fp:
# json.dump(event, fp, indent=2, cls=ScheduleEncoder)
#
#full_schedule.foreach_event(export_event)
print('\nDone')
print(' version: ' + full_schedule.version())
print('\n rooms of day 1: ')
for room in full_schedule.day(1)['rooms']:
print(' - ' + room)
if not local or options.git:
content_did_not_change = os.system(
'/usr/bin/env git diff -U0 --no-prefix | grep -e "^[+-] " | grep -v version > /dev/null'
)
def git(args):
os.system('/usr/bin/env git {}'.format(args))
if content_did_not_change:
print('nothing relevant changed, reverting to previous state')
git('reset --hard')
else:
git('add *.json *.xml')
git('commit -m "version {}"'.format(full_schedule.version()))
def main():
#main_schedule = get_schedule('main_rooms', main_schedule_url)
try:
full_schedule = Schedule.from_url(main_schedule_url)
except:
full_schedule = Schedule.from_XC3_template(None, congress_nr, 27, 4)
print(' version: ' + full_schedule.version())
print(
' contains {events_count} events, with local ids from {min_id} to {max_id}'
.format(**full_schedule.stats.__dict__))
# add addional rooms from this local config now, so they are in the correct order
for key in rooms:
full_schedule.add_rooms(rooms[key])
previous_max_id = 0
# add events from additional_schedule's to full_schedule
for entry in additional_schedule_urls:
try:
#other_schedule = get_schedule(entry['name'], entry['url'])
other_schedule = Schedule.from_url(entry['url'])
if 'version' in other_schedule.schedule():
full_schedule._schedule['schedule'][
'version'] += "; {}".format(entry['name'])
print(' version: ' + other_schedule.version())
else:
print(
' WARNING: schedule "{}" does not have a version number'.
format(entry['name']))
print(
' contains {events_count} events, with local ids from {min_id} to {max_id}'
.format(**other_schedule.stats.__dict__))
id_offset = entry.get('id_offset')
if not id_offset:
id_offset = 0
min_id = other_schedule.stats.min_id + id_offset
max_id = other_schedule.stats.max_id + id_offset
print(
' after adding the offset, ids reach from {} to {}'.format(
min_id, max_id))
if previous_max_id >= min_id:
print(
' WARNING: schedule "{}" has ID overlap with previous schedule'
.format(entry['name']))
previous_max_id = max_id
if full_schedule.add_events_from(other_schedule,
id_offset=id_offset,
options=entry.get('options')):
print(' success')
except KeyboardInterrupt:
exit()
except Exception as e:
print(' UNEXPECTED ERROR:' + str(sys.exc_info()[1]))
if options.exit_when_exception_occours:
raise e
# remove breaks from lightning talk schedule import
full_schedule.remove_event(guid='bca1ec84-e62d-528a-b254-68401ece6c7c')
full_schedule.remove_event(guid='cda64c9e-b230-589a-ace0-6beca2693eff')
full_schedule.remove_event(guid='f33dd7b7-99d6-574b-9282-26986b5a0ea0')
# write all events from the stages to a own schedule.json/xml
write('\nExporting main stages... ')
stages = full_schedule.copy('Stages')
for day in stages._schedule['schedule']['conference']['days']:
i = 0
room_keys = list(day['rooms'].keys())
for room_key in room_keys:
if not (i < 5 or room_key in rooms['stages'] or 'Stage' in room_key
or 'Bühne' in room_key):
del day['rooms'][room_key]
i += 1
print('\n stages of day 1: ')
for room in stages.day(1)['rooms']:
print(' - ' + room)
stages.export('stages')
del stages
print('\nBuilding wiki schedule...')
# wiki
wiki_schedule = generate_wiki_schedule(wiki_url, full_schedule)
full_schedule._schedule['schedule']['version'] += "; wiki"
full_schedule.add_events_from(wiki_schedule)
# remove rooms from wiki import, which we already have in more detail as pretalx rooms
full_schedule.remove_room('Assembly:Art-and-Play')
full_schedule.remove_room('Assembly:ChaosZone')
full_schedule.remove_room('Assembly:WikipakaWG')
# remove lighthing talk slot to fill with individual small events per lighthing talk
#full_schedule.remove_event(id=10380)
# remove talks starting before 9 am
def remove_too_early_events(room):
for event in room:
start_time = Event(event).start
if start_time.hour > 4 and start_time.hour < 9:
print(
'removing {} from full schedule, as it takes place at {} which is too early in the morning'
.format(event['title'], start_time.strftime('%H:%M')))
room.remove(event)
else:
break
full_schedule.foreach_day_room(remove_too_early_events)
# write all events to one big schedule.json/xml
write('\nExporting... ')
full_schedule.export('everything')
# write seperate file for each event, to get better git diffs
#full_schedule.foreach_event(lambda event: event.export('events/'))
def export_event(event):
with open("events/{}.json".format(event['guid']), "w") as fp:
json.dump(event, fp, indent=2, cls=ScheduleEncoder)
full_schedule.foreach_event(export_event)
print('\nDone')
print(' version: ' + full_schedule.version())
print('\n rooms of day 1: ')
for room in full_schedule.day(1)['rooms']:
print(' - ' + room)
if not local or options.git:
content_did_not_change = os.system(
'/usr/bin/env git diff -U0 --no-prefix | grep -e "^[+-] " | grep -v version > /dev/null'
)
def git(args):
os.system('/usr/bin/env git {}'.format(args))
if content_did_not_change:
print('nothing relevant changed, reverting to previous state')
git('reset --hard')
else:
git('add *.json *.xml events/*.json')
git('commit -m "version {}"'.format(full_schedule.version()))
git('push')