def generate_wiki_schedules(wiki_url): global wiki_schedule, workshop_schedule data = Wiki(wiki_url) print("Processing...") wiki_schedule = Schedule.from_XC3_template('Wiki', congress_nr, 27, 4) wiki_schedule.add_rooms(rooms) # workshops are all events from the wiki, which are in workshop rooms – starting from day 0 (the 26.) workshop_schedule = Schedule.from_XC3_template('Workshops', congress_nr, 26, 5) workshop_schedule.add_rooms(rooms) print("Combining data...") global sessions_complete sessions_complete = OrderedDict() # process_wiki_events() fills global variables: out, wiki_schedule, workshop_schedule process_wiki_events(data, wiki_schedule, workshop_schedule) # write imported data from wiki to one merged file with open("sessions_complete.json", "w") as fp: json.dump(sessions_complete, fp, indent=2) wiki_schedule.export("wiki") # write all sessions in workshop rooms to an additional schedule.json/xml workshop_schedule.export("workshops") print('done') return wiki_schedule
def export_schedule(filename, schedule): with open("{}.schedule.json".format(filename), "w") as fp: json.dump(schedule, fp, indent=4) with open('{}.schedule.xml'.format(filename), 'w') as fp: fp.write(Schedule(json=schedule).xml()) # validate xml os.system('{} {}.schedule.xml'.format(validator, filename))
def generate_wiki_schedule(wiki_url: str, full_schedule: Schedule): data = Wiki(wiki_url) write('Wiki: Processing...') wiki_schedule = Schedule.empty_copy_of(full_schedule, 'Wiki') wiki_schedule.add_rooms(rooms) load_sos_ids() # process_wiki_events() fills global variables: out, wiki_schedule, workshop_schedule process_wiki_events(data, wiki_schedule, timestamp_offset=-7200, options=options) store_sos_ids() # remove rooms from wiki schedule we already got via schedule.xml – looking at you THMS! wiki_schedule.remove_room('Village:Three Headed Monkey') write('Exporting... ') wiki_schedule.export('wiki') print('Wiki: done \n') return wiki_schedule
def generate_wiki_schedule(wiki_url: str, full_schedule: Schedule): data = Wiki(wiki_url) write('Wiki: Processing...') wiki_schedule = Schedule.empty_copy_of(full_schedule, 'Wiki', start_hour=9) wiki_schedule.add_rooms(rooms) load_sos_ids() # process_wiki_events() fills global variables: out, wiki_schedule, workshop_schedule process_wiki_events(data, wiki_schedule, timestamp_offset=-3600, options=options) store_sos_ids() write('Exporting... ') wiki_schedule.export('wiki') print('Wiki: done \n') return wiki_schedule
def main(): #main_schedule = get_schedule('main_rooms', main_schedule_url) full_schedule = Schedule.from_url(main_schedule_url) print(' version: ' + full_schedule.version()) # add addional rooms from this local config now, so they are in the correct order full_schedule.add_rooms(rooms) # add events from additional_schedule's to full_schedule for entry in additional_schedule_urls: try: #other_schedule = get_schedule(entry['name'], entry['url']) other_schedule = Schedule.from_url(entry['url']) if 'version' in other_schedule.schedule(): full_schedule._schedule['schedule']['version'] += "; {}".format(entry['name']) print(' version: ' + other_schedule.version()) else: print(' WARNING: schedule "{}" does not have a version number'.format(entry['name'])) if full_schedule.add_events_from(other_schedule, id_offset=entry.get('id_offset'), options=entry.get('options')): print(' success') except: print(' UNEXPECTED ERROR:' + str(sys.exc_info()[1])) # write all events from the three big stages to a own schedule.json/xml write('\nExporting three main stages... ') full_schedule.export('stages') print('\Building wiki schedule...') # wiki wiki_schedule = generate_wiki_schedule(wiki_url, full_schedule) full_schedule._schedule['schedule']['version'] += "; wiki" full_schedule.add_events_from(wiki_schedule) # remove lighthing talk slot to fill with individual small events per lighthing talk full_schedule.remove_event(id=10380) # write all events to one big schedule.json/xml write('\nExporting... ') full_schedule.export('everything') # write seperate file for each event, to get better git diffs #full_schedule.foreach_event(lambda event: event.export('events/')) def export_event(event): with open("events/{}.json".format(event['guid']), "w") as fp: json.dump(event, fp, indent=2, cls=ScheduleEncoder) full_schedule.foreach_event(export_event) print('\nDone') print(' version: ' + full_schedule.version()) print('\n rooms of day 1: ') for room in full_schedule.day(1)['rooms']: print(' - ' + room) if not local or options.git: content_did_not_change = os.system('/usr/bin/env git diff -U0 --no-prefix | grep -e "^[+-] " | grep -v version > /dev/null') def git(args): os.system('/usr/bin/env git {}'.format(args)) if content_did_not_change: print('nothing relevant changed, reverting to previous state') git('reset --hard') else: git('add *.json *.xml events/*.json') git('commit -m "version {}"'.format(full_schedule.version())) git('push')
def fetch_schedule(wiki_url): global template, days # TODO refactor schedule class, to allow more generic templates out = template tz = pytz.timezone(out['schedule']['conference']['time_zone_name']) conference_start_date = tz.localize( dateutil.parser.parse(out['schedule']['conference']['start'] + "T00:00:00")) for i in range(out['schedule']['conference']['daysCount']): date = conference_start_date + timedelta(days=i) start = date + timedelta(hours=9) # conference day starts at 10:00 end = start + timedelta(hours=20) # conference day lasts 17 hours days.append( OrderedDict([ ('index', i), ('date', date), ('start', start), ('end', end), ])) out['schedule']['conference']['days'].append( OrderedDict([('index', i + 1), ('date', date.strftime('%Y-%m-%d')), ('day_start', start.isoformat()), ('day_end', end.isoformat()), ('rooms', OrderedDict())])) print("Requesting wiki events") soup = BeautifulSoup(requests.get(wiki_url).text, 'html5lib') #soup = BeautifulSoup(open("divoc-sessions.xhtml"), 'lxml') #sections = soup.find_all('h3') elements = soup.select('h3, table.inline') print('Processing sections') section_title = None for element in elements: if element.name == 'h3': section_title = element continue # ignore some sections if element.name == 'table': if section_title.attrs['id'] in [ 'durchgehende_treffpunkte_und_assemblies', 'wochentag_datum' ]: continue day = section_title.text.split(',')[1].strip() + "{}".format(year) day_dt = tz.localize(datetime.strptime(day, '%d.%m.%Y')) # ignore sections which are not in target time span if day_dt < conference_start_date: continue rows = element.find_all('tr') # skip header row rows_iter = iter(rows) next(rows_iter) for row in rows_iter: data = {} for td in row.find_all('td'): #if type(td) != NoneType: key = td.attrs['class'][0] data[key] = re.compile(r'\s*\n\s*').split( td.get_text().strip()) try: [start, end] = [ tz.localize(datetime.strptime(day + x, '%d.%m.%Y%H:%M')) for x in re.compile(r'\s*(?:-|–)\s*').split(data['col0'] [0]) ] title = data['col1'][0] abstract = "\n".join(data['col1'][1:]) persons = data['col2'][0] links = data['col2'][1:] guid = voc.tools.gen_uuid('{}-{}'.format(start, links[0])) local_id = voc.tools.get_id(guid) duration = (end - start).total_seconds() / 60 room = 'Kidspace' if 'Kidspace' in persons else 'Self-organized' event_n = OrderedDict([ ('id', local_id), ('guid', guid), # ('logo', None), ('date', start.isoformat()), ('start', start.strftime('%H:%M')), ('duration', '%d:%02d' % divmod(duration, 60)), ('room', room), ('slug', '{slug}-{id}-{name}'.format( slug=out['schedule']['conference']['acronym'].lower(), id=local_id, name=voc.tools.normalise_string(title.lower()))), ('url', wiki_url.split('?')[0]), ('title', title), ('subtitle', ''), ('track', 'Workshop'), ('type', 'Workshop'), ('language', 'de'), ('abstract', abstract or ''), ('description', ''), ( 'persons', [ OrderedDict([ ('id', 0), ('public_name', p.strip()), #('#text', p), ]) for p in persons.split(',') ]), ('links', [{ 'url': url, 'title': url } for url in links]) ]) day_rooms = out['schedule']['conference']['days'][get_day( start)]['rooms'] if room not in day_rooms: day_rooms[room] = [] day_rooms[room].append(event_n) sys.stdout.write('.') except Exception as e: print(e) print(data) print(json.dumps(event_n, indent=2)) print() #print(json.dumps(out, indent=2)) print() print() schedule = Schedule(json=out) return schedule
def main(): try: full_schedule = Schedule.from_url(main_schedule_url) print(' version: ' + full_schedule.version()) #print(' contains {events_count} events, with local ids from {min_id} to {max_id}'.format(**full_schedule.stats.__dict__)) except: full_schedule = Schedule.from_XC3_template(None, 37, 27, 4) conference = full_schedule.conference() conference['acronym'] = 'rC3' conference['title'] = 'Remote Chaos Experience' loaded_schedules = {main_schedule_url: True, 'https://frab.cccv.de/': True} # add addional rooms from this local config now, so they are in the correct order for key in rooms: full_schedule.add_rooms(rooms[key]) previous_max_id = 0 # add events from additional_schedule's to full_schedule for entry in additional_schedule_urls: try: print('\n== Channel ' + entry['name']) url = entry['url'].replace('schedule.xml', 'schedule.json') if entry.get('room_guid'): full_schedule._room_ids[entry['schedule_room'] or entry['name']] = entry['room_guid'] if not url: print(' has no schedule_url yet – ignoring') continue if url in loaded_schedules: print(' schedule ' + url + ' was already loaded – ignoring') continue other_schedule = Schedule.from_url(url) loaded_schedules[url] = True if 'version' in other_schedule.schedule(): full_schedule._schedule['schedule'][ 'version'] += "; {}".format(entry['name']) print(' version: ' + other_schedule.version()) else: print( ' WARNING: schedule "{}" does not have a version number'. format(entry['name'])) id_offset = entry.get('id_offset') or id_offsets.get( entry['name']) or 0 ''' print(' contains {events_count} events, with local ids from {min_id} to {max_id}'.format(**other_schedule.stats.__dict__)) min_id = other_schedule.stats.min_id + id_offset max_id = other_schedule.stats.max_id + id_offset print(' after adding the offset, ids reach from {} to {}'.format(min_id, max_id)) # TODO improve error message and check actual intervals if previous_max_id >= min_id: print(' WARNING: schedule "{}" might have ID overlap with other schedules'.format(entry['name'])) previous_max_id = max_id ''' if full_schedule.add_events_from(other_schedule, id_offset=id_offset, options=entry.get('options')): print(' success') except KeyboardInterrupt: exit() except Exception as e: print(' UNEXPECTED ERROR:' + str(sys.exc_info()[1])) if options.exit_when_exception_occours: raise e # remove breaks from lightning talk schedule import # full_schedule.remove_event(guid='bca1ec84-e62d-528a-b254-68401ece6c7c') # write all events from the channels to a own schedule.json/xml write('\nExporting channels... ') channels = full_schedule.copy('Channels') for day in channels._schedule['schedule']['conference']['days']: i = 0 room_keys = list(day['rooms'].keys()) for room_key in room_keys: if ('Workshop' in room_key or 'Meetup' in room_key) and \ not(i < 4 or room_key in rooms['channels']): del day['rooms'][room_key] i += 1 print('\n channels: ') for room in channels.rooms(): print(' - ' + room) channels.export('channels') del channels # remove talks starting before 9 am def remove_too_early_events(room): for event in room: start_time = Event(event).start if start_time.hour > 4 and start_time.hour < 9: print( 'removing {} from full schedule, as it takes place at {} which is too early in the morning' .format(event['title'], start_time.strftime('%H:%M'))) room.remove(event) else: break full_schedule.foreach_day_room(remove_too_early_events) # write all events to one big schedule.json/xml write('\nExporting... ') full_schedule.export('everything') # write seperate file for each event, to get better git diffs def export_event(event): with open("events/{}.json".format(event['guid']), "w") as fp: json.dump( { **event, 'room_guid': full_schedule._room_ids.get(event['room'], None) }, fp, indent=2, cls=ScheduleEncoder) full_schedule.foreach_event(export_event) print('\nDone') print(' version: ' + full_schedule.version()) print('\n rooms: ') for room in full_schedule.rooms(): print(' - ' + room) if not local or options.git: content_did_not_change = os.system( '/usr/bin/env git diff -U0 --no-prefix | grep -e "^[+-] " | grep -v version > /dev/null' ) def git(args): os.system('/usr/bin/env git {}'.format(args)) if content_did_not_change: print('nothing relevant changed, reverting to previous state') git('reset --hard') else: git('add *.json *.xml events/*.json') git('commit -m "version {}"'.format(full_schedule.version())) git('push')
def main(): global local, options full_schedule = Schedule.from_url(main_schedule_url) print(' version: ' + full_schedule.version()) print( ' contains {events_count} events, with local ids from {min_id} to {max_id}' .format(**full_schedule.stats.__dict__)) # add addional rooms from this local config now, so they are in the correct order for key in rooms: full_schedule.add_rooms(rooms[key]) previous_max_id = 0 # add events from additional_schedule's to full_schedule for entry in additional_schedule_urls: try: #other_schedule = get_schedule(entry['name'], entry['url']) other_schedule = Schedule.from_url(entry['url']) if 'version' in other_schedule.schedule(): full_schedule._schedule['schedule'][ 'version'] += "; {}".format(entry['name']) print(' version: ' + other_schedule.version()) else: print( ' WARNING: schedule "{}" does not have a version number'. format(entry['name'])) print( ' contains {events_count} events, with local ids from {min_id} to {max_id}' .format(**other_schedule.stats.__dict__)) id_offset = entry.get('id_offset') if not id_offset: id_offset = 0 min_id = other_schedule.stats.min_id + id_offset max_id = other_schedule.stats.max_id + id_offset print( ' after adding the offset, ids reach from {} to {}'.format( min_id, max_id)) if previous_max_id >= min_id: print( ' WARNING: schedule "{}" has ID overlap with previous schedule' .format(entry['name'])) previous_max_id = max_id if full_schedule.add_events_from(other_schedule, id_offset=id_offset, options=entry.get('options')): print(' success') except KeyboardInterrupt: exit() except Exception as e: print(' UNEXPECTED ERROR:' + str(sys.exc_info()[1])) if options.exit_when_exception_occours: raise e print('\nBuilding wiki schedule...') # wiki wiki_schedule = generate_wiki_schedule(wiki_url, full_schedule) full_schedule._schedule['schedule']['version'] += "; wiki" full_schedule.add_events_from(wiki_schedule) # write all events to one big schedule.json/xml write('\nExporting... ') full_schedule.export('everything') # write seperate file for each event, to get better git diffs #full_schedule.foreach_event(lambda event: event.export('events/')) #def export_event(event): # with open("events/{}.json".format(event['guid']), "w") as fp: # json.dump(event, fp, indent=2, cls=ScheduleEncoder) # #full_schedule.foreach_event(export_event) print('\nDone') print(' version: ' + full_schedule.version()) print('\n rooms of day 1: ') for room in full_schedule.day(1)['rooms']: print(' - ' + room) if not local or options.git: content_did_not_change = os.system( '/usr/bin/env git diff -U0 --no-prefix | grep -e "^[+-] " | grep -v version > /dev/null' ) def git(args): os.system('/usr/bin/env git {}'.format(args)) if content_did_not_change: print('nothing relevant changed, reverting to previous state') git('reset --hard') else: git('add *.json *.xml') git('commit -m "version {}"'.format(full_schedule.version()))
def main(): #main_schedule = get_schedule('main_rooms', main_schedule_url) try: full_schedule = Schedule.from_url(main_schedule_url) except: full_schedule = Schedule.from_XC3_template(None, congress_nr, 27, 4) print(' version: ' + full_schedule.version()) print( ' contains {events_count} events, with local ids from {min_id} to {max_id}' .format(**full_schedule.stats.__dict__)) # add addional rooms from this local config now, so they are in the correct order for key in rooms: full_schedule.add_rooms(rooms[key]) previous_max_id = 0 # add events from additional_schedule's to full_schedule for entry in additional_schedule_urls: try: #other_schedule = get_schedule(entry['name'], entry['url']) other_schedule = Schedule.from_url(entry['url']) if 'version' in other_schedule.schedule(): full_schedule._schedule['schedule'][ 'version'] += "; {}".format(entry['name']) print(' version: ' + other_schedule.version()) else: print( ' WARNING: schedule "{}" does not have a version number'. format(entry['name'])) print( ' contains {events_count} events, with local ids from {min_id} to {max_id}' .format(**other_schedule.stats.__dict__)) id_offset = entry.get('id_offset') if not id_offset: id_offset = 0 min_id = other_schedule.stats.min_id + id_offset max_id = other_schedule.stats.max_id + id_offset print( ' after adding the offset, ids reach from {} to {}'.format( min_id, max_id)) if previous_max_id >= min_id: print( ' WARNING: schedule "{}" has ID overlap with previous schedule' .format(entry['name'])) previous_max_id = max_id if full_schedule.add_events_from(other_schedule, id_offset=id_offset, options=entry.get('options')): print(' success') except KeyboardInterrupt: exit() except Exception as e: print(' UNEXPECTED ERROR:' + str(sys.exc_info()[1])) if options.exit_when_exception_occours: raise e # remove breaks from lightning talk schedule import full_schedule.remove_event(guid='bca1ec84-e62d-528a-b254-68401ece6c7c') full_schedule.remove_event(guid='cda64c9e-b230-589a-ace0-6beca2693eff') full_schedule.remove_event(guid='f33dd7b7-99d6-574b-9282-26986b5a0ea0') # write all events from the stages to a own schedule.json/xml write('\nExporting main stages... ') stages = full_schedule.copy('Stages') for day in stages._schedule['schedule']['conference']['days']: i = 0 room_keys = list(day['rooms'].keys()) for room_key in room_keys: if not (i < 5 or room_key in rooms['stages'] or 'Stage' in room_key or 'Bühne' in room_key): del day['rooms'][room_key] i += 1 print('\n stages of day 1: ') for room in stages.day(1)['rooms']: print(' - ' + room) stages.export('stages') del stages print('\nBuilding wiki schedule...') # wiki wiki_schedule = generate_wiki_schedule(wiki_url, full_schedule) full_schedule._schedule['schedule']['version'] += "; wiki" full_schedule.add_events_from(wiki_schedule) # remove rooms from wiki import, which we already have in more detail as pretalx rooms full_schedule.remove_room('Assembly:Art-and-Play') full_schedule.remove_room('Assembly:ChaosZone') full_schedule.remove_room('Assembly:WikipakaWG') # remove lighthing talk slot to fill with individual small events per lighthing talk #full_schedule.remove_event(id=10380) # remove talks starting before 9 am def remove_too_early_events(room): for event in room: start_time = Event(event).start if start_time.hour > 4 and start_time.hour < 9: print( 'removing {} from full schedule, as it takes place at {} which is too early in the morning' .format(event['title'], start_time.strftime('%H:%M'))) room.remove(event) else: break full_schedule.foreach_day_room(remove_too_early_events) # write all events to one big schedule.json/xml write('\nExporting... ') full_schedule.export('everything') # write seperate file for each event, to get better git diffs #full_schedule.foreach_event(lambda event: event.export('events/')) def export_event(event): with open("events/{}.json".format(event['guid']), "w") as fp: json.dump(event, fp, indent=2, cls=ScheduleEncoder) full_schedule.foreach_event(export_event) print('\nDone') print(' version: ' + full_schedule.version()) print('\n rooms of day 1: ') for room in full_schedule.day(1)['rooms']: print(' - ' + room) if not local or options.git: content_did_not_change = os.system( '/usr/bin/env git diff -U0 --no-prefix | grep -e "^[+-] " | grep -v version > /dev/null' ) def git(args): os.system('/usr/bin/env git {}'.format(args)) if content_did_not_change: print('nothing relevant changed, reverting to previous state') git('reset --hard') else: git('add *.json *.xml events/*.json') git('commit -m "version {}"'.format(full_schedule.version())) git('push')