def harvest_html_agendas(): settings = get_settings() client = YTMigration().clientGet(settings["email"]) s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"], client["s3_bucket"]) granicus = GranicusUtils(client, s3) sessions = YTMigration().sessionsGet(client["id"]) for session in sessions: if session["status"] == "uploaded": agenda_html = granicus.htmlAgendaTransform(session["session_id"]) if agenda_html: storage_folder = "/nvme/client_files/" + client["email"] agenda_folder = storage_folder + "/documents/agenda_html" agenda_filename = session["session_id"] + "_agenda.html" agenda_path = agenda_folder + "/" + agenda_filename agenda_url = "https://s3-youtube.open.media/client_files/" + client[ "email"] + "/documents/agenda_html/" + agenda_filename os.makedirs(agenda_folder, exist_ok=True) html_file = open(agenda_path, "w") html_file.write(agenda_html) html_file.close() print("Harvested: " + agenda_path)
def harvest_html_minutes(): settings = get_settings("Store and transform Granicus html minutes") client = YTMigration().clientGet(settings["email"]) s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"], client["s3_bucket"]) granicus = GranicusUtils(client, s3) sessions = YTMigration().sessionsGet(client["id"]) counter = 0 for session in sessions: if counter >= settings["limit"]: break if settings["status"]: if session["status"] != settings["status"]: continue key_parts = session["s3_key"].split("/") del key_parts[-1] session_folder = '/'.join(key_parts) metadata = granicus.metadataGet(session_folder, 'document') minutes_docs = metadata_minutes_docs(metadata) if minutes_docs: if len(minutes_docs) > 1: print("Multiple minutes documents found for: " + session["session_id"]) minutes_doc = None else: minutes_doc = minutes_docs[0] else: minutes_doc = None if minutes_doc: minutes_html = granicus.htmlMinutesTransform( session["session_id"], minutes_doc["id"]) if minutes_html: print("Minutes Generated: " + session["session_id"] + ": " + session["title"]) if not settings["commit"]: counter += 1 continue storage_folder = "/nvme/client_files/" + client["email"] minutes_folder = storage_folder + "/documents/minutes_html" minutes_filename = session["session_id"] + "_minutes.html" minutes_path = minutes_folder + "/" + minutes_filename minutes_url = "https://s3-youtube.open.media/client_files/" + client[ "email"] + "/documents/minutes_html/" + minutes_filename os.makedirs(minutes_folder, exist_ok=True) html_file = open(minutes_path, "w") html_file.write(minutes_html) html_file.close() print("Stored: " + minutes_path) counter += 1
def granicus_html(): client = YTMigration().clientGet("*****@*****.**") pprint(client) s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"], client["s3_bucket"]) #clip_id = 1933 clip_id = 39432423 granicus = GranicusUtils(client, s3) html_agenda = granicus.htmlAgendaTransform(clip_id) print(html_agenda)
def get_meeting(): client = YTMigration().clientGet("*****@*****.**") s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"], client["s3_bucket"]) granicus = GranicusUtils(client, s3) meeting = granicus.apiFetchObject("1933") pprint(meeting)
def get_metadata(): client = YTMigration().clientGet("*****@*****.**") s3 = S3Utils( client["s3_access_key"], client["s3_secret_key"], client["s3_bucket"]) granicus = GranicusUtils(client, s3) session = granicus.apiFetchObject("2729") pprint(session)
def debug_agendas(): client = YTMigration().clientGet("*****@*****.**") s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"], client["s3_bucket"]) granicus = GranicusUtils(client, s3) sessions = YTMigration().sessionsGet(client["id"]) for index, session in enumerate(sessions): folder = granicus.keyFolderGet(session["s3_key"]) pprint(folder) agenda_items = granicus.metadataGet(folder, "agenda") if agenda_items: pprint(session) pprint(agenda_items) break
def harvest_docs(): settings = get_settings() client = YTMigration().clientGet(settings["email"]) s3 = S3Utils( client["s3_access_key"], client["s3_secret_key"], client["s3_bucket"]) granicus = GranicusUtils(client, s3) if settings["uploaded"]: sessions = YTMigration().sessionsGet(client["id"], "uploaded") else: sessions = YTMigration().sessionsGet(client["id"]) sessions = attach_documents(client, granicus, s3, settings, sessions) for sindex, session in enumerate(sessions): if 'documents' in session and session["documents"]: session_docs = { 's3_key': session["s3_key"], 'client_id': session["client_id"], 'documents': session["documents"]} YTMigration().sessionUpdate(session_docs) if 'agenda_items' in session and session["agenda_items"]: for aindex, agenda_item in enumerate(session["agenda_items"]): agenda_item_id = YTMigration().agendaItemCreate(agenda_item) sessions[sindex]["agenda_items"][aindex]["id"] = agenda_item_id sessions[sindex]["agenda_items"] = finalize_agenda_items(session["id"], sessions[sindex]["agenda_items"])
def harvest_html_agendas(): settings = get_settings() client = YTMigration().clientGet(settings["email"]) s3 = S3Utils( client["s3_access_key"], client["s3_secret_key"], client["s3_bucket"]) granicus = GranicusUtils(client, s3) sessions = YTMigration().sessionsGet(client["id"]) for session in sessions: agenda_html = granicus.htmlAgendaTransform(session["session_id"]) if agenda_html: storage_folder = "/nvme/client_files/"+client["email"] agenda_folder = storage_folder+"/documents/agenda_html" agenda_filename = session["session_id"]+"_agenda.html" agenda_path = agenda_folder+"/"+agenda_filename agenda_url = "https://s3-youtube.open.media/client_files/"+client["email"]+"/documents/agenda_html/"+agenda_filename os.makedirs(agenda_folder, exist_ok=True) html_file = open(agenda_path,"w") html_file.write(agenda_html) html_file.close() if not session["documents"]: session["documents"] = [] session["documents"].append({ "type": "agenda_html", "location": "internal", "s3_key": None, "filename": agenda_filename, "path": agenda_path, "url": agenda_url}) session_docs = { 's3_key': session["s3_key"], 'client_id': session["client_id"], 'documents': session["documents"]} YTMigration().sessionUpdate(session_docs) print("Harvested: "+agenda_path)
def harvest(): settings = get_settings( "Harvest basic session information from Granicus S3 export") client = YTMigration().clientGet(settings["email"]) s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"], client["s3_bucket"]) granicus = GranicusUtils(client, s3) folders = granicus.foldersGet() sessions = sessions_get(s3, client, granicus, folders, settings["limit"]) print("Total sessions found: " + str(len(sessions))) sessions = sessions_filter_settings(settings, sessions) print("Total sessions to be stored after filtering by settings: " + str(len(sessions))) if settings["commit"]: sessions_store(client, sessions) print("Sessions stored: " + str(len(sessions))) else: print("Add --commit to command line store sessions")