def harvest_html_agendas():
settings = get_settings()
client = YTMigration().clientGet(settings["email"])
s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"],
client["s3_bucket"])
granicus = GranicusUtils(client, s3)
sessions = YTMigration().sessionsGet(client["id"])
for session in sessions:
if session["status"] == "uploaded":
agenda_html = granicus.htmlAgendaTransform(session["session_id"])
if agenda_html:
storage_folder = "/nvme/client_files/" + client["email"]
agenda_folder = storage_folder + "/documents/agenda_html"
agenda_filename = session["session_id"] + "_agenda.html"
agenda_path = agenda_folder + "/" + agenda_filename
agenda_url = "https://s3-youtube.open.media/client_files/" + client[
"email"] + "/documents/agenda_html/" + agenda_filename
os.makedirs(agenda_folder, exist_ok=True)
html_file = open(agenda_path, "w")
html_file.write(agenda_html)
html_file.close()
print("Harvested: " + agenda_path)
def harvest_html_minutes():
settings = get_settings("Store and transform Granicus html minutes")
client = YTMigration().clientGet(settings["email"])
s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"],
client["s3_bucket"])
granicus = GranicusUtils(client, s3)
sessions = YTMigration().sessionsGet(client["id"])
counter = 0
for session in sessions:
if counter >= settings["limit"]:
break
if settings["status"]:
if session["status"] != settings["status"]:
continue
key_parts = session["s3_key"].split("/")
del key_parts[-1]
session_folder = '/'.join(key_parts)
metadata = granicus.metadataGet(session_folder, 'document')
minutes_docs = metadata_minutes_docs(metadata)
if minutes_docs:
if len(minutes_docs) > 1:
print("Multiple minutes documents found for: " +
session["session_id"])
minutes_doc = None
else:
minutes_doc = minutes_docs[0]
else:
minutes_doc = None
if minutes_doc:
minutes_html = granicus.htmlMinutesTransform(
session["session_id"], minutes_doc["id"])
if minutes_html:
print("Minutes Generated: " + session["session_id"] + ": " +
session["title"])
if not settings["commit"]:
counter += 1
continue
storage_folder = "/nvme/client_files/" + client["email"]
minutes_folder = storage_folder + "/documents/minutes_html"
minutes_filename = session["session_id"] + "_minutes.html"
minutes_path = minutes_folder + "/" + minutes_filename
minutes_url = "https://s3-youtube.open.media/client_files/" + client[
"email"] + "/documents/minutes_html/" + minutes_filename
os.makedirs(minutes_folder, exist_ok=True)
html_file = open(minutes_path, "w")
html_file.write(minutes_html)
html_file.close()
print("Stored: " + minutes_path)
counter += 1
def granicus_html():
client = YTMigration().clientGet("*****@*****.**")
pprint(client)
s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"],
client["s3_bucket"])
#clip_id = 1933
clip_id = 39432423
granicus = GranicusUtils(client, s3)
html_agenda = granicus.htmlAgendaTransform(clip_id)
print(html_agenda)
def get_meeting():
client = YTMigration().clientGet("*****@*****.**")
s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"],
client["s3_bucket"])
granicus = GranicusUtils(client, s3)
meeting = granicus.apiFetchObject("1933")
pprint(meeting)
def get_metadata():
client = YTMigration().clientGet("*****@*****.**")
s3 = S3Utils(
client["s3_access_key"],
client["s3_secret_key"],
client["s3_bucket"])
granicus = GranicusUtils(client, s3)
session = granicus.apiFetchObject("2729")
pprint(session)
def debug_agendas():
client = YTMigration().clientGet("*****@*****.**")
s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"],
client["s3_bucket"])
granicus = GranicusUtils(client, s3)
sessions = YTMigration().sessionsGet(client["id"])
for index, session in enumerate(sessions):
folder = granicus.keyFolderGet(session["s3_key"])
pprint(folder)
agenda_items = granicus.metadataGet(folder, "agenda")
if agenda_items:
pprint(session)
pprint(agenda_items)
break
def harvest_docs():
settings = get_settings()
client = YTMigration().clientGet(settings["email"])
s3 = S3Utils(
client["s3_access_key"],
client["s3_secret_key"],
client["s3_bucket"])
granicus = GranicusUtils(client, s3)
if settings["uploaded"]:
sessions = YTMigration().sessionsGet(client["id"], "uploaded")
else:
sessions = YTMigration().sessionsGet(client["id"])
sessions = attach_documents(client, granicus, s3, settings, sessions)
for sindex, session in enumerate(sessions):
if 'documents' in session and session["documents"]:
session_docs = {
's3_key': session["s3_key"],
'client_id': session["client_id"],
'documents': session["documents"]}
YTMigration().sessionUpdate(session_docs)
if 'agenda_items' in session and session["agenda_items"]:
for aindex, agenda_item in enumerate(session["agenda_items"]):
agenda_item_id = YTMigration().agendaItemCreate(agenda_item)
sessions[sindex]["agenda_items"][aindex]["id"] = agenda_item_id
sessions[sindex]["agenda_items"] = finalize_agenda_items(session["id"], sessions[sindex]["agenda_items"])
def harvest_html_agendas():
settings = get_settings()
client = YTMigration().clientGet(settings["email"])
s3 = S3Utils(
client["s3_access_key"],
client["s3_secret_key"],
client["s3_bucket"])
granicus = GranicusUtils(client, s3)
sessions = YTMigration().sessionsGet(client["id"])
for session in sessions:
agenda_html = granicus.htmlAgendaTransform(session["session_id"])
if agenda_html:
storage_folder = "/nvme/client_files/"+client["email"]
agenda_folder = storage_folder+"/documents/agenda_html"
agenda_filename = session["session_id"]+"_agenda.html"
agenda_path = agenda_folder+"/"+agenda_filename
agenda_url = "https://s3-youtube.open.media/client_files/"+client["email"]+"/documents/agenda_html/"+agenda_filename
os.makedirs(agenda_folder, exist_ok=True)
html_file = open(agenda_path,"w")
html_file.write(agenda_html)
html_file.close()
if not session["documents"]:
session["documents"] = []
session["documents"].append({
"type": "agenda_html",
"location": "internal",
"s3_key": None,
"filename": agenda_filename,
"path": agenda_path,
"url": agenda_url})
session_docs = {
's3_key': session["s3_key"],
'client_id': session["client_id"],
'documents': session["documents"]}
YTMigration().sessionUpdate(session_docs)
print("Harvested: "+agenda_path)
def harvest():
settings = get_settings(
"Harvest basic session information from Granicus S3 export")
client = YTMigration().clientGet(settings["email"])
s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"],
client["s3_bucket"])
granicus = GranicusUtils(client, s3)
folders = granicus.foldersGet()
sessions = sessions_get(s3, client, granicus, folders, settings["limit"])
print("Total sessions found: " + str(len(sessions)))
sessions = sessions_filter_settings(settings, sessions)
print("Total sessions to be stored after filtering by settings: " +
str(len(sessions)))
if settings["commit"]:
sessions_store(client, sessions)
print("Sessions stored: " + str(len(sessions)))
else:
print("Add --commit to command line store sessions")