def harvest_html_agendas():
settings = get_settings()
client = YTMigration().clientGet(settings["email"])
s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"],
client["s3_bucket"])
granicus = GranicusUtils(client, s3)
sessions = YTMigration().sessionsGet(client["id"])
for session in sessions:
if session["status"] == "uploaded":
agenda_html = granicus.htmlAgendaTransform(session["session_id"])
if agenda_html:
storage_folder = "/nvme/client_files/" + client["email"]
agenda_folder = storage_folder + "/documents/agenda_html"
agenda_filename = session["session_id"] + "_agenda.html"
agenda_path = agenda_folder + "/" + agenda_filename
agenda_url = "https://s3-youtube.open.media/client_files/" + client[
"email"] + "/documents/agenda_html/" + agenda_filename
os.makedirs(agenda_folder, exist_ok=True)
html_file = open(agenda_path, "w")
html_file.write(agenda_html)
html_file.close()
print("Harvested: " + agenda_path)
def fix_tacoma_docs():
migration_client = YTMigration().clientGet(
"*****@*****.**")
s3 = S3Utils(migration_client["s3_access_key"],
migration_client["s3_secret_key"],
migration_client["s3_bucket"])
sessions = YTMigration().sessionsGet(migration_client["id"], 'uploaded')
for session in sessions:
if session["documents"]:
docUpdate = False
for index, document in enumerate(session["documents"]):
if "filename" in document and document["filename"]:
ext_agenda_item = YTMigration(
).agendaItemGetBySessionFilename(session["id"],
document["filename"])
if ext_agenda_item:
session["documents"][index][
"external_agenda_item_id"] = ext_agenda_item["id"]
docUpdate = True
if docUpdate:
session_docs = {
's3_key': session["s3_key"],
'client_id': session["client_id"],
'documents': session["documents"]
}
pprint(session_docs)
def harvest_docs():
settings = get_settings()
client = YTMigration().clientGet(settings["email"])
s3 = S3Utils(
client["s3_access_key"],
client["s3_secret_key"],
client["s3_bucket"])
granicus = GranicusUtils(client, s3)
if settings["uploaded"]:
sessions = YTMigration().sessionsGet(client["id"], "uploaded")
else:
sessions = YTMigration().sessionsGet(client["id"])
sessions = attach_documents(client, granicus, s3, settings, sessions)
for sindex, session in enumerate(sessions):
if 'documents' in session and session["documents"]:
session_docs = {
's3_key': session["s3_key"],
'client_id': session["client_id"],
'documents': session["documents"]}
YTMigration().sessionUpdate(session_docs)
if 'agenda_items' in session and session["agenda_items"]:
for aindex, agenda_item in enumerate(session["agenda_items"]):
agenda_item_id = YTMigration().agendaItemCreate(agenda_item)
sessions[sindex]["agenda_items"][aindex]["id"] = agenda_item_id
sessions[sindex]["agenda_items"] = finalize_agenda_items(session["id"], sessions[sindex]["agenda_items"])
def harvest_html_minutes():
settings = get_settings("Store and transform Granicus html minutes")
client = YTMigration().clientGet(settings["email"])
s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"],
client["s3_bucket"])
granicus = GranicusUtils(client, s3)
sessions = YTMigration().sessionsGet(client["id"])
counter = 0
for session in sessions:
if counter >= settings["limit"]:
break
if settings["status"]:
if session["status"] != settings["status"]:
continue
key_parts = session["s3_key"].split("/")
del key_parts[-1]
session_folder = '/'.join(key_parts)
metadata = granicus.metadataGet(session_folder, 'document')
minutes_docs = metadata_minutes_docs(metadata)
if minutes_docs:
if len(minutes_docs) > 1:
print("Multiple minutes documents found for: " +
session["session_id"])
minutes_doc = None
else:
minutes_doc = minutes_docs[0]
else:
minutes_doc = None
if minutes_doc:
minutes_html = granicus.htmlMinutesTransform(
session["session_id"], minutes_doc["id"])
if minutes_html:
print("Minutes Generated: " + session["session_id"] + ": " +
session["title"])
if not settings["commit"]:
counter += 1
continue
storage_folder = "/nvme/client_files/" + client["email"]
minutes_folder = storage_folder + "/documents/minutes_html"
minutes_filename = session["session_id"] + "_minutes.html"
minutes_path = minutes_folder + "/" + minutes_filename
minutes_url = "https://s3-youtube.open.media/client_files/" + client[
"email"] + "/documents/minutes_html/" + minutes_filename
os.makedirs(minutes_folder, exist_ok=True)
html_file = open(minutes_path, "w")
html_file.write(minutes_html)
html_file.close()
print("Stored: " + minutes_path)
counter += 1
def granicus_html():
client = YTMigration().clientGet("*****@*****.**")
pprint(client)
s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"],
client["s3_bucket"])
#clip_id = 1933
clip_id = 39432423
granicus = GranicusUtils(client, s3)
html_agenda = granicus.htmlAgendaTransform(clip_id)
print(html_agenda)
def get_meeting():
client = YTMigration().clientGet("*****@*****.**")
s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"],
client["s3_bucket"])
granicus = GranicusUtils(client, s3)
meeting = granicus.apiFetchObject("1933")
pprint(meeting)
def harvest():
settings = get_settings()
migration_client = YTMigration().clientGet(settings["email"])
s3 = S3Utils(
migration_client["s3_access_key"],
migration_client["s3_secret_key"],
migration_client["s3_bucket"])
sessions = prepare_sessions(s3)
sessions = attach_agendas(s3, sessions)
sessions_store(settings, migration_client, sessions)
def get_metadata():
client = YTMigration().clientGet("*****@*****.**")
s3 = S3Utils(
client["s3_access_key"],
client["s3_secret_key"],
client["s3_bucket"])
granicus = GranicusUtils(client, s3)
session = granicus.apiFetchObject("2729")
pprint(session)
def debug_agendas():
client = YTMigration().clientGet("*****@*****.**")
s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"],
client["s3_bucket"])
granicus = GranicusUtils(client, s3)
sessions = YTMigration().sessionsGet(client["id"])
for index, session in enumerate(sessions):
folder = granicus.keyFolderGet(session["s3_key"])
pprint(folder)
agenda_items = granicus.metadataGet(folder, "agenda")
if agenda_items:
pprint(session)
pprint(agenda_items)
break
def fix():
settings = get_settings()
migration_client = YTMigration().clientGet(settings["email"])
s3 = S3Utils(migration_client["s3_access_key"],
migration_client["s3_secret_key"],
migration_client["s3_bucket"])
sessions = YTMigration().sessionsGet(migration_client["id"])
sessions = attach_agendas(s3, sessions)
for session in sessions:
if 'documents' in session and session["documents"]:
session_docs = {
's3_key': session["s3_key"],
'client_id': session["client_id"],
'documents': session["documents"]
}
YTMigration().sessionUpdate(session_docs)
def harvest_html_agendas():
settings = get_settings()
client = YTMigration().clientGet(settings["email"])
s3 = S3Utils(
client["s3_access_key"],
client["s3_secret_key"],
client["s3_bucket"])
granicus = GranicusUtils(client, s3)
sessions = YTMigration().sessionsGet(client["id"])
for session in sessions:
agenda_html = granicus.htmlAgendaTransform(session["session_id"])
if agenda_html:
storage_folder = "/nvme/client_files/"+client["email"]
agenda_folder = storage_folder+"/documents/agenda_html"
agenda_filename = session["session_id"]+"_agenda.html"
agenda_path = agenda_folder+"/"+agenda_filename
agenda_url = "https://s3-youtube.open.media/client_files/"+client["email"]+"/documents/agenda_html/"+agenda_filename
os.makedirs(agenda_folder, exist_ok=True)
html_file = open(agenda_path,"w")
html_file.write(agenda_html)
html_file.close()
if not session["documents"]:
session["documents"] = []
session["documents"].append({
"type": "agenda_html",
"location": "internal",
"s3_key": None,
"filename": agenda_filename,
"path": agenda_path,
"url": agenda_url})
session_docs = {
's3_key': session["s3_key"],
'client_id': session["client_id"],
'documents': session["documents"]}
YTMigration().sessionUpdate(session_docs)
print("Harvested: "+agenda_path)
def takoma_minutes_report():
client = YTMigration().clientGet("*****@*****.**")
s3 = S3Utils(
client["s3_access_key"],
client["s3_secret_key"],
client["s3_bucket"])
sessions = YTMigration().sessionsGet(client["id"])
for session in sessions:
if session["status"] == "uploaded":
has_minutes = False
if session["documents"]:
for index, doc in enumerate(session["documents"]):
if doc["type"] == "minutes_html" or doc["type"] == "minutes":
has_minutes = True
if not has_minutes:
if session["session_date"] and session["title"]:
print(session["session_date"]+": "+session["title"])
else:
print("NO DATE: "+session["title"])
def download_documents():
settings = get_settings()
migration_client = YTMigration().clientGet(settings["email"])
sessions = YTMigration().sessionsGet(migration_client["id"], 'uploaded')
s3 = S3Utils(migration_client["s3_access_key"],
migration_client["s3_secret_key"],
migration_client["s3_bucket"])
storage_path = "/nvme/client_files/" + migration_client["email"]
doc_path = storage_path + "/documents"
doc_url = "https://s3-youtube.open.media/client_files/" + migration_client[
"email"] + "/documents"
counter = 0
for session in sessions:
if counter < settings["limit"]:
if session["documents"]:
doc_counter = 0
for index, document in enumerate(session["documents"]):
if "path" not in document or not document["path"]:
if document["s3_key"]:
destination_path = doc_path + "/" + document[
"type"] + "/" + document["filename"]
destination_url = doc_url + "/" + document[
"type"] + "/" + document["filename"]
s3.fileDownload(document["s3_key"],
destination_path)
session["documents"][index][
"path"] = destination_path
session["documents"][index][
"url"] = destination_url
doc_counter += 1
YTMigration().sessionUpdate(session)
print("Downloaded " + str(doc_counter) + " documents for " +
session["title"])
counter += 1
def fix_tacoma_doc():
client = YTMigration().clientGet("*****@*****.**")
s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"],
client["s3_bucket"])
sessions = YTMigration().sessionsGet(client["id"])
for session in sessions:
update_docs = False
if session["documents"]:
for index, doc in enumerate(session["documents"]):
if doc["type"] == "minutes_html":
del (session["documents"][index])
update_docs = True
if update_docs:
session_docs = {
's3_key': session["s3_key"],
'client_id': session["client_id"],
'documents': session["documents"]
}
pprint(session_docs)
YTMigration().sessionUpdate(session_docs)
def harvest():
settings = get_settings(
"Harvest basic session information from Granicus S3 export")
client = YTMigration().clientGet(settings["email"])
s3 = S3Utils(client["s3_access_key"], client["s3_secret_key"],
client["s3_bucket"])
granicus = GranicusUtils(client, s3)
folders = granicus.foldersGet()
sessions = sessions_get(s3, client, granicus, folders, settings["limit"])
print("Total sessions found: " + str(len(sessions)))
sessions = sessions_filter_settings(settings, sessions)
print("Total sessions to be stored after filtering by settings: " +
str(len(sessions)))
if settings["commit"]:
sessions_store(client, sessions)
print("Sessions stored: " + str(len(sessions)))
else:
print("Add --commit to command line store sessions")
def process_sessions():
settings = get_settings("Copy videos from a Granicus S3 export to YouTube")
migration = YTMigration()
client_data = migration.clientGet(settings["email"])
s3 = S3Utils(client_data["s3_access_key"], client_data["s3_secret_key"],
client_data["s3_bucket"])
migration_info = {
'summary': {
'attempted': 0,
'uploaded': 0,
'skipped': 0,
'failed': 0
},
'sessions': {
'attempted': [],
'uploaded': [],
'skipped': [],
'failed': []
}
}
if not client_data:
pprint(
"There are no client credentials in the system for the email address provided"
)
return
if 'token' not in client_data or not client_data["token"]:
pprint(
"Please authenticate this client first at https://s3-youtube.open.media"
)
return
youtube_client = yt_api_utils.youtube_client_get(client_data)
sessions = migration.sessionsGet(client_data["id"], 'harvested')
inclusions = granicus_manual_csv(client_data, "include")
print("Using manual inclusions file, " + str(len(inclusions)) +
" sessions")
exclusions = granicus_manual_csv(client_data, "exclude")
if sessions:
counter = 0
limit = settings["limit"]
for session in sessions:
if counter >= limit:
break
manual_inclusion = False
#if there are manual inclusions we only do those
if inclusions and session["session_id"] and int(
session["session_id"]) in inclusions:
manual_inclusion = True
elif inclusions:
migration_info["sessions"]["skipped"].append(sessions)
continue
if exclusions and session["session_id"] in exclusions:
migration_info["sessions"]["skipped"].append(sessions)
continue
if not manual_inclusion and settings["start_timestamp"]:
if not session["session_timestamp"] or session[
"session_timestamp"] < settings["start_timestamp"]:
migration_info["sessions"]["skipped"].append(session)
continue
if not manual_inclusion and settings["end_timestamp"]:
if not session["session_timestamp"] or session[
"session_timestamp"] > settings["end_timestamp"]:
migration_info["sessions"]["skipped"].append(session)
continue
if not manual_inclusion and settings["folder_ids"]:
if not session["session_folder"] or session[
"session_folder"] not in settings["folder_ids"]:
migration_info["sessions"]["skipped"].append(session)
continue
if settings["commit"]:
print("Downloading " + session["title"])
session = s3_download_file(client_data, s3, session)
response = yt_api_utils.youtube_upload_session(
youtube_client, session)
if response["id"]:
session["youtube_id"] = response["id"]
session["status"] = 'uploaded'
os.remove(session["local_path"])
migration.sessionUpdate(session)
migration_info["sessions"]["uploaded"].append(session)
pprint("Session uploaded: " + session["title"])
if "youtube_playlist" in session:
yt_api_utils.youtube_add_session_to_playlist(
youtube_client, session)
else:
session["status"] = 'upload failed'
migration.sessionUpdate(session)
pprint("Session upload failed: " + session["title"])
migration_info["sessions"]["failed"].append(session)
else:
migration_info["sessions"]["attempted"].append(session)
counter += 1
migration_info["summary"]["attempted"] = len(
migration_info["sessions"]["attempted"])
migration_info["summary"]["skipped"] = len(
migration_info["sessions"]["skipped"])
migration_info["summary"]["uploaded"] = len(
migration_info["sessions"]["uploaded"])
migration_info["summary"]["failed"] = len(
migration_info["sessions"]["failed"])
if settings["verbose"]:
print("Sessions:")
for session in migration_info["sessions"]["attempted"]:
if session["session_date"]:
datetime = parser.parse(session["session_date"])
utcDate = datetime.astimezone(timezone('UTC'))
title_date = utcDate.strftime("%m/%d/%Y")
else:
title_date = "NO DATE"
print(
str(session["session_folder"]) + ": " + session["title"] +
" - " + title_date)
print("Summary:")
pprint(migration_info["summary"])