def __init__(self):
udpKeyFileName: str = ENV.get('MIVIDEO',
{}).get('service_account_json_filename')
if (udpKeyFileName is None):
errorMessage: str = (f'"MIVIDEO.service_account_json_filename" '
f'was not found in {CONFIG_PATH}')
logger.error(errorMessage)
raise ValueError(errorMessage)
self.udpKeyFilePath: str = os.path.join(CONFIG_DIR, udpKeyFileName)
logger.debug(f'udpKeyFilePath: "{self.udpKeyFilePath}"')
self.credentials: service_account.Credentials = (
service_account.Credentials.from_service_account_file(
self.udpKeyFilePath,
scopes=['https://www.googleapis.com/auth/cloud-platform']))
self.udpDb: bigquery.Client = bigquery.Client(
credentials=self.credentials, project=self.credentials.project_id)
logger.info(f'Connected to BigQuery project: "{self.udpDb.project}"')
dbParams: Dict = ENV['INVENTORY_DB']
appendTableNames: Sequence[str] = ENV.get(
'APPEND_TABLE_NAMES', ['mivideo_media_started_hourly'])
self.appDb: DBCreator = DBCreator(dbParams, appendTableNames)
def main() -> Sequence[DataSourceStatus]:
'''
This method is invoked when its module is executed as a standalone program.
'''
# Get ids for tools in lti_type table as supported tools
canvas_env = ENV.get('CANVAS', {})
lti_processor = CanvasLtiPlacementProcessor(canvas_env.get("CANVAS_URL"),
canvas_env.get("CANVAS_TOKEN"))
lti_processor.generate_lti_course_report(
canvas_env.get("CANVAS_ACCOUNT_ID", 1),
canvas_env.get("CANVAS_TERM_IDS", []),
canvas_env.get("ADD_COURSE_IDS", []), True)
lti_processor.output_report()
return [DataSourceStatus(ValidDataSourceName.CANVAS_LTI)]
def output_report(self) -> None:
lti_placement_df = pd.DataFrame(self.lti_placements)
lti_placement_df = lti_placement_df.set_index("id")
lti_zoom_meeting_df = pd.DataFrame(self.zoom_courses_meetings)
lti_zoom_meeting_df.index.name = "id"
if ENV.get('CREATE_CSVS', False):
logger.info(
f'Writing {len(lti_placement_df)} lti_placement records to CSV'
)
lti_placement_df.to_csv(os.path.join(DATA_DIR,
"lti_placement.csv"))
logger.info(
f'Writing {len(lti_zoom_meeting_df)} lti_zoom_meeting records to CSV'
)
lti_zoom_meeting_df.to_csv(
os.path.join(DATA_DIR, "lti_zoom_meeting.csv"))
# For now until this process is improved just remove all the previous records
logger.info('Emptying Canvas LTI data tables in DB')
self.db_creator.drop_records(['lti_placement', 'lti_zoom_meeting'])
logger.info(
f'Inserting {len(lti_placement_df)} lti_placement records to DB')
lti_placement_df.to_sql("lti_placement",
self.db_creator.engine,
if_exists="append",
index=True)
logger.info(
f'Inserted data into lti_placement table in {self.db_creator.db_name}'
)
logger.info(
f'Inserting {len(lti_zoom_meeting_df)} lti_zoom_meeting records to DB'
)
lti_zoom_meeting_df.to_sql("lti_zoom_meeting",
self.db_creator.engine,
if_exists="append",
index=True)
logger.info(
f'Inserted data into lti_zoom_meeting table in {self.db_creator.db_name}'
)
# third-party libraries
import pandas as pd
import sqlalchemy
# local libraries
from db.db_creator import DBCreator
from environ import ENV
from vocab import ValidJobName, ValidDataSourceName
# Initialize settings and global variables
logger = logging.getLogger(__name__)
logging.basicConfig(
level=ENV.get('LOG_LEVEL', 'DEBUG'),
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
# Class(es)
class Job:
def __init__(self, job_name: ValidJobName) -> None:
self.name: str = job_name.name
self.import_path: str = '.'.join(job_name.value.split('.')[:-1])
self.method_name: str = job_name.value.split('.')[-1]
self.started_at: Union[float, None] = None
self.finished_at: Union[float, None] = None
self.data_sources: Sequence[Dict[str, Union[ValidDataSourceName, pd.Timestamp]]] = []
# standard libraries
import logging
# local libraries
from db.db_creator import DBCreator
from environ import ENV
# Initializing settings and global variables
logger = logging.getLogger(__name__)
DB_PARAMS = ENV['INVENTORY_DB']
APPEND_TABLE_NAMES = ENV.get('APPEND_TABLE_NAMES', [])
# Main Program
if __name__ == '__main__':
logging.basicConfig(level=ENV.get('LOG_LEVEL', 'DEBUG'))
db_creator_obj = DBCreator(DB_PARAMS, APPEND_TABLE_NAMES)
db_creator_obj.set_up_database()
from umich_api.api_utils import ApiUtil
# local libraries
from course_inventory.async_enroll_gatherer import AsyncEnrollGatherer
from course_inventory.canvas_course_usage import CanvasCourseUsage
from course_inventory.gql_queries import queries as QUERIES
from course_inventory.published_date import FetchPublishedDate
from db.db_creator import DBCreator
from environ import DATA_DIR, ENV
from vocab import DataSourceStatus, JobError, ValidDataSourceName
# Initialize settings and globals
logger = logging.getLogger(__name__)
CANVAS = ENV.get('CANVAS', {})
ACCOUNT_ID = CANVAS.get('CANVAS_ACCOUNT_ID', 1)
TERM_IDS = CANVAS['CANVAS_TERM_IDS']
API_UTIL = ApiUtil(CANVAS['API_BASE_URL'], CANVAS['API_CLIENT_ID'],
CANVAS['API_CLIENT_SECRET'])
SUBSCRIPTION_NAME = CANVAS['API_SUBSCRIPTION_NAME']
API_SCOPE_PREFIX = CANVAS['API_SCOPE_PREFIX']
CANVAS_TOKEN = CANVAS['CANVAS_TOKEN']
CANVAS_URL = CANVAS['CANVAS_URL']
MAX_REQ_ATTEMPTS = ENV.get('MAX_REQ_ATTEMPTS', 3)
NUM_ASYNC_WORKERS = ENV.get('NUM_ASYNC_WORKERS', 8)
CREATE_CSVS = ENV.get('CREATE_CSVS', False)
# standard libraries
import logging
# local libraries
from db.db_creator import DBCreator
from environ import ENV
# Initializing settings and global variables
logger = logging.getLogger(__name__)
DB_PARAMS = ENV['INVENTORY_DB']
# Main Program
if __name__ == '__main__':
logging.basicConfig(level=ENV.get('LOG_LEVEL', 'DEBUG'))
db_creator_obj = DBCreator(DB_PARAMS)
db_creator_obj.reset_database()
# local libraries
from course_inventory.async_enroll_gatherer import AsyncEnrollGatherer
from course_inventory.canvas_course_usage import CanvasCourseUsage
from course_inventory.gql_queries import queries as QUERIES
from course_inventory.published_date import FetchPublishedDate
from db.db_creator import DBCreator
from environ import ENV
from vocab import ValidDataSourceName
# Initialize settings and globals
logger = logging.getLogger(__name__)
ACCOUNT_ID = ENV.get('CANVAS_ACCOUNT_ID', 1)
TERM_IDS = ENV['CANVAS_TERM_IDS']
API_UTIL = ApiUtil(ENV['API_BASE_URL'], ENV['API_CLIENT_ID'], ENV['API_CLIENT_SECRET'])
SUBSCRIPTION_NAME = ENV['API_SUBSCRIPTION_NAME']
API_SCOPE_PREFIX = ENV['API_SCOPE_PREFIX']
MAX_REQ_ATTEMPTS = ENV['MAX_REQ_ATTEMPTS']
CANVAS_TOKEN = ENV['CANVAS_TOKEN']
CANVAS_URL = ENV['CANVAS_URL']
NUM_ASYNC_WORKERS = ENV.get('NUM_ASYNC_WORKERS', 8)
CREATE_CSVS = ENV.get('CREATE_CSVS', False)
INVENTORY_DB = ENV['INVENTORY_DB']
APPEND_TABLE_NAMES = ENV.get('APPEND_TABLE_NAMES', ['job_run', 'data_source_status'])
CANVAS_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
import os
import re
import time
from typing import Dict, List, Optional, Sequence, Union
import canvasapi
import pandas as pd
import requests
from bs4 import BeautifulSoup as bs
from environ import ENV, DATA_DIR
from vocab import ValidDataSourceName
logger = logging.getLogger(__name__)
CANVAS_ENV = ENV.get('CANVAS', {})
class ZoomPlacements:
zoom_courses: List[Dict] = []
zoom_courses_meetings: List[Dict] = []
def __init__(self):
self.zoom_session = requests.Session()
self.canvas = canvasapi.Canvas(CANVAS_ENV.get("CANVAS_URL"), CANVAS_ENV.get("CANVAS_TOKEN"))
def get_zoom_json(self, **kwargs) -> Optional[Dict]:
"""Retrieves data directly from Zoom. You need to have zoom_session already setup
:param kwargs: Supplied additional parameters to pass to the API. Should at least supply page and lti_scid.
:type kwargs: Dict
from datetime import datetime, timedelta
from typing import Dict, List, Union
import dateparser
import pandas as pd
import requests
from requests.auth import AuthBase
from environ import ENV
from .requests_zoom_jwt import ZoomJWT
# Initialize settings and globals
logger = logging.getLogger(__name__)
logging.basicConfig(level=ENV.get('LOG_LEVEL', 'DEBUG'))
DEFAULT_SLEEP_TIME = ENV.get('DEFAULT_SLEEP_TIME', 10)
def to_seconds(duration: str) -> int:
"""Returns a value of a string duration in seconds
This string is expected in HH:MM:SS or MM:SS or SS
:param timestr: String in a duration format like HH:MM:SS
:type timestr: str
:return: Value in seconds
:rtype: int
"""
total_seconds = 0
for second in duration.split(':'):