def main(): placement_exam = os.getenv('placement_exam') path_properties = os.getenv('path_properties') path_persistence = os.getenv('path_persistence') print(f"Path to properties {path_properties}") print(f"Placement exam: {placement_exam}") persistance_file = f"{path_persistence}persistence_{placement_exam}.txt" print(f"Persistance file: {persistance_file}") with open(path_properties, 'r') as yml_file: sf = yaml.load(yml_file, Loader=yaml.FullLoader) print(sf['course'][placement_exam]['id']) client_ = sf['api']['client'] secret_ = sf['api']['secret'] url_ = sf['api']['url'] print(client_) print(secret_) print(url_) api_json: str = pkg_resources.resource_filename(__name__, 'apis.json') api_handler = ApiUtil(url_, client_, secret_, api_json) print( api_handler.get_access_token('aa/oauth2/token', 'spanishplacementscores')) with open(f'{path_persistence}/{placement_exam}.txt') as f: latest_exam_date: str = f.read() print(latest_exam_date)
def setUpClass(self): self.url = os.getenv("url") self.client_id = os.getenv("client_id") self.secret = os.getenv("secret") self.canvas_api = ApiUtil(self.url, self.client_id, self.secret, "canvasreadonly") self.mcommunity_api = ApiUtil(self.url, self.client_id, self.secret, "mcommunity") print ("URL is %s" % self.url)
def setUp(self): """Sets up ApiUtil instance and custom fixtures to be used by ScoresOrchestration tests.""" self.api_handler: ApiUtil = ApiUtil( os.getenv('API_DIR_URL', ''), os.getenv('API_DIR_CLIENT_ID', ''), os.getenv('API_DIR_SECRET', ''), os.path.join(ROOT_DIR, 'config', 'apis.json')) with open(os.path.join(API_FIXTURES_DIR, 'canvas_subs.json'), 'r') as test_canvas_subs_file: canvas_subs_dict: Dict[str, List[Dict[str, Any]]] = json.loads( test_canvas_subs_file.read()) self.canvas_potions_val_subs: List[Dict[ str, Any]] = canvas_subs_dict['Potions_Validation_1'] self.canvas_dada_place_subs_one: List[Dict[ str, Any]] = canvas_subs_dict['DADA_Placement_1'] self.canvas_dada_place_subs_two: List[Dict[ str, Any]] = canvas_subs_dict['DADA_Placement_2'] self.canvas_potions_place_subs_three: List[Dict[ str, Any]] = canvas_subs_dict['Potions_Placement_3'] with open(os.path.join(API_FIXTURES_DIR, 'mpathways_resp_data.json'), 'r') as mpathways_resp_data_file: self.mpathways_resp_data: List[Dict[str, Any]] = json.loads( mpathways_resp_data_file.read()) self.dup_get_mocks: List[MagicMock] = [ MagicMock(spec=Response, status_code=200, text=json.dumps( canvas_subs_dict['Potions_Placement_1'])), MagicMock(spec=Response, status_code=200, text=json.dumps(canvas_subs_dict['Potions_Placement_2'])) ]
def setUp(self): """Set up ApiUtil instance and data needed by test methods.""" self.api_handler: ApiUtil = ApiUtil( os.getenv('API_DIR_URL', ''), os.getenv('API_DIR_CLIENT_ID', ''), os.getenv('API_DIR_SECRET', ''), os.path.join(ROOT_DIR, 'config', 'apis.json')) with open(os.path.join(API_FIXTURES_DIR, 'canvas_subs.json'), 'r') as test_canvas_subs_file: canvas_subs_dict: dict[str, list[dict[str, Any]]] = json.loads( test_canvas_subs_file.read()) self.canvas_potions_val_subs: list[dict[ str, Any]] = canvas_subs_dict['Potions_Validation_1'] # "Potions Validation" from test_04.json some_course_id: int = 888888 some_assignment_id: int = 111112 some_filter: datetime = datetime(2020, 6, 1, 0, 0, 0, tzinfo=utc) self.get_scores_url: str = ( f'{CANVAS_URL_BEGIN}/courses/{some_course_id}/students/submissions' ) self.canvas_params: dict[str, Any] = { 'student_ids': ['all'], 'assignment_ids': [str(some_assignment_id)], 'per_page': '50', 'include': ['user'], 'graded_since': some_filter.strftime(ISO8601_FORMAT) }
def setUpClass(cls): cls.url = os.getenv("url") cls.client_id = os.getenv("client_id") cls.secret = os.getenv("secret") cls.apiutil = ApiUtil(cls.url, cls.client_id, cls.secret) # This line is needed so pylint doesn't complain about this variable not existing. cls.__log = cls.__log cls.__log.info("URL is %s" % cls.url)
def setUp(self): """ Initializes report; runs orchestrations, gathering time metadata; and sets up other shared variables. """ api_handler: ApiUtil = ApiUtil( os.getenv('API_DIR_URL', ''), os.getenv('API_DIR_CLIENT_ID', ''), os.getenv('API_DIR_SECRET', ''), os.path.join(ROOT_DIR, 'config', 'apis.json') ) with open(os.path.join(API_FIXTURES_DIR, 'canvas_subs.json'), 'r') as test_canvas_subs_file: canvas_subs_dict: Dict[str, List[Dict[str, Any]]] = json.loads(test_canvas_subs_file.read()) canvas_potions_val_subs: List[Dict[str, Any]] = canvas_subs_dict['Potions_Validation_1'] with open(os.path.join(API_FIXTURES_DIR, 'mpathways_resp_data.json'), 'r') as mpathways_resp_data_file: mpathways_resp_data: List[Dict[str, Any]] = json.loads(mpathways_resp_data_file.read()) self.potions_report = Report.objects.get(id=1) with patch('pe.orchestration.api_call_with_retries', autospec=True) as mock_get: with patch.object(ApiUtil, 'api_call', autospec=True) as mock_send: mock_get.side_effect = [ # Potions Placement - no more new submissions MagicMock(spec=Response, status_code=200, text=json.dumps([])), # Potions Validation - two more new submissions MagicMock(spec=Response, status_code=200, text=json.dumps(canvas_potions_val_subs)) ] mock_send.side_effect = [ # Potions Placement - Only rweasley sub from test_04.json, fails to send MagicMock(spec=Response, status_code=200, text=json.dumps(mpathways_resp_data[5])), # Potions Validation - Four subs, two from canvas_subs.json, two from test_04.json, all send MagicMock(spec=Response, status_code=200, text=json.dumps(mpathways_resp_data[2])) ] fake_running_dt: datetime = datetime(2020, 6, 25, 16, 0, 0, tzinfo=utc) self.exams_time_metadata: Dict[int, Dict[str, datetime]] = dict() for exam in self.potions_report.exams.all(): start: datetime = fake_running_dt exam_orca: ScoresOrchestration = ScoresOrchestration(api_handler, exam) exam_orca.main() fake_running_dt += timedelta(seconds=5) end: datetime = fake_running_dt self.exams_time_metadata[exam.id] = { 'start_time': start, 'end_time': end, 'sub_time_filter': exam_orca.sub_time_filter } fake_running_dt += timedelta(seconds=1) self.fake_finished_at: datetime = fake_running_dt self.expected_subject: str = ( 'Placement Exams Report - Potions - Success: 4, Failure: 1, New: 2 - Run finished at 2020-06-25 12:00 PM' )
class TestApiCalls(unittest.TestCase): @classmethod def setUpClass(self): self.url = os.getenv("url") self.client_id = os.getenv("client_id") self.secret = os.getenv("secret") self.canvas_api = ApiUtil(self.url, self.client_id, self.secret, "canvasreadonly") self.mcommunity_api = ApiUtil(self.url, self.client_id, self.secret, "mcommunity") print ("URL is %s" % self.url) def test_mcommunity(self): uniqname = "uniqname" self.assertEqual(self.mcommunity_api.api_call(f"MCommunity/People/{uniqname}").status_code, 200) def test_pagination(self): resp = self.canvas_api.api_call(f"aa/CanvasReadOnly/courses/1/users") logger.info(resp.headers) def test_canvasreadonly(self): resp = self.canvas_api.api_call(f"aa/CanvasReadOnly/brand_variables") self.assertEqual(resp.status_code, 200)
def api_call_with_retries( api_handler: ApiUtil, url: str, subscription: str, method: str, payload: Union[Dict[str, Any], None] = None, max_req_attempts: int = 3, ) -> Union[Response, None]: """ Pulls data from the UM API Directory, handling errors and retrying if necessary. When the maximum number of request attempts is reached, the function logs an error and returns None. :param api_handler: Instance of ApiUtil :type api_handler: ApiUtil :param url: URL ending for request :type url: string :param subscription: Name of the subscription or scope the request should use :type subscription: string :param method: Request method that should be used (e.g. "GET", "PUT") :type method: string :param payload: Dictionary to include in the request body :type payload: Dictionary with string keys or None, optional :param max_req_attempts: Number of request attempts to make before logging an error :type max_req_attempts: int, optional :return: Either a Response object or None :rtype: Response or None """ if payload is None: request_payload = dict() else: request_payload = payload LOGGER.debug('Making a request for data...') for i in range(1, max_req_attempts + 1): LOGGER.debug(f'Attempt #{i}') response = api_handler.api_call(url, subscription, method, request_payload) LOGGER.debug(f'Response URL: {response.url}') if not check_if_response_successful(response): LOGGER.info('Beginning next_attempt') else: return response LOGGER.error( 'The maximum number of request attempts was reached; returning None') return None
def handle(self, *args, **options) -> None: """ Entrypoint method required by BaseCommand class (see Django docs). Checks whether the ApiUtil instance is properly configured, invoking the main function if so and exiting if not. """ try: api_util: ApiUtil = ApiUtil(os.getenv('API_DIR_URL', ''), os.getenv('API_DIR_CLIENT_ID', ''), os.getenv('API_DIR_SECRET', ''), API_CONFIG_PATH) except Exception as e: LOGGER.error(e) LOGGER.error( 'api_util was improperly configured; the program will exit.') sys.exit(1) main(api_util)
def setUp(self): """ Initializes api_handler and API response data used for patching. """ self.api_handler: ApiUtil = ApiUtil( os.getenv('API_DIR_URL', ''), os.getenv('API_DIR_CLIENT_ID', ''), os.getenv('API_DIR_SECRET', ''), os.path.join(ROOT_DIR, 'config', 'apis.json')) with open(os.path.join(API_FIXTURES_DIR, 'canvas_subs.json'), 'r') as test_canvas_subs_file: canvas_subs_dict: Dict[str, List[Dict[str, Any]]] = json.loads( test_canvas_subs_file.read()) self.canvas_dada_place_subs: List[Dict[ str, Any]] = canvas_subs_dict['DADA_Placement_1'] with open(os.path.join(API_FIXTURES_DIR, 'mpathways_resp_data.json'), 'r') as mpathways_resp_data_file: self.mpathways_resp_data: List[Dict[str, Any]] = json.loads( mpathways_resp_data_file.read())
# Related documentation # https://virtualenv.pypa.io/en/latest/ # https://docs.python.org/3/library/logging.html # https://requests.readthedocs.io/en/master/api/#requests.Response # # Load configuration dictionary, set up logging and constants with open('config/apis.json', 'r', encoding='utf-8') as config_file: CONFIG_DICT = json.loads(config_file.read()) # Better to use logging here so you can see all the output from umich_api by specifying DEBUG logging.basicConfig(level=CONFIG_DICT['LOG_LEVEL']) logger = logging.getLogger(__name__) API_UTIL = ApiUtil(CONFIG_DICT['BASE_URL'], CONFIG_DICT['CLIENT_ID'], CONFIG_DICT['SECRET']) print(CONFIG_DICT['BASE_URL']) print(CONFIG_DICT['CLIENT_ID']) print(CONFIG_DICT['SECRET']) MCOMM_URL_ENDING = 'MCommunity/People/' MCOMM_SCOPE = 'mcommunity' # Function(s) def get_role_info_using_uniqname(uniqname): url_ending = MCOMM_URL_ENDING + uniqname response = API_UTIL.api_call(url_ending, MCOMM_SCOPE) logger.debug(response.status_code) data = json.loads(response.text) person_data = data['person']
from umich_api.api_utils import ApiUtil from seumich.models import Mentor from management.models import MentorStudentCourseObserver import dateutil.parser as parser from decouple import config, Csv logger = logging.getLogger(__name__) # use ApiUtil for all API directory calls apiUtil = ApiUtil(config("APIUTIL_URL", default=""), config("APIUTIL_KEY", default=""), config("APIUTIL_SECRET", default="")) # cron job to populate course and user tables class StudentExplorerCronJob(CronJobBase): schedule = Schedule(run_at_times=settings.RUN_AT_TIMES) code = 'student_explorer.StudentExplorerCronJob' # a unique code def get_user_canvas_id(self, user_uniqname): """ This process gets the canvas user id for user :param user_uniqname: :return: """
from course_inventory.gql_queries import queries as QUERIES from course_inventory.published_date import FetchPublishedDate from db.db_creator import DBCreator from environ import DATA_DIR, ENV from vocab import DataSourceStatus, JobError, ValidDataSourceName # Initialize settings and globals logger = logging.getLogger(__name__) CANVAS = ENV.get('CANVAS', {}) ACCOUNT_ID = CANVAS.get('CANVAS_ACCOUNT_ID', 1) TERM_IDS = CANVAS['CANVAS_TERM_IDS'] API_UTIL = ApiUtil(CANVAS['API_BASE_URL'], CANVAS['API_CLIENT_ID'], CANVAS['API_CLIENT_SECRET']) SUBSCRIPTION_NAME = CANVAS['API_SUBSCRIPTION_NAME'] API_SCOPE_PREFIX = CANVAS['API_SCOPE_PREFIX'] CANVAS_TOKEN = CANVAS['CANVAS_TOKEN'] CANVAS_URL = CANVAS['CANVAS_URL'] MAX_REQ_ATTEMPTS = ENV.get('MAX_REQ_ATTEMPTS', 3) NUM_ASYNC_WORKERS = ENV.get('NUM_ASYNC_WORKERS', 8) CREATE_CSVS = ENV.get('CREATE_CSVS', False) INVENTORY_DB = ENV['INVENTORY_DB'] CANVAS_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ' # Function(s) - Canvas
from course_inventory.canvas_course_usage import CanvasCourseUsage from course_inventory.gql_queries import queries as QUERIES from course_inventory.published_date import FetchPublishedDate from db.db_creator import DBCreator from environ import ENV from vocab import ValidDataSourceName # Initialize settings and globals logger = logging.getLogger(__name__) ACCOUNT_ID = ENV.get('CANVAS_ACCOUNT_ID', 1) TERM_IDS = ENV['CANVAS_TERM_IDS'] API_UTIL = ApiUtil(ENV['API_BASE_URL'], ENV['API_CLIENT_ID'], ENV['API_CLIENT_SECRET']) SUBSCRIPTION_NAME = ENV['API_SUBSCRIPTION_NAME'] API_SCOPE_PREFIX = ENV['API_SCOPE_PREFIX'] MAX_REQ_ATTEMPTS = ENV['MAX_REQ_ATTEMPTS'] CANVAS_TOKEN = ENV['CANVAS_TOKEN'] CANVAS_URL = ENV['CANVAS_URL'] NUM_ASYNC_WORKERS = ENV.get('NUM_ASYNC_WORKERS', 8) CREATE_CSVS = ENV.get('CREATE_CSVS', False) INVENTORY_DB = ENV['INVENTORY_DB'] APPEND_TABLE_NAMES = ENV.get('APPEND_TABLE_NAMES', ['job_run', 'data_source_status']) CANVAS_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ' # Function(s) - Canvas