def main():
placement_exam = os.getenv('placement_exam')
path_properties = os.getenv('path_properties')
path_persistence = os.getenv('path_persistence')
print(f"Path to properties {path_properties}")
print(f"Placement exam: {placement_exam}")
persistance_file = f"{path_persistence}persistence_{placement_exam}.txt"
print(f"Persistance file: {persistance_file}")
with open(path_properties, 'r') as yml_file:
sf = yaml.load(yml_file, Loader=yaml.FullLoader)
print(sf['course'][placement_exam]['id'])
client_ = sf['api']['client']
secret_ = sf['api']['secret']
url_ = sf['api']['url']
print(client_)
print(secret_)
print(url_)
api_json: str = pkg_resources.resource_filename(__name__, 'apis.json')
api_handler = ApiUtil(url_, client_, secret_, api_json)
print(
api_handler.get_access_token('aa/oauth2/token',
'spanishplacementscores'))
with open(f'{path_persistence}/{placement_exam}.txt') as f:
latest_exam_date: str = f.read()
print(latest_exam_date)
def setUpClass(self):
self.url = os.getenv("url")
self.client_id = os.getenv("client_id")
self.secret = os.getenv("secret")
self.canvas_api = ApiUtil(self.url, self.client_id, self.secret, "canvasreadonly")
self.mcommunity_api = ApiUtil(self.url, self.client_id, self.secret, "mcommunity")
print ("URL is %s" % self.url)
def setUp(self):
"""Sets up ApiUtil instance and custom fixtures to be used by ScoresOrchestration tests."""
self.api_handler: ApiUtil = ApiUtil(
os.getenv('API_DIR_URL', ''), os.getenv('API_DIR_CLIENT_ID', ''),
os.getenv('API_DIR_SECRET', ''),
os.path.join(ROOT_DIR, 'config', 'apis.json'))
with open(os.path.join(API_FIXTURES_DIR, 'canvas_subs.json'),
'r') as test_canvas_subs_file:
canvas_subs_dict: Dict[str, List[Dict[str, Any]]] = json.loads(
test_canvas_subs_file.read())
self.canvas_potions_val_subs: List[Dict[
str, Any]] = canvas_subs_dict['Potions_Validation_1']
self.canvas_dada_place_subs_one: List[Dict[
str, Any]] = canvas_subs_dict['DADA_Placement_1']
self.canvas_dada_place_subs_two: List[Dict[
str, Any]] = canvas_subs_dict['DADA_Placement_2']
self.canvas_potions_place_subs_three: List[Dict[
str, Any]] = canvas_subs_dict['Potions_Placement_3']
with open(os.path.join(API_FIXTURES_DIR, 'mpathways_resp_data.json'),
'r') as mpathways_resp_data_file:
self.mpathways_resp_data: List[Dict[str, Any]] = json.loads(
mpathways_resp_data_file.read())
self.dup_get_mocks: List[MagicMock] = [
MagicMock(spec=Response,
status_code=200,
text=json.dumps(
canvas_subs_dict['Potions_Placement_1'])),
MagicMock(spec=Response,
status_code=200,
text=json.dumps(canvas_subs_dict['Potions_Placement_2']))
]
def setUp(self):
"""Set up ApiUtil instance and data needed by test methods."""
self.api_handler: ApiUtil = ApiUtil(
os.getenv('API_DIR_URL', ''), os.getenv('API_DIR_CLIENT_ID', ''),
os.getenv('API_DIR_SECRET', ''),
os.path.join(ROOT_DIR, 'config', 'apis.json'))
with open(os.path.join(API_FIXTURES_DIR, 'canvas_subs.json'),
'r') as test_canvas_subs_file:
canvas_subs_dict: dict[str, list[dict[str, Any]]] = json.loads(
test_canvas_subs_file.read())
self.canvas_potions_val_subs: list[dict[
str, Any]] = canvas_subs_dict['Potions_Validation_1']
# "Potions Validation" from test_04.json
some_course_id: int = 888888
some_assignment_id: int = 111112
some_filter: datetime = datetime(2020, 6, 1, 0, 0, 0, tzinfo=utc)
self.get_scores_url: str = (
f'{CANVAS_URL_BEGIN}/courses/{some_course_id}/students/submissions'
)
self.canvas_params: dict[str, Any] = {
'student_ids': ['all'],
'assignment_ids': [str(some_assignment_id)],
'per_page': '50',
'include': ['user'],
'graded_since': some_filter.strftime(ISO8601_FORMAT)
}
def setUpClass(cls):
cls.url = os.getenv("url")
cls.client_id = os.getenv("client_id")
cls.secret = os.getenv("secret")
cls.apiutil = ApiUtil(cls.url, cls.client_id, cls.secret)
# This line is needed so pylint doesn't complain about this variable not existing.
cls.__log = cls.__log
cls.__log.info("URL is %s" % cls.url)
def setUp(self):
"""
Initializes report; runs orchestrations, gathering time metadata; and sets up other shared variables.
"""
api_handler: ApiUtil = ApiUtil(
os.getenv('API_DIR_URL', ''),
os.getenv('API_DIR_CLIENT_ID', ''),
os.getenv('API_DIR_SECRET', ''),
os.path.join(ROOT_DIR, 'config', 'apis.json')
)
with open(os.path.join(API_FIXTURES_DIR, 'canvas_subs.json'), 'r') as test_canvas_subs_file:
canvas_subs_dict: Dict[str, List[Dict[str, Any]]] = json.loads(test_canvas_subs_file.read())
canvas_potions_val_subs: List[Dict[str, Any]] = canvas_subs_dict['Potions_Validation_1']
with open(os.path.join(API_FIXTURES_DIR, 'mpathways_resp_data.json'), 'r') as mpathways_resp_data_file:
mpathways_resp_data: List[Dict[str, Any]] = json.loads(mpathways_resp_data_file.read())
self.potions_report = Report.objects.get(id=1)
with patch('pe.orchestration.api_call_with_retries', autospec=True) as mock_get:
with patch.object(ApiUtil, 'api_call', autospec=True) as mock_send:
mock_get.side_effect = [
# Potions Placement - no more new submissions
MagicMock(spec=Response, status_code=200, text=json.dumps([])),
# Potions Validation - two more new submissions
MagicMock(spec=Response, status_code=200, text=json.dumps(canvas_potions_val_subs))
]
mock_send.side_effect = [
# Potions Placement - Only rweasley sub from test_04.json, fails to send
MagicMock(spec=Response, status_code=200, text=json.dumps(mpathways_resp_data[5])),
# Potions Validation - Four subs, two from canvas_subs.json, two from test_04.json, all send
MagicMock(spec=Response, status_code=200, text=json.dumps(mpathways_resp_data[2]))
]
fake_running_dt: datetime = datetime(2020, 6, 25, 16, 0, 0, tzinfo=utc)
self.exams_time_metadata: Dict[int, Dict[str, datetime]] = dict()
for exam in self.potions_report.exams.all():
start: datetime = fake_running_dt
exam_orca: ScoresOrchestration = ScoresOrchestration(api_handler, exam)
exam_orca.main()
fake_running_dt += timedelta(seconds=5)
end: datetime = fake_running_dt
self.exams_time_metadata[exam.id] = {
'start_time': start,
'end_time': end,
'sub_time_filter': exam_orca.sub_time_filter
}
fake_running_dt += timedelta(seconds=1)
self.fake_finished_at: datetime = fake_running_dt
self.expected_subject: str = (
'Placement Exams Report - Potions - Success: 4, Failure: 1, New: 2 - Run finished at 2020-06-25 12:00 PM'
)
class TestApiCalls(unittest.TestCase):
@classmethod
def setUpClass(self):
self.url = os.getenv("url")
self.client_id = os.getenv("client_id")
self.secret = os.getenv("secret")
self.canvas_api = ApiUtil(self.url, self.client_id, self.secret, "canvasreadonly")
self.mcommunity_api = ApiUtil(self.url, self.client_id, self.secret, "mcommunity")
print ("URL is %s" % self.url)
def test_mcommunity(self):
uniqname = "uniqname"
self.assertEqual(self.mcommunity_api.api_call(f"MCommunity/People/{uniqname}").status_code, 200)
def test_pagination(self):
resp = self.canvas_api.api_call(f"aa/CanvasReadOnly/courses/1/users")
logger.info(resp.headers)
def test_canvasreadonly(self):
resp = self.canvas_api.api_call(f"aa/CanvasReadOnly/brand_variables")
self.assertEqual(resp.status_code, 200)
def api_call_with_retries(
api_handler: ApiUtil,
url: str,
subscription: str,
method: str,
payload: Union[Dict[str, Any], None] = None,
max_req_attempts: int = 3,
) -> Union[Response, None]:
"""
Pulls data from the UM API Directory, handling errors and retrying if necessary.
When the maximum number of request attempts is reached, the function logs an error and returns None.
:param api_handler: Instance of ApiUtil
:type api_handler: ApiUtil
:param url: URL ending for request
:type url: string
:param subscription: Name of the subscription or scope the request should use
:type subscription: string
:param method: Request method that should be used (e.g. "GET", "PUT")
:type method: string
:param payload: Dictionary to include in the request body
:type payload: Dictionary with string keys or None, optional
:param max_req_attempts: Number of request attempts to make before logging an error
:type max_req_attempts: int, optional
:return: Either a Response object or None
:rtype: Response or None
"""
if payload is None:
request_payload = dict()
else:
request_payload = payload
LOGGER.debug('Making a request for data...')
for i in range(1, max_req_attempts + 1):
LOGGER.debug(f'Attempt #{i}')
response = api_handler.api_call(url, subscription, method,
request_payload)
LOGGER.debug(f'Response URL: {response.url}')
if not check_if_response_successful(response):
LOGGER.info('Beginning next_attempt')
else:
return response
LOGGER.error(
'The maximum number of request attempts was reached; returning None')
return None
def handle(self, *args, **options) -> None:
"""
Entrypoint method required by BaseCommand class (see Django docs).
Checks whether the ApiUtil instance is properly configured, invoking the main function if so
and exiting if not.
"""
try:
api_util: ApiUtil = ApiUtil(os.getenv('API_DIR_URL', ''),
os.getenv('API_DIR_CLIENT_ID', ''),
os.getenv('API_DIR_SECRET', ''),
API_CONFIG_PATH)
except Exception as e:
LOGGER.error(e)
LOGGER.error(
'api_util was improperly configured; the program will exit.')
sys.exit(1)
main(api_util)
def setUp(self):
"""
Initializes api_handler and API response data used for patching.
"""
self.api_handler: ApiUtil = ApiUtil(
os.getenv('API_DIR_URL', ''), os.getenv('API_DIR_CLIENT_ID', ''),
os.getenv('API_DIR_SECRET', ''),
os.path.join(ROOT_DIR, 'config', 'apis.json'))
with open(os.path.join(API_FIXTURES_DIR, 'canvas_subs.json'),
'r') as test_canvas_subs_file:
canvas_subs_dict: Dict[str, List[Dict[str, Any]]] = json.loads(
test_canvas_subs_file.read())
self.canvas_dada_place_subs: List[Dict[
str, Any]] = canvas_subs_dict['DADA_Placement_1']
with open(os.path.join(API_FIXTURES_DIR, 'mpathways_resp_data.json'),
'r') as mpathways_resp_data_file:
self.mpathways_resp_data: List[Dict[str, Any]] = json.loads(
mpathways_resp_data_file.read())
# Related documentation
# https://virtualenv.pypa.io/en/latest/
# https://docs.python.org/3/library/logging.html
# https://requests.readthedocs.io/en/master/api/#requests.Response
#
# Load configuration dictionary, set up logging and constants
with open('config/apis.json', 'r', encoding='utf-8') as config_file:
CONFIG_DICT = json.loads(config_file.read())
# Better to use logging here so you can see all the output from umich_api by specifying DEBUG
logging.basicConfig(level=CONFIG_DICT['LOG_LEVEL'])
logger = logging.getLogger(__name__)
API_UTIL = ApiUtil(CONFIG_DICT['BASE_URL'], CONFIG_DICT['CLIENT_ID'],
CONFIG_DICT['SECRET'])
print(CONFIG_DICT['BASE_URL'])
print(CONFIG_DICT['CLIENT_ID'])
print(CONFIG_DICT['SECRET'])
MCOMM_URL_ENDING = 'MCommunity/People/'
MCOMM_SCOPE = 'mcommunity'
# Function(s)
def get_role_info_using_uniqname(uniqname):
url_ending = MCOMM_URL_ENDING + uniqname
response = API_UTIL.api_call(url_ending, MCOMM_SCOPE)
logger.debug(response.status_code)
data = json.loads(response.text)
person_data = data['person']
from umich_api.api_utils import ApiUtil
from seumich.models import Mentor
from management.models import MentorStudentCourseObserver
import dateutil.parser as parser
from decouple import config, Csv
logger = logging.getLogger(__name__)
# use ApiUtil for all API directory calls
apiUtil = ApiUtil(config("APIUTIL_URL", default=""),
config("APIUTIL_KEY", default=""),
config("APIUTIL_SECRET", default=""))
# cron job to populate course and user tables
class StudentExplorerCronJob(CronJobBase):
schedule = Schedule(run_at_times=settings.RUN_AT_TIMES)
code = 'student_explorer.StudentExplorerCronJob' # a unique code
def get_user_canvas_id(self, user_uniqname):
"""
This process gets the canvas user id for user
:param user_uniqname:
:return:
"""
from course_inventory.gql_queries import queries as QUERIES
from course_inventory.published_date import FetchPublishedDate
from db.db_creator import DBCreator
from environ import DATA_DIR, ENV
from vocab import DataSourceStatus, JobError, ValidDataSourceName
# Initialize settings and globals
logger = logging.getLogger(__name__)
CANVAS = ENV.get('CANVAS', {})
ACCOUNT_ID = CANVAS.get('CANVAS_ACCOUNT_ID', 1)
TERM_IDS = CANVAS['CANVAS_TERM_IDS']
API_UTIL = ApiUtil(CANVAS['API_BASE_URL'], CANVAS['API_CLIENT_ID'],
CANVAS['API_CLIENT_SECRET'])
SUBSCRIPTION_NAME = CANVAS['API_SUBSCRIPTION_NAME']
API_SCOPE_PREFIX = CANVAS['API_SCOPE_PREFIX']
CANVAS_TOKEN = CANVAS['CANVAS_TOKEN']
CANVAS_URL = CANVAS['CANVAS_URL']
MAX_REQ_ATTEMPTS = ENV.get('MAX_REQ_ATTEMPTS', 3)
NUM_ASYNC_WORKERS = ENV.get('NUM_ASYNC_WORKERS', 8)
CREATE_CSVS = ENV.get('CREATE_CSVS', False)
INVENTORY_DB = ENV['INVENTORY_DB']
CANVAS_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
# Function(s) - Canvas
from course_inventory.canvas_course_usage import CanvasCourseUsage
from course_inventory.gql_queries import queries as QUERIES
from course_inventory.published_date import FetchPublishedDate
from db.db_creator import DBCreator
from environ import ENV
from vocab import ValidDataSourceName
# Initialize settings and globals
logger = logging.getLogger(__name__)
ACCOUNT_ID = ENV.get('CANVAS_ACCOUNT_ID', 1)
TERM_IDS = ENV['CANVAS_TERM_IDS']
API_UTIL = ApiUtil(ENV['API_BASE_URL'], ENV['API_CLIENT_ID'], ENV['API_CLIENT_SECRET'])
SUBSCRIPTION_NAME = ENV['API_SUBSCRIPTION_NAME']
API_SCOPE_PREFIX = ENV['API_SCOPE_PREFIX']
MAX_REQ_ATTEMPTS = ENV['MAX_REQ_ATTEMPTS']
CANVAS_TOKEN = ENV['CANVAS_TOKEN']
CANVAS_URL = ENV['CANVAS_URL']
NUM_ASYNC_WORKERS = ENV.get('NUM_ASYNC_WORKERS', 8)
CREATE_CSVS = ENV.get('CREATE_CSVS', False)
INVENTORY_DB = ENV['INVENTORY_DB']
APPEND_TABLE_NAMES = ENV.get('APPEND_TABLE_NAMES', ['job_run', 'data_source_status'])
CANVAS_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
# Function(s) - Canvas