def pull(self, db, course_, dry_run): cid = canvas_id.CanvasID(self.filename, course_.canvas_id) cid.find_id(db) quiz_fields = dict(self.gen_fields()) quiz_fields['id'] = cid.canvas_id pulled, _ = pull(db, course_, quiz_fields, dry_run) return pulled
def remove(self, db, course_, dry_run): course_id = course_.canvas_id cid = canvas_id.CanvasID(self.filename, course_id) cid.find_id(db) if not cid.canvas_id: print(f"failed to delete {self} from course {course_}") print("No canvas information found for the given component. If the" " component still exists in Canvas we may have lost track " "of it so you will have to manually delete it. Sorry!") return # TODO: confirm they want to delete it? path = self.format_update_path(db, course_id, cid.canvas_id) resp = helpers.delete(path, dry_run=dry_run) err = False if "errors" in resp: print(f"canvas failed to delete the component {self}") for error in resp['errors']: if "does not exist" in error['message']: print("But that's ok because you were probably just " "removing the local canvas info for it.") else: print("CANVAS ERROR:", error['message']) err = True if err: print("local remove action aborted") print( "Canvas may or may not have successfully deleted the component" ) return if dry_run: print(f"DRYRUN - deleting the canvas relationship for {self}") else: cid.remove(db)
def removefile(db, course_, full_path, dry_run): cid = canvas_id.CanvasID(full_path, course_.canvas_id) cid.find_id(db) if cid.canvas_id: r = helpers.delete(FILE_PATH.format(cid.canvas_id), dry_run=dry_run) if r.get('upload_status') == 'success': cid.remove(db)
def pull(self, db, course_, dry_run): cid = canvas_id.CanvasID(self.filename, course_.canvas_id) cid.find_id(db) path = self.format_update_path(db, course_.canvas_id, cid.canvas_id) resp = helpers.get(path, dry_run=dry_run) remote = self.__class__.build(resp) remote.filename = self.filename return remote
def pushfile(db, course_, full_path, hidden, dry_run): parent_folder_path, name = os.path.split(full_path) if parent_folder_path[:5] == "files": parent_folder_path = parent_folder_path[5:] if parent_folder_path[:1] == "/": parent_folder_path = parent_folder_path[1:] # https://canvas.instructure.com/doc/api/file.file_uploads.html params = { "name": name, "size": os.path.getsize(full_path), "parent_folder_path": parent_folder_path, "publish": False, } # Step 1: Telling Canvas about the file upload and getting a token resp = helpers.post(COURSE_FILES_PATH.format(course_.canvas_id), params, dry_run=dry_run) # Step 2: Upload the file data to the URL given in the previous response if not dry_run and ("upload_url" not in resp or "upload_params" not in resp): logging.error("Invalid response received for file upload") print(resp) return file_ = {'file': open(full_path, 'rb')} req_url = resp.get('upload_url') params = resp.get("upload_params") logging.info(f"POST {req_url}") logging.debug(f"Params: {params}") logging.debug(f"File: {file_}") if dry_run: print( "DRYRUN - making request (use --api or --api-dump for more details)" ) else: resp = requests.request("POST", req_url, params=params, files=file_) if resp.text: r = resp.json() logging.debug(json.dumps(r, sort_keys=True, indent=4)) file_id = r.get('id') cid = canvas_id.CanvasID(full_path, course_.canvas_id) cid.canvas_id = file_id cid.save(db) # Step 3: Confirm the upload's success (requests follows redirects by # default, so far this has been enough to satisfy canvas) if hidden: r = helpers.put(FILE_PATH.format(file_id), {"hidden": True}) if not r.get('hidden'): logging.error("TODO: failed to hide the file")
def get_assignment_group_id(self, db, course_id): if self.assignment_group: ags = db.table(assignment_group.ASSIGN_GROUPS_TABLE) results = ags.search(tinydb.Query().name == self.assignment_group) if not results: raise ValueError(f"failed to find AssignmentGroup called '{self.assignment_group}'") # assumes assignment group names will be unique fname = assignment_group.AssignmentGroup(**dict(results[0])).filename cid = canvas_id.CanvasID(fname, course_id) cid.find_id(db) self.assignment_group_id = cid.canvas_id
def postprocess(self, db, course_, dry_run): course_id = course_.canvas_id cid = canvas_id.CanvasID(self.filename, course_id) cid.find_id(db) if not cid.canvas_id: print(f"failed to add ModuleItems to Module {self}, we don't " "have a canvas id for it") print("make sure the module has first been pushed to Canvas") return for i in range(len(self.items)): item = self.items[i] if isinstance(item, dict): # yaml objects (dictionaries) should be properly formatted if self.filename and 'item' in item: item["filename"] = f"{self.filename}--{item['item']}" elif'external_url' in item: # URL item["filename"] = f"{self.filename}--{item['external_url']}" if 'published' not in item: item['published'] = True elif 'title' in item: # SubHeader item["filename"] = f"{self.filename}--{item['title']}" if 'published' not in item: item['published'] = True item["position"] = i+1 item_component = component.build("ModuleItem", item) elif isinstance(item, str): # possible string-only options: filename, url, SubHeader base_item = { "filename": f"{self.filename}--{item}", "position": i+1 } if os.path.isfile(item): base_item['item'] = item elif helpers.isurl(item): base_item["external_url"] = item # if title is blank, canvas says "No Title" on the module # item, displaying the url instead seems like a better # default base_item["title"] = item base_item["published"] = True else: # assume SubHeader base_item["title"] = item base_item["published"] = True item_component = component.build("ModuleItem", base_item) else: raise TypeError(f"Invalid item specification on module {self}") print(f"\tpushing {item_component} to {self}") item_component.push(db, course_, dry_run, parent_component=self)
def pull_page(db, course_id, page_url, dry_run): page_ = helpers.get(PAGE_PATH.format(course_id, page_url), dry_run=dry_run) cid = canvas_id.find_by_id(db, course_id, page_.get('url')) if cid: page_['filename'] = cid.filename else: page_['filename'] = component.gen_filename(PAGES_DIR, page_.get('title', '')) cid = canvas_id.CanvasID(page_['filename'], course_id) cid.canvas_id = page_.get('url') cid.save(db) return Page.build(page_), cid
def pull(db, course_, quiz_, dry_run): course_id = course_.canvas_id quiz_id = quiz_.get('id') if not quiz_id: logging.error(f"Quiz {quiz_id} does not exist for course {course_id}") return None, None cid = canvas_id.find_by_id(db, course_id, quiz_id) if cid: quiz_['filename'] = cid.filename else: quiz_['filename'] = component.gen_filename(QUIZZES_DIR, quiz_.get('title', '')) cid = canvas_id.CanvasID(quiz_['filename'], course_id) cid.canvas_id = quiz_.get('id') cid.save(db) # check assignment_group_id to fill in assignment_group by name agid = quiz_.get('assignment_group_id') if agid: # first check if we have a cid for the assignment group agcid = canvas_id.find_by_id(db, course_id, agid) if agcid: ag = helpers_yaml.read(agcid.filename) if ag: quiz_['assignment_group'] = ag.name else: logging.error("failed to find the assignment group for " f"the assignment group with id {agid}. Your " ".easeldb may be out of sync") else: # we could look at all the local assignment group files if we # don't have a cid for it but chances are there isn't a file. # so might as well just go back to canvas and ask for it agpath = assignment_group.ASSIGN_GROUP_PATH.format(course_id, agid) r = helpers.get(agpath, dry_run=dry_run) if 'name' in r: quiz_['assignment_group'] = r['name'] else: logging.error("TODO: invalid response from canvas for " "the assignment group: " + json.dumps(r, indent=4)) # quiz questions quiz_questions_path = QUIZ_PATH.format(course_.canvas_id, quiz_id) + "/questions" quiz_questions = helpers.get(quiz_questions_path) quiz_['quiz_questions'] = quiz_questions return Quiz.build(quiz_), cid
def pull_all(db, course_, dry_run): r = helpers.get(ASSIGN_GROUPS_PATH.format(course_.canvas_id), dry_run=dry_run) ags = [] for ag in tqdm(r): cid = canvas_id.find_by_id(db, course_.canvas_id, ag.get('id')) if cid: ag['filename'] = cid.filename else: ag['filename'] = component.gen_filename(ASSIGN_GROUPS_DIR, ag.get('name', '')) cid = canvas_id.CanvasID(ag['filename'], course_.canvas_id) cid.canvas_id = ag.get('id') cid.save(db) ags.append(AssignmentGroup.build(ag)) return ags
def pull(db, course_, assignment_id, dry_run): course_id = course_.canvas_id a = helpers.get(ASSIGNMENT_PATH.format(course_id, assignment_id), dry_run=dry_run) if not a.get('id'): logging.error(f"Assignment {assignment_id} does not exist for course {course_id}") return None, None cid = canvas_id.find_by_id(db, course_id, a.get('id')) if cid: a['filename'] = cid.filename else: a['filename'] = component.gen_filename(ASSIGNMENTS_DIR, a.get('name','')) cid = canvas_id.CanvasID(a['filename'], course_id) cid.canvas_id = a.get('id') cid.save(db) # check assignment_group_id to fill in assignment_group by name agid = a.get('assignment_group_id') if agid: # first check if we have a cid for the assignment group agcid = canvas_id.find_by_id(db, course_id, agid) if agcid: ag = helpers_yaml.read(agcid.filename) if ag: a['assignment_group'] = ag.name else: logging.error("failed to find the assignment group for " f"the assignment group with id {agid}. Your " ".easeldb may be out of sync") else: # we could look at all the local assignment group files if we # don't have a cid for it but chances are there isn't a file. # so might as well just go back to canvas and ask for it agpath = assignment_group.ASSIGN_GROUP_PATH.format(course_id, agid) r = helpers.get(agpath, dry_run=dry_run) if 'name' in r: a['assignment_group'] = r['name'] else: logging.error("TODO: invalid response from canvas for " "the assignment group: " + json.dumps(r, indent=4)) return Assignment.build(a), cid
def preprocess(self, db, course_, dry_run): if not self.item: # it should be a url or SubHeader so we only have to set the type # and filename if self.external_url: self.type = "ExternalUrl" else: self.type = "SubHeader" return cid = canvas_id.CanvasID(self.item, course_.canvas_id) cid.find_id(db) if self.item.startswith("files"): # TODO: need a better way to detect that this is a literal File to # link to and not just a yaml file containing item info such as for # a Page or Assignment self.type = "File" if not cid.canvas_id: # the file probably doesn't exist in the course so we need to # push it first files.push(db, course_, self.item, True, dry_run) return item = helpers_yaml.read(self.item) item.filename = self.item if not cid.canvas_id: # the component probably doesn't exist in the course so we need to # push it first item.push(db, course_, dry_run) if not self.title: self.title = getattr(item, "name", getattr(item, "title", self.item)) type_ = type(item).__name__ if type_ not in VALID_TYPES: raise ValueError(f"Cannot add an item of type {type_} to a module." " Can be one of {VALID_TYPES}") self.type = type_ cid = item.get_canvas_id(db, course_.canvas_id) if self.type == "Page": self.page_url = cid else: self.content_id = cid
def postprocess(self, db, course_id, dry_run): cid = canvas_id.CanvasID(self.filename, course_id) cid.find_id(db) if not cid.canvas_id: print(f"failed to add ModuleItems to Module {self}, we don't " "have a canvas id for it") print("make sure the module has first been pushed to Canvas") return for item in self.items: if isinstance(item, dict): item["filename"] = f"{self.filename}--{item['item']}" item_component = component.build("ModuleItem", item) elif isinstance(item, str): item_component = component.build("ModuleItem", { "item": item, "filename": f"{self.filename}--{item}"}) else: raise TypeError(f"Invalid item specification on module {self}") print(f"\tpushing ModuleItem {item_component} to Module {self}") item_component.push(db, course_id, dry_run, parent_component=self)
def postprocess(self, db, course_, dry_run): course_id = course_.canvas_id cid = canvas_id.CanvasID(self.filename, course_id) cid.find_id(db) if not cid.canvas_id: print(f"failed to add QuizQuestions to {self}, we don't " "have a canvas id for it") print("make sure the quiz has first been pushed to Canvas") return # delete any existing questions on the quiz # we'll use the local file as the source of truth and always update # canvas to match it quiz_path = QUIZ_PATH.format(course_id, cid.canvas_id) questions_path = quiz_path + "/questions" quiz_questions = helpers.get(questions_path, dry_run) for question in quiz_questions: if 'id' in question: path = questions_path + "/{}".format(question['id']) helpers.delete(path) # prepare actual QuizQuestion objects to be pushed to canvas questions = build_questions(self.quiz_questions) # push the questions for question in questions: print(f"\tpushing {question} to Quiz {self}") question.push(db, course_, dry_run, parent_component=self) # once I push the questions, canvas doesn't seem to update the # quiz's points possible until I save the entire quiz again... # https://community.canvaslms.com/t5/Question-Forum/Saving-Quizzes-w-API/td-p/226406 # turns out that canvas won't do this if the quiz is unpublished when # you create the questions. so I'm hackily unpublishing and then # publishing (if the user wants to publish it) if self.remember_published: helpers.put(quiz_path, {"quiz": {"published": True}})
def postprocess(self, db, course_, dry_run): course_id = course_.canvas_id if self.type not in ["ExternalUrl", "SubHeader"]: return cid = canvas_id.CanvasID(self.filename, course_id) cid.find_id(db) if not cid.canvas_id: print(f"Failed to publish {self}, we don't " "have a canvas id for it") print("Make sure the module item was succesfully pushed to Canvas") return # since Canvas doesn't allow you to publish some ModuleItems # (ExternalUrl, SubHeader) on creation, we'll update the same item here # to make sure it gets published (if desired; when 'published' is False # it won't be published) parent_module = self.load_module(db, course_id) path = self.format_update_path(db, course_id, cid.canvas_id, parent_module) self.preprocess(db, course_id, dry_run) resp = helpers.put(path, self, dry_run=dry_run) if "errors" in resp: print(f"failed to publish {self}") for error in resp['errors']: logging.error(error['message'])
def pull_all(db, course_, dry_run): modules = [] m = helpers.get(MODULES_PATH.format(course_.canvas_id), params={'include': ['items']}, dry_run=dry_run) for module_ in m: if 'items' not in module_ or not module_['items']: items_path = urllib.parse.urlparse(module_['items_url']).path module_['items'] = helpers.get(items_path, dry_run=dry_run) cid = canvas_id.find_by_id(db, course_.canvas_id, module_.get('id')) if cid: module_['filename'] = cid.filename else: module_['filename'] = component.gen_filename(MODULES_DIR, str(module_.get('position')) + '-' + module_.get('name','')) cid = canvas_id.CanvasID(module_['filename'], course_.canvas_id) cid.canvas_id = module_.get('id') cid.save(db) items = {} for item in module_['items']: item_id = 0 if item['type'] == "Page": item_id = item['page_url'] elif item['type'] in ['File', 'Assignment', 'Quiz']: item_id = item['content_id'] # SubHeaders and ExternalUrls won't be in the db anyway icid = canvas_id.find_by_id(db, course_.canvas_id, item_id) if item_id == 0 or not icid: # we don't have the item locally so try to pull it if item['type'] == "Page": _, icid = page.pull_page(db, course_.canvas_id, item['page_url'], dry_run) elif item['type'] == "File": # get file info file_path = urllib.parse.urlparse(item['url']).path file_ = helpers.get(file_path, dry_run=dry_run) url = file_['url'] # get parent folder info folder_path = files.COURSE_FOLDERS_PATH.format(course_.canvas_id)+"/"+str(file_['folder_id']) folder = helpers.get(folder_path, dry_run=dry_run) path = folder['full_name'][len("course "):] filepath = path + file_['filename'] # download the file icid = files.pull_file(db, course_.canvas_id, url, file_['id'], filepath) elif item['type'] == 'Assignment': _, icid = assignment.pull(db, course_, item_id, dry_run) elif item['type'] == 'Quiz': _, icid = quiz.pull(db, course_, item, dry_run) elif item['type'] == 'SubHeader': built_item = {'title': item['title']} if not item['published']: built_item['published'] = False elif item['type'] == 'ExternalUrl': built_item = { 'title': item.get('title', ''), 'external_url': item['external_url']} if not item['published']: built_item['published'] = False else: logging.warn("I can't find the module item " f"'{item['title']}' locally and I couldn't figure " "out how to pull it because it is not an " "easel-supported type: " + item['type']) continue position = item.get('position', 0) if position not in items: items[position] = [] if icid: built_item = {"item": icid.filename, "indent": item.get('indent', 0)} if item['type'] == 'File' and 'title' in item: built_item['title'] = item['title'] if 'indent' in built_item and built_item['indent'] == 0: del built_item['indent'] items[position].append(built_item) # order items by position all_items = [] for position in sorted(items.keys()): if position == 0: # we used 0 as default above so skip that for now continue all_items += items[position] # add positionless items to the end module_['items'] = all_items + items.get(0, []) modules.append(Module.build(module_)) return modules
def format_update_path(self, db, *path_args): """3 args -> course_id, quiz_id, quiz_question_id""" course_id = path_args[0] cid = canvas_id.CanvasID(path_args[2].filename, course_id) cid.find_id(db) return self.update_path.format(course_id, cid.canvas_id, path_args[1])
def format_create_path(self, db, *path_args): """2 args -> course_id, quiz_id""" course_id = path_args[0] cid = canvas_id.CanvasID(path_args[1].filename, course_id) cid.find_id(db) return self.create_path.format(course_id, cid.canvas_id)
def cmd_push(db, args): if not args.components: # push everything args.components = ["syllabus.md", "grading_scheme.yaml", "navigation.yaml", "assignment_groups", "assignments", "files", "pages", "quizzes", "modules"] if not args.course: args.course = course.find_all(db) else: args.course = course.match_courses(db, args.course) for component_filepath in args.components: if component_filepath.endswith("*"): component_filepath = component_filepath[:-1] if component_filepath.endswith("/"): component_filepath = component_filepath[:-1] if os.path.isdir(component_filepath) and not component_filepath.startswith("files"): if component_filepath not in helpers.DIRS: logging.error("Invalid directory: "+component_filepath) continue for child_path in os.listdir(component_filepath): full_child_path = component_filepath + '/' + child_path component = helpers_yaml.read(full_child_path) if component and not isinstance(component, str): component.filename = full_child_path for course_ in args.course: print(f"pushing {component} to {course_.name} ({course_.canvas_id})") component.push(db, course_, args.dry_run) elif component_filepath.startswith("files"): for course_ in args.course: files.push(db, course_, component_filepath, args.hidden, args.dry_run) else: if not os.path.isfile(component_filepath): logging.error("Cannot find file: " + component_filepath) continue for course_ in args.course: if component_filepath == "syllabus.md": print(f"pushing syllabus to {course_.name} ({course_.canvas_id})") course.push_syllabus(db, course_.canvas_id, args.dry_run) elif component_filepath == "grading_scheme.yaml": print(f"pushing grading scheme to {course_.name} ({course_.canvas_id})") cid = canvas_id.CanvasID(component_filepath, course_.canvas_id) cid.find_id(db) if cid.canvas_id == "": # Don't try to create a scheme if it doesn't already # exist. Ideally, we update the scheme but Canvas # apparently doesn't allow for that. component = helpers_yaml.read(component_filepath) component.push(db, course_, args.dry_run) cid.find_id(db) course.update_grading_scheme(db, course_.canvas_id, cid.canvas_id, args.dry_run) elif component_filepath == "navigation.yaml": print(f"pushing navigation tabs to {course_.name} ({course_.canvas_id})") navigation_tab.push(db, course_, args.dry_run) else: component = helpers_yaml.read(component_filepath) if component and not isinstance(component, str): component.filename = component_filepath print(f"pushing {component} to {course_.name} ({course_.canvas_id})") component.push(db, course_, args.dry_run) else: # not a yaml file so assume it's a file/dir to upload files.push(db, course_, component_filepath, args.hidden, args.dry_run)
def push(self, db, course_, dry_run, parent_component=None): """ push the component to the given canvas course The parent_component is used for the more complex scenarios when canvas nests components inside of others (e.g., ModuleItems inside of Modules). Usually in these cases, the child component's API endpoint paths will be an extension of the parent's path, so it parent_component allows us to extend the default behavior. In these cases, push() will be called in a different location than the typical execution path in commands.py. However, this other location should mimic that typical behavior (see preprocess() in module.py). """ course_id = course_.canvas_id found = self.find(db) cid = canvas_id.CanvasID(self.filename, course_id) cid.find_id(db) # possible scenarios: # - record not found, no canvas id -> create # - record found, no canvas id -> yaml overrules record, right? # that's what it's doing now -> create # - record not found, canvas id found -> this one's weird.. should we # pull the canvas version and merge with the local version? right # now it just overwrites the canvas version -> update # - record found, canvas id found -> update # - TODO: what about if we have a canvas id but the component has # been deleted in canvas? maybe just delete the canvas id record # and try again? Since deleting components might be rare anyway, # for now we'll just inform the user and they can remove the # component themselves before proceeding. This will mainly happen # when we delete a component that tracks children (e.g., an # assignment group because it will delete the # assignments that belong to that group, a module with its items) # so we might want to be proactive when we delete and then go # delete the assignments, but that requires even deeper tracking # ahead of time. # conclusion: whether we create or update only depends on the canvas id # but later on we might end up needing to do other stuff depending on # whether or not we have a db record? if cid.canvas_id == "": # create path = self.format_create_path(db, course_id, parent_component) self.preprocess(db, course_, dry_run) resp = helpers.post(path, self, dry_run=dry_run) if dry_run: print("DRYRUN - saving the canvas_id for the component " "(assuming the request worked)") return if self.filename: # only save the canvas id if we have a filename because we # don't want to save some components (e.g., quiz questions) if 'id' in resp: self.save(db) cid.canvas_id = resp['id'] cid.save(db) elif 'url' in resp: # pages use a url instead of an id but we can use them # interchangably for this self.save(db) cid.canvas_id = resp['url'] cid.save(db) elif "message" in resp: logging.error(resp['message']) else: raise ValueError( "TODO: handle unexpected response when creating component" ) self.postprocess(db, course_, dry_run) else: # update if not found: found = self elif len(found) > 1: raise ValueError( "TODO: handle too many results, means the filename was not unique" ) else: found = build(type(self).__name__, dict(found[0])) found.merge(self) path = self.format_update_path(db, course_id, cid.canvas_id, parent_component) found.preprocess(db, course_, dry_run) resp = helpers.put(path, found, dry_run=dry_run) if "errors" in resp: print(f"failed to update the component {found}") for error in resp['errors']: if isinstance(error, dict): if "does not exist" in error['message']: print("The component was deleted in canvas " "without us knowing about it. You should " "remove the component here and then try " "pushing it again.") else: print("CANVAS ERROR:", error['message']) else: print("CANVAS ERROR:", error) if dry_run: print(f"DRYRUN - saving the component {found}") else: found.save(db) found.postprocess(db, course_, dry_run)
def pull_file(db, course_id, url, id_, filepath): helpers.download_file(url, filepath) cid = canvas_id.CanvasID(filepath, course_id) cid.canvas_id = id_ cid.save(db) return cid
def get_canvas_id(self, db, course_id): cid = canvas_id.CanvasID(self.filename, course_id) cid.find_id(db) return cid.canvas_id