def pull(self, db, course_, dry_run):
cid = canvas_id.CanvasID(self.filename, course_.canvas_id)
cid.find_id(db)
quiz_fields = dict(self.gen_fields())
quiz_fields['id'] = cid.canvas_id
pulled, _ = pull(db, course_, quiz_fields, dry_run)
return pulled
def remove(self, db, course_, dry_run):
course_id = course_.canvas_id
cid = canvas_id.CanvasID(self.filename, course_id)
cid.find_id(db)
if not cid.canvas_id:
print(f"failed to delete {self} from course {course_}")
print("No canvas information found for the given component. If the"
" component still exists in Canvas we may have lost track "
"of it so you will have to manually delete it. Sorry!")
return
# TODO: confirm they want to delete it?
path = self.format_update_path(db, course_id, cid.canvas_id)
resp = helpers.delete(path, dry_run=dry_run)
err = False
if "errors" in resp:
print(f"canvas failed to delete the component {self}")
for error in resp['errors']:
if "does not exist" in error['message']:
print("But that's ok because you were probably just "
"removing the local canvas info for it.")
else:
print("CANVAS ERROR:", error['message'])
err = True
if err:
print("local remove action aborted")
print(
"Canvas may or may not have successfully deleted the component"
)
return
if dry_run:
print(f"DRYRUN - deleting the canvas relationship for {self}")
else:
cid.remove(db)
def removefile(db, course_, full_path, dry_run):
cid = canvas_id.CanvasID(full_path, course_.canvas_id)
cid.find_id(db)
if cid.canvas_id:
r = helpers.delete(FILE_PATH.format(cid.canvas_id), dry_run=dry_run)
if r.get('upload_status') == 'success':
cid.remove(db)
def pull(self, db, course_, dry_run):
cid = canvas_id.CanvasID(self.filename, course_.canvas_id)
cid.find_id(db)
path = self.format_update_path(db, course_.canvas_id, cid.canvas_id)
resp = helpers.get(path, dry_run=dry_run)
remote = self.__class__.build(resp)
remote.filename = self.filename
return remote
def pushfile(db, course_, full_path, hidden, dry_run):
parent_folder_path, name = os.path.split(full_path)
if parent_folder_path[:5] == "files":
parent_folder_path = parent_folder_path[5:]
if parent_folder_path[:1] == "/":
parent_folder_path = parent_folder_path[1:]
# https://canvas.instructure.com/doc/api/file.file_uploads.html
params = {
"name": name,
"size": os.path.getsize(full_path),
"parent_folder_path": parent_folder_path,
"publish": False,
}
# Step 1: Telling Canvas about the file upload and getting a token
resp = helpers.post(COURSE_FILES_PATH.format(course_.canvas_id),
params,
dry_run=dry_run)
# Step 2: Upload the file data to the URL given in the previous response
if not dry_run and ("upload_url" not in resp
or "upload_params" not in resp):
logging.error("Invalid response received for file upload")
print(resp)
return
file_ = {'file': open(full_path, 'rb')}
req_url = resp.get('upload_url')
params = resp.get("upload_params")
logging.info(f"POST {req_url}")
logging.debug(f"Params: {params}")
logging.debug(f"File: {file_}")
if dry_run:
print(
"DRYRUN - making request (use --api or --api-dump for more details)"
)
else:
resp = requests.request("POST", req_url, params=params, files=file_)
if resp.text:
r = resp.json()
logging.debug(json.dumps(r, sort_keys=True, indent=4))
file_id = r.get('id')
cid = canvas_id.CanvasID(full_path, course_.canvas_id)
cid.canvas_id = file_id
cid.save(db)
# Step 3: Confirm the upload's success (requests follows redirects by
# default, so far this has been enough to satisfy canvas)
if hidden:
r = helpers.put(FILE_PATH.format(file_id), {"hidden": True})
if not r.get('hidden'):
logging.error("TODO: failed to hide the file")
def get_assignment_group_id(self, db, course_id):
if self.assignment_group:
ags = db.table(assignment_group.ASSIGN_GROUPS_TABLE)
results = ags.search(tinydb.Query().name == self.assignment_group)
if not results:
raise ValueError(f"failed to find AssignmentGroup called '{self.assignment_group}'")
# assumes assignment group names will be unique
fname = assignment_group.AssignmentGroup(**dict(results[0])).filename
cid = canvas_id.CanvasID(fname, course_id)
cid.find_id(db)
self.assignment_group_id = cid.canvas_id
def postprocess(self, db, course_, dry_run):
course_id = course_.canvas_id
cid = canvas_id.CanvasID(self.filename, course_id)
cid.find_id(db)
if not cid.canvas_id:
print(f"failed to add ModuleItems to Module {self}, we don't "
"have a canvas id for it")
print("make sure the module has first been pushed to Canvas")
return
for i in range(len(self.items)):
item = self.items[i]
if isinstance(item, dict):
# yaml objects (dictionaries) should be properly formatted
if self.filename and 'item' in item:
item["filename"] = f"{self.filename}--{item['item']}"
elif'external_url' in item:
# URL
item["filename"] = f"{self.filename}--{item['external_url']}"
if 'published' not in item:
item['published'] = True
elif 'title' in item:
# SubHeader
item["filename"] = f"{self.filename}--{item['title']}"
if 'published' not in item:
item['published'] = True
item["position"] = i+1
item_component = component.build("ModuleItem", item)
elif isinstance(item, str):
# possible string-only options: filename, url, SubHeader
base_item = {
"filename": f"{self.filename}--{item}",
"position": i+1
}
if os.path.isfile(item):
base_item['item'] = item
elif helpers.isurl(item):
base_item["external_url"] = item
# if title is blank, canvas says "No Title" on the module
# item, displaying the url instead seems like a better
# default
base_item["title"] = item
base_item["published"] = True
else:
# assume SubHeader
base_item["title"] = item
base_item["published"] = True
item_component = component.build("ModuleItem", base_item)
else:
raise TypeError(f"Invalid item specification on module {self}")
print(f"\tpushing {item_component} to {self}")
item_component.push(db, course_, dry_run, parent_component=self)
def pull_page(db, course_id, page_url, dry_run):
page_ = helpers.get(PAGE_PATH.format(course_id, page_url), dry_run=dry_run)
cid = canvas_id.find_by_id(db, course_id, page_.get('url'))
if cid:
page_['filename'] = cid.filename
else:
page_['filename'] = component.gen_filename(PAGES_DIR,
page_.get('title', ''))
cid = canvas_id.CanvasID(page_['filename'], course_id)
cid.canvas_id = page_.get('url')
cid.save(db)
return Page.build(page_), cid
def pull(db, course_, quiz_, dry_run):
course_id = course_.canvas_id
quiz_id = quiz_.get('id')
if not quiz_id:
logging.error(f"Quiz {quiz_id} does not exist for course {course_id}")
return None, None
cid = canvas_id.find_by_id(db, course_id, quiz_id)
if cid:
quiz_['filename'] = cid.filename
else:
quiz_['filename'] = component.gen_filename(QUIZZES_DIR,
quiz_.get('title', ''))
cid = canvas_id.CanvasID(quiz_['filename'], course_id)
cid.canvas_id = quiz_.get('id')
cid.save(db)
# check assignment_group_id to fill in assignment_group by name
agid = quiz_.get('assignment_group_id')
if agid:
# first check if we have a cid for the assignment group
agcid = canvas_id.find_by_id(db, course_id, agid)
if agcid:
ag = helpers_yaml.read(agcid.filename)
if ag:
quiz_['assignment_group'] = ag.name
else:
logging.error("failed to find the assignment group for "
f"the assignment group with id {agid}. Your "
".easeldb may be out of sync")
else:
# we could look at all the local assignment group files if we
# don't have a cid for it but chances are there isn't a file.
# so might as well just go back to canvas and ask for it
agpath = assignment_group.ASSIGN_GROUP_PATH.format(course_id, agid)
r = helpers.get(agpath, dry_run=dry_run)
if 'name' in r:
quiz_['assignment_group'] = r['name']
else:
logging.error("TODO: invalid response from canvas for "
"the assignment group: " +
json.dumps(r, indent=4))
# quiz questions
quiz_questions_path = QUIZ_PATH.format(course_.canvas_id,
quiz_id) + "/questions"
quiz_questions = helpers.get(quiz_questions_path)
quiz_['quiz_questions'] = quiz_questions
return Quiz.build(quiz_), cid
def pull_all(db, course_, dry_run):
r = helpers.get(ASSIGN_GROUPS_PATH.format(course_.canvas_id),
dry_run=dry_run)
ags = []
for ag in tqdm(r):
cid = canvas_id.find_by_id(db, course_.canvas_id, ag.get('id'))
if cid:
ag['filename'] = cid.filename
else:
ag['filename'] = component.gen_filename(ASSIGN_GROUPS_DIR,
ag.get('name', ''))
cid = canvas_id.CanvasID(ag['filename'], course_.canvas_id)
cid.canvas_id = ag.get('id')
cid.save(db)
ags.append(AssignmentGroup.build(ag))
return ags
def pull(db, course_, assignment_id, dry_run):
course_id = course_.canvas_id
a = helpers.get(ASSIGNMENT_PATH.format(course_id,
assignment_id), dry_run=dry_run)
if not a.get('id'):
logging.error(f"Assignment {assignment_id} does not exist for course {course_id}")
return None, None
cid = canvas_id.find_by_id(db, course_id, a.get('id'))
if cid:
a['filename'] = cid.filename
else:
a['filename'] = component.gen_filename(ASSIGNMENTS_DIR, a.get('name',''))
cid = canvas_id.CanvasID(a['filename'], course_id)
cid.canvas_id = a.get('id')
cid.save(db)
# check assignment_group_id to fill in assignment_group by name
agid = a.get('assignment_group_id')
if agid:
# first check if we have a cid for the assignment group
agcid = canvas_id.find_by_id(db, course_id, agid)
if agcid:
ag = helpers_yaml.read(agcid.filename)
if ag:
a['assignment_group'] = ag.name
else:
logging.error("failed to find the assignment group for "
f"the assignment group with id {agid}. Your "
".easeldb may be out of sync")
else:
# we could look at all the local assignment group files if we
# don't have a cid for it but chances are there isn't a file.
# so might as well just go back to canvas and ask for it
agpath = assignment_group.ASSIGN_GROUP_PATH.format(course_id, agid)
r = helpers.get(agpath, dry_run=dry_run)
if 'name' in r:
a['assignment_group'] = r['name']
else:
logging.error("TODO: invalid response from canvas for "
"the assignment group: " + json.dumps(r, indent=4))
return Assignment.build(a), cid
def preprocess(self, db, course_, dry_run):
if not self.item:
# it should be a url or SubHeader so we only have to set the type
# and filename
if self.external_url:
self.type = "ExternalUrl"
else:
self.type = "SubHeader"
return
cid = canvas_id.CanvasID(self.item, course_.canvas_id)
cid.find_id(db)
if self.item.startswith("files"):
# TODO: need a better way to detect that this is a literal File to
# link to and not just a yaml file containing item info such as for
# a Page or Assignment
self.type = "File"
if not cid.canvas_id:
# the file probably doesn't exist in the course so we need to
# push it first
files.push(db, course_, self.item, True, dry_run)
return
item = helpers_yaml.read(self.item)
item.filename = self.item
if not cid.canvas_id:
# the component probably doesn't exist in the course so we need to
# push it first
item.push(db, course_, dry_run)
if not self.title:
self.title = getattr(item, "name", getattr(item, "title",
self.item))
type_ = type(item).__name__
if type_ not in VALID_TYPES:
raise ValueError(f"Cannot add an item of type {type_} to a module."
" Can be one of {VALID_TYPES}")
self.type = type_
cid = item.get_canvas_id(db, course_.canvas_id)
if self.type == "Page":
self.page_url = cid
else:
self.content_id = cid
def postprocess(self, db, course_id, dry_run):
cid = canvas_id.CanvasID(self.filename, course_id)
cid.find_id(db)
if not cid.canvas_id:
print(f"failed to add ModuleItems to Module {self}, we don't "
"have a canvas id for it")
print("make sure the module has first been pushed to Canvas")
return
for item in self.items:
if isinstance(item, dict):
item["filename"] = f"{self.filename}--{item['item']}"
item_component = component.build("ModuleItem", item)
elif isinstance(item, str):
item_component = component.build("ModuleItem", {
"item": item,
"filename": f"{self.filename}--{item}"})
else:
raise TypeError(f"Invalid item specification on module {self}")
print(f"\tpushing ModuleItem {item_component} to Module {self}")
item_component.push(db, course_id, dry_run, parent_component=self)
def postprocess(self, db, course_, dry_run):
course_id = course_.canvas_id
cid = canvas_id.CanvasID(self.filename, course_id)
cid.find_id(db)
if not cid.canvas_id:
print(f"failed to add QuizQuestions to {self}, we don't "
"have a canvas id for it")
print("make sure the quiz has first been pushed to Canvas")
return
# delete any existing questions on the quiz
# we'll use the local file as the source of truth and always update
# canvas to match it
quiz_path = QUIZ_PATH.format(course_id, cid.canvas_id)
questions_path = quiz_path + "/questions"
quiz_questions = helpers.get(questions_path, dry_run)
for question in quiz_questions:
if 'id' in question:
path = questions_path + "/{}".format(question['id'])
helpers.delete(path)
# prepare actual QuizQuestion objects to be pushed to canvas
questions = build_questions(self.quiz_questions)
# push the questions
for question in questions:
print(f"\tpushing {question} to Quiz {self}")
question.push(db, course_, dry_run, parent_component=self)
# once I push the questions, canvas doesn't seem to update the
# quiz's points possible until I save the entire quiz again...
# https://community.canvaslms.com/t5/Question-Forum/Saving-Quizzes-w-API/td-p/226406
# turns out that canvas won't do this if the quiz is unpublished when
# you create the questions. so I'm hackily unpublishing and then
# publishing (if the user wants to publish it)
if self.remember_published:
helpers.put(quiz_path, {"quiz": {"published": True}})
def postprocess(self, db, course_, dry_run):
course_id = course_.canvas_id
if self.type not in ["ExternalUrl", "SubHeader"]:
return
cid = canvas_id.CanvasID(self.filename, course_id)
cid.find_id(db)
if not cid.canvas_id:
print(f"Failed to publish {self}, we don't "
"have a canvas id for it")
print("Make sure the module item was succesfully pushed to Canvas")
return
# since Canvas doesn't allow you to publish some ModuleItems
# (ExternalUrl, SubHeader) on creation, we'll update the same item here
# to make sure it gets published (if desired; when 'published' is False
# it won't be published)
parent_module = self.load_module(db, course_id)
path = self.format_update_path(db, course_id, cid.canvas_id,
parent_module)
self.preprocess(db, course_id, dry_run)
resp = helpers.put(path, self, dry_run=dry_run)
if "errors" in resp:
print(f"failed to publish {self}")
for error in resp['errors']:
logging.error(error['message'])
def pull_all(db, course_, dry_run):
modules = []
m = helpers.get(MODULES_PATH.format(course_.canvas_id),
params={'include': ['items']}, dry_run=dry_run)
for module_ in m:
if 'items' not in module_ or not module_['items']:
items_path = urllib.parse.urlparse(module_['items_url']).path
module_['items'] = helpers.get(items_path, dry_run=dry_run)
cid = canvas_id.find_by_id(db, course_.canvas_id, module_.get('id'))
if cid:
module_['filename'] = cid.filename
else:
module_['filename'] = component.gen_filename(MODULES_DIR,
str(module_.get('position')) + '-' + module_.get('name',''))
cid = canvas_id.CanvasID(module_['filename'], course_.canvas_id)
cid.canvas_id = module_.get('id')
cid.save(db)
items = {}
for item in module_['items']:
item_id = 0
if item['type'] == "Page":
item_id = item['page_url']
elif item['type'] in ['File', 'Assignment', 'Quiz']:
item_id = item['content_id']
# SubHeaders and ExternalUrls won't be in the db anyway
icid = canvas_id.find_by_id(db, course_.canvas_id, item_id)
if item_id == 0 or not icid:
# we don't have the item locally so try to pull it
if item['type'] == "Page":
_, icid = page.pull_page(db, course_.canvas_id,
item['page_url'], dry_run)
elif item['type'] == "File":
# get file info
file_path = urllib.parse.urlparse(item['url']).path
file_ = helpers.get(file_path, dry_run=dry_run)
url = file_['url']
# get parent folder info
folder_path = files.COURSE_FOLDERS_PATH.format(course_.canvas_id)+"/"+str(file_['folder_id'])
folder = helpers.get(folder_path, dry_run=dry_run)
path = folder['full_name'][len("course "):]
filepath = path + file_['filename']
# download the file
icid = files.pull_file(db, course_.canvas_id, url, file_['id'], filepath)
elif item['type'] == 'Assignment':
_, icid = assignment.pull(db, course_, item_id, dry_run)
elif item['type'] == 'Quiz':
_, icid = quiz.pull(db, course_, item, dry_run)
elif item['type'] == 'SubHeader':
built_item = {'title': item['title']}
if not item['published']:
built_item['published'] = False
elif item['type'] == 'ExternalUrl':
built_item = {
'title': item.get('title', ''),
'external_url': item['external_url']}
if not item['published']:
built_item['published'] = False
else:
logging.warn("I can't find the module item "
f"'{item['title']}' locally and I couldn't figure "
"out how to pull it because it is not an "
"easel-supported type: " + item['type'])
continue
position = item.get('position', 0)
if position not in items:
items[position] = []
if icid:
built_item = {"item": icid.filename, "indent": item.get('indent', 0)}
if item['type'] == 'File' and 'title' in item:
built_item['title'] = item['title']
if 'indent' in built_item and built_item['indent'] == 0:
del built_item['indent']
items[position].append(built_item)
# order items by position
all_items = []
for position in sorted(items.keys()):
if position == 0:
# we used 0 as default above so skip that for now
continue
all_items += items[position]
# add positionless items to the end
module_['items'] = all_items + items.get(0, [])
modules.append(Module.build(module_))
return modules
def format_update_path(self, db, *path_args):
"""3 args -> course_id, quiz_id, quiz_question_id"""
course_id = path_args[0]
cid = canvas_id.CanvasID(path_args[2].filename, course_id)
cid.find_id(db)
return self.update_path.format(course_id, cid.canvas_id, path_args[1])
def format_create_path(self, db, *path_args):
"""2 args -> course_id, quiz_id"""
course_id = path_args[0]
cid = canvas_id.CanvasID(path_args[1].filename, course_id)
cid.find_id(db)
return self.create_path.format(course_id, cid.canvas_id)
def cmd_push(db, args):
if not args.components:
# push everything
args.components = ["syllabus.md", "grading_scheme.yaml",
"navigation.yaml", "assignment_groups", "assignments", "files",
"pages", "quizzes", "modules"]
if not args.course:
args.course = course.find_all(db)
else:
args.course = course.match_courses(db, args.course)
for component_filepath in args.components:
if component_filepath.endswith("*"):
component_filepath = component_filepath[:-1]
if component_filepath.endswith("/"):
component_filepath = component_filepath[:-1]
if os.path.isdir(component_filepath) and not component_filepath.startswith("files"):
if component_filepath not in helpers.DIRS:
logging.error("Invalid directory: "+component_filepath)
continue
for child_path in os.listdir(component_filepath):
full_child_path = component_filepath + '/' + child_path
component = helpers_yaml.read(full_child_path)
if component and not isinstance(component, str):
component.filename = full_child_path
for course_ in args.course:
print(f"pushing {component} to {course_.name} ({course_.canvas_id})")
component.push(db, course_, args.dry_run)
elif component_filepath.startswith("files"):
for course_ in args.course:
files.push(db, course_, component_filepath, args.hidden,
args.dry_run)
else:
if not os.path.isfile(component_filepath):
logging.error("Cannot find file: " + component_filepath)
continue
for course_ in args.course:
if component_filepath == "syllabus.md":
print(f"pushing syllabus to {course_.name} ({course_.canvas_id})")
course.push_syllabus(db, course_.canvas_id, args.dry_run)
elif component_filepath == "grading_scheme.yaml":
print(f"pushing grading scheme to {course_.name} ({course_.canvas_id})")
cid = canvas_id.CanvasID(component_filepath, course_.canvas_id)
cid.find_id(db)
if cid.canvas_id == "":
# Don't try to create a scheme if it doesn't already
# exist. Ideally, we update the scheme but Canvas
# apparently doesn't allow for that.
component = helpers_yaml.read(component_filepath)
component.push(db, course_, args.dry_run)
cid.find_id(db)
course.update_grading_scheme(db, course_.canvas_id,
cid.canvas_id, args.dry_run)
elif component_filepath == "navigation.yaml":
print(f"pushing navigation tabs to {course_.name} ({course_.canvas_id})")
navigation_tab.push(db, course_, args.dry_run)
else:
component = helpers_yaml.read(component_filepath)
if component and not isinstance(component, str):
component.filename = component_filepath
print(f"pushing {component} to {course_.name} ({course_.canvas_id})")
component.push(db, course_, args.dry_run)
else:
# not a yaml file so assume it's a file/dir to upload
files.push(db, course_, component_filepath,
args.hidden, args.dry_run)
def push(self, db, course_, dry_run, parent_component=None):
"""
push the component to the given canvas course
The parent_component is used for the more complex scenarios when canvas
nests components inside of others (e.g., ModuleItems inside of
Modules). Usually in these cases, the child component's API endpoint
paths will be an extension of the parent's path, so it parent_component
allows us to extend the default behavior. In these cases, push() will
be called in a different location than the typical execution path in
commands.py. However, this other location should mimic that typical
behavior (see preprocess() in module.py).
"""
course_id = course_.canvas_id
found = self.find(db)
cid = canvas_id.CanvasID(self.filename, course_id)
cid.find_id(db)
# possible scenarios:
# - record not found, no canvas id -> create
# - record found, no canvas id -> yaml overrules record, right?
# that's what it's doing now -> create
# - record not found, canvas id found -> this one's weird.. should we
# pull the canvas version and merge with the local version? right
# now it just overwrites the canvas version -> update
# - record found, canvas id found -> update
# - TODO: what about if we have a canvas id but the component has
# been deleted in canvas? maybe just delete the canvas id record
# and try again? Since deleting components might be rare anyway,
# for now we'll just inform the user and they can remove the
# component themselves before proceeding. This will mainly happen
# when we delete a component that tracks children (e.g., an
# assignment group because it will delete the
# assignments that belong to that group, a module with its items)
# so we might want to be proactive when we delete and then go
# delete the assignments, but that requires even deeper tracking
# ahead of time.
# conclusion: whether we create or update only depends on the canvas id
# but later on we might end up needing to do other stuff depending on
# whether or not we have a db record?
if cid.canvas_id == "":
# create
path = self.format_create_path(db, course_id, parent_component)
self.preprocess(db, course_, dry_run)
resp = helpers.post(path, self, dry_run=dry_run)
if dry_run:
print("DRYRUN - saving the canvas_id for the component "
"(assuming the request worked)")
return
if self.filename:
# only save the canvas id if we have a filename because we
# don't want to save some components (e.g., quiz questions)
if 'id' in resp:
self.save(db)
cid.canvas_id = resp['id']
cid.save(db)
elif 'url' in resp:
# pages use a url instead of an id but we can use them
# interchangably for this
self.save(db)
cid.canvas_id = resp['url']
cid.save(db)
elif "message" in resp:
logging.error(resp['message'])
else:
raise ValueError(
"TODO: handle unexpected response when creating component"
)
self.postprocess(db, course_, dry_run)
else:
# update
if not found:
found = self
elif len(found) > 1:
raise ValueError(
"TODO: handle too many results, means the filename was not unique"
)
else:
found = build(type(self).__name__, dict(found[0]))
found.merge(self)
path = self.format_update_path(db, course_id, cid.canvas_id,
parent_component)
found.preprocess(db, course_, dry_run)
resp = helpers.put(path, found, dry_run=dry_run)
if "errors" in resp:
print(f"failed to update the component {found}")
for error in resp['errors']:
if isinstance(error, dict):
if "does not exist" in error['message']:
print("The component was deleted in canvas "
"without us knowing about it. You should "
"remove the component here and then try "
"pushing it again.")
else:
print("CANVAS ERROR:", error['message'])
else:
print("CANVAS ERROR:", error)
if dry_run:
print(f"DRYRUN - saving the component {found}")
else:
found.save(db)
found.postprocess(db, course_, dry_run)
def pull_file(db, course_id, url, id_, filepath):
helpers.download_file(url, filepath)
cid = canvas_id.CanvasID(filepath, course_id)
cid.canvas_id = id_
cid.save(db)
return cid
def get_canvas_id(self, db, course_id):
cid = canvas_id.CanvasID(self.filename, course_id)
cid.find_id(db)
return cid.canvas_id
