def test_good_zip_archive(self):
"""
A good zip archive container passes validation.
:return:
"""
parent = Container(id=41, file_type=Container.SIMG)
container = Container(id=42, file_type=Container.ZIP, parent=parent)
with open(self.zip_archive, "rb") as zip_archive:
container.file = File(zip_archive)
container.clean()
def test_good_tar_archive(self):
"""
A good tar archive container passes validation.
:return:
"""
parent = Container(id=41, file_type=Container.SIMG)
container = Container(id=42, file_type=Container.TAR, parent=parent)
with open(self.tar_archive, "rb") as tar_archive:
container.file = File(tar_archive)
container.clean()
def empty_archive_test_helper(self, archive_type):
"""
Helper for testing archive containers that are just empty files (as opposed to empty archives).
:return:
"""
parent = Container(id=41, file_type=Container.SIMG)
container = Container(id=42, file_type=archive_type, parent=parent)
with open(self.empty_file, "rb") as f:
container.file = File(f)
with self.assertRaisesMessage(
ValidationError,
Container.DEFAULT_ERROR_MESSAGES["invalid_archive"]):
container.clean()
def bad_archive_test_helper(self, archive_type):
"""
Helper for testing bad archive containers.
:return:
"""
parent = Container(id=41, file_type=Container.SIMG)
container = Container(id=42, file_type=archive_type, parent=parent)
with open(self.useless_file, "rb") as f:
container.file = File(f)
with self.assertRaisesMessage(
ValidationError,
Container.DEFAULT_ERROR_MESSAGES["invalid_archive"]):
container.clean()
def admissible_driver_test_helper(self, archive_type, driver_type):
"""
Helper for testing archive containers' drivers' admissibility.
:return:
"""
pipeline_dict = {"steps": [{"driver": "foobarbaz"}]}
# Archives that contain a mix of files, including one driver.
fd, archive = tempfile.mkstemp()
try:
with open(archive, mode="wb") as f:
if archive_type == Container.ZIP:
archive_handler = ZipHandler(f, mode="w")
else:
archive_handler = TarHandler(f, mode="w")
archive_handler.write(u"hello_world.sh",
self.hello_world_script)
archive_handler.write(u"useless.lib", self.useless)
archive_handler.write(u"not_a_script", self.not_a_script)
if driver_type == "good":
pipeline_dict["steps"][0]["driver"] = "hello_world.sh"
elif driver_type == "bad":
pipeline_dict["steps"][0]["driver"] = "not_a_script"
archive_handler.write("kive/pipeline0.json",
json.dumps(pipeline_dict))
archive_handler.close()
parent = Container(id=41, file_type=Container.SIMG)
container = Container(id=42, file_type=archive_type, parent=parent)
with open(archive, "rb") as f:
container.file = File(f)
if driver_type == "good":
container.clean()
elif driver_type == "bad":
with self.assertRaisesMessage(
ValidationError, Container.
DEFAULT_ERROR_MESSAGES["inadmissible_driver"]):
container.clean()
elif driver_type == "nonexistent":
with self.assertRaisesMessage(
ValidationError, Container.
DEFAULT_ERROR_MESSAGES["driver_not_in_archive"]):
container.clean()
finally:
os.remove(archive)
def test_clean_singularity_image_with_parent(self):
"""
A Singularity container should not have a parent.
:return:
"""
parent = Container(id=41)
container = Container(id=42, parent=parent, file_type=Container.SIMG)
with open(self.alpine_path, "rb") as alpine_file:
container.file = File(alpine_file)
with self.assertRaisesMessage(
ValidationError, Container.
DEFAULT_ERROR_MESSAGES["singularity_cannot_have_parent"]):
container.clean()
def test_str(self):
family = ContainerFamily(name='Spline Reticulator')
container = Container(tag='v1.0.7', family=family)
s = str(container)
self.assertEqual("Spline Reticulator:v1.0.7", s)
def test_removal(self):
container = Container(id=42)
expected_plan = {'Containers': {container}}
plan = container.build_removal_plan()
self.assertEqual(expected_plan, strip_removal_plan(plan))
def test_removal_with_method(self):
container = Container(id=42)
method = container.methods.create(transformation_ptr_id=43)
expected_plan = {'Containers': {container}, 'Methods': {method}}
plan = container.build_removal_plan()
self.assertEqual(expected_plan, strip_removal_plan(plan))
def test_removal_with_app(self):
container = Container(id=42)
app = container.apps.create(id=43)
expected_plan = {'Containers': {container}, 'ContainerApps': {app}}
plan = container.build_removal_plan()
self.assertEqual(expected_plan, strip_removal_plan(plan))
def test_removal_with_child(self):
container = Container(id=42)
child_container = container.children.create(id=43)
expected_plan = {'Containers': {container, child_container}}
plan = container.build_removal_plan()
self.assertEqual(expected_plan, strip_removal_plan(plan))
def build_run(self):
run = ContainerRun()
run.app = ContainerApp()
run.app.container = Container()
run.app.container.file = Namespace(path='/tmp/foo.simg')
run.sandbox_path = '/tmp/box23'
run.app.arguments.create(type=ContainerArgument.INPUT, name='in_csv')
run.app.arguments.create(type=ContainerArgument.OUTPUT, name='out_csv')
return run
def test_display_default(self):
app = ContainerApp(name='') # default app
app.container = Container(tag='v1.0')
app.container.family = ContainerFamily(name='Splines')
expected_display_name = 'Splines:v1.0'
display_name = app.display_name
self.assertEqual(expected_display_name, display_name)
def test_clean_good_singularity_image(self):
"""
A proper Singularity container should pass validation.
:return:
"""
container = Container(id=42)
container.file_type = Container.SIMG
with open(self.alpine_path, 'rb') as alpine_file:
container.file = File(alpine_file)
container.clean()
def test_display_name(self):
app = ContainerApp(name='reticulate')
app.container = Container(tag='v1.0')
app.container.family = ContainerFamily(name='Splines')
expected_display_name = 'Splines:v1.0 / reticulate'
display_name = app.display_name
app_str = str(app)
self.assertEqual(expected_display_name, display_name)
self.assertEqual(expected_display_name, app_str)
def test_skip_singularity_validation(self, mock_val):
"""
Skip Singularity validation if it's marked as having already been done.
:param mock_val:
:return:
"""
container = Container(id=42, file_type=Container.SIMG)
container.singularity_validated = True
with open(self.alpine_path, 'rb') as alpine_file:
container.file = File(alpine_file)
container.clean()
mock_val.assert_not_called()
def no_driver_archive_test_helper(self, archive_type, empty=True):
"""
Helper for testing archive containers with no driver.
:return:
"""
parent = Container(id=41, file_type=Container.SIMG)
container = Container(id=42, file_type=archive_type, parent=parent)
archive_file = self.empty_zip_archive
if empty and archive_type == Container.TAR:
archive_file = self.empty_tar_archive
elif not empty:
if archive_type == Container.ZIP:
archive_file = self.no_driver_zip
else:
archive_file = self.no_driver_tar
with open(archive_file, "rb") as f:
container.file = File(f)
with self.assertRaisesMessage(
ValidationError, Container.
DEFAULT_ERROR_MESSAGES["archive_has_no_drivers"]):
container.clean()
def save(self, data, element_object=None):
if element_object == None:
container = Container()
uid = self.generateUID()
container.uid = uid
else:
container = element_object
vars = container.variable.all()
for vr in vars:
container.variable.remove(vr)
uid_deploy = self.parseData(data, "parent")
deployment = self.getDeployment(uid_deploy)
if not deployment == None:
container.name = self.parseData(data, "name")
container.type = self.parseData(data, "type")
container.base_image = self.parseData(data, "base_image")
container.image_name = self.parseData(data, "image_name")
container.source = self.parseData(data, "source")
container.port = self.parseData(data, "port")
container.commands = self.parseData(data, "commands")
container.cmd = self.parseData(data, "cmd")
container.volume_path = self.parseData(data, "volume_path")
container.save()
list_variables = self.parseData(data, "variables")
for variable_uid in list_variables:
variable_item = self.getVariable(variable_uid)
if not variable_item == None:
container.variable.add(variable_item)
list_variables_items = self.parseData(data, "variables_items")
print(list_variables_items)
for var_item in list_variables_items:
var_item_object = self.saveVariable(var_item)
if not var_item_object == None:
container.variable.add(var_item_object)
container.deployment.add(deployment)
return { "status" : "success", "message" : "Saved", "uid" : container.uid, "element" : "container" }
else:
return {"status": "error", "message": "Deployment Not exist", "element" : "container"}
def test_archive_with_no_parent(self):
"""
An archive container must have a parent.
:return:
"""
container = Container(id=42, file_type=Container.ZIP)
with open(self.zip_archive, "rb") as zip_archive:
container.file = File(zip_archive)
with self.assertRaisesMessage(
ValidationError, Container.
DEFAULT_ERROR_MESSAGES["archive_must_have_parent"]):
container.clean()
def test_slurm_command_custom_memory(self):
run = ContainerRun(pk=99)
run.user = User(username='******')
run.app = ContainerApp(threads=3, memory=100)
run.app.container = Container()
run.app.container.family = ContainerFamily(name='my container')
run.sandbox_path = 'run23'
expected_command = [
'sbatch', '-J', 'r99 my container', '--parsable', '--output',
'/tmp/kive_media/run23/logs/job%J_node%N_stdout.txt', '--error',
'/tmp/kive_media/run23/logs/job%J_node%N_stderr.txt', '-c', '3',
'--mem', '100', EXPECTED_MANAGE_PATH, 'runcontainer', '99'
]
command = run.build_slurm_command()
self.assertListEqual(expected_command, command)
def test_slurm_command_priority(self):
run = ContainerRun(pk=99)
run.user = User(username='******')
run.app = ContainerApp()
run.app.container = Container()
run.app.container.family = ContainerFamily(name='my container')
slurm_queues = (('low', 'kive-low'), ('medium', 'kive-medium'),
('high', 'kive-high'))
run.priority = 2
run.sandbox_path = 'run23'
expected_command = [
'sbatch', '-J', 'r99 my container', '--parsable', '--output',
'/tmp/kive_media/run23/logs/job%J_node%N_stdout.txt', '--error',
'/tmp/kive_media/run23/logs/job%J_node%N_stderr.txt', '-c', '1',
'--mem', '6000', '-p', 'kive-high', EXPECTED_MANAGE_PATH,
'runcontainer', '99'
]
command = run.build_slurm_command(slurm_queues)
self.assertListEqual(expected_command, command)
def seed_data(user):
# Create container
demo_container = Container(
owner=user.email,
code="DEMO",
description="A demo container to explore OnTask.")
demo_container.save()
# Create datasources
def generate_s3_csv_datasource(name):
connection = Connection(
dbType="s3BucketFile",
bucket=DEMO_BUCKET,
fileName=f"{name}.csv",
delimiter=",",
)
datasource = Datasource(container=demo_container,
name=name,
connection=connection)
datasource.data = datasource.retrieve_data()
datasource.fields = [field for field in datasource.data[0]]
datasource.types = TYPES[name]
datasource.save()
return datasource
students_datasource, classes_datasource, tutors_datasource = [
generate_s3_csv_datasource(demo_datasource)
for demo_datasource in ["Students", "Classes", "Tutors"]
]
# Create datalab modules
demo_modules = []
demo_modules.append(
Module(
type="datasource",
datasource=DatasourceModule(
id=str(students_datasource.id),
primary="zid",
fields=["zid", "first_name", "last_name", "email"],
labels={
"zid": "zId",
"first_name": "first_name",
"last_name": "last_name",
"email": "email",
},
types=TYPES["Students"],
),
))
demo_modules.append(
Module(
type="datasource",
datasource=DatasourceModule(
id=str(classes_datasource.id),
primary="ID",
matching="zId",
fields=["CLASS"],
labels={"CLASS": "class"},
discrepencies={"primary": False},
types=TYPES["Classes"],
),
))
demo_modules.append(
Module(
type="datasource",
datasource=DatasourceModule(
id=str(tutors_datasource.id),
primary="class",
matching="class",
fields=["first_name", "last_name", "email"],
labels={
"first_name": "tutor_first_name",
"last_name": "tutor_last_name",
"email": "tutor_email",
},
types=TYPES["Tutors"],
),
))
# attendance_form_fields = [
# FormField(name=f"attendance_w{i+1}", type="checkbox") for i in range(4)
# ]
# grade_form_fields = [
# FormField(
# name=field,
# type="number",
# minimum=0,
# maximum=10,
# precision=2,
# interval=0.25,
# numberDisplay="input",
# )
# for field in ["grade_midterm", "grade_final"]
# ]
# demo_form_data = [
# {
# "zId": student["zid"],
# **{field.name: randint(1, 100) < 90 for field in attendance_form_fields},
# # **{field.name: randint(1, 40) * 0.25 for field in grade_form_fields},
# }
# for student in students_datasource.data
# ]
# demo_modules.append(
# Module(
# type="form",
# form=FormModule(
# primary="zId",
# name="Grades",
# # fields=attendance_form_fields + grade_form_fields,
# fields=attendance_form_fields,
# data=demo_form_data,
# ),
# )
# )
demo_average = ComputedField(
name="average_attendance",
type="number",
formula={
"object": "value",
"document": {
"object":
"document",
"data": {},
"nodes": [{
"object": "block",
"type": "aggregation",
"isVoid": False,
"data": {
"type": "average",
"columns": ["3_0", "3_1", "3_2", "3_3"],
},
}],
},
},
)
demo_modules.append(
Module(type="computed",
computed=ComputedModule(fields=[demo_average])))
demo_order = [
Column(stepIndex=0, field="zid", pinned=True),
Column(stepIndex=0, field="first_name"),
Column(stepIndex=0, field="last_name"),
Column(stepIndex=0, field="email"),
Column(stepIndex=1, field="CLASS"),
Column(stepIndex=2, field="first_name"),
Column(stepIndex=2, field="last_name"),
Column(stepIndex=2, field="email"),
]
# demo_order += [
# Column(stepIndex=3, field=field.name) for field in attendance_form_fields
# ]
demo_order.append(Column(stepIndex=4, field=demo_average.name))
def post(self, request, format=None):
url = f"{BACKEND_DOMAIN}{reverse('lti')}"
consumers = LTI_CONFIG["consumers"]
method = request.method
headers = request.META
payload = request.POST.dict()
# Verify that the payload received from LTI is valid
try:
verify_request_common(consumers, url, method, headers, payload)
# If not, redirect to the error page
# This page would be displayed in an iframe on Moodle
except LTIException:
# TODO: Implement logging of this error
return redirect(FRONTEND_DOMAIN + "/forbidden")
current_user_email = payload["lis_person_contact_email_primary"]
data = {
"email": current_user_email,
"first_name": payload["lis_person_name_given"],
"last_name": payload["lis_person_name_family"],
}
user = get_or_create_user(data)
# Elevate the user to instructor group if they have a staff role in LTI
# If they are already instructor or admin, then do nothing
user_groups = [group.name for group in user.groups.all()]
is_lti_instructor = (LTI_CONFIG["staff_role"]
in payload["roles"].split(",")
if LTI_CONFIG.get("staff_role") else False)
if "user" in user_groups and is_lti_instructor:
user.groups.set([Group.objects.get(name="instructor")])
token = generate_one_time_token(user)
user.last_login = dt.now(timezone.utc)
user.save()
logger.info("authentication.login",
extra={
"user": user.email,
"type": "LTI"
})
# Store the important LTI fields for this user
# These fields be used to grant permissions in containers
lti_payload = {
"lti_id": payload["user_id"],
"lti_email": current_user_email,
"user_id": payload.get(LTI_CONFIG.get("username_field")),
}
lti.objects(user=user.id).update_one(payload=lti_payload, upsert=True)
# If any containers have been bound to this LTI resource, then
# redirect to that container
# Otherwise, prompt the user to choose a container for binding
lti_resource_id = payload["resource_link_id"]
try:
container = Container.objects.get(lti_resource=lti_resource_id)
if LTI_CONFIG.get("auto_create_share_containers"):
if container.owner != current_user_email and current_user_email not in container.sharing:
container.sharing.append(current_user_email)
container.save()
return redirect(FRONTEND_DOMAIN + "?tkn=" + token + "&container=" +
str(container.id))
except:
if LTI_CONFIG.get("auto_create_share_containers"):
container = Container(owner=current_user_email,
code=payload["context_title"],
lti_resource=payload["resource_link_id"],
lti_context=payload["context_id"])
container.save()
return redirect(FRONTEND_DOMAIN + "?tkn=" + token + "<i=" +
lti_resource_id)
def mutate(self, info, container_id, tracker_id):
container = Container(container_id=container_id, tracker_id=tracker_id)
container.save()
return RegisterContainerMutation(ok=True, container=container)