def test_good_zip_archive(self): """ A good zip archive container passes validation. :return: """ parent = Container(id=41, file_type=Container.SIMG) container = Container(id=42, file_type=Container.ZIP, parent=parent) with open(self.zip_archive, "rb") as zip_archive: container.file = File(zip_archive) container.clean()
def test_good_tar_archive(self): """ A good tar archive container passes validation. :return: """ parent = Container(id=41, file_type=Container.SIMG) container = Container(id=42, file_type=Container.TAR, parent=parent) with open(self.tar_archive, "rb") as tar_archive: container.file = File(tar_archive) container.clean()
def empty_archive_test_helper(self, archive_type): """ Helper for testing archive containers that are just empty files (as opposed to empty archives). :return: """ parent = Container(id=41, file_type=Container.SIMG) container = Container(id=42, file_type=archive_type, parent=parent) with open(self.empty_file, "rb") as f: container.file = File(f) with self.assertRaisesMessage( ValidationError, Container.DEFAULT_ERROR_MESSAGES["invalid_archive"]): container.clean()
def bad_archive_test_helper(self, archive_type): """ Helper for testing bad archive containers. :return: """ parent = Container(id=41, file_type=Container.SIMG) container = Container(id=42, file_type=archive_type, parent=parent) with open(self.useless_file, "rb") as f: container.file = File(f) with self.assertRaisesMessage( ValidationError, Container.DEFAULT_ERROR_MESSAGES["invalid_archive"]): container.clean()
def admissible_driver_test_helper(self, archive_type, driver_type): """ Helper for testing archive containers' drivers' admissibility. :return: """ pipeline_dict = {"steps": [{"driver": "foobarbaz"}]} # Archives that contain a mix of files, including one driver. fd, archive = tempfile.mkstemp() try: with open(archive, mode="wb") as f: if archive_type == Container.ZIP: archive_handler = ZipHandler(f, mode="w") else: archive_handler = TarHandler(f, mode="w") archive_handler.write(u"hello_world.sh", self.hello_world_script) archive_handler.write(u"useless.lib", self.useless) archive_handler.write(u"not_a_script", self.not_a_script) if driver_type == "good": pipeline_dict["steps"][0]["driver"] = "hello_world.sh" elif driver_type == "bad": pipeline_dict["steps"][0]["driver"] = "not_a_script" archive_handler.write("kive/pipeline0.json", json.dumps(pipeline_dict)) archive_handler.close() parent = Container(id=41, file_type=Container.SIMG) container = Container(id=42, file_type=archive_type, parent=parent) with open(archive, "rb") as f: container.file = File(f) if driver_type == "good": container.clean() elif driver_type == "bad": with self.assertRaisesMessage( ValidationError, Container. DEFAULT_ERROR_MESSAGES["inadmissible_driver"]): container.clean() elif driver_type == "nonexistent": with self.assertRaisesMessage( ValidationError, Container. DEFAULT_ERROR_MESSAGES["driver_not_in_archive"]): container.clean() finally: os.remove(archive)
def test_clean_singularity_image_with_parent(self): """ A Singularity container should not have a parent. :return: """ parent = Container(id=41) container = Container(id=42, parent=parent, file_type=Container.SIMG) with open(self.alpine_path, "rb") as alpine_file: container.file = File(alpine_file) with self.assertRaisesMessage( ValidationError, Container. DEFAULT_ERROR_MESSAGES["singularity_cannot_have_parent"]): container.clean()
def test_str(self): family = ContainerFamily(name='Spline Reticulator') container = Container(tag='v1.0.7', family=family) s = str(container) self.assertEqual("Spline Reticulator:v1.0.7", s)
def test_removal(self): container = Container(id=42) expected_plan = {'Containers': {container}} plan = container.build_removal_plan() self.assertEqual(expected_plan, strip_removal_plan(plan))
def test_removal_with_method(self): container = Container(id=42) method = container.methods.create(transformation_ptr_id=43) expected_plan = {'Containers': {container}, 'Methods': {method}} plan = container.build_removal_plan() self.assertEqual(expected_plan, strip_removal_plan(plan))
def test_removal_with_app(self): container = Container(id=42) app = container.apps.create(id=43) expected_plan = {'Containers': {container}, 'ContainerApps': {app}} plan = container.build_removal_plan() self.assertEqual(expected_plan, strip_removal_plan(plan))
def test_removal_with_child(self): container = Container(id=42) child_container = container.children.create(id=43) expected_plan = {'Containers': {container, child_container}} plan = container.build_removal_plan() self.assertEqual(expected_plan, strip_removal_plan(plan))
def build_run(self): run = ContainerRun() run.app = ContainerApp() run.app.container = Container() run.app.container.file = Namespace(path='/tmp/foo.simg') run.sandbox_path = '/tmp/box23' run.app.arguments.create(type=ContainerArgument.INPUT, name='in_csv') run.app.arguments.create(type=ContainerArgument.OUTPUT, name='out_csv') return run
def test_display_default(self): app = ContainerApp(name='') # default app app.container = Container(tag='v1.0') app.container.family = ContainerFamily(name='Splines') expected_display_name = 'Splines:v1.0' display_name = app.display_name self.assertEqual(expected_display_name, display_name)
def test_clean_good_singularity_image(self): """ A proper Singularity container should pass validation. :return: """ container = Container(id=42) container.file_type = Container.SIMG with open(self.alpine_path, 'rb') as alpine_file: container.file = File(alpine_file) container.clean()
def test_display_name(self): app = ContainerApp(name='reticulate') app.container = Container(tag='v1.0') app.container.family = ContainerFamily(name='Splines') expected_display_name = 'Splines:v1.0 / reticulate' display_name = app.display_name app_str = str(app) self.assertEqual(expected_display_name, display_name) self.assertEqual(expected_display_name, app_str)
def test_skip_singularity_validation(self, mock_val): """ Skip Singularity validation if it's marked as having already been done. :param mock_val: :return: """ container = Container(id=42, file_type=Container.SIMG) container.singularity_validated = True with open(self.alpine_path, 'rb') as alpine_file: container.file = File(alpine_file) container.clean() mock_val.assert_not_called()
def no_driver_archive_test_helper(self, archive_type, empty=True): """ Helper for testing archive containers with no driver. :return: """ parent = Container(id=41, file_type=Container.SIMG) container = Container(id=42, file_type=archive_type, parent=parent) archive_file = self.empty_zip_archive if empty and archive_type == Container.TAR: archive_file = self.empty_tar_archive elif not empty: if archive_type == Container.ZIP: archive_file = self.no_driver_zip else: archive_file = self.no_driver_tar with open(archive_file, "rb") as f: container.file = File(f) with self.assertRaisesMessage( ValidationError, Container. DEFAULT_ERROR_MESSAGES["archive_has_no_drivers"]): container.clean()
def save(self, data, element_object=None): if element_object == None: container = Container() uid = self.generateUID() container.uid = uid else: container = element_object vars = container.variable.all() for vr in vars: container.variable.remove(vr) uid_deploy = self.parseData(data, "parent") deployment = self.getDeployment(uid_deploy) if not deployment == None: container.name = self.parseData(data, "name") container.type = self.parseData(data, "type") container.base_image = self.parseData(data, "base_image") container.image_name = self.parseData(data, "image_name") container.source = self.parseData(data, "source") container.port = self.parseData(data, "port") container.commands = self.parseData(data, "commands") container.cmd = self.parseData(data, "cmd") container.volume_path = self.parseData(data, "volume_path") container.save() list_variables = self.parseData(data, "variables") for variable_uid in list_variables: variable_item = self.getVariable(variable_uid) if not variable_item == None: container.variable.add(variable_item) list_variables_items = self.parseData(data, "variables_items") print(list_variables_items) for var_item in list_variables_items: var_item_object = self.saveVariable(var_item) if not var_item_object == None: container.variable.add(var_item_object) container.deployment.add(deployment) return { "status" : "success", "message" : "Saved", "uid" : container.uid, "element" : "container" } else: return {"status": "error", "message": "Deployment Not exist", "element" : "container"}
def test_archive_with_no_parent(self): """ An archive container must have a parent. :return: """ container = Container(id=42, file_type=Container.ZIP) with open(self.zip_archive, "rb") as zip_archive: container.file = File(zip_archive) with self.assertRaisesMessage( ValidationError, Container. DEFAULT_ERROR_MESSAGES["archive_must_have_parent"]): container.clean()
def test_slurm_command_custom_memory(self): run = ContainerRun(pk=99) run.user = User(username='******') run.app = ContainerApp(threads=3, memory=100) run.app.container = Container() run.app.container.family = ContainerFamily(name='my container') run.sandbox_path = 'run23' expected_command = [ 'sbatch', '-J', 'r99 my container', '--parsable', '--output', '/tmp/kive_media/run23/logs/job%J_node%N_stdout.txt', '--error', '/tmp/kive_media/run23/logs/job%J_node%N_stderr.txt', '-c', '3', '--mem', '100', EXPECTED_MANAGE_PATH, 'runcontainer', '99' ] command = run.build_slurm_command() self.assertListEqual(expected_command, command)
def test_slurm_command_priority(self): run = ContainerRun(pk=99) run.user = User(username='******') run.app = ContainerApp() run.app.container = Container() run.app.container.family = ContainerFamily(name='my container') slurm_queues = (('low', 'kive-low'), ('medium', 'kive-medium'), ('high', 'kive-high')) run.priority = 2 run.sandbox_path = 'run23' expected_command = [ 'sbatch', '-J', 'r99 my container', '--parsable', '--output', '/tmp/kive_media/run23/logs/job%J_node%N_stdout.txt', '--error', '/tmp/kive_media/run23/logs/job%J_node%N_stderr.txt', '-c', '1', '--mem', '6000', '-p', 'kive-high', EXPECTED_MANAGE_PATH, 'runcontainer', '99' ] command = run.build_slurm_command(slurm_queues) self.assertListEqual(expected_command, command)
def seed_data(user): # Create container demo_container = Container( owner=user.email, code="DEMO", description="A demo container to explore OnTask.") demo_container.save() # Create datasources def generate_s3_csv_datasource(name): connection = Connection( dbType="s3BucketFile", bucket=DEMO_BUCKET, fileName=f"{name}.csv", delimiter=",", ) datasource = Datasource(container=demo_container, name=name, connection=connection) datasource.data = datasource.retrieve_data() datasource.fields = [field for field in datasource.data[0]] datasource.types = TYPES[name] datasource.save() return datasource students_datasource, classes_datasource, tutors_datasource = [ generate_s3_csv_datasource(demo_datasource) for demo_datasource in ["Students", "Classes", "Tutors"] ] # Create datalab modules demo_modules = [] demo_modules.append( Module( type="datasource", datasource=DatasourceModule( id=str(students_datasource.id), primary="zid", fields=["zid", "first_name", "last_name", "email"], labels={ "zid": "zId", "first_name": "first_name", "last_name": "last_name", "email": "email", }, types=TYPES["Students"], ), )) demo_modules.append( Module( type="datasource", datasource=DatasourceModule( id=str(classes_datasource.id), primary="ID", matching="zId", fields=["CLASS"], labels={"CLASS": "class"}, discrepencies={"primary": False}, types=TYPES["Classes"], ), )) demo_modules.append( Module( type="datasource", datasource=DatasourceModule( id=str(tutors_datasource.id), primary="class", matching="class", fields=["first_name", "last_name", "email"], labels={ "first_name": "tutor_first_name", "last_name": "tutor_last_name", "email": "tutor_email", }, types=TYPES["Tutors"], ), )) # attendance_form_fields = [ # FormField(name=f"attendance_w{i+1}", type="checkbox") for i in range(4) # ] # grade_form_fields = [ # FormField( # name=field, # type="number", # minimum=0, # maximum=10, # precision=2, # interval=0.25, # numberDisplay="input", # ) # for field in ["grade_midterm", "grade_final"] # ] # demo_form_data = [ # { # "zId": student["zid"], # **{field.name: randint(1, 100) < 90 for field in attendance_form_fields}, # # **{field.name: randint(1, 40) * 0.25 for field in grade_form_fields}, # } # for student in students_datasource.data # ] # demo_modules.append( # Module( # type="form", # form=FormModule( # primary="zId", # name="Grades", # # fields=attendance_form_fields + grade_form_fields, # fields=attendance_form_fields, # data=demo_form_data, # ), # ) # ) demo_average = ComputedField( name="average_attendance", type="number", formula={ "object": "value", "document": { "object": "document", "data": {}, "nodes": [{ "object": "block", "type": "aggregation", "isVoid": False, "data": { "type": "average", "columns": ["3_0", "3_1", "3_2", "3_3"], }, }], }, }, ) demo_modules.append( Module(type="computed", computed=ComputedModule(fields=[demo_average]))) demo_order = [ Column(stepIndex=0, field="zid", pinned=True), Column(stepIndex=0, field="first_name"), Column(stepIndex=0, field="last_name"), Column(stepIndex=0, field="email"), Column(stepIndex=1, field="CLASS"), Column(stepIndex=2, field="first_name"), Column(stepIndex=2, field="last_name"), Column(stepIndex=2, field="email"), ] # demo_order += [ # Column(stepIndex=3, field=field.name) for field in attendance_form_fields # ] demo_order.append(Column(stepIndex=4, field=demo_average.name))
def post(self, request, format=None): url = f"{BACKEND_DOMAIN}{reverse('lti')}" consumers = LTI_CONFIG["consumers"] method = request.method headers = request.META payload = request.POST.dict() # Verify that the payload received from LTI is valid try: verify_request_common(consumers, url, method, headers, payload) # If not, redirect to the error page # This page would be displayed in an iframe on Moodle except LTIException: # TODO: Implement logging of this error return redirect(FRONTEND_DOMAIN + "/forbidden") current_user_email = payload["lis_person_contact_email_primary"] data = { "email": current_user_email, "first_name": payload["lis_person_name_given"], "last_name": payload["lis_person_name_family"], } user = get_or_create_user(data) # Elevate the user to instructor group if they have a staff role in LTI # If they are already instructor or admin, then do nothing user_groups = [group.name for group in user.groups.all()] is_lti_instructor = (LTI_CONFIG["staff_role"] in payload["roles"].split(",") if LTI_CONFIG.get("staff_role") else False) if "user" in user_groups and is_lti_instructor: user.groups.set([Group.objects.get(name="instructor")]) token = generate_one_time_token(user) user.last_login = dt.now(timezone.utc) user.save() logger.info("authentication.login", extra={ "user": user.email, "type": "LTI" }) # Store the important LTI fields for this user # These fields be used to grant permissions in containers lti_payload = { "lti_id": payload["user_id"], "lti_email": current_user_email, "user_id": payload.get(LTI_CONFIG.get("username_field")), } lti.objects(user=user.id).update_one(payload=lti_payload, upsert=True) # If any containers have been bound to this LTI resource, then # redirect to that container # Otherwise, prompt the user to choose a container for binding lti_resource_id = payload["resource_link_id"] try: container = Container.objects.get(lti_resource=lti_resource_id) if LTI_CONFIG.get("auto_create_share_containers"): if container.owner != current_user_email and current_user_email not in container.sharing: container.sharing.append(current_user_email) container.save() return redirect(FRONTEND_DOMAIN + "?tkn=" + token + "&container=" + str(container.id)) except: if LTI_CONFIG.get("auto_create_share_containers"): container = Container(owner=current_user_email, code=payload["context_title"], lti_resource=payload["resource_link_id"], lti_context=payload["context_id"]) container.save() return redirect(FRONTEND_DOMAIN + "?tkn=" + token + "<i=" + lti_resource_id)
def mutate(self, info, container_id, tracker_id): container = Container(container_id=container_id, tracker_id=tracker_id) container.save() return RegisterContainerMutation(ok=True, container=container)