def test_bundled_app_lines(self, mock_labbook): """Test if the Dockerfile builds with bundled app ports""" lb = mock_labbook[2] bam = BundledAppManager(lb) bam.add_bundled_app(8050, 'dash 1', 'a demo dash app 1', 'python app1.py') bam.add_bundled_app(9000, 'dash 2', 'a demo dash app 2', 'python app2.py') bam.add_bundled_app(9001, 'dash 3', 'a demo dash app 3', 'python app3.py') erm = RepositoryManager(mock_labbook[0]) erm.update_repositories() erm.index_repositories() cm = ComponentManager(lb) cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV) cm.add_packages("pip", [{ "manager": "pip", "package": "requests", "version": "2.18.4" }]) ib = ImageBuilder(lb) dockerfile_text = ib.assemble_dockerfile(write=False) test_lines = [ '# Bundled Application Ports', 'EXPOSE 8050', 'EXPOSE 9000', 'EXPOSE 9001' ] docker_lines = dockerfile_text.split(os.linesep) for line in test_lines: assert line in docker_lines
def test_get_component_list_packages(self, mock_config_with_repo): """Test listing packages added a to labbook""" lb = create_tmp_labbook(mock_config_with_repo[0]) labbook_dir = lb.root_dir cm = ComponentManager(lb) # mock_config_with_repo is a ComponentManager Instance pkgs = [{ "manager": "pip3", "package": "requests", "version": "2.18.2" }, { "manager": "pip3", "package": "gigantum", "version": "0.5" }] cm.add_packages('pip3', pkgs) packages = cm.get_component_list('package_manager') assert len(packages) == 2 assert packages[1]['manager'] == 'pip3' assert packages[1]['package'] == 'requests' assert packages[1]['version'] == '2.18.2' assert packages[0]['manager'] == 'pip3' assert packages[0]['package'] == 'gigantum' assert packages[0]['version'] == '0.5'
def build_lb_image_for_env_conda(mock_config_with_repo): """A fixture that installs an old version of matplotlib and latest version of requests to increase code coverage""" im = InventoryManager(mock_config_with_repo[0]) lb = im.create_labbook('unittester', 'unittester', "containerunittestbookenvconda", description="Testing environment functions.") cm = ComponentManager(lb) cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV) cm.add_packages('conda3', [{'package': 'python-coveralls', 'version': '2.7.0'}]) ib = ImageBuilder(lb) ib.assemble_dockerfile(write=True) client = get_docker_client() client.containers.prune() try: lb, docker_image_id = ContainerOperations.build_image(labbook=lb, username="******") yield lb, 'unittester' finally: shutil.rmtree(lb.root_dir) try: client.images.remove(docker_image_id, force=True, noprune=False) except: pass
def _add_package_components(cls, lb, packages): manager = list(set([x['manager'] for x in packages])) if len(manager) != 1: raise ValueError( "Only batch add packages via 1 package manager at a time.") manager = manager[0] # Set the cursor to the end of the collection of packages glob_path = os.path.join(lb.root_dir, '.gigantum', 'env', 'package_manager', f"{manager}*") cursor = len(glob.glob(glob_path)) # Create Component Manager cm = ComponentManager(lb) cm.add_packages(package_manager=manager, packages=packages, from_base=False, force=True) new_edges = list() for cnt, pkg in enumerate(packages): new_edges.append( PackageComponentConnection.Edge( node=PackageComponent(manager=manager, package=pkg["package"], version=pkg["version"], schema=CURRENT_SCHEMA), cursor=base64.b64encode(str(cursor + cnt).encode()).decode())) return new_edges
def test_add_package(self, mock_config_with_repo): """Test adding a package such as one from apt-get or pip3. """ # Create a labook lb = create_tmp_labbook(mock_config_with_repo[0]) labbook_dir = lb.root_dir # Create Component Manager cm = ComponentManager(lb) # Add some sample components pkgs = [{ "manager": "pip3", "package": "requests", "version": "2.18.2" }, { "manager": "pip3", "package": "gigantum", "version": "0.5" }] cm.add_packages('pip3', pkgs) pkgs = [{ "manager": "apt", "package": "ack", "version": "1.0" }, { "manager": "apt", "package": "docker", "version": "3.5" }] cm.add_packages('apt', pkgs) package_path = os.path.join(lb._root_dir, '.gigantum', 'env', 'package_manager') assert os.path.exists(package_path) # Ensure all four packages exist. package_files = [f for f in os.listdir(package_path)] package_files = [p for p in package_files if p != '.gitkeep'] assert len(package_files) == 4 # Ensure the fields in each of the 4 packages exist. for file in package_files: full_path = os.path.join(package_path, file) with open(full_path) as package_yaml: fields_dict = yaml.safe_load(package_yaml.read()) for required_field in [ 'manager', 'package', 'from_base', 'version' ]: assert required_field in fields_dict.keys() # Verify git/activity log = lb.git.log() print(log) assert len(log) == 7 assert "_GTM_ACTIVITY_START_" in log[0]["message"] assert 'Added 2 apt package(s)' in log[0]["message"] assert "_GTM_ACTIVITY_START_" in log[2]["message"] assert 'Added 2 pip3 package(s)' in log[2]["message"]
def build_lb_image_for_jupyterlab(mock_config_with_repo): with patch.object(Configuration, 'find_default_config', lambda self: mock_config_with_repo[0]): im = InventoryManager(mock_config_with_repo[0]) lb = im.create_labbook('unittester', 'unittester', "containerunittestbook") # Create Component Manager cm = ComponentManager(lb) # Add a component cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV) cm.add_packages("pip", [{"manager": "pip", "package": "requests", "version": "2.18.4"}]) ib = ImageBuilder(lb) docker_lines = ib.assemble_dockerfile(write=True) assert 'RUN pip install requests==2.18.4' in docker_lines assert all(['==None' not in l for l in docker_lines.split()]) assert all(['=None' not in l for l in docker_lines.split()]) client = get_docker_client() client.containers.prune() assert os.path.exists(os.path.join(lb.root_dir, '.gigantum', 'env', 'entrypoint.sh')) try: lb, docker_image_id = ContainerOperations.build_image(labbook=lb, username="******") lb, container_id = ContainerOperations.start_container(lb, username="******") assert isinstance(container_id, str) yield lb, ib, client, docker_image_id, container_id, None, 'unittester' try: _, s = ContainerOperations.stop_container(labbook=lb, username="******") except docker.errors.APIError: client.containers.get(container_id=container_id).stop(timeout=2) s = False finally: shutil.rmtree(lb.root_dir) # Stop and remove container if it's still there try: client.containers.get(container_id=container_id).stop(timeout=2) client.containers.get(container_id=container_id).remove() except: pass # Remove image if it's still there try: ContainerOperations.delete_image(labbook=lb, username='******') client.images.remove(docker_image_id, force=True, noprune=False) except: pass try: client.images.remove(docker_image_id, force=True, noprune=False) except: pass
def test_docker_snippet(self, mock_labbook): lb = mock_labbook[2] package_manager_dir = os.path.join(lb.root_dir, '.gigantum', 'env', 'custom') erm = RepositoryManager(mock_labbook[0]) erm.update_repositories() erm.index_repositories() cm = ComponentManager(lb) custom = ['RUN true', 'RUN touch /tmp/cat', 'RUN rm /tmp/cat'] cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV) cm.add_packages("pip", [{"manager": "pip", "package": "requests", "version": "2.18.4"}]) cm.add_docker_snippet('test-docker', custom, description="Apostrophe's and wėįrd çhårāčtêrś") ib = ImageBuilder(lb) l = ib.assemble_dockerfile() assert all([any([i in l for i in custom]) for n in custom])
def test_remove_package_errors(self, mock_config_with_repo): """Test removing a package with expected errors""" lb = create_tmp_labbook(mock_config_with_repo[0]) cm = ComponentManager(lb) # Try removing package that doesn't exist with pytest.raises(ValueError): cm.remove_packages('apt', ['ack']) # Add a package as if it's from the base pkgs = [{ "manager": "pip3", "package": "requests", "version": "2.18.2" }] cm.add_packages('pip3', pkgs, from_base=True) # Try removing package that you can't because it comes from a base with pytest.raises(ValueError): cm.remove_packages('pip3', ['requests'])
def test_add_duplicate_package(self, mock_config_with_repo): """Test adding a duplicate package to a labbook""" lb = create_tmp_labbook(mock_config_with_repo[0]) labbook_dir = lb.root_dir cm = ComponentManager(lb) # Add a component; pkgs = [{ "manager": "pip3", "package": "requests", "version": "2.18.2" }] cm.add_packages('pip3', pkgs) # Verify file package_file = os.path.join(labbook_dir, '.gigantum', 'env', 'package_manager', 'pip3_requests.yaml') assert os.path.exists(package_file) is True # Add a component with pytest.raises(ValueError): cm.add_packages('pip3', pkgs) # Force add a component cm.add_packages('pip3', pkgs, force=True) assert os.path.exists(package_file) is True with open(package_file, 'rt') as pf: data = yaml.safe_load(pf) assert data['version'] == '2.18.2'
def test_package_counts(self, fixture_working_dir_env_repo_scoped, snapshot): """Test getting the a LabBook's package manager dependencies""" # Create labbook im = InventoryManager(fixture_working_dir_env_repo_scoped[0]) lb = im.create_labbook("default", "default", "labbook5", description="my first labbook10000") cm = ComponentManager(lb) # Add packages cm.add_packages("apt", [{ "manager": "apt", "package": "docker", "version": "" }]) pkgs = [{ "manager": "pip", "package": "requests", "version": "1.3" }, { "manager": "pip", "package": "numpy", "version": "1.12" }] cm.add_packages('pip', pkgs) pkgs = [{ "manager": "conda2", "package": "requests", "version": "1.3" }, { "manager": "conda2", "package": "numpy", "version": "1.12" }, { "manager": "conda2", "package": "matplotlib", "version": "1.12" }, { "manager": "conda2", "package": "plotly", "version": "1.12" }] cm.add_packages('conda2', pkgs) pkgs = [{ "manager": "conda3", "package": "networkx", "version": "1.3" }, { "manager": "conda3", "package": "nibabel", "version": "1.3" }, { "manager": "conda3", "package": "scipy", "version": "1.12" }] cm.add_packages('conda3', pkgs) query = """ { labbook(owner: "default", name: "labbook5") { overview { numAptPackages numConda2Packages numConda3Packages numPipPackages } } } """ snapshot.assert_match( fixture_working_dir_env_repo_scoped[2].execute(query))
def test_remove_package(self, mock_config_with_repo): """Test removing a package such as one from apt-get or pip3. """ lb = create_tmp_labbook(mock_config_with_repo[0]) labbook_dir = lb.root_dir cm = ComponentManager(lb) # Add some sample components pkgs = [{ "manager": "pip3", "package": "requests", "version": "2.18.2" }, { "manager": "pip3", "package": "docker", "version": "0.5" }] cm.add_packages('pip3', pkgs) pkgs = [{ "manager": "apt", "package": "ack", "version": "1.5" }, { "manager": "apt", "package": "docker", "version": "1.3" }] cm.add_packages('apt', pkgs) pkgs = [{ "manager": "pip3", "package": "matplotlib", "version": "2.0.0" }] cm.add_packages('pip3', pkgs, from_base=True) package_path = os.path.join(lb._root_dir, '.gigantum', 'env', 'package_manager') assert os.path.exists(package_path) # Ensure all four packages exist assert os.path.exists(os.path.join(package_path, "apt_ack.yaml")) assert os.path.exists(os.path.join(package_path, "pip3_requests.yaml")) assert os.path.exists(os.path.join(package_path, "apt_docker.yaml")) assert os.path.exists(os.path.join(package_path, "pip3_docker.yaml")) assert os.path.exists( os.path.join(package_path, "pip3_matplotlib.yaml")) # Remove packages cm.remove_packages("apt", ["ack", "docker"]) cm.remove_packages("pip3", ["requests", "docker"]) with pytest.raises(ValueError): cm.remove_packages("pip3", ["matplotlib"]) # Ensure files are gone assert not os.path.exists(os.path.join(package_path, "apt_ack.yaml")) assert not os.path.exists( os.path.join(package_path, "pip3_requests.yaml")) assert not os.path.exists(os.path.join(package_path, "apt_docker.yaml")) assert not os.path.exists( os.path.join(package_path, "pip3_docker.yaml")) assert os.path.exists( os.path.join(package_path, "pip3_matplotlib.yaml")) # Ensure git is clean status = lb.git.status() assert status['untracked'] == [] assert status['staged'] == [] assert status['unstaged'] == [] # Ensure activity is being written log = lb.git.log() assert "_GTM_ACTIVITY_START_" in log[0]["message"] assert 'Removed 2 pip3 managed package(s)' in log[0]["message"]
def test_change_base(self, mock_labbook): """change_base is used both for updating versions and truly changing the base""" conf_file, root_dir, lb = mock_labbook # Initial configuration for the labbook - base config taken from `test_add_base` and package config from # `test_add_package` - so we don't test assertions (again) on this part cm = ComponentManager(lb) # We "misconfigure" a package that is not part of the base as if it was from a base # This shouldn't happen, but we address it just in case pkgs = [ { "manager": "pip3", "package": "gigantum", "version": "0.5" }, # pandas *is* part of the quickstart-juypter base, but we specify a different version here { "package": "pandas", "version": "0.21" } ] cm.add_packages('pip3', pkgs, force=True, from_base=True) packages = [p for p in cm.get_component_list('package_manager')] assert (len(packages) == 2) assert (all(p['from_base'] for p in packages)) cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO, 'quickstart-jupyterlab', 1) # After installing the base, we should have one version of matplotlib installed packages = [ p for p in cm.get_component_list('package_manager') if p['package'] == 'matplotlib' ] assert (len(packages) == 1) assert (packages[0]['version'] == '2.1.1') # add_base() should have converted these to user-installed packages = [ p for p in cm.get_component_list('package_manager') if p['package'] in ['gigantum', 'pandas'] ] # If we had redundancy from fake base-installed pandas plus real base-installed pandas, this would be 3 assert (len(packages) == 2) for p in packages: # Fake base-installed is converted to user ("not from_base") installed assert (not p['from_base']) if p['package'] == 'pandas': # we should still have the fake base-installed version assert (p['version'] == '0.21') pkgs = [ { "manager": "pip3", "package": "requests", "version": "2.18.2" }, # This will override an already installed package { "manager": "pip3", "package": "matplotlib", "version": "2.2" } ] cm.add_packages('pip3', pkgs, force=True) pkgs = [{ "manager": "apt", "package": "ack", "version": "1.0" }, { "manager": "apt", "package": "docker", "version": "3.5" }] cm.add_packages('apt', pkgs) # Installing a customized version of matplotlib is a new package compared to other tests, # and is a critical piece of testing cm.change_base packages = [ p for p in cm.get_component_list('package_manager') if p['package'] == 'matplotlib' ] assert (len(packages) == 1) assert (packages[0]['version'] == '2.2') # We upgrade our base cm.change_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO, 'quickstart-jupyterlab', 2) # matplotlib still set up per "user" update? packages = [ p for p in cm.get_component_list('package_manager') if p['package'] == 'matplotlib' ] assert (len(packages) == 1) assert (packages[0]['version'] == '2.2') # Base revision now 2? assert (cm.base_fields['revision'] == 2)
def test_get_package_manager(self, fixture_working_dir_env_repo_scoped, snapshot): """Test getting the a LabBook's package manager dependencies""" # Create labbook im = InventoryManager(fixture_working_dir_env_repo_scoped[0]) lb = im.create_labbook("default", "default", "labbook4", description="my first labbook10000") query = """ { labbook(owner: "default", name: "labbook4") { environment { packageDependencies { edges { node { id manager package version fromBase } cursor } pageInfo { hasNextPage } } } } } """ # should be null snapshot.assert_match( fixture_working_dir_env_repo_scoped[2].execute(query)) # Add a base image cm = ComponentManager(lb) pkgs = [{ "manager": "pip", "package": "requests", "version": "1.3" }, { "manager": "pip", "package": "numpy", "version": "1.12" }, { "manager": "pip", "package": "gtmunit1", "version": "0.2.4" }] cm.add_packages('pip', pkgs) pkgs = [{ "manager": "conda3", "package": "cdutil", "version": "8.1" }, { "manager": "conda3", "package": "nltk", "version": '3.2.5' }] cm.add_packages('conda3', pkgs) # Add one package without a version, which should cause an error in the API since version is required pkgs = [{"manager": "apt", "package": "lxml", "version": "3.4"}] cm.add_packages('apt', pkgs) query = """ { labbook(owner: "default", name: "labbook4") { environment { packageDependencies { edges { node { id manager package version fromBase } cursor } pageInfo { hasNextPage } } } } } """ r1 = fixture_working_dir_env_repo_scoped[2].execute(query) assert 'errors' not in r1 snapshot.assert_match(r1) query = """ { labbook(owner: "default", name: "labbook4") { environment { packageDependencies(first: 2, after: "MA==") { edges { node { id manager package version fromBase } cursor } pageInfo { hasNextPage } } } } } """ r1 = fixture_working_dir_env_repo_scoped[2].execute(query) assert 'errors' not in r1 snapshot.assert_match(r1)
def test_get_package_manager_metadata(self, fixture_working_dir_env_repo_scoped, snapshot): """Test getting the a LabBook's package manager dependencies""" # Create labbook im = InventoryManager(fixture_working_dir_env_repo_scoped[0]) lb = im.create_labbook("default", "default", "labbook4meta", description="my first asdf") query = """ { labbook(owner: "default", name: "labbook4meta") { environment { packageDependencies { edges { node { id manager package version fromBase description docsUrl latestVersion } cursor } pageInfo { hasNextPage } } } } } """ # should be null snapshot.assert_match( fixture_working_dir_env_repo_scoped[2].execute(query)) # Add a base image cm = ComponentManager(lb) pkgs = [{ "manager": "pip", "package": "gtmunit3", "version": "5.0" }, { "manager": "pip", "package": "gtmunit2", "version": "12.2" }, { "manager": "pip", "package": "gtmunit1", "version": '0.2.1' }] cm.add_packages('pip', pkgs) pkgs = [{ "manager": "conda3", "package": "cdutil", "version": "8.1" }, { "manager": "conda3", "package": "python-coveralls", "version": "2.5.0" }] cm.add_packages('conda3', pkgs) r1 = fixture_working_dir_env_repo_scoped[2].execute(query) assert 'errors' not in r1 snapshot.assert_match(r1)
def build_image_for_jupyterlab(): # Create temp dir config_file, temp_dir = _create_temp_work_dir() # Create user identity insert_cached_identity(temp_dir) # Create test client schema = graphene.Schema(query=LabbookQuery, mutation=LabbookMutations) # get environment data and index erm = RepositoryManager(config_file) erm.update_repositories() erm.index_repositories() with patch.object(Configuration, 'find_default_config', lambda self: config_file): # Load User identity into app context app = Flask("lmsrvlabbook") app.config["LABMGR_CONFIG"] = Configuration() app.config["LABMGR_ID_MGR"] = get_identity_manager(Configuration()) with app.app_context(): # within this block, current_app points to app. Set current user explicitly (this is done in the middleware) flask.g.user_obj = app.config["LABMGR_ID_MGR"].get_user_profile() # Create a test client client = Client( schema, middleware=[DataloaderMiddleware(), error_middleware], context_value=ContextMock()) # Create a labook im = InventoryManager(config_file) lb = im.create_labbook('default', 'unittester', "containerunittestbook", description="Testing docker building.") cm = ComponentManager(lb) cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV) cm.add_packages("pip3", [{ "manager": "pip3", "package": "requests", "version": "2.18.4" }]) bam = BundledAppManager(lb) bam.add_bundled_app(9999, 'share', 'A bundled app for testing', "cd /mnt; python3 -m http.server 9999") ib = ImageBuilder(lb) ib.assemble_dockerfile(write=True) docker_client = get_docker_client() try: lb, docker_image_id = ContainerOperations.build_image( labbook=lb, username="******") # Note: The final field is the owner yield lb, ib, docker_client, docker_image_id, client, "unittester" finally: try: docker_client.containers.get(docker_image_id).stop() docker_client.containers.get(docker_image_id).remove() except: pass try: docker_client.images.remove(docker_image_id, force=True, noprune=False) except: pass shutil.rmtree(lb.root_dir)