def mock_create_labbooks(fixture_working_dir): # Create a labbook in the temporary directory config_file = fixture_working_dir[0] im = InventoryManager(fixture_working_dir[0]) lb = im.create_labbook(UT_USERNAME, UT_USERNAME, UT_LBNAME, description="Cats labbook 1") # Create a file in the dir with open(os.path.join(fixture_working_dir[1], 'unittest-examplefile'), 'w') as sf: sf.write("test data") sf.seek(0) FileOperations.insert_file(lb, 'code', sf.name) assert os.path.isfile( os.path.join(lb.root_dir, 'code', 'unittest-examplefile')) # Create test client schema = graphene.Schema(query=LabbookQuery, mutation=LabbookMutations) with patch.object(Configuration, 'find_default_config', lambda self: config_file): app = Flask("lmsrvlabbook") app.config["LABMGR_CONFIG"] = Configuration() app.config["LABMGR_ID_MGR"] = get_identity_manager(Configuration()) with app.app_context(): flask.g.user_obj = app.config["LABMGR_ID_MGR"].get_user_profile() client = Client( schema, middleware=[DataloaderMiddleware(), error_middleware], context_value=ContextMock()) yield lb, client, schema shutil.rmtree(fixture_working_dir, ignore_errors=True)
def fixture_working_dir_lfs_disabled(): """A pytest fixture that creates a temporary working directory, config file, schema, and local user identity """ # Create temp dir config_file, temp_dir = _create_temp_work_dir(lfs_enabled=False) # Create user identity insert_cached_identity(temp_dir) # Create test client schema = graphene.Schema(query=LabbookQuery, mutation=LabbookMutations) with patch.object(Configuration, 'find_default_config', lambda self: config_file): # Load User identity into app context app = Flask("lmsrvlabbook") app.config["LABMGR_CONFIG"] = Configuration() app.config["LABMGR_ID_MGR"] = get_identity_manager(Configuration()) with app.app_context(): # within this block, current_app points to app. Set current usert explicitly(this is done in the middleware) flask.g.user_obj = app.config["LABMGR_ID_MGR"].get_user_profile() # Create a test client client = Client(schema, middleware=[DataloaderMiddleware()], context_value=ContextMock()) yield config_file, temp_dir, client, schema # name of the config file, temporary working directory, the schema # Remove the temp_dir shutil.rmtree(temp_dir)
def resolve_repository_name_is_available(self, info, name: str): """Resolver to check if a repository name is in use locally or remotely Args: info: name: desired name for the repository Returns: """ # Check if repository exists locally logged_in_username = get_logged_in_username() im = InventoryManager() if im.repository_exists(logged_in_username, logged_in_username, name): return False # Check if repository exists remotely remote_config = Configuration().get_remote_configuration() auth_service = None remote = None if remote_config: auth_service = remote_config.get('admin_service') remote = remote_config.get('git_remote') # Get collaborators from remote service mgr = GitLabManager(remote, auth_service, flask.g.access_token) if mgr.repository_exists(logged_in_username, name): return False # If you get here the name is available return True
def _create_temp_work_dir(lfs_enabled: bool = True): """Helper method to create a temporary working directory and associated config file""" # Create a temporary working directory temp_dir = os.path.join(tempfile.gettempdir(), uuid.uuid4().hex) os.makedirs(temp_dir) config = Configuration() # Make sure the "test" environment components are always used config.config["environment"]["repo_url"] = [ "https://github.com/gigantum/base-images-testing.git" ] config.config["flask"]["DEBUG"] = False # Set the working dir to the new temp dir config.config["git"]["working_directory"] = temp_dir config.config["git"]["lfs_enabled"] = lfs_enabled # Set the auth0 client to the test client (only contains 1 test user and is partitioned from prod) config.config["auth"]["audience"] = "io.gigantum.api.dev" config.config["auth"]["client_id"] = "Z6Wl854wqCjNY0D4uJx8SyPyySyfKmAy" config_file = os.path.join(temp_dir, "temp_config.yaml") config.save(config_file) os.environ['HOST_WORK_DIR'] = temp_dir # Create upload folder if not os.path.exists(config.upload_dir): os.makedirs(config.upload_dir) return config_file, temp_dir
def _create_temp_work_dir(override_dict: dict = None, lfs_enabled: bool = True): """Helper method to create a temporary working directory and associated config file""" def merge_dict(d1, d2) -> None: """Method to merge 1 dictionary into another, updating and adding key/values as needed """ for k, v2 in d2.items(): v1 = d1.get(k) # returns None if v1 has no value for this key if (isinstance(v1, collections.Mapping) and isinstance(v2, collections.Mapping)): merge_dict(v1, v2) else: d1[k] = v2 # Create a temporary working directory unit_test_working_dir = os.path.join(tempfile.gettempdir(), uuid.uuid4().hex) os.makedirs(unit_test_working_dir) os.makedirs(os.path.join(unit_test_working_dir, '.labmanager', 'upload'), exist_ok=True) default_override_config = { 'core': { 'team_mode': False, 'import_demo_on_first_login': False }, 'environment': { 'repo_url': ["https://github.com/gigantum/base-images-testing.git"] }, 'flask': { 'DEBUG': False }, 'git': { 'working_directory': unit_test_working_dir, 'backend': 'filesystem-shim', 'lfs_enabled': lfs_enabled }, 'auth': { 'audience': "io.gigantum.api.dev" }, 'lock': { 'redis': { 'strict': False, } } } os.environ['HOST_WORK_DIR'] = unit_test_working_dir config = Configuration() merge_dict(config.config, default_override_config) if override_dict: config.config.update(override_dict) config_file = os.path.join(unit_test_working_dir, "temp_config.yaml") config.save(config_file) # Return (path-to-config-file, ephemeral-working-directory). return config_file, unit_test_working_dir
def fixture_single_dataset(): """A pytest fixture that creates a temporary working directory, a config file to match, creates the schema, and populates the environment component repository. Class scope modifier attached """ # Create temp dir config_file, temp_dir = _create_temp_work_dir() # Create user identity insert_cached_identity(temp_dir) # Create test client schema = graphene.Schema(query=LabbookQuery, mutation=LabbookMutations) # Create a bunch of lab books im = InventoryManager(config_file) ds = im.create_dataset('default', 'default', "test-dataset", storage_type="gigantum_object_v1", description="Cats 2") m = Manifest(ds, 'default') cm_class = get_cache_manager_class(ds.client_config) cache_mgr = cm_class(ds, 'default') revision = ds.git.repo.head.commit.hexsha os.makedirs(os.path.join(cache_mgr.cache_root, revision, "other_dir")) helper_append_file(cache_mgr.cache_root, revision, "test1.txt", "asdfasdf") helper_append_file(cache_mgr.cache_root, revision, "test2.txt", "rtg") helper_append_file(cache_mgr.cache_root, revision, "test3.txt", "wer") helper_append_file(cache_mgr.cache_root, revision, "other_dir/test4.txt", "dfasdfhfgjhg") helper_append_file(cache_mgr.cache_root, revision, "other_dir/test5.txt", "fdghdfgsa") m.update() with patch.object(Configuration, 'find_default_config', lambda self: config_file): # Load User identity into app context app = Flask("lmsrvlabbook") app.config["LABMGR_CONFIG"] = Configuration() app.config["LABMGR_ID_MGR"] = get_identity_manager(Configuration()) with app.app_context(): # within this block, current_app points to app. Set current user explicitly (this is done in the middleware) flask.g.user_obj = app.config["LABMGR_ID_MGR"].get_user_profile() # Create a test client client = Client(schema, middleware=[DataloaderMiddleware()], context_value=ContextMock()) yield config_file, temp_dir, client, ds, cache_mgr # Remove the temp_dir shutil.rmtree(temp_dir)
def test_multiple_acquires(self, mock_config_file): """Test trying to lock around multiple writes""" conf_file, working_dir = mock_config_file config = Configuration(config_file=conf_file) filename = os.path.join(working_dir, "testfile1.dat") lock = FileWriteLock(filename, config) proc1 = Process(target=write_function, args=(filename, 1, "1", lock)) proc1.start() proc2 = Process(target=write_function, args=(filename, 0, "2", lock)) proc2.start() proc3 = Process(target=write_function, args=(filename, .5, "3", lock)) proc3.start() time.sleep(7) proc1.join() proc1.terminate() proc2.join() proc2.terminate() proc3.join() proc3.terminate() with open(filename, 'rt') as f: data = f.read() assert data == "231"
def test_is_authenticated_token(self, mock_config_file_with_auth): """test checking if the user is authenticated via a token""" config = Configuration(mock_config_file_with_auth[0]) mgr = get_identity_manager(config) assert type(mgr) == LocalIdentityManager # Don't check at_hash claim due to password grant not setting it in the token mgr.validate_at_hash_claim = False # Invalid with no token assert mgr.is_authenticated() is False assert mgr.is_authenticated(None) is False assert mgr.is_authenticated("asdfasdfa") is False assert mgr.is_authenticated("asdfasdfa", "asdfasdffdgfgh") is False assert mgr.is_authenticated( mock_config_file_with_auth[1]['access_token'], mock_config_file_with_auth[1]['id_token']) is True # Second access should load from disk and not need a token mgr2 = get_identity_manager(config) assert mgr2.is_authenticated() is True assert mgr2.is_authenticated( "asdfasdfa") is True # An "expired" token will essentially do this # Double check logging out un-authenticates mgr2.logout() assert mgr.is_authenticated() is False assert mgr2.is_authenticated() is False clean_local_cache(mgr)
def test_get_profile_attribute(self, mock_config_file_with_auth): """test getting profile attributes safely from the profile dictionary""" config = Configuration(mock_config_file_with_auth[0]) mgr = get_identity_manager(config) profile_data = {"username": "", "email": "*****@*****.**"} assert mgr._get_profile_attribute(profile_data, "email") == "*****@*****.**" assert mgr._get_profile_attribute(profile_data, "email", False) == "*****@*****.**" assert mgr._get_profile_attribute(profile_data, "username", False) is None with pytest.raises(AuthenticationError): mgr._get_profile_attribute(profile_data, "username") with pytest.raises(AuthenticationError): mgr._get_profile_attribute(profile_data, "username", True) with pytest.raises(AuthenticationError): mgr._get_profile_attribute(profile_data, "first_name") assert mgr._get_profile_attribute(profile_data, "first_name", False) is None clean_local_cache(mgr)
def test_authenticate_user_exists_token(self, mock_config_file_with_auth): """test getting a user after stored locally already""" config = Configuration(mock_config_file_with_auth[0]) mgr = get_identity_manager(config) assert type(mgr) == LocalIdentityManager # Save User assert os.path.exists(os.path.join(mgr.auth_dir, 'cached_id_jwt')) is False mgr._safe_cached_id_access(mock_config_file_with_auth[1]['id_token']) assert os.path.exists(os.path.join(mgr.auth_dir, 'cached_id_jwt')) is True # Load User u2 = mgr.get_user_profile( mock_config_file_with_auth[1]['access_token'], mock_config_file_with_auth[1]['id_token']) assert type(u2) == User assert "johndoe" == u2.username assert "*****@*****.**" == u2.email assert "John" == u2.given_name assert "Doe" == u2.family_name clean_local_cache(mgr)
def test_load_user_mismatch(self, mock_config_file_with_auth): """handling a new token for the same user""" config = Configuration(mock_config_file_with_auth[0]) with mock.patch.object(jose.jwt, 'get_unverified_claims', lambda x: mock_jwt_claims(x)): with mock.patch.object(LocalIdentityManager, 'validate_jwt_token', mock_jwt_validate): mgr = get_identity_manager(config) assert type(mgr) == LocalIdentityManager # Save User assert os.path.exists( os.path.join(mgr.auth_dir, 'cached_id_jwt')) is False mgr._safe_cached_id_access( json.dumps({ 'nickname': 'olduser', 'dummy': '1' })) assert os.path.exists( os.path.join(mgr.auth_dir, 'cached_id_jwt')) is True # Load User u2 = mgr._load_user( json.dumps({ 'nickname': 'testuser', 'dummy': '1' })) assert os.path.exists( os.path.join(mgr.auth_dir, 'cached_id_jwt')) is False clean_local_cache(mgr)
def clone_repo(remote_url: str, username: str, owner: str, load_repository: Callable[[str], Any], put_repository: Callable[[str, str, str], Any], make_owner: bool = False) -> Repository: # Clone into a temporary directory, such that if anything # gets messed up, then this directory will be cleaned up. tempdir = os.path.join( Configuration().upload_dir, f"{username}_{owner}_clone_{uuid.uuid4().hex[0:10]}") os.makedirs(tempdir) path = _clone(remote_url=remote_url, working_dir=tempdir) candidate_repo = load_repository(path) if os.environ.get('WINDOWS_HOST'): logger.warning("Imported on Windows host - set fileMode to false") call_subprocess("git config core.fileMode false".split(), cwd=candidate_repo.root_dir) repository = put_repository(candidate_repo.root_dir, username, owner) shutil.rmtree(tempdir) return repository
def test_lock_independence(self, mock_config_file): """Test to verify different files have different locks automatically""" conf_file, working_dir = mock_config_file config = Configuration(config_file=conf_file) filename1 = os.path.join(working_dir, "testfile1.dat") lock1 = FileWriteLock(filename1, config) filename2 = os.path.join(working_dir, "testfile2.dat") lock2 = FileWriteLock(filename2, config) proc1 = Process(target=write_function, args=(filename1, 1, "1", lock1)) proc1.start() proc2 = Process(target=write_function, args=(filename1, 6, "2", lock1)) proc2.start() proc3 = Process(target=write_function, args=(filename2, 0, "1", lock2)) proc3.start() proc4 = Process(target=write_function, args=(filename2, 1, "2", lock2)) proc4.start() time.sleep(3) with open(filename1, 'rt') as f: assert f.read() == '1' with open(filename2, 'rt') as f: assert f.read() == '12' proc1.join() proc1.terminate() proc2.join() proc2.terminate() proc3.join() proc3.terminate() proc4.join() proc4.terminate()
def test_check_first_login_no_user_locally_no_repo( self, mock_import, mock_config_file_with_auth_first_login, cleanup_auto_import): """Test login with the user in the repo alread""" # Add mock for call to auth service responses.add(responses.GET, 'https://usersrv.gigantum.io/user', json={'exists': False}, status=404) responses.add(responses.POST, 'https://usersrv.gigantum.io/user', status=201) config = Configuration(mock_config_file_with_auth_first_login[0]) mgr = get_identity_manager(config) # Don't check at_hash claim due to password grant not setting it in the token mgr.validate_at_hash_claim = False mgr._check_first_login( "johndoe", access_token=mock_config_file_with_auth_first_login[2] ['access_token']) # Should import labbook - note we aren't mocking all the way to the workers time.sleep(5) assert os.path.exists( os.path.join('/mnt', 'gigantum', "johndoe", "johndoe", "labbooks", "my-first-project")) is True
def test_load_user_refresh(self, mock_config_file_with_auth): """handling a new token for the same user""" config = Configuration(mock_config_file_with_auth[0]) with mock.patch.object(jose.jwt, 'get_unverified_claims', lambda x: mock_jwt_claims(x)): with mock.patch.object(LocalIdentityManager, 'validate_jwt_token', mock_jwt_validate): mgr = get_identity_manager(config) assert type(mgr) == LocalIdentityManager # Save User assert os.path.exists(os.path.join(mgr.auth_dir, 'cached_id_jwt')) is False mgr._safe_cached_id_access(json.dumps({'nickname': 'testuser', 'dummy': '1'})) assert os.path.exists(os.path.join(mgr.auth_dir, 'cached_id_jwt')) is True with open(os.path.join(mgr.auth_dir, 'cached_id_jwt'), 'rt') as cf: data = json.loads(json.load(cf)) assert data['dummy'] == '1' # Load User u2 = mgr._load_user(json.dumps({'nickname': 'testuser', 'dummy': '2'})) assert type(u2) == User assert "testuser" == u2.username assert "*****@*****.**" == u2.email assert "test" == u2.given_name assert "user" == u2.family_name with open(os.path.join(mgr.auth_dir, 'cached_id_jwt'), 'rt') as cf: data = json.loads(json.load(cf)) assert data['dummy'] == '2' clean_local_cache(mgr)
def test_get_user_profile(self, mock_config_file_with_auth_browser): """test getting a user profile from Auth0""" config = Configuration(mock_config_file_with_auth_browser[0]) mgr = get_identity_manager(config) assert type(mgr) == BrowserIdentityManager # Don't check at_hash claim due to password grant not setting it in the token mgr.validate_at_hash_claim = False # Load User with pytest.raises(AuthenticationError): # Should fail without a token mgr.get_user_profile() # Load User u = mgr.get_user_profile(mock_config_file_with_auth_browser[2]['access_token'], mock_config_file_with_auth_browser[2]['id_token']) assert type(u) == User assert u.username == "johndoe" assert u.email == "*****@*****.**" assert u.given_name == "John" assert u.family_name == "Doe" # Second access should fail since not cached mgr2 = get_identity_manager(config) with pytest.raises(AuthenticationError): # Should fail without a token mgr2.get_user_profile()
def test_get_remote_configuration(self): """Test get_remote_configuration""" configuration = Configuration() remote = configuration.get_remote_configuration() assert remote['git_remote'] == 'repo.gigantum.io' assert remote['remote_type'] == 'gitlab' assert remote['admin_service'] == 'usersrv.gigantum.io' assert remote['index_service'] == 'api.gigantum.com/read' assert remote['object_service'] == 'api.gigantum.com/object-v1' remote = configuration.get_remote_configuration("repo.gigantum.io") assert remote['git_remote'] == 'repo.gigantum.io' assert remote['remote_type'] == 'gitlab' assert remote['admin_service'] == 'usersrv.gigantum.io' assert remote['index_service'] == 'api.gigantum.com/read' assert remote['object_service'] == 'api.gigantum.com/object-v1'
def test_init_load_from_install(self, mock_config_file): """Test loading the default file from the installed location""" with patch('gtmcore.configuration.Configuration.INSTALLED_LOCATION', new_callable=PropertyMock, return_value=mock_config_file[0]): configuration = Configuration() assert 'core' in configuration.config assert 'git' in configuration.config
def test_get_identity_manager_errors(self, mock_config_file_with_auth): """Testing get_identity_manager error handling""" config = Configuration(mock_config_file_with_auth[0]) config.config['auth']['identity_manager'] = "asdfasdf" with pytest.raises(ValueError): get_identity_manager(config) del config.config['auth']['identity_manager'] with pytest.raises(ValueError): get_identity_manager(config) del config.config['auth'] with pytest.raises(ValueError): get_identity_manager(config)
def test_init(self, mock_config_file_team): """Test loading a config file explicitly""" configuration = Configuration(mock_config_file_team[0]) assert 'core' in configuration.config assert 'team_mode' in configuration.config["core"] assert configuration.config["core"]["team_mode"] is True assert 'git' in configuration.config
def ping(): """Unauthorized endpoint for validating the API is up""" config = Configuration() app_name, built_on, revision = config.config['build_info'].split(' :: ') return jsonify({ "application": app_name, "built_on": built_on, "revision": revision })
def test_load_user_no_user(self, mock_config_file_with_auth): """test getting an identity manager""" config = Configuration(mock_config_file_with_auth[0]) mgr = get_identity_manager(config) assert type(mgr) == LocalIdentityManager # Load User assert mgr._load_user(None) is None clean_local_cache(mgr)
def test_load_user_config(self, mock_config_file): """ Test loading configuration override items from a user's custom config """ with tempfile.TemporaryDirectory() as tempdir: override_config_path = os.path.join(tempdir, 'user-config.yaml') with open(override_config_path, 'w') as yf: override_dict = {'container': {'memory': 99}} yf.write(yaml.safe_dump(override_dict, default_flow_style=False)) with patch('gtmcore.configuration.Configuration.USER_LOCATION', new_callable=PropertyMock, return_value=yf.name): conf = Configuration() assert conf.USER_LOCATION == yf.name assert conf.user_config['container']['memory'] == 99 assert conf.config['container']['memory'] == 99 # If we give an explicit config file, then we IGNORE any user overrides conf2 = Configuration(mock_config_file[0]) assert conf2.config['container']['memory'] is None assert len(conf2.user_config.keys()) == 0
def __init__(self, config_file: str = None) -> None: """Constructor Args: config_file(str): Optional config file location if don't want to load from default location """ self.config = Configuration(config_file=config_file) self.local_repo_directory = os.path.expanduser( os.path.join(self.config.config["git"]['working_directory'], ".labmanager", "environment_repositories")) self.git = get_git_interface(self.config.config['git'])
def test_init_inherit(self, mock_config_file_inherit): """Test loading a config file explicitly from a file that inherits properties""" configuration = Configuration(mock_config_file_inherit) assert 'core' in configuration.config assert 'team_mode' in configuration.config["core"] assert configuration.config["core"]["team_mode"] is False assert 'git' in configuration.config assert 'test' in configuration.config assert 'from' in configuration.config assert configuration.config["test"] == 'new field'
def fixture_working_dir_env_repo_scoped(): """A pytest fixture that creates a temporary working directory, a config file to match, creates the schema, and populates the environment component repository. Class scope modifier attached """ # Create temp dir config_file, temp_dir = _create_temp_work_dir() # Create user identity insert_cached_identity(temp_dir) # Create test client schema = graphene.Schema(query=LabbookQuery, mutation=LabbookMutations) # get environment data and index erm = RepositoryManager(config_file) erm.update_repositories() erm.index_repositories() with patch.object(Configuration, 'find_default_config', lambda self: config_file): # Load User identity into app context app = Flask("lmsrvlabbook") app.config["LABMGR_CONFIG"] = Configuration() app.config["LABMGR_ID_MGR"] = get_identity_manager(Configuration()) with app.app_context(): # within this block, current_app points to app. Set current user explicitly (this is done in the middleware) flask.g.user_obj = app.config["LABMGR_ID_MGR"].get_user_profile() # Create a test client client = Client( schema, middleware=[DataloaderMiddleware(), error_middleware], context_value=ContextMock()) # name of the config file, temporary working directory, the schema yield config_file, temp_dir, client, schema # Remove the temp_dir shutil.rmtree(temp_dir)
def test_get_identity_manager(self, mock_config_file_with_auth): """test getting an identity manager""" config = Configuration(mock_config_file_with_auth[0]) mgr = get_identity_manager(config) assert type(mgr) == LocalIdentityManager assert mgr.config == config assert mgr.auth_dir == os.path.join(mock_config_file_with_auth[2], '.labmanager', 'identity') assert mgr.user is None assert mgr.rsa_key is None assert mgr._user is None
def fixture_working_dir_with_cached_user(): """A pytest fixture that creates a temporary working directory, config file, schema, and local user identity """ # Create temp dir config_file, temp_dir = _create_temp_work_dir() insert_cached_identity(temp_dir) with patch.object(Configuration, 'find_default_config', lambda self: config_file): app = Flask("lmsrvlabbook") # Load configuration class into the flask application app.config["LABMGR_CONFIG"] = Configuration() app.config["LABMGR_ID_MGR"] = get_identity_manager(Configuration()) with app.app_context(): # within this block, current_app points to app. yield config_file, temp_dir # name of the config file, temporary working directory # Remove the temp_dir shutil.rmtree(temp_dir)
def _get_local_data_dir(self) -> str: """Method to get the local data directory inside the current container Returns: str """ working_dir = Configuration().config['git']['working_directory'] data_dir = self.configuration.get("Data Directory") if not data_dir: raise ValueError("Data Directory must be specified.") return os.path.join(working_dir, 'local_data', data_dir)
def test_success_import_valid_labbook_from_windows(self, mock_config_file): import_zip = os.path.join(resource_filename('gtmcore','workflows/tests'), 'test_from_windows.zip') dup_import = shutil.copy(import_zip, '/tmp/copy-of-test_from_windows.zip') workspace = Configuration(mock_config_file[0]).config['git']['working_directory'] # Snapshots of directories before and after import - assert different pre_snapshot = str(list(sorted(os.walk(workspace)))) z = ZipExporter() x = z.import_labbook(dup_import, 'test', 'test', mock_config_file[0]) post_snapshot = str(list(sorted(os.walk(workspace)))) assert pre_snapshot != post_snapshot