def test_cleanup_old_files( datapack_zip, svc_client_with_repo, service_job, mock_redis ): """Upload archive and add its contents to a dataset.""" svc_client, headers, _, _ = svc_client_with_repo headers.pop('Content-Type') response = svc_client.post( '/cache.files_upload', data=dict( file=(io.BytesIO(datapack_zip.read_bytes()), datapack_zip.name), ), query_string={ 'unpack_archive': True, 'override_existing': True, }, headers=headers ) assert response assert_rpc_response(response) assert 200 == response.status_code assert 3 == len(response.json['result']['files']) cache_files_cleanup() response = svc_client.get('/cache.files_list', headers=headers) assert response assert_rpc_response(response) assert 200 == response.status_code assert 0 == len(response.json['result']['files'])
def test_dataset_import_twice_job(doi, svc_client_with_repo): """Test dataset import.""" svc_client, headers, project_id, url_components = svc_client_with_repo user = {'user_id': headers['Renku-User-Id']} payload = { 'project_id': project_id, 'dataset_uri': doi, } response = svc_client.post( '/datasets.import', data=json.dumps(payload), headers=headers, ) assert response assert_rpc_response(response) assert {'job_id', 'created_at'} == set(response.json['result'].keys()) dest = make_project_path( user, { 'owner': url_components.owner, 'name': url_components.name } ) old_commit = Repo(dest).head.commit job_id = response.json['result']['job_id'] dataset_import( user, job_id, project_id, doi, ) new_commit = Repo(dest).head.commit assert old_commit.hexsha != new_commit.hexsha with pytest.raises(DatasetExistsError): dataset_import( user, job_id, project_id, doi, ) new_commit2 = Repo(dest).head.commit assert new_commit.hexsha == new_commit2.hexsha response = svc_client.get( f'/jobs/{job_id}', data=json.dumps(payload), headers=headers, ) assert_rpc_response(response) extras = response.json['result']['extras'] assert 'error' in extras assert 'Dataset exists' in extras['error']
def test_dataset_import_junk_job(doi, expected_err, svc_client_with_repo): """Test dataset import.""" svc_client, headers, project_id, url_components = svc_client_with_repo user = {"user_id": headers["Renku-User-Id"]} payload = { "project_id": project_id, "dataset_uri": doi, } response = svc_client.post("/datasets.import", data=json.dumps(payload), headers=headers,) assert response assert_rpc_response(response) assert {"job_id", "created_at"} == set(response.json["result"].keys()) dest = make_project_path(user, {"owner": url_components.owner, "name": url_components.name}) old_commit = Repo(dest).head.commit job_id = response.json["result"]["job_id"] with pytest.raises(ParameterError): dataset_import( user, job_id, project_id, doi, ) new_commit = Repo(dest).head.commit assert old_commit.hexsha == new_commit.hexsha response = svc_client.get(f"/jobs/{job_id}", data=json.dumps(payload), headers=headers,) assert_rpc_response(response) extras = response.json["result"]["extras"] assert "error" in extras assert expected_err in extras["error"]
def test_dataset_import_job(doi, svc_client_with_repo): """Test dataset import via doi.""" svc_client, headers, project_id, url_components = svc_client_with_repo user = {"user_id": headers["Renku-User-Id"]} payload = { "project_id": project_id, "dataset_uri": doi, } response = svc_client.post("/datasets.import", data=json.dumps(payload), headers=headers,) assert response assert_rpc_response(response) assert {"job_id", "created_at"} == set(response.json["result"].keys()) dest = make_project_path(user, {"owner": url_components.owner, "name": url_components.name}) old_commit = Repo(dest).head.commit job_id = response.json["result"]["job_id"] dataset_import( user, job_id, project_id, doi, ) new_commit = Repo(dest).head.commit assert old_commit.hexsha != new_commit.hexsha assert f"service: dataset import {doi}" == new_commit.message response = svc_client.get(f"/jobs/{job_id}", headers=headers,) assert response assert_rpc_response(response) assert "COMPLETED" == response.json["result"]["state"]
def test_cleanup_files_old_keys(svc_client_cache, service_job, tmp_path): """Cleanup old project.""" svc_client, headers, cache = svc_client_cache user = cache.ensure_user({'user_id': 'user'}) mydata = tmp_path / 'mydata.json' mydata.write_text('1,2,3') file_upload = { 'file_id': uuid.uuid4().hex, 'content_type': 'application/json', 'file_name': 'mydata.json', 'file_size': 6, 'is_archive': False, 'is_dir': False, 'unpack_archive': False, 'relative_path': str(mydata) } cache.set_file(user, file_upload) response = svc_client.get('/cache.files_list', headers=headers) assert response assert_rpc_response(response) assert 200 == response.status_code assert 1 == len(response.json['result']['files']) cache_files_cleanup() response = svc_client.get('/cache.files_list', headers=headers) assert response assert_rpc_response(response) assert 200 == response.status_code assert 0 == len(response.json['result']['files'])
def test_cleanup_project_old_keys(svc_client_cache, service_job): """Cleanup old project with old hset keys.""" svc_client, headers, cache = svc_client_cache user = cache.ensure_user({'user_id': 'user'}) project = { 'project_id': uuid.uuid4().hex, 'name': 'my-project', 'fullname': 'full project name', 'email': '*****@*****.**', 'owner': 'me', 'token': 'awesome token', 'git_url': '*****@*****.**' } project = cache.make_project(user, project) os.makedirs(str(project.abs_path), exist_ok=True) response = svc_client.get('/cache.project_list', headers=headers) assert response assert_rpc_response(response) assert 200 == response.status_code assert 1 == len(response.json['result']['projects']) cache_project_cleanup() response = svc_client.get('/cache.project_list', headers=headers) assert response assert_rpc_response(response) assert 200 == response.status_code assert 0 == len(response.json['result']['projects'])
def test_dataset_project_lock(doi, svc_client_with_repo): """Test dataset project lock.""" svc_client, headers, project_id, url_components = svc_client_with_repo user = {'user_id': headers['Renku-User-Id']} payload = { 'project_id': project_id, 'dataset_uri': doi, } response = svc_client.post( '/datasets.import', data=json.dumps(payload), headers=headers, ) assert response assert_rpc_response(response) assert {'job_id', 'created_at'} == set(response.json['result'].keys()) dest = make_project_path(user, { 'owner': url_components.owner, 'name': url_components.name }) old_commit = Repo(dest).head.commit cache_project_cleanup() new_commit = Repo(dest).head.commit assert old_commit.hexsha == new_commit.hexsha assert dest.exists() and [file for file in dest.glob('*')]
def test_dataset_url_import_job(url, svc_client_with_repo): """Test dataset import via url.""" svc_client, headers, project_id, url_components = svc_client_with_repo user = {'user_id': headers['Renku-User-Id']} payload = { 'project_id': project_id, 'dataset_uri': url, } response = svc_client.post( '/datasets.import', data=json.dumps(payload), headers=headers, ) assert response assert_rpc_response(response) assert {'job_id', 'created_at'} == set(response.json['result'].keys()) dest = make_project_path( user, { 'owner': url_components.owner, 'name': url_components.name } ) old_commit = Repo(dest).head.commit job_id = response.json['result']['job_id'] dataset_import( user, job_id, project_id, url, ) new_commit = Repo(dest).head.commit assert old_commit.hexsha != new_commit.hexsha assert f'service: dataset import {url}' == new_commit.message response = svc_client.get( f'/jobs/{job_id}', headers=headers, ) assert response assert_rpc_response(response) assert 'COMPLETED' == response.json['result']['state']
def test_dataset_add_remote_file(url, svc_client_with_repo): """Test dataset add a remote file.""" svc_client, headers, project_id, url_components = svc_client_with_repo user = {'user_id': headers['Renku-User-Id']} payload = { 'project_id': project_id, 'short_name': uuid.uuid4().hex, 'create_dataset': True, 'files': [{ 'file_url': url }] } response = svc_client.post( '/datasets.add', data=json.dumps(payload), headers=headers, ) assert response assert_rpc_response(response) assert {'files', 'short_name', 'project_id'} == set(response.json['result'].keys()) dest = make_project_path(user, { 'owner': url_components.owner, 'name': url_components.name }) old_commit = Repo(dest).head.commit job_id = response.json['result']['files'][0]['job_id'] commit_message = 'service: dataset add remote file' dataset_add_remote_file(user, job_id, project_id, True, commit_message, payload['short_name'], url) new_commit = Repo(dest).head.commit assert old_commit.hexsha != new_commit.hexsha assert commit_message == new_commit.message
def test_dataset_project_lock(doi, svc_client_with_repo): """Test dataset project lock.""" svc_client, headers, project_id, url_components = svc_client_with_repo user = {"user_id": headers["Renku-User-Id"]} payload = { "project_id": project_id, "dataset_uri": doi, } response = svc_client.post("/datasets.import", data=json.dumps(payload), headers=headers,) assert response assert_rpc_response(response) assert {"job_id", "created_at"} == set(response.json["result"].keys()) dest = make_project_path(user, {"owner": url_components.owner, "name": url_components.name}) old_commit = Repo(dest).head.commit cache_project_cleanup() new_commit = Repo(dest).head.commit assert old_commit.hexsha == new_commit.hexsha assert dest.exists() and [file for file in dest.glob("*")]
def test_dataset_add_remote_file(url, svc_client_with_repo): """Test dataset add a remote file.""" svc_client, headers, project_id, url_components = svc_client_with_repo user = {"user_id": headers["Renku-User-Id"]} payload = {"project_id": project_id, "name": uuid.uuid4().hex, "create_dataset": True, "files": [{"file_url": url}]} response = svc_client.post("/datasets.add", data=json.dumps(payload), headers=headers,) assert response assert_rpc_response(response) assert {"files", "name", "project_id"} == set(response.json["result"].keys()) dest = make_project_path(user, {"owner": url_components.owner, "name": url_components.name}) old_commit = Repo(dest).head.commit job_id = response.json["result"]["files"][0]["job_id"] commit_message = "service: dataset add remote file" dataset_add_remote_file(user, job_id, project_id, True, commit_message, payload["name"], url) new_commit = Repo(dest).head.commit assert old_commit.hexsha != new_commit.hexsha assert commit_message == new_commit.message
def test_cleanup_old_project(datapack_zip, svc_client_with_repo, service_job, mock_redis): """Upload archive and add its contents to a dataset.""" svc_client, headers, _, _ = svc_client_with_repo headers.pop('Content-Type') response = svc_client.get('/cache.project_list', headers=headers) assert response assert_rpc_response(response) assert 200 == response.status_code assert 1 == len(response.json['result']['projects']) cache_project_cleanup() response = svc_client.get('/cache.project_list', headers=headers) assert response assert_rpc_response(response) assert 200 == response.status_code assert 0 == len(response.json['result']['projects'])