def test_cleanup_old_files(
datapack_zip, svc_client_with_repo, service_job, mock_redis
):
"""Upload archive and add its contents to a dataset."""
svc_client, headers, _, _ = svc_client_with_repo
headers.pop('Content-Type')
response = svc_client.post(
'/cache.files_upload',
data=dict(
file=(io.BytesIO(datapack_zip.read_bytes()), datapack_zip.name),
),
query_string={
'unpack_archive': True,
'override_existing': True,
},
headers=headers
)
assert response
assert_rpc_response(response)
assert 200 == response.status_code
assert 3 == len(response.json['result']['files'])
cache_files_cleanup()
response = svc_client.get('/cache.files_list', headers=headers)
assert response
assert_rpc_response(response)
assert 200 == response.status_code
assert 0 == len(response.json['result']['files'])
def test_dataset_import_twice_job(doi, svc_client_with_repo):
"""Test dataset import."""
svc_client, headers, project_id, url_components = svc_client_with_repo
user = {'user_id': headers['Renku-User-Id']}
payload = {
'project_id': project_id,
'dataset_uri': doi,
}
response = svc_client.post(
'/datasets.import',
data=json.dumps(payload),
headers=headers,
)
assert response
assert_rpc_response(response)
assert {'job_id', 'created_at'} == set(response.json['result'].keys())
dest = make_project_path(
user, {
'owner': url_components.owner,
'name': url_components.name
}
)
old_commit = Repo(dest).head.commit
job_id = response.json['result']['job_id']
dataset_import(
user,
job_id,
project_id,
doi,
)
new_commit = Repo(dest).head.commit
assert old_commit.hexsha != new_commit.hexsha
with pytest.raises(DatasetExistsError):
dataset_import(
user,
job_id,
project_id,
doi,
)
new_commit2 = Repo(dest).head.commit
assert new_commit.hexsha == new_commit2.hexsha
response = svc_client.get(
f'/jobs/{job_id}',
data=json.dumps(payload),
headers=headers,
)
assert_rpc_response(response)
extras = response.json['result']['extras']
assert 'error' in extras
assert 'Dataset exists' in extras['error']
def test_dataset_import_junk_job(doi, expected_err, svc_client_with_repo):
"""Test dataset import."""
svc_client, headers, project_id, url_components = svc_client_with_repo
user = {"user_id": headers["Renku-User-Id"]}
payload = {
"project_id": project_id,
"dataset_uri": doi,
}
response = svc_client.post("/datasets.import", data=json.dumps(payload), headers=headers,)
assert response
assert_rpc_response(response)
assert {"job_id", "created_at"} == set(response.json["result"].keys())
dest = make_project_path(user, {"owner": url_components.owner, "name": url_components.name})
old_commit = Repo(dest).head.commit
job_id = response.json["result"]["job_id"]
with pytest.raises(ParameterError):
dataset_import(
user, job_id, project_id, doi,
)
new_commit = Repo(dest).head.commit
assert old_commit.hexsha == new_commit.hexsha
response = svc_client.get(f"/jobs/{job_id}", data=json.dumps(payload), headers=headers,)
assert_rpc_response(response)
extras = response.json["result"]["extras"]
assert "error" in extras
assert expected_err in extras["error"]
def test_dataset_import_job(doi, svc_client_with_repo):
"""Test dataset import via doi."""
svc_client, headers, project_id, url_components = svc_client_with_repo
user = {"user_id": headers["Renku-User-Id"]}
payload = {
"project_id": project_id,
"dataset_uri": doi,
}
response = svc_client.post("/datasets.import", data=json.dumps(payload), headers=headers,)
assert response
assert_rpc_response(response)
assert {"job_id", "created_at"} == set(response.json["result"].keys())
dest = make_project_path(user, {"owner": url_components.owner, "name": url_components.name})
old_commit = Repo(dest).head.commit
job_id = response.json["result"]["job_id"]
dataset_import(
user, job_id, project_id, doi,
)
new_commit = Repo(dest).head.commit
assert old_commit.hexsha != new_commit.hexsha
assert f"service: dataset import {doi}" == new_commit.message
response = svc_client.get(f"/jobs/{job_id}", headers=headers,)
assert response
assert_rpc_response(response)
assert "COMPLETED" == response.json["result"]["state"]
def test_cleanup_files_old_keys(svc_client_cache, service_job, tmp_path):
"""Cleanup old project."""
svc_client, headers, cache = svc_client_cache
user = cache.ensure_user({'user_id': 'user'})
mydata = tmp_path / 'mydata.json'
mydata.write_text('1,2,3')
file_upload = {
'file_id': uuid.uuid4().hex,
'content_type': 'application/json',
'file_name': 'mydata.json',
'file_size': 6,
'is_archive': False,
'is_dir': False,
'unpack_archive': False,
'relative_path': str(mydata)
}
cache.set_file(user, file_upload)
response = svc_client.get('/cache.files_list', headers=headers)
assert response
assert_rpc_response(response)
assert 200 == response.status_code
assert 1 == len(response.json['result']['files'])
cache_files_cleanup()
response = svc_client.get('/cache.files_list', headers=headers)
assert response
assert_rpc_response(response)
assert 200 == response.status_code
assert 0 == len(response.json['result']['files'])
def test_cleanup_project_old_keys(svc_client_cache, service_job):
"""Cleanup old project with old hset keys."""
svc_client, headers, cache = svc_client_cache
user = cache.ensure_user({'user_id': 'user'})
project = {
'project_id': uuid.uuid4().hex,
'name': 'my-project',
'fullname': 'full project name',
'email': '*****@*****.**',
'owner': 'me',
'token': 'awesome token',
'git_url': '*****@*****.**'
}
project = cache.make_project(user, project)
os.makedirs(str(project.abs_path), exist_ok=True)
response = svc_client.get('/cache.project_list', headers=headers)
assert response
assert_rpc_response(response)
assert 200 == response.status_code
assert 1 == len(response.json['result']['projects'])
cache_project_cleanup()
response = svc_client.get('/cache.project_list', headers=headers)
assert response
assert_rpc_response(response)
assert 200 == response.status_code
assert 0 == len(response.json['result']['projects'])
def test_dataset_project_lock(doi, svc_client_with_repo):
"""Test dataset project lock."""
svc_client, headers, project_id, url_components = svc_client_with_repo
user = {'user_id': headers['Renku-User-Id']}
payload = {
'project_id': project_id,
'dataset_uri': doi,
}
response = svc_client.post(
'/datasets.import',
data=json.dumps(payload),
headers=headers,
)
assert response
assert_rpc_response(response)
assert {'job_id', 'created_at'} == set(response.json['result'].keys())
dest = make_project_path(user, {
'owner': url_components.owner,
'name': url_components.name
})
old_commit = Repo(dest).head.commit
cache_project_cleanup()
new_commit = Repo(dest).head.commit
assert old_commit.hexsha == new_commit.hexsha
assert dest.exists() and [file for file in dest.glob('*')]
def test_dataset_url_import_job(url, svc_client_with_repo):
"""Test dataset import via url."""
svc_client, headers, project_id, url_components = svc_client_with_repo
user = {'user_id': headers['Renku-User-Id']}
payload = {
'project_id': project_id,
'dataset_uri': url,
}
response = svc_client.post(
'/datasets.import',
data=json.dumps(payload),
headers=headers,
)
assert response
assert_rpc_response(response)
assert {'job_id', 'created_at'} == set(response.json['result'].keys())
dest = make_project_path(
user, {
'owner': url_components.owner,
'name': url_components.name
}
)
old_commit = Repo(dest).head.commit
job_id = response.json['result']['job_id']
dataset_import(
user,
job_id,
project_id,
url,
)
new_commit = Repo(dest).head.commit
assert old_commit.hexsha != new_commit.hexsha
assert f'service: dataset import {url}' == new_commit.message
response = svc_client.get(
f'/jobs/{job_id}',
headers=headers,
)
assert response
assert_rpc_response(response)
assert 'COMPLETED' == response.json['result']['state']
def test_dataset_add_remote_file(url, svc_client_with_repo):
"""Test dataset add a remote file."""
svc_client, headers, project_id, url_components = svc_client_with_repo
user = {'user_id': headers['Renku-User-Id']}
payload = {
'project_id': project_id,
'short_name': uuid.uuid4().hex,
'create_dataset': True,
'files': [{
'file_url': url
}]
}
response = svc_client.post(
'/datasets.add',
data=json.dumps(payload),
headers=headers,
)
assert response
assert_rpc_response(response)
assert {'files', 'short_name',
'project_id'} == set(response.json['result'].keys())
dest = make_project_path(user, {
'owner': url_components.owner,
'name': url_components.name
})
old_commit = Repo(dest).head.commit
job_id = response.json['result']['files'][0]['job_id']
commit_message = 'service: dataset add remote file'
dataset_add_remote_file(user, job_id, project_id, True, commit_message,
payload['short_name'], url)
new_commit = Repo(dest).head.commit
assert old_commit.hexsha != new_commit.hexsha
assert commit_message == new_commit.message
def test_dataset_project_lock(doi, svc_client_with_repo):
"""Test dataset project lock."""
svc_client, headers, project_id, url_components = svc_client_with_repo
user = {"user_id": headers["Renku-User-Id"]}
payload = {
"project_id": project_id,
"dataset_uri": doi,
}
response = svc_client.post("/datasets.import", data=json.dumps(payload), headers=headers,)
assert response
assert_rpc_response(response)
assert {"job_id", "created_at"} == set(response.json["result"].keys())
dest = make_project_path(user, {"owner": url_components.owner, "name": url_components.name})
old_commit = Repo(dest).head.commit
cache_project_cleanup()
new_commit = Repo(dest).head.commit
assert old_commit.hexsha == new_commit.hexsha
assert dest.exists() and [file for file in dest.glob("*")]
def test_dataset_add_remote_file(url, svc_client_with_repo):
"""Test dataset add a remote file."""
svc_client, headers, project_id, url_components = svc_client_with_repo
user = {"user_id": headers["Renku-User-Id"]}
payload = {"project_id": project_id, "name": uuid.uuid4().hex, "create_dataset": True, "files": [{"file_url": url}]}
response = svc_client.post("/datasets.add", data=json.dumps(payload), headers=headers,)
assert response
assert_rpc_response(response)
assert {"files", "name", "project_id"} == set(response.json["result"].keys())
dest = make_project_path(user, {"owner": url_components.owner, "name": url_components.name})
old_commit = Repo(dest).head.commit
job_id = response.json["result"]["files"][0]["job_id"]
commit_message = "service: dataset add remote file"
dataset_add_remote_file(user, job_id, project_id, True, commit_message, payload["name"], url)
new_commit = Repo(dest).head.commit
assert old_commit.hexsha != new_commit.hexsha
assert commit_message == new_commit.message
def test_cleanup_old_project(datapack_zip, svc_client_with_repo, service_job,
mock_redis):
"""Upload archive and add its contents to a dataset."""
svc_client, headers, _, _ = svc_client_with_repo
headers.pop('Content-Type')
response = svc_client.get('/cache.project_list', headers=headers)
assert response
assert_rpc_response(response)
assert 200 == response.status_code
assert 1 == len(response.json['result']['projects'])
cache_project_cleanup()
response = svc_client.get('/cache.project_list', headers=headers)
assert response
assert_rpc_response(response)
assert 200 == response.status_code
assert 0 == len(response.json['result']['projects'])
