def test_upload_artifacts_SHOULD_upload_files_to_buckets_properly_WHEN_multiple_keywords(
cwd):
bucket_paths = SimpleNamespace(main_bucket_path='test_cloud',
client_name='sicloudman_client',
project_name='sicloudman_project')
cloud_manager, artifacts_path = get_updated_cloud_manager(
cwd, bucket_paths, [
sicloudman.Bucket(name='release', keywords=['_release', '.whl']),
sicloudman.Bucket(name='client', keywords=['_client'])
])
Path(artifacts_path / 'test_1_release.txt').touch()
Path(artifacts_path / 'test_1.whl').touch()
Path(artifacts_path / 'test_1_client.txt').touch()
Path(artifacts_path / 'test_1_dev.txt').touch()
time.sleep(1)
Path(artifacts_path / 'test_2_release.txt').touch()
Path(artifacts_path / 'test_2.whl').touch()
Path(artifacts_path / 'test_2_client.txt').touch()
Path(artifacts_path / 'test_2_dev.txt').touch()
cloud_manager.upload_artifacts(prompt=False)
cloud_files = cloud_manager.list_cloud()
assert set(cloud_files.release) == {'test_2_release.txt', 'test_2.whl'}
assert cloud_files.release.__len__() == 2
assert set(cloud_files.client) == {'test_2_client.txt'}
assert cloud_files.client.__len__() == 1
with ftplib.FTP(cloud_manager.credentials.server,
cloud_manager.credentials.username,
cloud_manager.credentials.password) as ftp_conn:
ftp_rmtree(ftp_conn,
cloud_manager._get_project_bucket_path().parent.as_posix())
def test_upload_file_SHOULD_upload_file_properly(cwd):
bucket_paths = SimpleNamespace(main_bucket_path='test_cloud',
client_name='sicloudman_client',
project_name='sicloudman_project')
cloud_manager, artifacts_path = get_updated_cloud_manager(
cwd, bucket_paths, [
sicloudman.Bucket(name='release', keywords=['_release']),
sicloudman.Bucket(name='client', keywords=['_client'])
])
Path(artifacts_path / 'test_1_release.txt').touch()
Path(artifacts_path / 'test_1.whl').touch()
uploaded_file_path = cloud_manager.upload_file(file_path=artifacts_path /
'test_1_release.txt',
bucket_name='release',
prompt=False)
cloud_files = cloud_manager.list_cloud()
assert set(cloud_files.release) == {'test_1_release.txt'}
assert cloud_files.release.__len__() == 1
assert cloud_files.client.__len__() == 0
assert (cloud_manager._get_project_bucket_path() / 'release' /
'test_1_release.txt').as_posix() == uploaded_file_path
with ftplib.FTP(cloud_manager.credentials.server,
cloud_manager.credentials.username,
cloud_manager.credentials.password) as ftp_conn:
ftp_rmtree(ftp_conn,
cloud_manager._get_project_bucket_path().parent.as_posix())
def test_upload_file_SHOULD_print_warning_when_file_already_exists(
cwd, caplog):
bucket_paths = SimpleNamespace(main_bucket_path='test_cloud',
client_name='sicloudman_client',
project_name='sicloudman_project')
cloud_manager, artifacts_path = get_updated_cloud_manager(
cwd, bucket_paths, [
sicloudman.Bucket(name='release', keywords=['_release']),
sicloudman.Bucket(name='client', keywords=['_client'])
])
Path(artifacts_path / 'test_1_release.txt').touch()
Path(artifacts_path / 'test_1.whl').touch()
cloud_manager._logger.setLevel(logging.INFO)
cloud_manager.upload_file(file_path=artifacts_path / 'test_1_release.txt',
bucket_name='release',
prompt=False)
cloud_manager.upload_file(file_path=artifacts_path / 'test_1_release.txt',
bucket_name='release',
prompt=False)
assert 'already exists' in caplog.text
with ftplib.FTP(cloud_manager.credentials.server,
cloud_manager.credentials.username,
cloud_manager.credentials.password) as ftp_conn:
ftp_rmtree(ftp_conn,
cloud_manager._get_project_bucket_path().parent.as_posix())
def test_download_file_SHOULD_download_file_properly(cwd):
bucket_paths = SimpleNamespace(main_bucket_path='test_cloud',
client_name='sicloudman_client',
project_name='sicloudman_project')
cloud_manager, artifacts_path = get_updated_cloud_manager(
cwd, bucket_paths, [
sicloudman.Bucket(name='release', keywords=['_release', '.whl']),
sicloudman.Bucket(name='client', keywords=['_client'])
])
Path(artifacts_path / 'test_1_release.txt').touch()
Path(artifacts_path / 'test_1.whl').touch()
Path(artifacts_path / 'test_1_client.txt').touch()
Path(artifacts_path / 'test_1_dev.txt').touch()
cloud_manager.upload_artifacts(prompt=False)
shutil.rmtree(artifacts_path)
downloaded_file_path = cloud_manager.download_file(filename='test_1.whl')
assert (artifacts_path / 'test_1.whl') in set(artifacts_path.iterdir())
assert downloaded_file_path == (artifacts_path / 'test_1.whl').as_posix()
with ftplib.FTP(cloud_manager.credentials.server,
cloud_manager.credentials.username,
cloud_manager.credentials.password) as ftp_conn:
ftp_rmtree(ftp_conn,
cloud_manager._get_project_bucket_path().parent.as_posix())
def test_is_path_exists_SHOULD_return_true_if_exists(cwd):
cloud_manager = sicloudman.CloudManager(
'artifacts',
[sicloudman.Bucket(name='release', keywords=['_release'])],
credentials_path=TEST_CLOUD_CREDENTIALS_PATH,
cwd=cwd)
cloud_manager._get_project_bucket_path()
credentials = copy.copy(cloud_manager.credentials)
credentials.client_name = 'this_is_abstrac_client_path_173'
sicloudman.CloudManager.touch_credentials(cwd,
keywords=credentials.__dict__)
cloud_manager = sicloudman.CloudManager(
'artifacts',
[sicloudman.Bucket(name='release', keywords=['_release'])],
cwd=cwd)
cloud_manager._get_project_bucket_path()
with ftplib.FTP(cloud_manager.credentials.server,
cloud_manager.credentials.username,
cloud_manager.credentials.password) as ftp_conn:
project_bucket_path = cloud_manager._get_project_bucket_path()
assert cloud_manager._is_path_exists(ftp_conn,
project_bucket_path) == False
def test_download_file_SHOULD_raise_error_when_bucket_not_found(cwd):
bucket_paths = SimpleNamespace(main_bucket_path='test_cloud',
client_name='sicloudman_client',
project_name='sicloudman_project')
cloud_manager, _ = get_updated_cloud_manager(cwd, bucket_paths, [
sicloudman.Bucket(name='release', keywords=['_release']),
sicloudman.Bucket(name='client', keywords=['_client'])
])
with pytest.raises(sicloudman.FileNotFoundError):
cloud_manager.download_file(filename='file.txt')
def test_upload_file_SHOULD_raise_error_when_file_path_not_specified(cwd):
bucket_paths = SimpleNamespace(main_bucket_path='test_cloud',
client_name='sicloudman_client',
project_name='sicloudman_project')
cloud_manager, _ = get_updated_cloud_manager(cwd, bucket_paths, [
sicloudman.Bucket(name='release', keywords=['_release']),
sicloudman.Bucket(name='client', keywords=['_client'])
])
with pytest.raises(sicloudman.ValueError) as exc:
cloud_manager.upload_file(bucket_name='release', prompt=False)
assert 'file_path' in str(exc.value)
def test_list_cloud_SHOULD_print_proper_info_when_no_buckets(cwd, caplog):
bucket_paths = SimpleNamespace(main_bucket_path='main_bucket_dummy_dir',
client_name='sicloudman_client',
project_name='sicloudman_project')
cloud_manager, _ = get_updated_cloud_manager(cwd, bucket_paths, [
sicloudman.Bucket(name='release', keywords=['_release']),
sicloudman.Bucket(name='client', keywords=['_client'])
])
cloud_manager._logger.setLevel(logging.INFO)
cloud_manager.list_cloud()
assert 'no buckets' in caplog.text
def test_get_bucket_name_from_filename_SHOULD_return_bucket_name_WHEN_many_keywords(
cwd):
cloud_manager = sicloudman.CloudManager(
'artifacts', [
sicloudman.Bucket(name='release', keywords=['_release']),
sicloudman.Bucket(name='client', keywords=['_client', '.whl'])
],
credentials_path=TEST_CLOUD_CREDENTIALS_PATH,
cwd=cwd)
bucket = cloud_manager._get_bucket_name_from_filename('sample_file.whl')
assert bucket == 'client'
def test_create_buckets_tree_SHOULD_raise_error_when_main_bucket_not_exists(
cwd):
bucket_paths = SimpleNamespace(main_bucket_path='main_bucket_dummy_dir',
client_name='sicloudman_client',
project_name='sicloudman_project')
cloud_manager, _ = get_updated_cloud_manager(cwd, bucket_paths, [
sicloudman.Bucket(name='release', keywords=['_release']),
sicloudman.Bucket(name='client', keywords=['_client'])
])
with ftplib.FTP(cloud_manager.credentials.server,
cloud_manager.credentials.username,
cloud_manager.credentials.password) as ftp_conn:
with pytest.raises(sicloudman.BucketNotFoundError):
cloud_manager._create_buckets_tree(ftp_conn)
def get_updated_cloud_manager(cwd, bucket_paths, buckets):
artifacts_path = cwd / 'artifacts'
artifacts_path.mkdir()
cloud_manager = sicloudman.CloudManager(
artifacts_path,
[sicloudman.Bucket(name='release', keywords=['_release'])],
credentials_path=TEST_CLOUD_CREDENTIALS_PATH,
cwd=cwd)
cloud_manager._get_project_bucket_path()
credentials = copy.copy(cloud_manager.credentials)
credentials.main_bucket_path = bucket_paths.main_bucket_path
credentials.client_name = bucket_paths.client_name
credentials.project_name = bucket_paths.project_name
sicloudman.CloudManager.touch_credentials(cwd,
keywords=credentials.__dict__)
cloud_manager = sicloudman.CloudManager(artifacts_path, buckets, cwd=cwd)
cloud_manager._get_project_bucket_path()
with ftplib.FTP(cloud_manager.credentials.server,
cloud_manager.credentials.username,
cloud_manager.credentials.password) as ftp_conn:
main_bucket_path = cloud_manager.credentials.main_bucket_path
if main_bucket_path == cloud_manager._get_project_bucket_path().parent:
path_to_rm = cloud_manager._get_project_bucket_path()
else:
path_to_rm = cloud_manager._get_project_bucket_path().parent
if cloud_manager._is_path_exists(ftp_conn, path_to_rm):
ftp_rmtree(ftp_conn, path_to_rm.as_posix())
return cloud_manager, artifacts_path
def test_check_credentials_SHOULD_raise_error_WHEN_no_credentials_file(cwd):
cloud_manager = sicloudman.CloudManager(
'artifacts',
[sicloudman.Bucket(name='release', keywords=['_release'])],
cwd=cwd)
with pytest.raises(sicloudman.CredentialsNotFoundError):
cloud_manager._get_project_bucket_path()
def test_upload_file_SHOULD_raise_error_when_bucket_not_exists(cwd):
bucket_paths = SimpleNamespace(main_bucket_path='test_cloud',
client_name='sicloudman_client',
project_name='sicloudman_project')
cloud_manager, artifacts_path = get_updated_cloud_manager(
cwd, bucket_paths, [
sicloudman.Bucket(name='release', keywords=['_release']),
sicloudman.Bucket(name='client', keywords=['_client'])
])
Path(artifacts_path / 'test_1_release.txt').touch()
with pytest.raises(sicloudman.BucketNotFoundError):
cloud_manager.upload_file(file_path=artifacts_path /
'test_1_release.txt',
bucket_name='dummy_bucket',
prompt=False)
def test_CloudManager_init_SHOULD_not_raise_error_WHEN_no_credentials_file(
cwd):
try:
sicloudman.CloudManager(
'artifacts',
[sicloudman.Bucket(name='release', keywords=['_release'])],
cwd=cwd)
except Exception as e:
assert 0, f'Unexpected error occured: {e}'
def test_list_cloud_SHOULD_print_that_bucket_is_empty_WHEN_bucket_empty(
cwd, caplog):
bucket_paths = SimpleNamespace(main_bucket_path='test_cloud',
client_name='sicloudman_client',
project_name='sicloudman_project')
cloud_manager, artifacts_path = get_updated_cloud_manager(
cwd, bucket_paths, [
sicloudman.Bucket(name='release', keywords=['_release']),
sicloudman.Bucket(name='client', keywords=['_client'])
])
Path(artifacts_path / 'test_1_release.txt').touch()
Path(artifacts_path / 'test_1_dev.txt').touch()
cloud_manager._logger.setLevel(logging.INFO)
cloud_manager.upload_artifacts(prompt=False)
cloud_manager.list_cloud()
assert 'No files in bucket: client' in caplog.text
def test_CloudManager_init_SHOULD_raise_error_when_credentials_are_not_credentials(
cwd):
with pytest.raises(sicloudman.TypeError) as exc:
sicloudman.CloudManager(
'artifacts',
[sicloudman.Bucket(name='release', keywords=['_release'])],
credentials={'server': 'serverro'},
cwd=cwd)
assert 'not an instance' in str(exc.value)
def test_get_project_bucket_path(credentials, expected_path, cwd):
sicloudman.CloudManager.touch_credentials(cwd,
keywords=credentials.__dict__)
cloud_manager = sicloudman.CloudManager(
'artifacts',
[sicloudman.Bucket(name='release', keywords=['_release'])],
cwd=cwd)
path = cloud_manager._get_project_bucket_path().as_posix()
assert path == expected_path
def test_upload_artifacts_SHOULD_not_upload_files_to_buckets_properly_WHEN_no_match_files(
cwd, caplog):
bucket_paths = SimpleNamespace(main_bucket_path='test_cloud',
client_name='sicloudman_client',
project_name='sicloudman_project')
cloud_manager, artifacts_path = get_updated_cloud_manager(
cwd, bucket_paths, [
sicloudman.Bucket(name='release', keywords=['_release']),
sicloudman.Bucket(name='client', keywords=['_client'])
])
Path(artifacts_path / 'test_1.whl').touch()
Path(artifacts_path / 'test_1_dev.txt').touch()
time.sleep(1)
Path(artifacts_path / 'test_2.whl').touch()
Path(artifacts_path / 'test_2_dev.txt').touch()
cloud_manager._logger.setLevel(logging.INFO)
cloud_manager.upload_artifacts(prompt=False)
cloud_files = cloud_manager.list_cloud()
assert cloud_files is None
assert 'No files to upload' in caplog.text
def test_create_buckets_tree_SHOULD_create_tree_properly_when_complex_main_bucket_path(
cwd):
bucket_paths = SimpleNamespace(main_bucket_path='test_cloud/test_cloud',
client_name='sicloudman_client',
project_name='sicloudman_project')
cloud_manager, _ = get_updated_cloud_manager(cwd, bucket_paths, [
sicloudman.Bucket(name='release', keywords=['_release']),
sicloudman.Bucket(name='client', keywords=['_client'])
])
with ftplib.FTP(cloud_manager.credentials.server,
cloud_manager.credentials.username,
cloud_manager.credentials.password) as ftp_conn:
cloud_manager._create_buckets_tree(ftp_conn)
assert cloud_manager._is_path_exists(
ftp_conn,
cloud_manager._get_project_bucket_path() / 'release') == True
assert cloud_manager._is_path_exists(
ftp_conn,
cloud_manager._get_project_bucket_path() / 'client') == True
ftp_rmtree(ftp_conn,
cloud_manager._get_project_bucket_path().parent.as_posix())
def test_read_credentials_SHOULD_rise_error_when_field_has_invalid_value(cwd):
credentials_expected = sicloudman.Credentials(
server='',
username='******',
password='******',
main_bucket_path='/',
client_name='client',
project_name='project',
)
sicloudman.CloudManager.touch_credentials(
cwd, keywords=credentials_expected.__dict__)
with pytest.raises(sicloudman.CredentialsError):
cloud_manager = sicloudman.CloudManager(
'artifacts',
[sicloudman.Bucket(name='release', keywords=['_release'])],
cwd=cwd)
cloud_manager._get_project_bucket_path()
def test_check_credentials_SHOULD_not_raise_error_WHEN_credentials_file_is_empty_AND_only_CloudManager_init(
cwd):
sicloudman.CloudManager.touch_credentials(
cwd,
keywords=sicloudman.Credentials(
server='my_server',
username='******',
password='******',
main_bucket_path='main_bucket',
client_name='client',
project_name='',
).__dict__)
try:
sicloudman.CloudManager(
'artifacts',
[sicloudman.Bucket(name='release', keywords=['_release'])],
cwd=cwd)
except Exception as e:
assert 0, f'Unexpected error occured: {e}'
def test_read_credentials_SHOULD_not_rise_error_when_extra_field_empty(cwd):
credentials_expected = sicloudman.Credentials(
server='my_server',
username='******',
password='******',
main_bucket_path='main_bucket',
client_name='',
project_name='',
)
sicloudman.CloudManager.touch_credentials(
cwd, keywords=credentials_expected.__dict__)
cloud_manager = sicloudman.CloudManager(
'artifacts',
[sicloudman.Bucket(name='release', keywords=['_release'])],
cwd=cwd)
cloud_manager._get_project_bucket_path()
assert set(cloud_manager.credentials.__dict__) == set(
credentials_expected.__dict__)
def test_touch_and_read_credentials_WHEN_keywords(cwd):
credentials_expected = sicloudman.Credentials(
server='my_server',
username='******',
password='******',
main_bucket_path='main_bucket',
client_name='client',
project_name='project',
)
file_path = sicloudman.CloudManager.touch_credentials(
cwd, keywords=credentials_expected.__dict__)
file_content = file_path.read_text()
print(file_content)
cloud_manager = sicloudman.CloudManager(
'artifacts',
[sicloudman.Bucket(name='release', keywords=['_release'])],
cwd=cwd)
cloud_manager._get_project_bucket_path()
pprint(cloud_manager.credentials.__dict__)
assert set(cloud_manager.credentials.__dict__) == set(
credentials_expected.__dict__)
