def test_store(put_mock):
test_config = mc.Config()
test_config.logging_level = 'ERROR'
test_config.working_directory = '/tmp'
test_fqn = '/users/OpenData_DonneesOuvertes/pub/NEOSSAT/ASTRO/2019/' \
'268/NEOS_SCI_2019268004930_clean.fits'
test_storage_name = NEOSSatName(file_name=test_fqn, entry=test_fqn)
transferrer = Mock()
cred_param = Mock()
cadc_data_client = Mock()
caom_repo_client = Mock()
observable = mc.Observable(mc.Rejected('/tmp/rejected.yml'),
mc.Metrics(test_config))
test_subject = ec.Store(test_config, test_storage_name, APPLICATION,
cred_param, cadc_data_client, caom_repo_client,
observable, transferrer)
test_subject.execute(None)
assert put_mock.called, 'expect a call'
args, kwargs = put_mock.call_args
assert args[2] == test_storage_name.file_name, 'wrong file name'
assert transferrer.get.called, 'expect a transfer call'
args, kwargs = transferrer.get.call_args
import logging
logging.error(args)
assert args[0] == test_fqn, 'wrong source parameter'
assert args[1] == f'/tmp/{test_storage_name.obs_id}/' \
f'{test_storage_name.file_name}', \
'wrong destination parameter'
def test_store(put_mock):
test_config = mc.Config()
test_config.logging_level = 'ERROR'
test_config.working_directory = '/tmp'
test_url = 'https://archive-new.nrao.edu/vlass/quicklook/VLASS2.1/' \
'T10t12/VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1/' \
'VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.iter1.image.' \
'pbcor.tt0.rms.subim.fits'
test_storage_name = VlassName(url=test_url, entry=test_url)
transferrer = Mock()
cred_param = Mock()
cadc_data_client = Mock()
caom_repo_client = Mock()
observable = mc.Observable(mc.Rejected('/tmp/rejected.yml'),
mc.Metrics(test_config))
test_subject = ec.Store(test_config, test_storage_name, APPLICATION,
cred_param, cadc_data_client, caom_repo_client,
observable, transferrer)
test_subject.execute(None)
assert put_mock.called, 'expect a call'
args, kwargs = put_mock.call_args
assert args[2] == test_storage_name.file_name, 'wrong file name'
assert transferrer.get.called, 'expect a transfer call'
args, kwargs = transferrer.get.call_args
assert args[0] == test_url, 'wrong source parameter'
assert args[1] == f'/tmp/{test_storage_name.obs_id}/' \
f'{test_storage_name.file_name}',\
'wrong destination parameter'
def test_store(test_config):
test_config.working_directory = tc.TEST_DATA_DIR
test_sn = tc.TestStorageName()
test_sn.source_names = ['vos:goliaths/nonexistent.fits.gz']
test_command = 'collection2caom2'
test_data_client = Mock(autospec=True)
test_observable = Mock(autospec=True)
test_transferrer = Mock(autospec=True)
test_transferrer.get.side_effect = _transfer_get_mock
test_subject = ec.Store(
test_config,
test_sn,
test_command,
test_data_client,
test_observable,
test_transferrer,
)
assert test_subject is not None, 'expect construction'
assert test_subject.working_dir == os.path.join(
tc.TEST_DATA_DIR, 'test_obs_id'), 'wrong working directory'
assert (len(
test_subject._storage_name.destination_uris) == 1), 'wrong file count'
assert (test_subject._storage_name.destination_uris[0] ==
'cadc:TEST/test_file.fits.gz'), 'wrong destination'
test_subject.execute(None)
assert test_data_client.put.called, 'data put not called'
test_data_client.put.assert_called_with(
'/usr/src/app/caom2pipe/caom2pipe/tests/data/test_obs_id',
'cadc:TEST/test_file.fits.gz',
'TEST',
), 'wrong put call args'
def test_data_store(test_config):
test_dir = f'{tc.TEST_DATA_DIR}/test_obs_id'
if os.path.exists(test_dir):
os.rmdir(test_dir)
data_client_mock = Mock()
test_observer = Mock()
# stat mock is for CadcDataClient
stat_orig = os.stat
os.stat = Mock()
os.stat.st_size.return_value = 1243
# path.exists mock is because the stat mock causes os.path.exists to
# return True
path_orig = os.path.exists
os.path.exists = Mock(return_value=False)
test_config.features.supports_multiple_files = False
try:
test_config.working_directory = tc.TEST_DATA_DIR
test_executor = ec.Store(
test_config,
tc.TestStorageName(entry=f'{tc.TEST_DATA_DIR}/test_file.fits.gz'),
'command_name',
data_client_mock,
observable=test_observer,
transferrer=transfer_composable.Transfer(),
)
test_executor.execute(None)
# check that things worked as expected - no cleanup
assert data_client_mock.put.called, 'put_file call missed'
finally:
os.stat = stat_orig
os.path.exists = path_orig
def test_store(test_config):
test_config.working_directory = '/test_files/caom2pipe'
test_sn = tc.TestStorageName()
test_command = 'collection2caom2'
test_cred_param = ''
test_data_client = Mock(autospec=True)
test_repo_client = Mock(autospec=True)
test_observable = Mock(autospec=True)
test_transferrer = Mock(autospec=True)
test_transferrer.get.side_effect = _transfer_get_mock
test_subject = ec.Store(
test_config,
test_sn,
test_command,
test_cred_param,
test_data_client,
test_repo_client,
test_observable,
test_transferrer,
)
assert test_subject is not None, 'expect construction'
assert (
test_subject.working_dir == '/test_files/caom2pipe/test_obs_id'
), 'wrong working directory'
assert len(test_subject.multiple_files) == 1, 'wrong file count'
assert (
len(test_subject._destination_f_names) == 1
), 'wrong destination file count'
assert (
test_subject._destination_f_names[0] == 'test_obs_id.fits'
), 'wrong destination'
test_subject.execute(None)
assert test_data_client.put_file.called, 'data put not called'
assert (
test_data_client.put_file.call_args.args[0] is None
), 'archive not set for test_config'
assert (
test_data_client.put_file.call_args.args[1] == 'test_obs_id.fits'
), 'expect a file name'
assert (
test_data_client.put_file.call_args.kwargs['archive_stream'] == 'TEST'
), 'wrong archive stream'
assert (
test_data_client.put_file.call_args.kwargs['mime_type'] ==
'application/fits'
), 'wrong archive'
assert (
test_data_client.put_file.call_args.kwargs['mime_encoding'] is None
), 'wrong archive'
assert (
test_data_client.put_file.call_args.kwargs['md5_check'] is True
), 'wrong archive'
def test_store():
test_config = mc.Config()
test_config.logging_level = 'ERROR'
test_config.working_directory = '/tmp'
test_url = ('https://archive-new.nrao.edu/vlass/quicklook/VLASS2.1/'
'T10t12/VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1/'
'VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.iter1.image.'
'pbcor.tt0.rms.subim.fits')
test_storage_name = VlassName(test_url)
transferrer = Mock()
cadc_data_client = Mock()
observable = mc.Observable(mc.Rejected('/tmp/rejected.yml'),
mc.Metrics(test_config))
test_subject = ec.Store(
test_config,
test_storage_name,
cadc_data_client,
observable,
transferrer,
)
test_subject.execute(None)
assert cadc_data_client.put.called, 'expect a call'
cadc_data_client.put.assert_called_with(
'/tmp/VLASS2.1.T10t12.J073401-033000',
f'{SCHEME}:VLASS/VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.'
f'iter1.image.pbcor.tt0.rms.subim.fits',
None,
), 'wrong put args'
assert transferrer.get.called, 'expect a transfer call'
test_f_name = ('VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.iter1.'
'image.pbcor.tt0.rms.subim.fits')
transferrer.get.assert_called_with(
f'https://archive-new.nrao.edu/vlass/quicklook/VLASS2.1/T10t12/'
f'VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1/{test_f_name}',
f'/tmp/VLASS2.1.T10t12.J073401-033000/{test_f_name}',
), 'wrong transferrer args'