def test_put(config_mock):
config_mock.get = config_get
with patch('cadcetrans.utils.CadcDataClient') as data_mock:
# success
put_mock = Mock(return_value=None)
file = tempfile.NamedTemporaryFile(suffix='.fits')
data_mock.return_value.put_file = put_mock
put_cadc_file(file.name, None, Subject())
put_mock.assert_called_with('TESTPUT',
file.name,
archive_stream=None,
mime_type=None,
mime_encoding=None)
# to stream
put_mock = Mock(return_value=None)
data_mock.return_value.put_file = put_mock
put_cadc_file(file.name, 'raw', Subject())
put_mock.assert_called_with('TESTPUT',
file.name,
archive_stream='raw',
mime_type=None,
mime_encoding=None)
# MIME types and encoding
put_mock = Mock(return_value=None)
data_mock.return_value.put_file = put_mock
put_cadc_file(file.name, 'raw', Subject(), 'application/fits', 'gzip')
put_mock.assert_called_with('TESTPUT',
file.name,
archive_stream='raw',
mime_type='application/fits',
mime_encoding='gzip')
put_mock = Mock(return_value=None)
data_mock.return_value.put_file = put_mock
put_cadc_file(file.name, 'raw', Subject(), '', None)
put_mock.assert_called_with('TESTPUT',
file.name,
archive_stream='raw',
mime_type='',
mime_encoding=None)
# calling error
with pytest.raises(ProcError):
data_mock.return_value.put_file.side_effect = RuntimeError()
put_cadc_file(file.name, 'raw', Subject())
def test_astropytable():
# example of how to integrate with astropy table. Not sure it belongs here
client = cadctap.CadcTapClient(Subject(),
resource_id='ivo://cadc.nrc.ca/tap')
buffer = BytesIO()
client.query('select top 1000 * from caom2.Observation',
output_file=buffer)
tb = parse_single_table(buffer).to_table()
assert len(tb) == 1000
def test_info(config_mock):
config_mock.get.return_value = 'TESTINFO'
with patch('cadcetrans.utils.CadcDataClient') as data_mock:
# file found
info_mock = Mock(return_value={'md5sum': '0xbeef'})
data_mock.return_value.get_file_info = info_mock
result = fetch_cadc_file_info('foo.fits', Subject())
assert result
assert '0xbeef' == result['md5sum']
info_mock.assert_called_with('TESTINFO', 'foo.fits')
# file not found
data_mock.return_value.get_file_info.side_effect = NotFoundException()
assert not fetch_cadc_file_info('foo.fits', Subject())
# calling error
with pytest.raises(ProcError):
data_mock.return_value.get_file_info.side_effect = RuntimeError()
fetch_cadc_file_info('foo.fits', Subject())
def test_transfer_dryrun(get_files_mock, data_client_mock, config_mock):
config_mock.get = config_get
data_client_mock.return_value.get_file_info.return_value = None
# no files to transfer
transfer(PROC_DIR, 'new', True, Subject())
# copy all the files including the invalid ones:
src_files = os.listdir(TESTDATA_INPUT_DIR)
# mock _get_files_to_transfer so all the files are picked up with no
# staging time dalay
get_files_mock.return_value = [FileInfo(f, 'new') for f in src_files]
for file_name in src_files:
full_file_name = os.path.join(TESTDATA_INPUT_DIR, file_name)
if (os.path.isfile(full_file_name)):
shutil.copy(full_file_name, dest)
invalid_files = [f for f in os.listdir(dest) if 'invalid' in f]
with pytest.raises(CommandError) as e:
transfer(PROC_DIR, 'new', True, Subject())
assert 'Errors occurred during transfer ({} error(s))'\
.format(len(invalid_files)) in str(e)
# remove the "invalid" files
for f in invalid_files:
os.unlink(os.path.join(dest, f))
src_files.remove(f)
get_files_mock.return_value = [FileInfo(f, 'new') for f in src_files]
transfer(PROC_DIR, 'new', True, Subject())
with patch('cadcetrans.etrans_core.put_cadc_file'):
transfer(PROC_DIR, 'new', False, Subject()) # no more errors
# all files processed
assert not os.listdir(dest)
# stream required in dryrun mode
with pytest.raises(CommandError):
transfer(PROC_DIR, None, True, Subject())
def test_main(backup_mock, status_mock, cleanup_mock, transfer_mock,
subject_mock):
"""
Tests the main function etrans_core
:return:
"""
# data transfer
subject = Subject()
subject_mock.return_value = subject
sys.argv = ['cadc-etran', 'data', PROC_DIR]
main_app()
transfer_mock.assert_called_with(PROC_DIR,
stream_name=None,
dry_run=False,
subject=subject,
namecheck_file=None)
cleanup_mock.assert_called_with(PROC_DIR, dry_run=False)
backup_mock.assert_not_called()
status_mock.assert_not_called()
# status print
transfer_mock.reset_mock()
cleanup_mock.reset_mock()
sys.argv = ['cadc-etran', 'status', PROC_DIR]
main_app()
status_mock.assert_called_with(PROC_DIR)
cleanup_mock.assert_not_called()
transfer_mock.assert_not_called()
backup_mock.assert_not_called()
# status backup
status_mock.reset_mock()
sys.argv = ['cadc-etran', 'status', '-b', PROC_DIR]
main_app()
backup_mock.assert_called_with(subject, PROC_DIR)
cleanup_mock.assert_not_called()
transfer_mock.assert_not_called()
status_mock.assert_not_called()
def test_get_cert_main():
""" Test the getCert function """
value = "CERTVALUE"
# get certificate default location
m = mock_open() # use a mock to avoid overriding the original file
with patch('cadcutils.old_get_cert.netrc.netrc') as \
netrc_mock:
with patch('cadcutils.old_get_cert.open', m, create=True):
netrc_mock.return_value.authenticators.return_value = \
('someuser', None, 'somepass')
sys.argv = ["getCert"]
old_get_cert._main()
m.assert_called_once_with('/tmp/.ssl/cadcproxy.pem', 'w')
m().write.assert_called_once_with(value)
# save certificate in a file
m.reset_mock()
cert_file = tempfile.NamedTemporaryFile()
with patch('cadcutils.old_get_cert.netrc.netrc') as \
netrc_mock:
with patch('cadcutils.old_get_cert.open', m, create=True):
netrc_mock.return_value.authenticators.return_value = \
('user', None, 'somepass')
sys.argv = ['getCert', '--cert-filename', cert_file.name]
old_get_cert._main()
m.assert_called_once_with(cert_file.name, 'w')
m().write.assert_called_once_with(value)
# test when realm not in the .netrc
cert_file = tempfile.NamedTemporaryFile()
m.reset_mock()
with patch('cadcutils.old_get_cert.netrc.netrc') as netrc_mock:
with patch('cadcutils.old_get_cert.sys.stdin.readline') as stdin_mock:
with patch('cadcutils.old_get_cert.Subject') as subject_mock:
authenticators_mock = \
Mock(side_effect=[None, None, None, None, None])
netrc_mock.return_value.authenticators = authenticators_mock
stdin_mock.return_value = 'auser'
subject_mock.return_value = Subject()
sys.argv = ['getCert', '--cert-filename', cert_file.name]
old_get_cert._main()
calls = [call(), call(username='******')]
subject_mock.assert_has_calls(calls)
netrc_calls = [call('the.cadc.domain')]
for realm in old_get_cert.CADC_REALMS:
netrc_calls.append(call(realm))
authenticators_mock.assert_has_calls(netrc_calls)
with open(cert_file.name, 'r') as f:
assert value == f.read()
# test when realm not in the .netrc but other CADC realm (the first one) is
cert_file = tempfile.NamedTemporaryFile()
m.reset_mock()
with patch('cadcutils.old_get_cert.netrc.netrc') as netrc_mock:
with patch('cadcutils.old_get_cert.Subject') as subject_mock:
authenticators_mock = Mock(
side_effect=[None, ('user1', None, 'pwd1')])
netrc_mock.return_value.authenticators = authenticators_mock
user_subject = Subject()
subject_mock.return_value = user_subject
sys.argv = ['getCert', '--cert-filename', cert_file.name]
old_get_cert._main()
netrc_calls = [call('the.cadc.domain'), call(old_get_cert.CADC_REALMS[0])]
authenticators_mock.assert_has_calls(netrc_calls)
assert user_subject._hosts_auth['the.cadc.domain'] == ('user1', 'pwd1')
with open(cert_file.name, 'r') as f:
assert value == f.read()
def test_transfer(get_files_mock, put_mock, data_client_mock, config_mock):
config_mock.get = config_get
data_client_mock.return_value.get_file_info.return_value = None
# cleanup the test directory (dest)
for f in os.listdir(dest):
ff = os.path.join(dest, f)
if os.path.isfile(ff):
os.unlink(ff)
# copy all the files including the invalid ones:
src_files = os.listdir(TESTDATA_INPUT_DIR)
# mock _get_files_to_transfer so all the files are picked up with no
# staging time dalay
get_files_mock.return_value = [FileInfo(f, 'new') for f in src_files]
for file_name in src_files:
full_file_name = os.path.join(TESTDATA_INPUT_DIR, file_name)
if (os.path.isfile(full_file_name)):
shutil.copy(full_file_name, dest)
invalid_files = [
f for f in os.listdir(dest) if 'invalid' in f or 'bad' in f
]
valid_files = [f for f in os.listdir(dest) if 'invalid' not in f]
subject = Subject()
with pytest.raises(CommandError) as e:
transfer(PROC_DIR,
'new',
False,
subject,
namecheck_file=os.path.join(TESTDATA_DIR, 'namecheck.xml'))
assert 'Errors occurred during transfer ({} error(s))'\
.format(len(invalid_files)) in str(e)
assert put_mock.call_count == len(src_files) - len(invalid_files), 'Calls'
calls = []
for f in valid_files:
calls.append(
call(os.path.join(dest, 'new', f),
None,
subject,
mime_type=None,
mime_encoding=None))
put_mock.asses_has_calls(calls, any_order=True)
# check that the left files are all invalid
for f in os.listdir(dest):
assert f.startswith('invalid')
# check to see if rejected files have been moved to the right place
for f in invalid_files:
if f.startswith('bad'):
assert os.path.isfile(os.path.join(PROC_DIR, 'reject', 'name', f))
else:
_, file_extension = os.path.splitext(f)
file_extension = file_extension.strip('.')
if file_extension == 'gz':
# this corresponds to a tar file with a invalid fits file
file_extension = 'fits'
if file_extension == 'jpg':
file_extension = 'jpeg'
assert os.path.isfile(
os.path.join(PROC_DIR, 'reject',
'{} verify'.format(file_extension), f))
# run again with the invalid files only
put_mock.reset_mock()
transfer(PROC_DIR, 'new', False, subject)
assert 'Errors occurred during transfer ({} error(s))'\
.format(len(invalid_files)) in str(e)
assert put_mock.call_count == 0