def test_put(config_mock): config_mock.get = config_get with patch('cadcetrans.utils.CadcDataClient') as data_mock: # success put_mock = Mock(return_value=None) file = tempfile.NamedTemporaryFile(suffix='.fits') data_mock.return_value.put_file = put_mock put_cadc_file(file.name, None, Subject()) put_mock.assert_called_with('TESTPUT', file.name, archive_stream=None, mime_type=None, mime_encoding=None) # to stream put_mock = Mock(return_value=None) data_mock.return_value.put_file = put_mock put_cadc_file(file.name, 'raw', Subject()) put_mock.assert_called_with('TESTPUT', file.name, archive_stream='raw', mime_type=None, mime_encoding=None) # MIME types and encoding put_mock = Mock(return_value=None) data_mock.return_value.put_file = put_mock put_cadc_file(file.name, 'raw', Subject(), 'application/fits', 'gzip') put_mock.assert_called_with('TESTPUT', file.name, archive_stream='raw', mime_type='application/fits', mime_encoding='gzip') put_mock = Mock(return_value=None) data_mock.return_value.put_file = put_mock put_cadc_file(file.name, 'raw', Subject(), '', None) put_mock.assert_called_with('TESTPUT', file.name, archive_stream='raw', mime_type='', mime_encoding=None) # calling error with pytest.raises(ProcError): data_mock.return_value.put_file.side_effect = RuntimeError() put_cadc_file(file.name, 'raw', Subject())
def test_astropytable(): # example of how to integrate with astropy table. Not sure it belongs here client = cadctap.CadcTapClient(Subject(), resource_id='ivo://cadc.nrc.ca/tap') buffer = BytesIO() client.query('select top 1000 * from caom2.Observation', output_file=buffer) tb = parse_single_table(buffer).to_table() assert len(tb) == 1000
def test_info(config_mock): config_mock.get.return_value = 'TESTINFO' with patch('cadcetrans.utils.CadcDataClient') as data_mock: # file found info_mock = Mock(return_value={'md5sum': '0xbeef'}) data_mock.return_value.get_file_info = info_mock result = fetch_cadc_file_info('foo.fits', Subject()) assert result assert '0xbeef' == result['md5sum'] info_mock.assert_called_with('TESTINFO', 'foo.fits') # file not found data_mock.return_value.get_file_info.side_effect = NotFoundException() assert not fetch_cadc_file_info('foo.fits', Subject()) # calling error with pytest.raises(ProcError): data_mock.return_value.get_file_info.side_effect = RuntimeError() fetch_cadc_file_info('foo.fits', Subject())
def test_transfer_dryrun(get_files_mock, data_client_mock, config_mock): config_mock.get = config_get data_client_mock.return_value.get_file_info.return_value = None # no files to transfer transfer(PROC_DIR, 'new', True, Subject()) # copy all the files including the invalid ones: src_files = os.listdir(TESTDATA_INPUT_DIR) # mock _get_files_to_transfer so all the files are picked up with no # staging time dalay get_files_mock.return_value = [FileInfo(f, 'new') for f in src_files] for file_name in src_files: full_file_name = os.path.join(TESTDATA_INPUT_DIR, file_name) if (os.path.isfile(full_file_name)): shutil.copy(full_file_name, dest) invalid_files = [f for f in os.listdir(dest) if 'invalid' in f] with pytest.raises(CommandError) as e: transfer(PROC_DIR, 'new', True, Subject()) assert 'Errors occurred during transfer ({} error(s))'\ .format(len(invalid_files)) in str(e) # remove the "invalid" files for f in invalid_files: os.unlink(os.path.join(dest, f)) src_files.remove(f) get_files_mock.return_value = [FileInfo(f, 'new') for f in src_files] transfer(PROC_DIR, 'new', True, Subject()) with patch('cadcetrans.etrans_core.put_cadc_file'): transfer(PROC_DIR, 'new', False, Subject()) # no more errors # all files processed assert not os.listdir(dest) # stream required in dryrun mode with pytest.raises(CommandError): transfer(PROC_DIR, None, True, Subject())
def test_main(backup_mock, status_mock, cleanup_mock, transfer_mock, subject_mock): """ Tests the main function etrans_core :return: """ # data transfer subject = Subject() subject_mock.return_value = subject sys.argv = ['cadc-etran', 'data', PROC_DIR] main_app() transfer_mock.assert_called_with(PROC_DIR, stream_name=None, dry_run=False, subject=subject, namecheck_file=None) cleanup_mock.assert_called_with(PROC_DIR, dry_run=False) backup_mock.assert_not_called() status_mock.assert_not_called() # status print transfer_mock.reset_mock() cleanup_mock.reset_mock() sys.argv = ['cadc-etran', 'status', PROC_DIR] main_app() status_mock.assert_called_with(PROC_DIR) cleanup_mock.assert_not_called() transfer_mock.assert_not_called() backup_mock.assert_not_called() # status backup status_mock.reset_mock() sys.argv = ['cadc-etran', 'status', '-b', PROC_DIR] main_app() backup_mock.assert_called_with(subject, PROC_DIR) cleanup_mock.assert_not_called() transfer_mock.assert_not_called() status_mock.assert_not_called()
def test_get_cert_main(): """ Test the getCert function """ value = "CERTVALUE" # get certificate default location m = mock_open() # use a mock to avoid overriding the original file with patch('cadcutils.old_get_cert.netrc.netrc') as \ netrc_mock: with patch('cadcutils.old_get_cert.open', m, create=True): netrc_mock.return_value.authenticators.return_value = \ ('someuser', None, 'somepass') sys.argv = ["getCert"] old_get_cert._main() m.assert_called_once_with('/tmp/.ssl/cadcproxy.pem', 'w') m().write.assert_called_once_with(value) # save certificate in a file m.reset_mock() cert_file = tempfile.NamedTemporaryFile() with patch('cadcutils.old_get_cert.netrc.netrc') as \ netrc_mock: with patch('cadcutils.old_get_cert.open', m, create=True): netrc_mock.return_value.authenticators.return_value = \ ('user', None, 'somepass') sys.argv = ['getCert', '--cert-filename', cert_file.name] old_get_cert._main() m.assert_called_once_with(cert_file.name, 'w') m().write.assert_called_once_with(value) # test when realm not in the .netrc cert_file = tempfile.NamedTemporaryFile() m.reset_mock() with patch('cadcutils.old_get_cert.netrc.netrc') as netrc_mock: with patch('cadcutils.old_get_cert.sys.stdin.readline') as stdin_mock: with patch('cadcutils.old_get_cert.Subject') as subject_mock: authenticators_mock = \ Mock(side_effect=[None, None, None, None, None]) netrc_mock.return_value.authenticators = authenticators_mock stdin_mock.return_value = 'auser' subject_mock.return_value = Subject() sys.argv = ['getCert', '--cert-filename', cert_file.name] old_get_cert._main() calls = [call(), call(username='******')] subject_mock.assert_has_calls(calls) netrc_calls = [call('the.cadc.domain')] for realm in old_get_cert.CADC_REALMS: netrc_calls.append(call(realm)) authenticators_mock.assert_has_calls(netrc_calls) with open(cert_file.name, 'r') as f: assert value == f.read() # test when realm not in the .netrc but other CADC realm (the first one) is cert_file = tempfile.NamedTemporaryFile() m.reset_mock() with patch('cadcutils.old_get_cert.netrc.netrc') as netrc_mock: with patch('cadcutils.old_get_cert.Subject') as subject_mock: authenticators_mock = Mock( side_effect=[None, ('user1', None, 'pwd1')]) netrc_mock.return_value.authenticators = authenticators_mock user_subject = Subject() subject_mock.return_value = user_subject sys.argv = ['getCert', '--cert-filename', cert_file.name] old_get_cert._main() netrc_calls = [call('the.cadc.domain'), call(old_get_cert.CADC_REALMS[0])] authenticators_mock.assert_has_calls(netrc_calls) assert user_subject._hosts_auth['the.cadc.domain'] == ('user1', 'pwd1') with open(cert_file.name, 'r') as f: assert value == f.read()
def test_transfer(get_files_mock, put_mock, data_client_mock, config_mock): config_mock.get = config_get data_client_mock.return_value.get_file_info.return_value = None # cleanup the test directory (dest) for f in os.listdir(dest): ff = os.path.join(dest, f) if os.path.isfile(ff): os.unlink(ff) # copy all the files including the invalid ones: src_files = os.listdir(TESTDATA_INPUT_DIR) # mock _get_files_to_transfer so all the files are picked up with no # staging time dalay get_files_mock.return_value = [FileInfo(f, 'new') for f in src_files] for file_name in src_files: full_file_name = os.path.join(TESTDATA_INPUT_DIR, file_name) if (os.path.isfile(full_file_name)): shutil.copy(full_file_name, dest) invalid_files = [ f for f in os.listdir(dest) if 'invalid' in f or 'bad' in f ] valid_files = [f for f in os.listdir(dest) if 'invalid' not in f] subject = Subject() with pytest.raises(CommandError) as e: transfer(PROC_DIR, 'new', False, subject, namecheck_file=os.path.join(TESTDATA_DIR, 'namecheck.xml')) assert 'Errors occurred during transfer ({} error(s))'\ .format(len(invalid_files)) in str(e) assert put_mock.call_count == len(src_files) - len(invalid_files), 'Calls' calls = [] for f in valid_files: calls.append( call(os.path.join(dest, 'new', f), None, subject, mime_type=None, mime_encoding=None)) put_mock.asses_has_calls(calls, any_order=True) # check that the left files are all invalid for f in os.listdir(dest): assert f.startswith('invalid') # check to see if rejected files have been moved to the right place for f in invalid_files: if f.startswith('bad'): assert os.path.isfile(os.path.join(PROC_DIR, 'reject', 'name', f)) else: _, file_extension = os.path.splitext(f) file_extension = file_extension.strip('.') if file_extension == 'gz': # this corresponds to a tar file with a invalid fits file file_extension = 'fits' if file_extension == 'jpg': file_extension = 'jpeg' assert os.path.isfile( os.path.join(PROC_DIR, 'reject', '{} verify'.format(file_extension), f)) # run again with the invalid files only put_mock.reset_mock() transfer(PROC_DIR, 'new', False, subject) assert 'Errors occurred during transfer ({} error(s))'\ .format(len(invalid_files)) in str(e) assert put_mock.call_count == 0