def test_upload_files_multiple_valid_files(self): mockftp = MockFtp() mockftp.put = Mock(return_value=3) mockftp.close = Mock(return_value=None) foo = FtpFileTransfer(None) foo.set_connection(mockftp) foo.set_remote_dir('/remote') foo.set_host('hosty') temp_dir = tempfile.mkdtemp() try: foo.connect() valid_file = os.path.join(temp_dir, 'valid') f = open(valid_file, 'a') f.write('hi') f.flush() f.close() afile = os.path.join(temp_dir, 'hi') open(afile, 'a').close() self.assertEqual(foo.upload_files([valid_file, afile]), True) self.assertEqual(foo.get_error_msg(), None) self.assertTrue('2 (6 bytes) files uploaded in ' in foo.get_upload_summary()) self.assertTrue(' seconds to host hosty:/remote' in foo.get_upload_summary()) foo.disconnect() self.assertEqual(mockftp.put.call_count, 2) mockftp.close.assert_not_called() finally: shutil.rmtree(temp_dir)
def test_upload_challenge_file_uploader_upload_fails(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() yeardir = os.path.join(temp_dir, '2016') os.mkdir(yeardir) weekdir = os.path.join(yeardir, 'dataset.week.50') os.mkdir(weekdir) chall = ChallengeDataTask(weekdir, params) chall.create_dir() mockftp = D3RParameters() mockftp.put = Mock(side_effect=IOError('hi')) ftp = FtpFileTransfer(None) ftp.set_remote_challenge_dir('/challenge') ftp.set_connection(mockftp) ftp.connect() chall.set_file_transfer(ftp) tarball = os.path.join(chall.get_dir(), 'celppweek50_2016.tar.gz') f = open(tarball, 'w') f.write('hi') f.flush() f.close() try: chall._upload_challenge_file(tarball) self.fail('Expected exception') except Exception as e: self.assertEqual(str(e), 'Unable to upload ' + tarball + ' to /challenge/celppweek50_2016.tar.gz : ' + 'hi') ftp.disconnect() finally: shutil.rmtree(temp_dir)
def test_upload_file_on_valid_file(self): temp_dir = tempfile.mkdtemp() try: mockftp = MockFtp() mockftp.put = Mock(return_value=3) foo = FtpFileTransfer(None) foo.set_connection(mockftp) foo.set_remote_dir('/remote') foo.connect() valid_file = os.path.join(temp_dir, 'file') f = open(valid_file, 'a') f.write('12') f.flush() f.close() foo._upload_file(valid_file) mockftp.put.assert_called_with(valid_file, os.path.normpath('/remote' + valid_file)) self.assertEqual(foo._bytes_transferred, 3) self.assertEqual(foo._files_transferred, 1) foo._upload_file(valid_file) self.assertEqual(foo._bytes_transferred, 6) self.assertEqual(foo._files_transferred, 2) finally: shutil.rmtree(temp_dir)
def test_upload_challenge_file_uploader_successful(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() yeardir = os.path.join(temp_dir, '2017') os.mkdir(yeardir) weekdir = os.path.join(yeardir, 'dataset.week.1') os.mkdir(weekdir) chall = ChallengeDataTask(weekdir, params) chall.create_dir() mockftp = D3RParameters() mockftp.put = Mock(side_effect=[3, 5]) ftp = FtpFileTransfer(None) ftp.set_remote_challenge_dir('/challenge') ftp.set_connection(mockftp) ftp.connect() chall.set_file_transfer(ftp) tarball = os.path.join(chall.get_dir(), 'celppweek1_2017.tar.gz') f = open(tarball, 'w') f.write('hi') f.flush() f.close() latest_file = os.path.join(chall.get_dir(), ChallengeDataTask.LATEST_TXT) chall._upload_challenge_file(tarball) f = open(latest_file, 'r') line = f.readline() self.assertEqual(line, 'celppweek1_2017.tar.gz') f.close() ftp.disconnect() finally: shutil.rmtree(temp_dir)
def test_upload_files_where_file_upload_raises_exception(self): mockftp = MockFtp() mockftp.put = Mock(side_effect=IOError('hi')) mockftp.close = Mock(return_value=None) foo = FtpFileTransfer(None) foo.set_connection(mockftp) foo.set_remote_dir('/remote') foo.set_host('hosty') temp_dir = tempfile.mkdtemp() try: foo.connect() valid_file = os.path.join(temp_dir, 'valid') f = open(valid_file, 'a') f.write('hi') f.flush() f.close() self.assertEqual(foo.upload_files([valid_file]), False) self.assertEqual(foo.get_error_msg(), 'Error during upload') self.assertEqual(foo.get_upload_summary(), 'Error during upload\n' '0 (0 bytes) files uploaded in 0 ' 'seconds to host hosty:/remote') foo.disconnect() mockftp.put.assert_called_with(valid_file, os.path.normpath('/remote' + valid_file)) finally: shutil.rmtree(temp_dir)
def test_upload_file_where_file_is_none(self): # test where file is None ftpspec = ["put"] mockftp = Mock(spec=ftpspec) foo = FtpFileTransfer(None) foo.set_connection(mockftp) foo.connect() foo._upload_file(None)
def test_download_file_success(self): mockftp = MockFtp() mockftp.get = Mock() mockftp.close = Mock(return_value=None) foo = FtpFileTransfer(None) foo.set_connection(mockftp) foo.connect() self.assertTrue(foo.download_file('/a/b', '/c/d')) self.assertEqual(foo.get_error_msg(), None) foo.disconnect() mockftp.get.assert_called_with('/a/b', local='/c/d')
def test_delete_file_fail(self): mockftp = MockFtp() mockftp.delete = Mock(side_effect=IOError('error')) mockftp.close = Mock(return_value=None) foo = FtpFileTransfer(None) foo.set_connection(mockftp) foo.connect() self.assertFalse(foo.delete_file('/a/b')) self.assertEqual(foo.get_error_msg(), 'Unable to delete /a/b : error') foo.disconnect() mockftp.delete.assert_called_with('/a/b')
def test_delete_file_success(self): mockftp = MockFtp() mockftp.delete = Mock(return_value='hello') mockftp.close = Mock(return_value=None) foo = FtpFileTransfer(None) foo.set_connection(mockftp) foo.connect() self.assertTrue(foo.delete_file('/a/b')) self.assertEqual(foo.get_error_msg(), None) foo.disconnect() mockftp.delete.assert_called_with('/a/b')
def test_download_file_fail(self): mockftp = MockFtp() mockftp.get = Mock(side_effect=IOError('error')) mockftp.close = Mock(return_value=None) foo = FtpFileTransfer(None) foo.set_connection(mockftp) foo.connect() self.assertFalse(foo.download_file('/a/b', '/c/d')) self.assertEqual(foo.get_error_msg(), 'Unable to download /a/b to ' '/c/d : error') foo.disconnect() mockftp.get.assert_called_with('/a/b', local='/c/d')
def test_list_files_fail(self): mockftp = MockFtp() mockftp.list = Mock(side_effect=IOError('error')) mockftp.close = Mock(return_value=None) foo = FtpFileTransfer(None) foo.set_connection(mockftp) foo.connect() filelist = foo.list_files('/foo2') self.assertTrue(filelist is None) foo.disconnect() self.assertEqual(foo.get_error_msg(), 'Unable to get file list ' 'for /foo2 : error') mockftp.list.assert_called_with('/foo2', extra=True)
def test_upload_file_where_file_does_not_exist(self): temp_dir = tempfile.mkdtemp() try: ftpspec = ["put"] mockftp = Mock(spec=ftpspec) foo = FtpFileTransfer(None) foo.set_connection(mockftp) foo.connect() foo._upload_file(os.path.join(temp_dir, 'nonexist')) self.assertEqual(mockftp.call_count, 0) finally: shutil.rmtree(temp_dir)
def test_list_files_success(self): mockftp = MockFtp() mockftp.list = Mock(return_value=[{'directory': 'd', 'name': '.'}, {'directory': 'd', 'name': '..'}, {'directory': 'd', 'name': 'foo'}, {'directory': '-', 'name': 'file'}]) mockftp.close = Mock(return_value=None) foo = FtpFileTransfer(None) foo.set_connection(mockftp) foo.connect() filelist = foo.list_files('/foo') self.assertTrue(len(filelist) == 1) self.assertEqual(filelist[0], 'file') foo.disconnect() mockftp.list.assert_called_with('/foo', extra=True)
def test_connect(self): # test where alt_ftp_con is None and we use ftpretty foo = FtpFileTransfer(None) foo.set_host('doesnotexist') foo.set_user('user') foo.set_password('') foo.set_remote_dir('/remote') foo.set_connect_timeout(0) self.assertFalse(foo.connect()) # test where alt_ftp_con is set foo = FtpFileTransfer(None) foo.set_connection('hi') foo.connect() self.assertEqual(foo._ftp, 'hi')
def test_upload_file_direct_put_throws_exception(self): temp_dir = tempfile.mkdtemp() try: mockftp = MockFtp() mockftp.put = Mock(side_effect=IOError('hi')) foo = FtpFileTransfer(None) foo.set_connection(mockftp) foo.connect() afile = os.path.join(temp_dir, 'afile') open(afile, 'a').close() self.assertEqual(foo.upload_file_direct(afile, '/foo', 'name'), False) self.assertEqual(foo.get_error_msg(), 'Unable to upload ' + afile + ' to /foo/name : hi') foo.disconnect() finally: shutil.rmtree(temp_dir)
def test_upload_file_direct_success(self): temp_dir = tempfile.mkdtemp() try: mockftp = MockFtp() mockftp.put = Mock(return_value=3) foo = FtpFileTransfer(None) foo.set_connection(mockftp) foo.connect() afile = os.path.join(temp_dir, 'afile') open(afile, 'a').close() self.assertEqual(foo.upload_file_direct(afile, '/foo', 'name'), True) self.assertEqual(foo.get_error_msg(), None) self.assertEqual(foo.get_upload_summary(), '1 (3 bytes) files ' 'uploaded in 0 seconds' ' to host Unset:') foo.disconnect() finally: shutil.rmtree(temp_dir)
def test_upload_file_that_raises_exception(self): temp_dir = tempfile.mkdtemp() try: mockftp = MockFtp() mockftp.put = Mock(side_effect=IOError('hi')) foo = FtpFileTransfer(None) foo.set_connection(mockftp) foo.connect() valid_file = os.path.join(temp_dir, 'file') f = open(valid_file, 'a') f.write('12') f.flush() f.close() try: foo._upload_file(valid_file) self.fail('Expected IOError') except IOError: pass finally: shutil.rmtree(temp_dir)
def test_disconnect(self): # test disconnect where _ftp is None foo = FtpFileTransfer(None) foo.disconnect() # test disconnect where _ftp is set and # so is _alt_ftp_con foo = FtpFileTransfer(None) foo.set_connection('hi') foo.connect() foo.disconnect() # test disconnect where _ftp is set and # _alt_ftp_con is None foo = FtpFileTransfer(None) mockftp = MockFtp() mockftp.close = Mock(side_effect=Exception()) foo.set_connection(mockftp) foo.connect() foo._alt_ftp_con = None foo.disconnect() mockftp.close.assert_any_call()
class ExternalDataSubmissionFactory(object): """Factory to create ExternalDataSubmissionObjects """ DOCKEDRESULTS = '_dockedresults_' def __init__(self, path, args): """Constructor """ try: logger.debug('ftpconfig set to ' + args.ftpconfig) self._file_transfer = FtpFileTransfer(args.ftpconfig) except Exception: logger.exception('Caught exception') self._file_transfer = None self._path = path self._args = args ctask = ChallengeDataTask(path, args) self._chall_dir_name = ctask.get_celpp_challenge_data_dir_name() def get_args(self): return self._args def get_path(self): return self._path def _get_challenge_package_results_file_name(self, dir_name): return (self._chall_dir_name + ExternalDataSubmissionFactory.DOCKEDRESULTS + dir_name + ChallengeDataTask.TAR_GZ_SUFFIX) def set_file_transfer(self, filetransfer): """Sets file transfer """ self._file_transfer = filetransfer def get_file_transfer(self): """Gets file transfer """ return self._file_transfer def _get_submission_dirs(self, remote_dir): """Gets list of directories under `remote_dir` :param remote_dir: path on remote server to examine :returns: list of directory names without path prefix :raises AttributeError: if `get_file_transfer()` is None """ dlist = self._file_transfer.list_dirs(remote_dir) if dlist is None: logger.debug('No directories returned') return [] logger.debug('Found ' + str(len(dlist)) + ' directories on ' + remote_dir) return dlist def _get_challenge_data_package_file(self, remote_dir, dir_name): """Gets challenge data package file under `remote_dir` / `dir_name` if it exists :returns: Path to remote challenge file upon success or None if not found :raises AttributeError if `get_file_transfer()` is None """ ft = self.get_file_transfer() flist = ft.list_files(os.path.normpath(os.path.join(remote_dir, dir_name))) if flist is None: logger.info('No files found in ' + dir_name) return None logger.debug('Found ' + str(len(flist)) + ' files in ' + dir_name) chall_fname = self._get_challenge_package_results_file_name(dir_name) logger.debug('Looking for ' + chall_fname + ' file in directory') for entry in flist: if entry == chall_fname: logger.info('Found matching entry ' + entry) return os.path.normpath(os.path.join(remote_dir, dir_name, chall_fname)) else: logger.debug('Encountered non challenge file: ' + entry) return None def _remove_latest_txt(self): """Removes the latest.txt file from ftp server if found """ try: ft = self.get_file_transfer() latest_txt = os.path.join(ft.get_remote_challenge_dir(), ChallengeDataTask.LATEST_TXT) logger.info('Attempting to remove ' + latest_txt) val = ft.delete_file(latest_txt) logger.info('Return value from delete call ' + str(val)) except Exception: logger.exception('Caught exception trying to remove latest.txt') def get_external_data_submissions(self): """Generate ExternalDataSubmission objects Method should examine the submission directory on the remote server via FtpFileTransfer(args.ftpconfig) for each directory under submission directory look for a celpp_weekXX_YYYY_dockedresults_ZZZZ.tar.gz file where ZZZZ matches name of directory under submission directory Regardless if its found or not create new ExternalDataSubmission object and pass ZZZZ for name and path to .tar.gz file as remotefile if the tar.gz file was found otherwise pass None along with args. Append this object to list and return it """ task_list = [] try: self._file_transfer.connect() self._remove_latest_txt() subdir = self._file_transfer.get_remote_submission_dir() dlist = self._get_submission_dirs(subdir) for d in dlist: chall_file = self._get_challenge_data_package_file(subdir, d) if chall_file is not None: et = ExternalDataSubmissionTask(self.get_path(), d, chall_file, self.get_args()) logger.info('Added ExternalData Submission Task: ' + et.get_dir_name()) task_list.append(et) except Exception: logger.exception('Caught exception') finally: try: self._file_transfer.disconnect() except Exception: logger.exception('Caught exception disconnecting') return task_list
def test_run_fails_cause_ftp_upload_fails(self): temp_dir = tempfile.mkdtemp() try: script = self.create_gen_challenge_script(temp_dir) params = D3RParameters() params.genchallenge = script params.pdbdb = '/foo' params.version = '1' blastnfilter = BlastNFilterTask(temp_dir, params) blastnfilter.create_dir() open(os.path.join(blastnfilter.get_dir(), D3RTask.COMPLETE_FILE), 'a').close() chall = ChallengeDataTask(temp_dir, params) mockftp = D3RParameters() mockftp.put = Mock(side_effect=[3, IOError('hi')]) ftp = FtpFileTransfer(None) ftp.set_remote_challenge_dir('/challenge') ftp.set_connection(mockftp) ftp.connect() chall.set_file_transfer(ftp) dimport = DataImportTask(temp_dir, params) dimport.create_dir() ctsv = dimport.get_crystalph_tsv() f = open(ctsv, 'w') f.write('crystal') f.flush() f.close() nonpoly = dimport.get_nonpolymer_tsv() f = open(nonpoly, 'w') f.write('nonpoly') f.flush() f.close() seq = dimport.get_sequence_tsv() f = open(seq, 'w') f.write('seq') f.flush() f.close() chall.run() latest_file = os.path.join(chall.get_dir(), ChallengeDataTask.LATEST_TXT) self.assertEqual(chall.get_error(), 'Caught exception ' + 'Unable to upload ' + latest_file + ' to ' + '/challenge/' + ChallengeDataTask.LATEST_TXT + ' : hi') # verify test files get created errfile = os.path.join(chall.get_dir(), D3RTask.ERROR_FILE) self.assertEqual(os.path.isfile(errfile), True) compfile = os.path.join(chall.get_dir(), D3RTask.COMPLETE_FILE) self.assertEqual(os.path.isfile(compfile), False) ftp.disconnect() finally: shutil.rmtree(temp_dir)