def test_upload_file_direct_none_passed_as_file(self):
foo = FtpFileTransfer(None)
self.assertEqual(foo.upload_file_direct(None, '/remote_dir',
'remote_filename'), False)
self.assertEqual(foo.get_error_msg(), 'File passed in is None')
def test_upload_challenge_file_uploader_successful(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
yeardir = os.path.join(temp_dir, '2017')
os.mkdir(yeardir)
weekdir = os.path.join(yeardir, 'dataset.week.1')
os.mkdir(weekdir)
chall = ChallengeDataTask(weekdir, params)
chall.create_dir()
mockftp = D3RParameters()
mockftp.put = Mock(side_effect=[3, 5])
ftp = FtpFileTransfer(None)
ftp.set_remote_challenge_dir('/challenge')
ftp.set_connection(mockftp)
ftp.connect()
chall.set_file_transfer(ftp)
tarball = os.path.join(chall.get_dir(), 'celppweek1_2017.tar.gz')
f = open(tarball, 'w')
f.write('hi')
f.flush()
f.close()
latest_file = os.path.join(chall.get_dir(),
ChallengeDataTask.LATEST_TXT)
chall._upload_challenge_file(tarball)
f = open(latest_file, 'r')
line = f.readline()
self.assertEqual(line, 'celppweek1_2017.tar.gz')
f.close()
ftp.disconnect()
finally:
shutil.rmtree(temp_dir)
def test_list_files_remote_dir_not_connected(self):
foo = FtpFileTransfer(None)
self.assertFalse(foo.list_files('/foo'))
self.assertEqual(
foo.get_error_msg(), "Unable to get file list "
"for /foo : 'NoneType' object "
"has no attribute 'list'")
def test_upload_challenge_file_uploader_upload_fails(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
yeardir = os.path.join(temp_dir, '2016')
os.mkdir(yeardir)
weekdir = os.path.join(yeardir, 'dataset.week.50')
os.mkdir(weekdir)
chall = ChallengeDataTask(weekdir, params)
chall.create_dir()
mockftp = D3RParameters()
mockftp.put = Mock(side_effect=IOError('hi'))
ftp = FtpFileTransfer(None)
ftp.set_remote_challenge_dir('/challenge')
ftp.set_connection(mockftp)
ftp.connect()
chall.set_file_transfer(ftp)
tarball = os.path.join(chall.get_dir(), 'celppweek50_2016.tar.gz')
f = open(tarball, 'w')
f.write('hi')
f.flush()
f.close()
try:
chall._upload_challenge_file(tarball)
self.fail('Expected exception')
except Exception as e:
self.assertEqual(str(e),
'Unable to upload ' + tarball +
' to /challenge/celppweek50_2016.tar.gz : ' +
'hi')
ftp.disconnect()
finally:
shutil.rmtree(temp_dir)
def test_download_file_not_connected(self):
foo = FtpFileTransfer(None)
self.assertFalse(foo.download_file('/remote/bye', '/local/hi'))
self.assertEqual(
foo.get_error_msg(), "Unable to download /remote/bye "
"to /local/hi : 'NoneType' "
"object has no attribute 'get'")
def test_delete_file_not_connected(self):
foo = FtpFileTransfer(None)
self.assertFalse(foo.delete_file('/./somefile'))
self.assertEqual(
foo.get_error_msg(), "Unable to delete /somefile : "
"'NoneType' object has no "
"attribute 'delete'")
def test_upload_files_file_list_is_empty(self):
foo = FtpFileTransfer(None)
self.assertEqual(foo.upload_files([]), True)
self.assertEqual(foo.get_error_msg(), 'No files to upload')
self.assertEqual(
foo.get_upload_summary(), 'No files to upload\n'
'0 (0 bytes) files uploaded in 0 '
'seconds to host Unset:')
def test_upload_files_file_list_is_none(self):
foo = FtpFileTransfer(None)
self.assertEqual(foo.upload_files(None), True)
self.assertEqual(foo.get_error_msg(),
'List of files passed in was None')
self.assertEqual(foo.get_upload_summary(),
'List of files passed in was None\n'
'0 (0 bytes) files uploaded in 0 '
'seconds to host Unset:')
def test_delete_file_success(self):
mockftp = MockFtp()
mockftp.delete = Mock(return_value='hello')
mockftp.close = Mock(return_value=None)
foo = FtpFileTransfer(None)
foo.set_connection(mockftp)
foo.connect()
self.assertTrue(foo.delete_file('/a/b'))
self.assertEqual(foo.get_error_msg(), None)
foo.disconnect()
mockftp.delete.assert_called_with('/a/b')
def test_upload_file_direct_remote_file_name_is_none(self):
temp_dir = tempfile.mkdtemp()
try:
foo = FtpFileTransfer(None)
afile = os.path.join(temp_dir, 'afile')
open(afile, 'a').close()
self.assertEqual(foo.upload_file_direct(afile, '/foo', None),
False)
self.assertEqual(foo.get_error_msg(), 'remote_file_name is None')
finally:
shutil.rmtree(temp_dir)
def test_delete_file_fail(self):
mockftp = MockFtp()
mockftp.delete = Mock(side_effect=IOError('error'))
mockftp.close = Mock(return_value=None)
foo = FtpFileTransfer(None)
foo.set_connection(mockftp)
foo.connect()
self.assertFalse(foo.delete_file('/a/b'))
self.assertEqual(foo.get_error_msg(), 'Unable to delete /a/b : error')
foo.disconnect()
mockftp.delete.assert_called_with('/a/b')
def test_download_file_success(self):
mockftp = MockFtp()
mockftp.get = Mock()
mockftp.close = Mock(return_value=None)
foo = FtpFileTransfer(None)
foo.set_connection(mockftp)
foo.connect()
self.assertTrue(foo.download_file('/a/b', '/c/d'))
self.assertEqual(foo.get_error_msg(), None)
foo.disconnect()
mockftp.get.assert_called_with('/a/b', local='/c/d')
def test_upload_file_direct_file_is_not_a_file(self):
temp_dir = tempfile.mkdtemp()
try:
foo = FtpFileTransfer(None)
noexist = os.path.join(temp_dir, 'noexist')
self.assertEqual(
foo.upload_file_direct(noexist, '/remote_dir',
'remote_filename'), False)
self.assertEqual(foo.get_error_msg(), noexist + ' is not a file')
finally:
shutil.rmtree(temp_dir)
def test_download_file_fail(self):
mockftp = MockFtp()
mockftp.get = Mock(side_effect=IOError('error'))
mockftp.close = Mock(return_value=None)
foo = FtpFileTransfer(None)
foo.set_connection(mockftp)
foo.connect()
self.assertFalse(foo.download_file('/a/b', '/c/d'))
self.assertEqual(foo.get_error_msg(), 'Unable to download /a/b to '
'/c/d : error')
foo.disconnect()
mockftp.get.assert_called_with('/a/b', local='/c/d')
def test_list_files_fail(self):
mockftp = MockFtp()
mockftp.list = Mock(side_effect=IOError('error'))
mockftp.close = Mock(return_value=None)
foo = FtpFileTransfer(None)
foo.set_connection(mockftp)
foo.connect()
filelist = foo.list_files('/foo2')
self.assertTrue(filelist is None)
foo.disconnect()
self.assertEqual(foo.get_error_msg(), 'Unable to get file list '
'for /foo2 : error')
mockftp.list.assert_called_with('/foo2', extra=True)
def __init__(self, path, args):
"""Constructor
"""
try:
logger.debug('ftpconfig set to ' + args.ftpconfig)
self._file_transfer = FtpFileTransfer(args.ftpconfig)
except Exception:
logger.exception('Caught exception')
self._file_transfer = None
self._path = path
self._args = args
ctask = ChallengeDataTask(path, args)
self._chall_dir_name = ctask.get_celpp_challenge_data_dir_name()
def test_run_success_with_ftp_upload(self):
temp_dir = tempfile.mkdtemp()
try:
script = self.create_gen_challenge_script(temp_dir)
params = D3RParameters()
params.genchallenge = script
params.pdbdb = '/foo'
params.version = '1'
blastnfilter = BlastNFilterTask(temp_dir, params)
blastnfilter.create_dir()
open(os.path.join(blastnfilter.get_dir(), D3RTask.COMPLETE_FILE),
'a').close()
chall = ChallengeDataTask(temp_dir, params)
mockftp = D3RParameters()
mockftp.put = Mock(side_effect=[3, 5])
ftp = FtpFileTransfer(None)
ftp.set_remote_challenge_dir('/challenge')
ftp.set_connection(mockftp)
chall.set_file_transfer(ftp)
dimport = DataImportTask(temp_dir, params)
dimport.create_dir()
ctsv = dimport.get_crystalph_tsv()
f = open(ctsv, 'w')
f.write('crystal')
f.flush()
f.close()
nonpoly = dimport.get_nonpolymer_tsv()
f = open(nonpoly, 'w')
f.write('nonpoly')
f.flush()
f.close()
seq = dimport.get_sequence_tsv()
f = open(seq, 'w')
f.write('seq')
f.flush()
f.close()
chall.run()
self.assertEqual(chall.get_error(), None)
# verify test files get created
errfile = os.path.join(chall.get_dir(),
D3RTask.ERROR_FILE)
self.assertEqual(os.path.isfile(errfile), False)
compfile = os.path.join(chall.get_dir(),
D3RTask.COMPLETE_FILE)
self.assertEqual(os.path.isfile(compfile), True)
ftp.disconnect()
finally:
shutil.rmtree(temp_dir)
def test_upload_file_where_file_is_none(self):
# test where file is None
ftpspec = ["put"]
mockftp = Mock(spec=ftpspec)
foo = FtpFileTransfer(None)
foo.set_connection(mockftp)
foo.connect()
foo._upload_file(None)
def test_upload_file_direct_put_throws_exception(self):
temp_dir = tempfile.mkdtemp()
try:
mockftp = MockFtp()
mockftp.put = Mock(side_effect=IOError('hi'))
foo = FtpFileTransfer(None)
foo.set_connection(mockftp)
foo.connect()
afile = os.path.join(temp_dir, 'afile')
open(afile, 'a').close()
self.assertEqual(foo.upload_file_direct(afile, '/foo', 'name'),
False)
self.assertEqual(foo.get_error_msg(), 'Unable to upload ' + afile +
' to /foo/name : hi')
foo.disconnect()
finally:
shutil.rmtree(temp_dir)
def test_list_files_success(self):
mockftp = MockFtp()
mockftp.list = Mock(return_value=[{'directory': 'd', 'name': '.'},
{'directory': 'd', 'name': '..'},
{'directory': 'd', 'name': 'foo'},
{'directory': '-', 'name': 'file'}])
mockftp.close = Mock(return_value=None)
foo = FtpFileTransfer(None)
foo.set_connection(mockftp)
foo.connect()
filelist = foo.list_files('/foo')
self.assertTrue(len(filelist) == 1)
self.assertEqual(filelist[0], 'file')
foo.disconnect()
mockftp.list.assert_called_with('/foo', extra=True)
def test_upload_file_where_file_does_not_exist(self):
temp_dir = tempfile.mkdtemp()
try:
ftpspec = ["put"]
mockftp = Mock(spec=ftpspec)
foo = FtpFileTransfer(None)
foo.set_connection(mockftp)
foo.connect()
foo._upload_file(os.path.join(temp_dir, 'nonexist'))
self.assertEqual(mockftp.call_count, 0)
finally:
shutil.rmtree(temp_dir)
def test_download_participant_list_file_not_found(self):
temp_dir = tempfile.mkdtemp()
try:
foo = FtpFileTransfer(None)
mockftp = D3RParameters()
mockftp.get = Mock()
foo.set_remote_dir('/foo')
foo.set_connection(mockftp)
params = D3RParameters()
task = DataImportTask(temp_dir, params)
task.set_file_transfer(foo)
task.create_dir()
task._download_participant_list_csv()
self.assertEqual(
task.get_email_log(),
'\nWARNING: participant_list.csv not downloaded '
'which means external users will NOT get '
'evaluation email\n')
mockftp.get\
.assert_called_with('/foo/' +
DataImportTask.PARTICIPANT_LIST_CSV,
local=task.get_participant_list_csv())
finally:
shutil.rmtree(temp_dir)
def test_download_file_remote_file_params_none(self):
foo = FtpFileTransfer(None)
self.assertFalse(foo.download_file(None, None))
self.assertEqual(foo.get_error_msg(), 'remote_file None')
self.assertFalse(foo.download_file('/hi', None))
self.assertEqual(foo.get_error_msg(), 'local_file None')
def test_upload_challenge_file_no_remote_challenge_dir(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
chall = ChallengeDataTask(temp_dir, params)
ftp = FtpFileTransfer(None)
chall.set_file_transfer(ftp)
chall._upload_challenge_file('/foo')
self.assertEqual(chall.get_email_log(), 'No remote challenge'
' directory set for '
'ftp upload\n')
finally:
shutil.rmtree(temp_dir)
def test_upload_file_that_raises_exception(self):
temp_dir = tempfile.mkdtemp()
try:
mockftp = MockFtp()
mockftp.put = Mock(side_effect=IOError('hi'))
foo = FtpFileTransfer(None)
foo.set_connection(mockftp)
foo.connect()
valid_file = os.path.join(temp_dir, 'file')
f = open(valid_file, 'a')
f.write('12')
f.flush()
f.close()
try:
foo._upload_file(valid_file)
self.fail('Expected IOError')
except IOError:
pass
finally:
shutil.rmtree(temp_dir)
def test_run_all_success(self):
temp_dir = tempfile.mkdtemp()
try:
fakeftp = FtpFileTransfer(None)
mftp = D3RParameters()
fakeftp.set_connection(mftp)
fakeftp.set_remote_dir('/foo2')
mftp.get = Mock()
params = D3RParameters()
params.pdbfileurl = 'file://' + temp_dir
params.compinchi = 'file://' + temp_dir
make_blast = MakeBlastDBTask(temp_dir, params)
make_blast.create_dir()
open(os.path.join(make_blast.get_dir(), D3RTask.COMPLETE_FILE),
'a').close()
task = DataImportTask(temp_dir, params)
task.set_file_transfer(fakeftp)
task._retrysleep = 0
open(os.path.join(temp_dir, task.NONPOLYMER_TSV), 'a').close()
open(os.path.join(temp_dir, task.SEQUENCE_TSV), 'a').close()
open(os.path.join(temp_dir, task.OLDSEQUENCE_TSV), 'a').close()
open(os.path.join(temp_dir, task.CRYSTALPH_TSV), 'a').close()
open(os.path.join(temp_dir, task.COMPINCHI_ICH), 'a').close()
task.run()
self.assertEquals(task.get_error(), None)
# check line count is 1 now which indicates
# standard was added
self.assertEqual(
util.get_file_line_count(task.get_nonpolymer_tsv()), 1)
self.assertEqual(util.get_file_line_count(task.get_sequence_tsv()),
1)
self.assertEqual(
util.get_file_line_count(task.get_oldsequence_tsv()), 1)
self.assertEqual(
util.get_file_line_count(task.get_crystalph_tsv()), 1)
mftp.get.assert_called_with('/foo2/' +
DataImportTask.PARTICIPANT_LIST_CSV,
local=task.get_participant_list_csv())
finally:
shutil.rmtree(temp_dir)
def __init__(self, path, args):
"""Constructor
Creates a `D3RTask` with `D3RTask.UNKNOWN_STATUS` status
with `path` and `args` set to values passed in
"""
self._path = path
self._name = None
self._stage = None
self._status = D3RTask.UNKNOWN_STATUS
self._error = None
self._args = args
self._can_run = None
self._start_time = None
self._duration = -1
self._email_log = None
try:
logger.debug('ftpconfig set to ' + args.ftpconfig)
self._file_uploader = FtpFileTransfer(args.ftpconfig)
except:
logger.debug('FtpFileUploader not set. This may not be '
'an error')
self._file_uploader = None
def test_download_participant_list_success(self):
temp_dir = tempfile.mkdtemp()
try:
foo = FtpFileTransfer(None)
mockftp = D3RParameters()
mockftp.get = Mock()
foo.set_remote_dir('/foo')
foo.set_connection(mockftp)
params = D3RParameters()
task = DataImportTask(temp_dir, params)
task.set_file_transfer(foo)
task.create_dir()
open(task.get_participant_list_csv(), 'a').close()
task._download_participant_list_csv()
self.assertEqual(task.get_email_log(), None)
mockftp.get\
.assert_called_with('/foo/' +
DataImportTask.PARTICIPANT_LIST_CSV,
local=task.get_participant_list_csv())
finally:
shutil.rmtree(temp_dir)
class ExternalDataSubmissionFactory(object):
"""Factory to create ExternalDataSubmissionObjects
"""
DOCKEDRESULTS = '_dockedresults_'
def __init__(self, path, args):
"""Constructor
"""
try:
logger.debug('ftpconfig set to ' + args.ftpconfig)
self._file_transfer = FtpFileTransfer(args.ftpconfig)
except Exception:
logger.exception('Caught exception')
self._file_transfer = None
self._path = path
self._args = args
ctask = ChallengeDataTask(path, args)
self._chall_dir_name = ctask.get_celpp_challenge_data_dir_name()
def get_args(self):
return self._args
def get_path(self):
return self._path
def _get_challenge_package_results_file_name(self, dir_name):
return (self._chall_dir_name +
ExternalDataSubmissionFactory.DOCKEDRESULTS +
dir_name + ChallengeDataTask.TAR_GZ_SUFFIX)
def set_file_transfer(self, filetransfer):
"""Sets file transfer
"""
self._file_transfer = filetransfer
def get_file_transfer(self):
"""Gets file transfer
"""
return self._file_transfer
def _get_submission_dirs(self, remote_dir):
"""Gets list of directories under `remote_dir`
:param remote_dir: path on remote server to examine
:returns: list of directory names without path prefix
:raises AttributeError: if `get_file_transfer()` is None
"""
dlist = self._file_transfer.list_dirs(remote_dir)
if dlist is None:
logger.debug('No directories returned')
return []
logger.debug('Found ' + str(len(dlist)) + ' directories on ' +
remote_dir)
return dlist
def _get_challenge_data_package_file(self, remote_dir, dir_name):
"""Gets challenge data package file under `remote_dir` / `dir_name`
if it exists
:returns: Path to remote challenge file upon success or None if
not found
:raises AttributeError if `get_file_transfer()` is None
"""
ft = self.get_file_transfer()
flist = ft.list_files(os.path.normpath(os.path.join(remote_dir,
dir_name)))
if flist is None:
logger.info('No files found in ' + dir_name)
return None
logger.debug('Found ' + str(len(flist)) + ' files in ' + dir_name)
chall_fname = self._get_challenge_package_results_file_name(dir_name)
logger.debug('Looking for ' + chall_fname + ' file in directory')
for entry in flist:
if entry == chall_fname:
logger.info('Found matching entry ' + entry)
return os.path.normpath(os.path.join(remote_dir,
dir_name,
chall_fname))
else:
logger.debug('Encountered non challenge file: ' + entry)
return None
def _remove_latest_txt(self):
"""Removes the latest.txt file from ftp server if found
"""
try:
ft = self.get_file_transfer()
latest_txt = os.path.join(ft.get_remote_challenge_dir(),
ChallengeDataTask.LATEST_TXT)
logger.info('Attempting to remove ' + latest_txt)
val = ft.delete_file(latest_txt)
logger.info('Return value from delete call ' + str(val))
except Exception:
logger.exception('Caught exception trying to remove latest.txt')
def get_external_data_submissions(self):
"""Generate ExternalDataSubmission objects
Method should examine the submission directory on the
remote server via FtpFileTransfer(args.ftpconfig)
for each directory under submission directory
look for a celpp_weekXX_YYYY_dockedresults_ZZZZ.tar.gz file
where ZZZZ matches name of directory under submission directory
Regardless if its found or not create new ExternalDataSubmission
object and pass ZZZZ for name and path to .tar.gz file as
remotefile if the tar.gz file was found otherwise pass None
along with args. Append this object to list and
return it
"""
task_list = []
try:
self._file_transfer.connect()
self._remove_latest_txt()
subdir = self._file_transfer.get_remote_submission_dir()
dlist = self._get_submission_dirs(subdir)
for d in dlist:
chall_file = self._get_challenge_data_package_file(subdir, d)
if chall_file is not None:
et = ExternalDataSubmissionTask(self.get_path(), d,
chall_file,
self.get_args())
logger.info('Added ExternalData Submission Task: ' +
et.get_dir_name())
task_list.append(et)
except Exception:
logger.exception('Caught exception')
finally:
try:
self._file_transfer.disconnect()
except Exception:
logger.exception('Caught exception disconnecting')
return task_list
def test_connect(self):
# test where alt_ftp_con is None and we use ftpretty
foo = FtpFileTransfer(None)
foo.set_host('doesnotexist')
foo.set_user('user')
foo.set_password('')
foo.set_remote_dir('/remote')
foo.set_connect_timeout(0)
self.assertFalse(foo.connect())
# test where alt_ftp_con is set
foo = FtpFileTransfer(None)
foo.set_connection('hi')
foo.connect()
self.assertEqual(foo._ftp, 'hi')
