def __init__(self, protocol, host, rootdir='', file_list=None):
'''
Initialize the files in list with today as last-modification date.
Size is also preset to zero, size will be set after download
:param file_list: list of files to download on server
:type file_list: list
'''
FTPDownload.__init__(self, protocol, host, rootdir)
if file_list is None:
file_list = []
today = datetime.date.today()
self.files_to_download = []
self.headers = {}
for file in file_list:
rfile = {}
rfile['root'] = self.rootdir
rfile['permissions'] = ''
rfile['group'] = ''
rfile['user'] = ''
rfile['size'] = 0
rfile['month'] = today.month
rfile['day'] = today.day
rfile['year'] = today.year
rfile['name'] = file
self.files_to_download.append(rfile)
def __init__(self, protocol, host, rootdir='', file_list=None):
'''
Initialize the files in list with today as last-modification date.
Size is also preset to zero, size will be set after download
:param file_list: list of files to download on server
:type file_list: list
'''
FTPDownload.__init__(self, protocol, host, rootdir)
if file_list is None:
file_list = []
today = datetime.date.today()
self.files_to_download = []
self.headers = {}
for file in file_list:
rfile = {}
rfile['root'] = self.rootdir
rfile['permissions'] = ''
rfile['group'] = ''
rfile['user'] = ''
rfile['size'] = 0
rfile['month'] = today.month
rfile['day'] = today.day
rfile['year'] = today.year
rfile['name'] = file
rfile['hash'] = None
self.files_to_download.append(rfile)
def test_download_in_subdir(self):
ftpd = FTPDownload('ftp', 'ftp.ncbi.nih.gov', '/blast/')
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^db/FASTA/alu.*\.gz$'], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 2)
def test_download_in_subdir(self):
ftpd = FTPDownload('ftp', 'ftp.ncbi.nih.gov', '/blast/')
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^db/FASTA/alu.*\.gz$'], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 2)
def get_handler(self, protocol, server, remote_dir, list_file=None):
"""
Get a protocol download handler
"""
if list_file is None:
list_file = []
downloader = None
if protocol == "ftp" or protocol == "sftp":
downloader = FTPDownload(protocol, server, remote_dir)
if protocol == "http":
downloader = HTTPDownload(protocol, server, remote_dir, self.bank.config)
if protocol == "local":
downloader = LocalDownload(remote_dir)
if protocol == "directftp":
downloader = DirectFTPDownload("ftp", server, remote_dir, list_file)
if protocol == "directhttp":
downloader = DirectHttpDownload("http", server, remote_dir, list_file)
if downloader is not None:
downloader.bank = self.bank.name
proxy = self.bank.config.get("proxy")
proxy_auth = self.bank.config.get("proxy_auth")
if proxy is not None and proxy:
downloader.set_proxy(proxy, proxy_auth)
return downloader
def get_handler(self, protocol, server, remote_dir, list_file=None):
'''
Get a protocol download handler
'''
if list_file is None:
list_file = []
downloader = None
if protocol == 'ftp' or protocol == 'sftp':
downloader = FTPDownload(protocol, server, remote_dir)
if protocol == 'http':
downloader = HTTPDownload(protocol, server, remote_dir, self.bank.config)
if protocol == 'local':
downloader = LocalDownload(remote_dir)
if protocol == 'directftp':
downloader = DirectFTPDownload('ftp', server, remote_dir, list_file)
if protocol == 'directhttp':
downloader = DirectHttpDownload('http', server, remote_dir, list_file)
if downloader is not None:
downloader.bank = self.bank.name
proxy = self.bank.config.get('proxy')
proxy_auth = self.bank.config.get('proxy_auth')
if proxy is not None and proxy:
downloader.set_proxy(proxy, proxy_auth)
return downloader
def test_download(self):
ftpd = FTPDownload("ftp", "ftp.ncbi.nih.gov", "/blast/db/FASTA/")
(file_list, dir_list) = ftpd.list()
ftpd.match([r"^alu.*\.gz$"], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 2)
def test_download_or_copy(self):
ftpd = FTPDownload('ftp', 'ftp.ncbi.nih.gov', '/blast/')
ftpd.files_to_download = [
{'name':'/test1', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},
{'name':'/test2', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},
{'name':'/test/test1', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},
{'name':'/test/test11', 'year': '2013', 'month': '11', 'day': '10', 'size': 10}
]
available_files = [
{'name':'/test1', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},
{'name':'/test12', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},
{'name':'/test3', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},
{'name':'/test/test1', 'year': '2013', 'month': '11', 'day': '10', 'size': 20},
{'name':'/test/test11', 'year': '2013', 'month': '11', 'day': '10', 'size': 10}
]
ftpd.download_or_copy(available_files, '/biomaj', False)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download)==2)
self.assertTrue(len(ftpd.files_to_copy)==2)
def test_download_or_copy(self):
ftpd = FTPDownload("ftp", "ftp.ncbi.nih.gov", "/blast/")
ftpd.files_to_download = [
{"name": "/test1", "year": "2013", "month": "11", "day": "10", "size": 10},
{"name": "/test2", "year": "2013", "month": "11", "day": "10", "size": 10},
{"name": "/test/test1", "year": "2013", "month": "11", "day": "10", "size": 10},
{"name": "/test/test11", "year": "2013", "month": "11", "day": "10", "size": 10},
]
available_files = [
{"name": "/test1", "year": "2013", "month": "11", "day": "10", "size": 10},
{"name": "/test12", "year": "2013", "month": "11", "day": "10", "size": 10},
{"name": "/test3", "year": "2013", "month": "11", "day": "10", "size": 10},
{"name": "/test/test1", "year": "2013", "month": "11", "day": "10", "size": 20},
{"name": "/test/test11", "year": "2013", "month": "11", "day": "10", "size": 10},
]
ftpd.download_or_copy(available_files, "/biomaj", False)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 2)
self.assertTrue(len(ftpd.files_to_copy) == 2)
def __init__(self, protocol, host, rootdir, config):
FTPDownload.__init__(self, protocol, host, rootdir)
self.config = config
def test_download_or_copy(self):
ftpd = FTPDownload('ftp', 'ftp.ncbi.nih.gov', '/blast/')
ftpd.files_to_download = [{
'name': '/test1',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test2',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test/test1',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test/test11',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}]
available_files = [{
'name': '/test1',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test12',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test3',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test/test1',
'year': '2013',
'month': '11',
'day': '10',
'size': 20
}, {
'name': '/test/test11',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}]
ftpd.download_or_copy(available_files, '/biomaj', False)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 2)
self.assertTrue(len(ftpd.files_to_copy) == 2)
def test_ftp_list(self):
ftpd = FTPDownload('ftp', 'ftp.ncbi.nih.gov', '/blast/db/FASTA/')
(file_list, dir_list) = ftpd.list()
ftpd.close()
self.assertTrue(len(file_list) > 1)
def test_ftp_list(self):
ftpd = FTPDownload('ftp', 'ftp.ncbi.nih.gov', '/blast/db/FASTA/')
(file_list, dir_list) = ftpd.list()
ftpd.close()
self.assertTrue(len(file_list) > 1)
def __init__(self, protocol, host, rootdir, config):
FTPDownload.__init__(self, protocol, host, rootdir)
self.config = config