def test_local_download_in_subdir(self):
locald = LocalDownload(self.curdir + '/')
(file_list, dir_list) = locald.list()
locald.match([r'^/bank/test.*\.gz$'], file_list, dir_list)
locald.download(self.utils.data_dir)
locald.close()
self.assertTrue(len(locald.files_to_download) == 1)
def test_local_download(self):
locald = LocalDownload(self.examples)
(file_list, dir_list) = locald.list()
locald.match([r'^test.*\.gz$'], file_list, dir_list)
locald.download(self.utils.data_dir)
locald.close()
self.assertTrue(len(locald.files_to_download) == 1)
def test_local_download_in_subdir(self): locald = LocalDownload(self.curdir+'/') (file_list, dir_list) = locald.list() locald.match([r'^/bank/test.*\.gz$'], file_list, dir_list) locald.download(self.utils.data_dir) locald.close() self.assertTrue(len(locald.files_to_download) == 1)
def test_local_download(self): locald = LocalDownload(self.examples) (file_list, dir_list) = locald.list() locald.match([r'^test.*\.gz$'], file_list, dir_list) locald.download(self.utils.data_dir) locald.close() self.assertTrue(len(locald.files_to_download) == 1)
def wf_copydepends(self):
"""
Copy files from dependent banks if needed
"""
logging.info("Workflow:wf_copydepends")
deps = self.bank.get_dependencies()
for dep in deps:
if self.bank.config.get(dep + ".files.move"):
logging.info("Worflow:wf_depends:Files:Move:" + self.bank.config.get(dep + ".files.move"))
bdir = None
for bdep in self.bank.depends:
if bdep.name == dep:
bdir = bdep.session.get_full_release_directory()
break
if bdir is None:
logging.error("Could not find a session update for bank " + dep)
return False
b = self.bank.get_bank(dep, no_log=True)
locald = LocalDownload(bdir)
(file_list, dir_list) = locald.list()
locald.match(self.bank.config.get(dep + ".files.move").split(), file_list, dir_list)
bankdepdir = self.bank.session.get_full_release_directory() + "/" + dep
if not os.path.exists(bankdepdir):
os.mkdir(bankdepdir)
downloadedfiles = locald.download(bankdepdir)
locald.close()
if not downloadedfiles:
logging.info("Workflow:wf_copydepends:no files to copy")
return False
return True
def test_parallel_local_download(self):
locald = LocalDownload(self.examples)
(file_list, dir_list) = locald.list()
locald.match([r'^test'], file_list, dir_list)
list1 = [locald.files_to_download[0]]
list2 = locald.files_to_download[1:]
locald.close()
locald1 = LocalDownload(self.examples)
locald1.files_to_download = list1
locald2 = LocalDownload(self.examples)
locald2.files_to_download = list2
t1 = DownloadThread(locald1, self.utils.data_dir)
t2 = DownloadThread(locald2, self.utils.data_dir)
t1.start()
t2.start()
t1.join()
t2.join()
self.assertTrue(len(t1.downloader.files_to_download) == 1)
self.assertTrue(os.path.exists(self.utils.data_dir + '/' +list1[0]['name']))
self.assertTrue(len(t2.downloader.files_to_download) == 2)
self.assertTrue(os.path.exists(self.utils.data_dir + '/' +list2[0]['name']))
self.assertTrue(os.path.exists(self.utils.data_dir + '/' +list2[1]['name']))
def test_parallel_local_download(self):
locald = LocalDownload(self.examples)
(file_list, dir_list) = locald.list()
locald.match([r'^test'], file_list, dir_list)
list1 = [locald.files_to_download[0]]
list2 = locald.files_to_download[1:]
locald.close()
locald1 = LocalDownload(self.examples)
locald1.files_to_download = list1
locald2 = LocalDownload(self.examples)
locald2.files_to_download = list2
t1 = DownloadThread(locald1, self.utils.data_dir)
t2 = DownloadThread(locald2, self.utils.data_dir)
t1.start()
t2.start()
t1.join()
t2.join()
self.assertTrue(len(t1.downloader.files_to_download) == 1)
self.assertTrue(
os.path.exists(self.utils.data_dir + '/' + list1[0]['name']))
self.assertTrue(len(t2.downloader.files_to_download) == 2)
self.assertTrue(
os.path.exists(self.utils.data_dir + '/' + list2[0]['name']))
self.assertTrue(
os.path.exists(self.utils.data_dir + '/' + list2[1]['name']))
def test_local_list(self):
locald = LocalDownload(self.examples)
(file_list, dir_list) = locald.list()
locald.close()
self.assertTrue(len(file_list) > 1)
def test_local_list(self): locald = LocalDownload(self.examples) (file_list, dir_list) = locald.list() locald.close() self.assertTrue(len(file_list) > 1)
