def test_local_download_in_subdir(self): locald = LocalDownload(self.curdir + '/') (file_list, dir_list) = locald.list() locald.match([r'^/bank/test.*\.gz$'], file_list, dir_list) locald.download(self.utils.data_dir) locald.close() self.assertTrue(len(locald.files_to_download) == 1)
def test_local_download(self): locald = LocalDownload(self.examples) (file_list, dir_list) = locald.list() locald.match([r'^test.*\.gz$'], file_list, dir_list) locald.download(self.utils.data_dir) locald.close() self.assertTrue(len(locald.files_to_download) == 1)
def test_local_download_in_subdir(self): locald = LocalDownload(self.curdir+'/') (file_list, dir_list) = locald.list() locald.match([r'^/bank/test.*\.gz$'], file_list, dir_list) locald.download(self.utils.data_dir) locald.close() self.assertTrue(len(locald.files_to_download) == 1)
def wf_copydepends(self): """ Copy files from dependent banks if needed """ logging.info("Workflow:wf_copydepends") deps = self.bank.get_dependencies() for dep in deps: if self.bank.config.get(dep + ".files.move"): logging.info("Worflow:wf_depends:Files:Move:" + self.bank.config.get(dep + ".files.move")) bdir = None for bdep in self.bank.depends: if bdep.name == dep: bdir = bdep.session.get_full_release_directory() break if bdir is None: logging.error("Could not find a session update for bank " + dep) return False b = self.bank.get_bank(dep, no_log=True) locald = LocalDownload(bdir) (file_list, dir_list) = locald.list() locald.match(self.bank.config.get(dep + ".files.move").split(), file_list, dir_list) bankdepdir = self.bank.session.get_full_release_directory() + "/" + dep if not os.path.exists(bankdepdir): os.mkdir(bankdepdir) downloadedfiles = locald.download(bankdepdir) locald.close() if not downloadedfiles: logging.info("Workflow:wf_copydepends:no files to copy") return False return True
def test_parallel_local_download(self): locald = LocalDownload(self.examples) (file_list, dir_list) = locald.list() locald.match([r'^test'], file_list, dir_list) list1 = [locald.files_to_download[0]] list2 = locald.files_to_download[1:] locald.close() locald1 = LocalDownload(self.examples) locald1.files_to_download = list1 locald2 = LocalDownload(self.examples) locald2.files_to_download = list2 t1 = DownloadThread(locald1, self.utils.data_dir) t2 = DownloadThread(locald2, self.utils.data_dir) t1.start() t2.start() t1.join() t2.join() self.assertTrue(len(t1.downloader.files_to_download) == 1) self.assertTrue(os.path.exists(self.utils.data_dir + '/' +list1[0]['name'])) self.assertTrue(len(t2.downloader.files_to_download) == 2) self.assertTrue(os.path.exists(self.utils.data_dir + '/' +list2[0]['name'])) self.assertTrue(os.path.exists(self.utils.data_dir + '/' +list2[1]['name']))
def test_parallel_local_download(self): locald = LocalDownload(self.examples) (file_list, dir_list) = locald.list() locald.match([r'^test'], file_list, dir_list) list1 = [locald.files_to_download[0]] list2 = locald.files_to_download[1:] locald.close() locald1 = LocalDownload(self.examples) locald1.files_to_download = list1 locald2 = LocalDownload(self.examples) locald2.files_to_download = list2 t1 = DownloadThread(locald1, self.utils.data_dir) t2 = DownloadThread(locald2, self.utils.data_dir) t1.start() t2.start() t1.join() t2.join() self.assertTrue(len(t1.downloader.files_to_download) == 1) self.assertTrue( os.path.exists(self.utils.data_dir + '/' + list1[0]['name'])) self.assertTrue(len(t2.downloader.files_to_download) == 2) self.assertTrue( os.path.exists(self.utils.data_dir + '/' + list2[0]['name'])) self.assertTrue( os.path.exists(self.utils.data_dir + '/' + list2[1]['name']))
def test_local_list(self): locald = LocalDownload(self.examples) (file_list, dir_list) = locald.list() locald.close() self.assertTrue(len(file_list) > 1)