Esempio n. 1
0
 def test_local_download_in_subdir(self):
     locald = LocalDownload(self.curdir + '/')
     (file_list, dir_list) = locald.list()
     locald.match([r'^/bank/test.*\.gz$'], file_list, dir_list)
     locald.download(self.utils.data_dir)
     locald.close()
     self.assertTrue(len(locald.files_to_download) == 1)
Esempio n. 2
0
    def test_parallel_local_download(self):
        locald = LocalDownload(self.examples)
        (file_list, dir_list) = locald.list()
        locald.match([r'^test'], file_list, dir_list)
        list1 = [locald.files_to_download[0]]
        list2 = locald.files_to_download[1:]
        locald.close()

        locald1 = LocalDownload(self.examples)
        locald1.files_to_download = list1
        locald2 = LocalDownload(self.examples)
        locald2.files_to_download = list2
        t1 = DownloadThread(locald1, self.utils.data_dir)
        t2 = DownloadThread(locald2, self.utils.data_dir)
        t1.start()
        t2.start()
        t1.join()
        t2.join()
        self.assertTrue(len(t1.downloader.files_to_download) == 1)
        self.assertTrue(
            os.path.exists(self.utils.data_dir + '/' + list1[0]['name']))
        self.assertTrue(len(t2.downloader.files_to_download) == 2)
        self.assertTrue(
            os.path.exists(self.utils.data_dir + '/' + list2[0]['name']))
        self.assertTrue(
            os.path.exists(self.utils.data_dir + '/' + list2[1]['name']))
Esempio n. 3
0
 def test_local_download_in_subdir(self):
   locald = LocalDownload(self.curdir+'/')
   (file_list, dir_list) = locald.list()
   locald.match([r'^/bank/test.*\.gz$'], file_list, dir_list)
   locald.download(self.utils.data_dir)
   locald.close()
   self.assertTrue(len(locald.files_to_download) == 1)
Esempio n. 4
0
 def test_local_download(self):
     locald = LocalDownload(self.examples)
     (file_list, dir_list) = locald.list()
     locald.match([r'^test.*\.gz$'], file_list, dir_list)
     locald.download(self.utils.data_dir)
     locald.close()
     self.assertTrue(len(locald.files_to_download) == 1)
Esempio n. 5
0
 def test_local_download(self):
   locald = LocalDownload(self.examples)
   (file_list, dir_list) = locald.list()
   locald.match([r'^test.*\.gz$'], file_list, dir_list)
   locald.download(self.utils.data_dir)
   locald.close()
   self.assertTrue(len(locald.files_to_download) == 1)
Esempio n. 6
0
 def wf_copydepends(self):
     """
     Copy files from dependent banks if needed
     """
     logging.info("Workflow:wf_copydepends")
     deps = self.bank.get_dependencies()
     for dep in deps:
         if self.bank.config.get(dep + ".files.move"):
             logging.info("Worflow:wf_depends:Files:Move:" + self.bank.config.get(dep + ".files.move"))
             bdir = None
             for bdep in self.bank.depends:
                 if bdep.name == dep:
                     bdir = bdep.session.get_full_release_directory()
                     break
             if bdir is None:
                 logging.error("Could not find a session update for bank " + dep)
                 return False
             b = self.bank.get_bank(dep, no_log=True)
             locald = LocalDownload(bdir)
             (file_list, dir_list) = locald.list()
             locald.match(self.bank.config.get(dep + ".files.move").split(), file_list, dir_list)
             bankdepdir = self.bank.session.get_full_release_directory() + "/" + dep
             if not os.path.exists(bankdepdir):
                 os.mkdir(bankdepdir)
             downloadedfiles = locald.download(bankdepdir)
             locald.close()
             if not downloadedfiles:
                 logging.info("Workflow:wf_copydepends:no files to copy")
                 return False
     return True
Esempio n. 7
0
  def test_parallel_local_download(self):
    locald = LocalDownload(self.examples)
    (file_list, dir_list) = locald.list()
    locald.match([r'^test'], file_list, dir_list)
    list1 = [locald.files_to_download[0]]
    list2 = locald.files_to_download[1:]
    locald.close()

    locald1 = LocalDownload(self.examples)
    locald1.files_to_download = list1
    locald2 = LocalDownload(self.examples)
    locald2.files_to_download = list2
    t1 = DownloadThread(locald1, self.utils.data_dir)
    t2 = DownloadThread(locald2, self.utils.data_dir)
    t1.start()
    t2.start()
    t1.join()
    t2.join()
    self.assertTrue(len(t1.downloader.files_to_download) == 1)
    self.assertTrue(os.path.exists(self.utils.data_dir + '/' +list1[0]['name']))
    self.assertTrue(len(t2.downloader.files_to_download) == 2)
    self.assertTrue(os.path.exists(self.utils.data_dir + '/' +list2[0]['name']))
    self.assertTrue(os.path.exists(self.utils.data_dir + '/' +list2[1]['name']))
Esempio n. 8
0
 def test_local_list(self):
     locald = LocalDownload(self.examples)
     (file_list, dir_list) = locald.list()
     locald.close()
     self.assertTrue(len(file_list) > 1)
Esempio n. 9
0
 def test_local_list(self):
   locald = LocalDownload(self.examples)
   (file_list, dir_list) = locald.list()
   locald.close()
   self.assertTrue(len(file_list) > 1)