def __init__(self, scidbUser, hosts, nInst, bkpFolder, allArrays): self._scidbUser = scidbUser self._hosts = hosts self._nInst = nInst self._createQuery = CREATE_800x40_ARRAYS self._cmdRunner = CommandRunner() self.__bkpFolder = bkpFolder self.__allArrays = allArrays
def __init__( self, scidbUser, hosts, nInst, bkpFolder, allArrays ): self._scidbUser = scidbUser self._hosts = hosts self._nInst = nInst self._createQuery = CREATE_200x200_ARRAYS self._cmdRunner = CommandRunner() self.__bkpFolder = bkpFolder self.__allArrays = allArrays
def test_remote_command(): c = CommandRunner("192.168.1.200", "frosty") c.invoke_command("ls")
def test_local_command(): c = CommandRunner() c.invoke_command("ls")
class BackupTest(unittest.TestCase): _cmdRunner = CommandRunner() def __init__(self,*args,**kwargs): super(BackupTest,self).__init__(*args,**kwargs) self._tempArrays = [] self._arrays = [] def setBackupHelper(self,bh): self.bkpHelper = bh def setArrays(self,AL,temp=False): if (temp): self._tempArrays = list(AL) else: self._arrays = list(AL) def setUp(self): self.bkpHelper.removeBackup(BACKUP) self.bkpHelper.removeArrays(self._arrays + self._tempArrays) self.bkpHelper.reCreateArrays(self._arrays,versions=4) self.bkpHelper.reCreateArrays(self._tempArrays,temp=True) def tearDown(self): self.bkpHelper.removeBackup(BACKUP) self.bkpHelper.removeArrays(self._arrays + self._tempArrays) def commonTestBody( self, toRestore, saveCmd, restoreCmd, versions=False ): sys.stderr.write('\n' + ' '.join(restoreCmd) + '\n') exits,outs=self._cmdRunner.waitForProcesses( [self._cmdRunner.runSubProcess(saveCmd,useShell=True)], True ) self.bkpHelper.removeArrays(self._arrays + self._tempArrays) exits,outs = self._cmdRunner.waitForProcesses( [self._cmdRunner.runSubProcess(restoreCmd,useShell=True)], True ) restored = self.bkpHelper.getDBArrays(versions) self.assertTrue( set(toRestore) == set(restored), 'Restored arrays do not match initial conditions!' ) self.assertTrue( self.bkpHelper.checkArrayData(toRestore), 'Restored array data does not match initial conditions!' ) #------------------------------------------------------------------------- @testWatchdog def test_tmp_text_parallel(self): toRestore = self.bkpHelper.getAllArrayNames() # All arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'text', '--parallel', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'text', '--parallel', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd) @testWatchdog def test_tmp_text_allVersions(self): toRestore = self.bkpHelper.getAllArrayNames(4) # All versions of arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] print 'toRestore',toRestore saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'text', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'text', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd,versions=True) @testWatchdog def test_tmp_text_zip(self): toRestore = self.bkpHelper.getAllArrayNames() # All arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'text', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'text', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd) @testWatchdog def test_tmp_text_parallel_allVersions(self): toRestore = self.bkpHelper.getAllArrayNames(4) # All versions of arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'text', '--parallel', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'text', '--parallel', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd,versions=True) @testWatchdog def test_tmp_text_parallel_zip(self): toRestore = self.bkpHelper.getAllArrayNames() # All arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'text', '--parallel', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'text', '--parallel', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd) @testWatchdog def test_tmp_text_allVersions_zip(self): toRestore = self.bkpHelper.getAllArrayNames(4) # All versions of arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'text', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'text', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd,versions=True) @testWatchdog def test_tmp_text_parallel_allVersions_zip(self): toRestore = self.bkpHelper.getAllArrayNames(4) # All versions of arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'text', '--parallel', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'text', '--parallel', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd,versions=True) @testWatchdog def test_tmp_binary_parallel(self): toRestore = self.bkpHelper.getAllArrayNames() # All arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'binary', '--parallel', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'binary', '--parallel', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd) @testWatchdog def test_tmp_binary_allVersions(self): toRestore = self.bkpHelper.getAllArrayNames(4) # All versions of arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save','binary', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'binary', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd,versions=True) @testWatchdog def test_tmp_binary_zip(self): toRestore = self.bkpHelper.getAllArrayNames() # All arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'binary', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'binary', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd) @testWatchdog def test_tmp_binary_parallel_allVersions(self): toRestore = self.bkpHelper.getAllArrayNames(4) # All versions of arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'binary', '--parallel', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'binary', '--parallel', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd,versions=True) @testWatchdog def test_tmp_binary_parallel_zip(self): toRestore = self.bkpHelper.getAllArrayNames() # All arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'binary', '--parallel', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'binary', '--parallel', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd) @testWatchdog def test_tmp_binary_allVersions_zip(self): toRestore = self.bkpHelper.getAllArrayNames(4) # All versions of arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'binary', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'binary', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd,versions=True) @testWatchdog def test_tmp_binary_parallel_allVersions_zip(self): toRestore = self.bkpHelper.getAllArrayNames(4) # All versions of arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'binary', '--parallel', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'binary', '--parallel', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd,versions=True) @testWatchdog def test_tmp_opaque_parallel(self): toRestore = self.bkpHelper.getAllArrayNames() # All arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'opaque', '--parallel', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'opaque', '--parallel', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd) @testWatchdog def test_tmp_opaque_allVersions(self): toRestore = self.bkpHelper.getAllArrayNames(4) # All versions of arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save','opaque', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'opaque', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd,versions=True) @testWatchdog def test_tmp_opaque_zip(self): toRestore = self.bkpHelper.getAllArrayNames() # All arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'opaque', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'opaque', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd) @testWatchdog def test_tmp_opaque_parallel_allVersions(self): toRestore = self.bkpHelper.getAllArrayNames(4) # All versions of arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'opaque', '--parallel', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'opaque', '--parallel', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd,versions=True) @testWatchdog def test_tmp_opaque_parallel_zip(self): toRestore = self.bkpHelper.getAllArrayNames() # All arrays should be restored toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'opaque', '--parallel', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'opaque', '--parallel', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd) @testWatchdog def test_tmp_opaque_allVersions_zip(self): toRestore = self.bkpHelper.getAllArrayNames(4) # All versions of arrays should be restored toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'opaque', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'opaque', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd,versions=True) @testWatchdog def test_tmp_opaque_parallel_allVersions_zip(self): toRestore = self.bkpHelper.getAllArrayNames(4) # All versions of arrays should be restored toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'opaque', '--parallel', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'opaque', '--parallel', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd,versions=True) @testWatchdog def test_tmp_text(self): toRestore = self.bkpHelper.getAllArrayNames() # All arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'text', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'text', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd) @testWatchdog def test_tmp_binary(self): toRestore = self.bkpHelper.getAllArrayNames() # All arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'binary', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'binary', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd) @testWatchdog def test_tmp_opaque(self): toRestore = self.bkpHelper.getAllArrayNames() # All arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'opaque', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'opaque', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore,saveCmd,restoreCmd) # "Litmus" test to make sure the testing infrastructure is working # as advertized: the test intentionally overwrites the restored # arrays and makes sure that they do NOT macth the original arrays. @testWatchdog def test_tmp_neg_binary(self): toRestore = self.bkpHelper.getAllArrayNames() # All arrays should be restored. toRestore = [ # Keep only non-temp arrays (temps are skipped). a_name for a_name in toRestore if not any([a_name in t_name for t_name in self._tempArrays]) ] saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'binary', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'binary', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] sys.stderr.write('\n' + ' '.join(restoreCmd) + '\n') exits,outs=self._cmdRunner.waitForProcesses( [self._cmdRunner.runSubProcess(saveCmd,useShell=True)], True ) self.bkpHelper.removeArrays(self._tempArrays + self._arrays) exits,outs = self._cmdRunner.waitForProcesses( [self._cmdRunner.runSubProcess(restoreCmd,useShell=True)], True ) restored = self.bkpHelper.getDBArrays(versions=False) print exits,outs print 'restored',restored self.assertTrue( set(toRestore) == set(restored), 'Restored arrays do not match initial conditions!' ) # Mess up the data in the restored arrays. iqueryCmd = ['iquery','-c','$IQUERY_HOST','-p','$IQUERY_PORT','-naq'] for a_pair in [('B1','A3'),('C2','B1'),('A3','C2')]: cmd = iqueryCmd + ['\"store(' + a_pair[0] + ',' + a_pair[1] + ')\"'] exits,outs = self._cmdRunner.waitForProcesses( [self._cmdRunner.runSubProcess(cmd,useShell=True)], True ) # Check that the arrays are NOT the same. self.assertTrue( not self.bkpHelper.checkArrayData(toRestore), 'Restored array data should not match initial conditions!' )
class BkpTestHelper: def __init__(self, scidbUser, hosts, nInst, bkpFolder, allArrays): self._scidbUser = scidbUser self._hosts = hosts self._nInst = nInst self._createQuery = CREATE_800x40_ARRAYS self._cmdRunner = CommandRunner() self.__bkpFolder = bkpFolder self.__allArrays = allArrays def setCreateQuery(self, q): self._createQuery = q def getAllArrayNames(self, versions=0): R = list(self.__allArrays) if (versions > 0): R = [a + '@' + str(i + 1) for a in R for i in range(versions)] return R def createArrays(self, arrays, iRange=[5], backup=False): storeQueries = [[ x.replace('<name>', n).replace('<i>', str(i)) for x in self._createQuery ] for n in arrays for i in iRange] bkpStoreQueries = [[ 'iquery', '-naq', 'store(' + n + ',' + n + '_bkp)' ] for n in arrays for i in iRange] for qz in zip(storeQueries, bkpStoreQueries): R = self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses([qz[0]]), outputs=False) if (backup): self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses([qz[1]]), outputs=False) def reCreateArrays(self, arrays, versions=0): bkpNames = [[name + '_bkp@' + str(v + 1) for v in range(versions)] for name in arrays] for pair in zip(arrays, bkpNames): name = pair[0] bkps = pair[1] for bkp in bkps: storeQ = ['iquery', '-naq', 'store(' + bkp + ',' + name + ')'] R = self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses([storeQ]), outputs=True) def removeArrays(self, arrays, backups=False): template = ['iquery', '-naq', 'remove(<name>)'] removeQueries = [[x.replace('<name>', n) for x in template] for n in arrays] bkpRemoveQueries = [ [x.replace('<name>', n + '_bkp') for x in template] for n in arrays ] self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses(removeQueries), outputs=False) if (backups): self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses(bkpRemoveQueries), outputs=False) def getDBArrays(self, versions=False): v = '' if (versions): v = ',true' q = ['iquery', '-o', 'csv', '-aq', 'list(\'arrays\'' + v + ')'] exits, outs = self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses([q], ), True) M = [ re.compile('\'[^\']+\'').match(x) for x in outs[0][0].split('\n')[1:] ] A = [m.group().replace('\'', '') for m in M if not (m is None)] # Filter out backups: A = [a for a in A if '_bkp' not in a] if (versions): A = [a for a in A if '@' in a] return A def removeBackup(self, bkpUtil): rmCmd = [bkpUtil, '-d', self.__bkpFolder] proc = self._cmdRunner.runSubProcess(rmCmd, si=subprocess.PIPE) proc.stdin.write('y' + '\n') proc.stdin.close() proc.stdin = None self._cmdRunner.waitForProcesses([proc]) def getBackupFolder(self): return self.__bkpFolder def strToNumList(self, data): L = [re.compile('[0-9\.\+\-Ee]+').search(x) for x in data.split('\n')] L = [m.group() for m in L if not m is None] return L def checkArrayData(self, arrays): compQ = [ 'iquery', '-c', os.environ['IQUERY_HOST'], '-p', os.environ['IQUERY_PORT'], '-ocsv', '-aq', 'aggregate(filter(apply(join(T1,T2),diff,T1.a - T2.a), diff<>0.0),count(*))' ] copyQ = [ 'iquery', '-c', os.environ['IQUERY_HOST'], '-p', os.environ['IQUERY_PORT'], '-ocsv', '-naq' ] removeQ = [ 'iquery', '-c', os.environ['IQUERY_HOST'], '-p', os.environ['IQUERY_PORT'], '-ocsv', '-naq', 'remove(<name>)' ] copyQueries = [copyQ + ['store(' + n + ',T1)'] for n in arrays] copyBkpQueries = [copyQ + ['store(' + n + '_bkp,T2)'] for n in arrays] if (len(arrays) > 0): if ('@' in arrays[0]): nVer = [x.split('@') for x in arrays] copyBkpQueries = [ copyQ + ['store(' + z[0] + '_bkp@' + z[1] + ',T2)'] for z in nVer ] counts = [] for i in range(len(arrays)): cp = copyQueries[i] cpBkp = copyBkpQueries[i] R = self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses([cp]), outputs=True) R = self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses([cpBkp]), outputs=True) R = self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses([compQ]), outputs=True) text = R[1][0][0] lines = text.split('\n') counts.append(int(lines[1]) == 0) rqs = [ [ 'iquery', '-c', os.environ['IQUERY_HOST'], '-p', os.environ['IQUERY_PORT'], '-naq', 'remove(T1)' ], [ 'iquery', '-c', os.environ['IQUERY_HOST'], '-p', os.environ['IQUERY_PORT'], '-naq', 'remove(T2)' ], ] self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses(rqs), outputs=False) return all(counts)
def run(self): self._stopped = False if not os.path.exists(self.CloneBaseDir): os.mkdir(self.CloneBaseDir) while not self._stopped: if len(self.PendingJobQueue) > 0: job = self.PendingJobQueue.pop() try: job.Status = "Active" self.ActiveJobQueue.append(job) print(job) cloneDir = "{}/{}".format(self.CloneBaseDir, job.JobId) os.mkdir(cloneDir) start = datetime.datetime.now() job.Start = str(start) output, err, p = gitClone(cloneDir, job) print(output) print(err) rc = p.returncode if rc != 0: job.Stop = str(datetime.datetime.now()) job.Status = "GitFailed" job.ReturnCode = rc job.StdErr = err job.StdOut = output self.CompletedJobs.add(job) self.ActiveJobQueue.pop() else: requirementsPath = os.path.abspath(cloneDir + "/requirements.txt") if os.path.exists(requirementsPath): p = CommandRunner.RunCommand( ['pip', "install", "-r", requirementsPath], job) p = CommandRunner.RunCommand([ 'python', '-m', job.Src.replace(".py", "").replace( "/", ".").replace("\\", "."), job.Parameters ], job, dir=cloneDir) print("JOB OUT-" + job.StdOut) print("JOB ERR-" + job.StdErr) job.Stop = str(datetime.datetime.now()) job.ReturnCode = p.poll() job.Status = "Complete" if job.ReturnCode == 0 else "Failed" self.CompletedJobs.add(job) self.ActiveJobQueue.pop() except Exception as e: job.Finish = str(datetime.datetime.now()) job.StdErr = "Internal OmniTask Failure " + str( traceback.format_exc()) job.Status = "Failed" job.Stop = str(datetime.datetime.now()) self.CompletedJobs.add(job) self.ActiveJobQueue.pop() sleep(1)
class BkpTestHelper: def __init__( self, scidbUser, hosts, nInst, bkpFolder, allArrays ): self._scidbUser = scidbUser self._hosts = hosts self._nInst = nInst self._createQuery = CREATE_200x200_ARRAYS self._cmdRunner = CommandRunner() self.__bkpFolder = bkpFolder self.__allArrays = allArrays def setCreateQuery(self,q): self._createQuery = q def getAllArrayNames(self,versions=0): R = list(self.__allArrays) if (versions > 0): R = [a + '@' + str(i+1) for a in R for i in range(versions)] return R def createArrays(self,arrays,iRange=[5],backup=False,temp=False): # For regular arrays - different store and backup queries storeQueries = [ [x.replace('<name>',n).replace('<i>',str(i)) for x in self._createQuery] for n in arrays for i in iRange ] bkpStoreQueries = [ ['iquery','-naq','store(' + n + ',' + n + '_bkp)'] for n in arrays for i in iRange ] createQueries = itertools.cycle([[]]) if (temp): # Create queries are strictly for temp arrays. schema = re.compile('\<[^\]]+\]').search(self._createQuery[-1]) createQueries = [ self._createQuery[:-1] + ['create temp array ' + n + ' ' + schema.group()] for n in arrays ] for qz in zip(storeQueries,bkpStoreQueries,createQueries): if (len(qz[2]) > 0): R0 = self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses( [qz[2]] ), outputs=False ) R = self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses( [qz[0]] ), outputs=False ) if (backup): self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses( [qz[1]] ), outputs=False ) def reCreateArrays(self,arrays,versions=0,temp=False): bkpNames = [[name + '_bkp@' + str(v+1) for v in range(versions)] for name in arrays] if (temp): bkpNames = [[name + '_bkp'] for name in arrays] # Create queries are strictly for temp arrays. schema = re.compile('\<[^\]]+\]').search(self._createQuery[-1]) for pair in zip(arrays,bkpNames): name = pair[0] bkps = pair[1] for bkp in bkps: if (temp): createQ = ['iquery','-aq','create temp array ' + name + ' ' + schema.group()] R0 = self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses([createQ]),outputs=True ) storeQ = ['iquery','-naq','store(' + bkp + ',' + name + ')'] R = self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses([storeQ]),outputs=True ) def removeArrays(self,arrays,backups=False): template = [ 'iquery', '-naq', 'remove(<name>)' ] removeQueries = [[x.replace('<name>',n) for x in template] for n in arrays] bkpRemoveQueries = [[x.replace('<name>',n + '_bkp') for x in template] for n in arrays] self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses( removeQueries ), outputs=False ) if (backups): self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses( bkpRemoveQueries ), outputs=False ) def getDBArrays(self,versions=False): v = '' if (versions): v = ',true' q = ['iquery','-o','csv','-aq','list(\'arrays\'' + v + ')'] exits,outs = self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses( [q], ), True ) M = [re.compile('\'[^\']+\'').match(x) for x in outs[0][0].split('\n')[1:]] A = [m.group().replace('\'','') for m in M if not (m is None)] # Filter out backups: A = [a for a in A if '_bkp' not in a] if (versions): A = [a for a in A if '@' in a] return A def removeBackup(self,bkpUtil): rmCmd = [bkpUtil,'-d',self.__bkpFolder] proc = self._cmdRunner.runSubProcess(rmCmd,si=subprocess.PIPE) proc.stdin.write('y' + '\n') proc.stdin.close() proc.stdin = None self._cmdRunner.waitForProcesses([proc]) def getBackupFolder(self): return self.__bkpFolder def strToNumList(self,data): L = [re.compile('[0-9\.\+\-Ee]+').search(x) for x in data.split('\n')] L = [m.group() for m in L if not m is None] return L def checkArrayData(self,arrays): compQ = [ 'iquery', '-c', os.environ['IQUERY_HOST'], '-p', os.environ['IQUERY_PORT'], '-ocsv', '-aq', 'aggregate(filter(apply(join(T1,T2),diff,T1.a - T2.a), diff<>0.0),count(*))' ] copyQ = [ 'iquery', '-c', os.environ['IQUERY_HOST'], '-p', os.environ['IQUERY_PORT'], '-ocsv', '-naq' ] removeQ = [ 'iquery', '-c', os.environ['IQUERY_HOST'], '-p', os.environ['IQUERY_PORT'], '-ocsv', '-naq', 'remove(<name>)' ] copyQueries = [copyQ + ['store(' + n + ',T1)'] for n in arrays] copyBkpQueries = [copyQ + ['store(' + n + '_bkp,T2)'] for n in arrays] if (len(arrays) > 0): if ('@' in arrays[0]): nVer = [x.split('@') for x in arrays] copyBkpQueries = [copyQ + ['store(' + z[0] + '_bkp@' + z[1] + ',T2)'] for z in nVer] counts = [] for i in range(len(arrays)): cp = copyQueries[i] cpBkp = copyBkpQueries[i] R=self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses( [cp] ), outputs=True ) R = self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses( [cpBkp] ), outputs=True ) R = self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses( [compQ] ), outputs=True ) text = R[1][0][0] lines = text.split('\n') counts.append(int(lines[1]) == 0) rqs = [ ['iquery','-c',os.environ['IQUERY_HOST'],'-p',os.environ['IQUERY_PORT'],'-naq','remove(T1)'], ['iquery','-c',os.environ['IQUERY_HOST'],'-p',os.environ['IQUERY_PORT'],'-naq','remove(T2)'], ] self._cmdRunner.waitForProcesses( self._cmdRunner.runSubProcesses( rqs ), outputs=False ) return all(counts)
class RestoreTest(unittest.TestCase): _cmdRunner = CommandRunner() def __init__(self, *args, **kwargs): super(RestoreTest, self).__init__(*args, **kwargs) def setBackupHelper(self, bh): self.bkpHelper = bh def setUp(self): self.bkpHelper.removeBackup(BACKUP) self.bkpHelper.removeArrays(self.bkpHelper.getAllArrayNames()) self.bkpHelper.reCreateArrays(self.bkpHelper.getAllArrayNames(), 4) def tearDown(self): self.bkpHelper.removeBackup(BACKUP) self.bkpHelper.removeArrays(self.bkpHelper.getAllArrayNames()) def setBackupHelper(self, bh): self.bkpHelper = bh def commonTestBody(self, toRestore, saveCmd, restoreCmd, versions=False): sys.stderr.write('\n' + ' '.join(restoreCmd) + '\n') exits, outs = self._cmdRunner.waitForProcesses( [self._cmdRunner.runSubProcess(saveCmd, useShell=True)], True) self.bkpHelper.removeArrays(self.bkpHelper.getAllArrayNames()) exits, outs = self._cmdRunner.waitForProcesses( [self._cmdRunner.runSubProcess(restoreCmd, useShell=True)], True) restored = self.bkpHelper.getDBArrays(versions) self.assertTrue( set(toRestore) == set(restored), 'Restored arrays do not match initial conditions!') self.assertTrue( self.bkpHelper.checkArrayData(toRestore), 'Restored array data does not match initial conditions!') @testWatchdog def test_text_parallel(self): toRestore = self.bkpHelper.getAllArrayNames( ) # All arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'text', '--parallel', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'text', '--parallel', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd) @testWatchdog def test_text_allVersions(self): toRestore = self.bkpHelper.getAllArrayNames( 4) # All versions of arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'text', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'text', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd, versions=True) @testWatchdog def test_text_zip(self): toRestore = self.bkpHelper.getAllArrayNames( ) # All arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'text', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'text', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd) @testWatchdog def test_text_parallel_allVersions(self): toRestore = self.bkpHelper.getAllArrayNames( 4) # All versions of arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'text', '--parallel', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'text', '--parallel', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd, versions=True) @testWatchdog def test_text_parallel_zip(self): toRestore = self.bkpHelper.getAllArrayNames( ) # All arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'text', '--parallel', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'text', '--parallel', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd) @testWatchdog def test_text_allVersions_zip(self): toRestore = self.bkpHelper.getAllArrayNames( 4) # All versions of arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'text', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'text', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd, versions=True) @testWatchdog def test_text_parallel_allVersions_zip(self): toRestore = self.bkpHelper.getAllArrayNames( 4) # All versions of arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'text', '--parallel', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'text', '--parallel', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd, versions=True) @testWatchdog def test_binary_parallel(self): toRestore = self.bkpHelper.getAllArrayNames( ) # All arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'binary', '--parallel', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'binary', '--parallel', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd) @testWatchdog def test_binary_allVersions(self): toRestore = self.bkpHelper.getAllArrayNames( 4) # All versions of arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'binary', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'binary', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd, versions=True) @testWatchdog def test_binary_zip(self): toRestore = self.bkpHelper.getAllArrayNames( ) # All arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'binary', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'binary', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd) @testWatchdog def test_binary_parallel_allVersions(self): toRestore = self.bkpHelper.getAllArrayNames( 4) # All versions of arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'binary', '--parallel', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'binary', '--parallel', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd, versions=True) @testWatchdog def test_binary_parallel_zip(self): toRestore = self.bkpHelper.getAllArrayNames( ) # All arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'binary', '--parallel', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'binary', '--parallel', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd) @testWatchdog def test_binary_allVersions_zip(self): toRestore = self.bkpHelper.getAllArrayNames( 4) # All versions of arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'binary', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'binary', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd, versions=True) @testWatchdog def test_binary_parallel_allVersions_zip(self): toRestore = self.bkpHelper.getAllArrayNames( 4) # All versions of arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'binary', '--parallel', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'binary', '--parallel', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd, versions=True) @testWatchdog def test_opaque_parallel(self): toRestore = self.bkpHelper.getAllArrayNames( ) # All arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'opaque', '--parallel', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'opaque', '--parallel', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd) @testWatchdog def test_opaque_allVersions(self): toRestore = self.bkpHelper.getAllArrayNames( 4) # All versions of arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'opaque', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'opaque', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd, versions=True) @testWatchdog def test_opaque_zip(self): toRestore = self.bkpHelper.getAllArrayNames( ) # All arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'opaque', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'opaque', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd) @testWatchdog def test_opaque_parallel_allVersions(self): toRestore = self.bkpHelper.getAllArrayNames( 4) # All versions of arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'opaque', '--parallel', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'opaque', '--parallel', '--allVersions', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd, versions=True) @testWatchdog def test_opaque_parallel_zip(self): toRestore = self.bkpHelper.getAllArrayNames( ) # All arrays should be restored saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'opaque', '--parallel', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'opaque', '--parallel', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd) @testWatchdog def test_opaque_allVersions_zip(self): toRestore = self.bkpHelper.getAllArrayNames( 4) # All versions of arrays should be restored saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'opaque', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'opaque', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd, versions=True) @testWatchdog def test_opaque_parallel_allVersions_zip(self): toRestore = self.bkpHelper.getAllArrayNames( 4) # All versions of arrays should be restored saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'opaque', '--parallel', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'opaque', '--parallel', '--allVersions', '-z', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd, versions=True) @testWatchdog def test_text(self): toRestore = self.bkpHelper.getAllArrayNames( ) # All arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'text', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'text', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd) @testWatchdog def test_binary(self): toRestore = self.bkpHelper.getAllArrayNames( ) # All arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'binary', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'binary', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd) @testWatchdog def test_opaque(self): toRestore = self.bkpHelper.getAllArrayNames( ) # All arrays should be restored. saveCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--save', 'opaque', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] restoreCmd = [ BACKUP, '--host', '$IQUERY_HOST', '--port', '$IQUERY_PORT', '--restore', 'opaque', self.bkpHelper.getBackupFolder(), '-f', '"[A|B|C][^_]+$"' ] self.commonTestBody(toRestore, saveCmd, restoreCmd)