def createClusterUtil(self, args=None): args = [] if args is None else args subprocess.check_call([ 'toil', 'launch-cluster', '-p=aws', '-z=us-west-2a', '--keyPairName=%s' % self.keyName, '--leaderNodeType=t2.medium', self.clusterName ] + args)
def createClusterUtil(self, args=None): if args is None: args = [] callCommand = ['toil', 'launch-cluster', '-p=aws', '-z=us-west-2a','--keyPairName=%s' % self.keyName, '--leaderNodeType=%s' % self.leaderInstanceType, self.clusterName] callCommand = callCommand + args if args else callCommand subprocess.check_call(callCommand)
def testJobStoreContents(self): """Test toilDebugFile.printContentsOfJobStore(). Runs a workflow that imports 'B.txt' and 'mkFile.py' into the jobStore. 'A.txt', 'C.txt', 'ABC.txt' are then created. This checks to make sure these contents are found in the jobStore and printed.""" contents = ['A.txt', 'B.txt', 'C.txt', 'ABC.txt', 'mkFile.py'] subprocess.check_call([ 'python', os.path.abspath('src/toil/utils/toilDebugFile.py'), self.jobStoreDir, '--listFilesInJobStore=True' ]) jobstoreFileContents = os.path.abspath('jobstore_files.txt') files = [] match = 0 with open(jobstoreFileContents, 'r') as f: for line in f: files.append(line.strip()) for xfile in files: for expected_file in contents: if xfile.endswith(expected_file): match = match + 1 logger.debug(files) logger.debug(contents) logger.debug(match) # C.txt will match twice (once with 'C.txt', and once with 'ABC.txt') assert match == 6 os.remove(jobstoreFileContents)
def testWriteGzipLogs(self): subprocess.check_call([ sys.executable, '-m', helloWorld.__name__, './toilTest', '--clean=always', '--logLevel=debug', '--writeLogsGzip=%s' % self.tempDir ]) self._assertFileTypeExists(self.tempDir, '.log.gz', 'gzip')
def _fixPermissions(tool, workDir): """ Deprecated. Fix permission of a mounted Docker directory by reusing the tool to change ownership. Docker natively runs as a root inside the container, and files written to the mounted directory are implicitly owned by root. :param list baseDockerCall: Docker run parameters :param str tool: Name of tool :param str workDir: Path of work directory to recursively chown """ if os.geteuid() == 0: # we're running as root so this chown is redundant return baseDockerCall = [ 'docker', 'run', '--log-driver=none', '-v', os.path.abspath(workDir) + ':/data', '--rm', '--entrypoint=chown' ] stat = os.stat(workDir) command = baseDockerCall + [tool] + [ '-R', '{}:{}'.format(stat.st_uid, stat.st_gid), '/data' ] for attempt in retry(predicate=dockerPredicate): with attempt: subprocess.check_call(command)
def fetchFiles(self, symLink): """ Fn for testFetchJobStoreFiles() and testFetchJobStoreFilesWSymlinks(). Runs a workflow that imports 'B.txt' and 'mkFile.py' into the jobStore. 'A.txt', 'C.txt', 'ABC.txt' are then created. This test then attempts to get a list of these files and copy them over into ./src from the jobStore, confirm that they are present, and then delete them. """ contents = ['A.txt', 'B.txt', 'C.txt', 'ABC.txt', 'mkFile.py'] outputDir = os.path.abspath('src') cmd = [ 'python', os.path.abspath('src/toil/utils/toilDebugFile.py'), self.jobStoreDir, '--fetch', '*A.txt', '*B.txt', '*C.txt', '*ABC.txt', '*mkFile.py', '--localFilePath=' + os.path.abspath('src'), '--useSymlinks=' + str(symLink) ] subprocess.check_call(cmd) for xfile in contents: matchingFilesFound = recursiveGlob(outputDir, '*' + xfile) self.assertGreaterEqual(len(matchingFilesFound), 1) for fileFound in matchingFilesFound: assert fileFound.endswith(xfile) and os.path.exists(fileFound) if fileFound.endswith('-' + xfile): os.remove(fileFound)
def setUp(self): """Initial set up of variables for the test.""" subprocess.check_call([ 'python', os.path.abspath( 'src/toil/test/utils/ABCWorkflowDebug/debugWorkflow.py') ]) self.jobStoreDir = os.path.abspath('toilWorkflowRun') self.tempDir = self._createTempDir(purpose='tempDir')
def fetch_and_unzip_from_s3(self, filename, data, data_dir): if not os.path.exists(data): s3_loc = os.path.join('http://toil-datasets.s3.amazonaws.com/', filename) fetch_from_s3_cmd = ["wget", "-P", self.test_directory, s3_loc] subprocess.check_call(fetch_from_s3_cmd) # extract the compressed data if not already extracted if not os.path.exists(data_dir): with zipfile.ZipFile(data, 'r') as zip_ref: zip_ref.extractall(self.test_directory)
def createClusterUtil(self, args=None): if args is None: args = [] callCommand = ['toil', 'launch-cluster', self.clusterName, '-p=gce', '--keyPairName=%s' % self.keyName, '--leaderNodeType=%s' % self.leaderInstanceType, '--zone=%s' % self.googleZone] if self.botoDir is not None: callCommand += ['--boto=%s' % self.botoDir] callCommand = callCommand + args if args else callCommand log.info("createClusterUtil: %s" % ''.join(callCommand)) subprocess.check_call(callCommand)
def system(command): """ A convenience wrapper around subprocess.check_call that logs the command before passing it on. The command can be either a string or a sequence of strings. If it is a string shell=True will be passed to subprocess.check_call. :type command: str | sequence[string] """ logger.debug('Running: %r', command) subprocess.check_call(command, shell=isinstance(command, string_types), bufsize=-1)
def testTut04(self): '''Test if toilwdl produces the same outputs as known good outputs for WDL's GATK tutorial #4.''' if self.manual_integration_tests: wdl = os.path.abspath("src/toil/test/wdl/wdl_templates/t04/jointCallingGenotypes.wdl") json = os.path.abspath("src/toil/test/wdl/wdl_templates/t04/jointCallingGenotypes_inputs.json") ref_dir = os.path.abspath("src/toil/test/wdl/wdl_templates/t04/output/") subprocess.check_call(['python', self.program, wdl, json, '-o', self.output_dir]) compare_runs(self.output_dir, ref_dir)
def testMD5sum(self): '''Test if toilwdl produces the same outputs as known good outputs for WDL's GATK tutorial #1.''' wdl = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.wdl') inputfile = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.input') json = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.json') subprocess.check_call(['python', self.program, wdl, json, '-o', self.output_dir]) md5sum_output = os.path.join(self.output_dir, 'md5sum.txt') assert os.path.exists(md5sum_output) os.unlink(md5sum_output)
def testPipe(self): '''Test basic bash input functionality with a pipe.''' wdl = os.path.abspath( "src/toil/test/wdl/wdl_templates/testPipe/call.wdl") json = os.path.abspath( "src/toil/test/wdl/wdl_templates/testPipe/call.json") ref_dir = os.path.abspath( "src/toil/test/wdl/wdl_templates/testPipe/output/") subprocess.check_call( ['python', self.program, wdl, json, '--out_dir', self.output_dir]) compare_runs(self.output_dir, ref_dir)
def testENCODE(self): '''Test if toilwdl produces the same outputs as known good outputs for a short ENCODE run.''' wdl = os.path.abspath( "src/toil/test/wdl/wdl_templates/testENCODE/encode_mapping_workflow.wdl") json = os.path.abspath( "src/toil/test/wdl/wdl_templates/testENCODE/encode_mapping_workflow.wdl.json") ref_dir = os.path.abspath( "src/toil/test/wdl/wdl_templates/testENCODE/output/") subprocess.check_call( ['python', self.program, wdl, json, '--docker_user=None', '--out_dir', self.output_dir]) compare_runs(self.output_dir, ref_dir)
def _dockerKill(containerName, action): """ Deprecated. Kills the specified container. :param str containerName: The name of the container created by docker_call :param int action: What action should be taken on the container? """ running = containerIsRunning(containerName) if running is None: # This means that the container doesn't exist. We will see this if the # container was run with --rm and has already exited before this call. logger.debug( 'The container with name "%s" appears to have already been ' 'removed. Nothing to ' 'do.', containerName) else: if action in (None, FORGO): logger.debug( 'The container with name %s continues to exist as we ' 'were asked to forgo a ' 'post-job action on it.', containerName) else: logger.debug( 'The container with name %s exists. Running ' 'user-specified defer functions.', containerName) if running and action >= STOP: logger.debug('Stopping container "%s".', containerName) for attempt in retry(predicate=dockerPredicate): with attempt: subprocess.check_call( ['docker', 'stop', containerName]) else: logger.debug('The container "%s" was not found to be running.', containerName) if action >= RM: # If the container was run with --rm, then stop will most likely # remove the container. We first check if it is running then # remove it. running = containerIsRunning(containerName) if running is not None: logger.debug('Removing container "%s".', containerName) for attempt in retry(predicate=dockerPredicate): with attempt: subprocess.check_call( ['docker', 'rm', '-f', containerName]) else: logger.debug( 'Container "%s" was not found on the system.' 'Nothing to remove.', containerName)
def test_bsubline_works(): command = lsf.build_bsub_line(cpu=1, mem=2147483648, runtime=1, jobname="Test Job") command.extend(["-K", "echo"]) assert subprocess.check_call(command) == 0
def _setSSH(self): if not os.path.exists('/root/.sshSuccess'): subprocess.check_call([ 'ssh-keygen', '-f', '/root/.ssh/id_rsa', '-t', 'rsa', '-N', '' ]) with open('/root/.sshSuccess', 'w') as f: f.write( 'written here because of restrictive permissions on .ssh dir' ) os.chmod('/root/.ssh', 0o700) subprocess.check_call( ['bash', '-c', 'eval $(ssh-agent) && ssh-add -k']) with open('/root/.ssh/id_rsa.pub') as f: masterPublicKey = f.read() masterPublicKey = masterPublicKey.split(' ')[1] # take 'body' of key # confirm it really is an RSA public key assert masterPublicKey.startswith('AAAAB3NzaC1yc2E'), masterPublicKey return masterPublicKey
def _setSSH(self): """ Generate a key pair, save it in /root/.ssh/id_rsa.pub, and return the public key. The file /root/.sshSuccess is used to prevent this operation from running twice. :return Public key. """ if not os.path.exists('/root/.sshSuccess'): subprocess.check_call([ 'ssh-keygen', '-f', '/root/.ssh/id_rsa', '-t', 'rsa', '-N', '' ]) with open('/root/.sshSuccess', 'w') as f: f.write( 'written here because of restrictive permissions on .ssh dir' ) os.chmod('/root/.ssh', 0o700) subprocess.check_call( ['bash', '-c', 'eval $(ssh-agent) && ssh-add -k']) with open('/root/.ssh/id_rsa.pub') as f: masterPublicKey = f.read() masterPublicKey = masterPublicKey.split(' ')[1] # take 'body' of key # confirm it really is an RSA public key assert masterPublicKey.startswith('AAAAB3NzaC1yc2E'), masterPublicKey return masterPublicKey
def _testExternal(self, moduleName, pyFiles, virtualenv=False): dirPath = self._createTempDir() if virtualenv: self.assertTrue(inVirtualEnv()) # --never-download prevents silent upgrades to pip, wheel and setuptools subprocess.check_call(['virtualenv', '--never-download', dirPath]) sitePackages = os.path.join(dirPath, 'lib', 'python2.7', 'site-packages') # tuple assignment is necessary to make this line immediately precede the try: oldPrefix, sys.prefix, dirPath = sys.prefix, dirPath, sitePackages else: oldPrefix = None try: for relPath in pyFiles: path = os.path.join(dirPath, relPath) mkdir_p(os.path.dirname(path)) with open(path, 'w') as f: f.write('pass\n') sys.path.append(dirPath) try: userScript = importlib.import_module(moduleName) try: self._test(userScript.__name__, expectedContents=pyFiles, allowExtraContents=True) finally: del userScript while moduleName: del sys.modules[moduleName] self.assertFalse(moduleName in sys.modules) moduleName = '.'.join(moduleName.split('.')[:-1]) finally: sys.path.remove(dirPath) finally: if oldPrefix: sys.prefix = oldPrefix
def _coreRsync(cls, ip, args, applianceName='toil_leader', **kwargs): remoteRsync = "docker exec -i %s rsync" % applianceName # Access rsync inside appliance parsedArgs = [] sshCommand = "ssh" strict = kwargs.pop('strict', False) if not strict: sshCommand = "ssh -oUserKnownHostsFile=/dev/null -oStrictHostKeyChecking=no" hostInserted = False # Insert remote host address for i in args: if i.startswith(":") and not hostInserted: i = ("core@%s" % ip) + i hostInserted = True elif i.startswith(":") and hostInserted: raise ValueError("Cannot rsync between two remote hosts") parsedArgs.append(i) if not hostInserted: raise ValueError("No remote host found in argument list") command = ['rsync', '-e', sshCommand, '--rsync-path', remoteRsync] log.debug("Running %r.", command + parsedArgs) return subprocess.check_call(command + parsedArgs)
def sshUtil(self, command): baseCommand = ['toil', 'ssh-cluster', '--insecure', '-p=gce', self.clusterName] callCommand = baseCommand + command subprocess.check_call(callCommand)
def killJob(self, jobID): subprocess.check_call(['bkill', self.getBatchSystemID(jobID)])
def _testFromMain(self): testMethodName = self.id().split('.')[-1] self.assertTrue(testMethodName.endswith('FromMain')) subprocess.check_call( [sys.executable, '-m', self.__module__, testMethodName[:-8]])
def greatGrandChild(cmd): subprocess.check_call(cmd, shell=True)
def stop(self, fileStore): subprocess.check_call(self.cmd + ' -1', shell=True)
def grandChildJob(job, cmd): job.addService(Service(cmd)) job.addChildFn(greatGrandChild, cmd) subprocess.check_call(cmd, shell=True)
def printUnicodeCharacter(): # We want to get a unicode character to stdout but we can't print it directly because of # Python encoding issues. To work around this we print in a separate Python process. See # http://stackoverflow.com/questions/492483/setting-the-correct-encoding-when-piping-stdout-in-python subprocess.check_call([sys.executable, '-c', "print('\\xc3\\xbc')"])
def rsyncUtil(self, src, dest): baseCommand = ['toil', 'rsync-cluster', '--insecure', '-p=gce', self.clusterName] callCommand = baseCommand + [src, dest] subprocess.check_call(callCommand)
def destroyClusterUtil(self): callCommand = ['toil', 'destroy-cluster', '-p=gce', self.clusterName] subprocess.check_call(callCommand)
def cleanJobStoreUtil(self): callCommand = ['toil', 'clean', self.jobStore] subprocess.check_call(callCommand)