def _do_git(self): # make sure we don't have any uncommitted changes print('Checking for Uncommitted Changes') out, err, res = shell.exec_cmd(['git', '--no-pager', 'diff']) if len(out) > 0: self._do_error('There are uncommitted changes. See `git status`.') # make sure we don't have any un-added files print('Checking for Untracked Files') out, err, res = shell.exec_cmd(['git', 'status']) if re.match('Untracked files', str(out)): self._do_error('There are untracked files. See `git status`.') # make sure there isn't an existing tag print("Checking for Duplicate Git Tag") out, err, res = shell.exec_cmd(['git', 'tag']) for ver in str(out).split('\n'): if ver == VERSION: self._do_error("Tag %s already exists" % VERSION) print("Tagging Git Release") out, err, res = shell.exec_cmd(['git', 'tag', '-a', '-m', VERSION, VERSION]) if res > 0: self._do_error("Unable to tag release with git.")
def _do_git(self): # make sure we don't have any uncommitted changes print('Checking for Uncommitted Changes') out, err, res = shell.exec_cmd(['git', '--no-pager', 'diff']) if len(out) > 0: self._do_error('There are uncommitted changes. See `git status`.') # make sure we don't have any un-added files print('Checking for Untracked Files') out, err, res = shell.exec_cmd(['git', 'status']) if re.match('Untracked files', out): self._do_error('There are untracked files. See `git status`.') # make sure there isn't an existing tag print("Checking for Duplicate Git Tag") out, err, res = shell.exec_cmd(['git', 'tag']) for ver in out.split('\n'): if ver == VERSION: self._do_error("Tag %s already exists" % VERSION) print("Tagging Git Release") out, err, res = shell.exec_cmd(['git', 'tag', '-a', '-m', VERSION, VERSION]) if res > 0: self._do_error("Unable to tag release with git.")
def test_4_from_pre_to_pre_casava_transfer(self): """Test pre_casava transfer to project directory""" self.app = self.make_app(argv = ['production', 'transfer', 'J.Doe_00_01', '-f', '120829_SN0001_0001_AA001AAAXX', '--from_pre_casava', '--to_pre_casava', '--quiet']) handler.register(ProductionController) self._run_app() ## Assert data output res = shell.exec_cmd(["ls", "-1", os.path.join(delivery_dir, "120829_AA001AAAXX", "1_120829_AA001AAAXX_barcode")]) res_files = ['1_120829_AA001AAAXX_nophix_10_1_fastq.txt', '1_120829_AA001AAAXX_nophix_10_2_fastq.txt', '1_120829_AA001AAAXX_nophix_12_1_fastq.txt','1_120829_AA001AAAXX_nophix_12_2_fastq.txt', '1_120829_AA001AAAXX_nophix_1_1_fastq.txt','1_120829_AA001AAAXX_nophix_1_2_fastq.txt', '1_120829_AA001AAAXX_nophix_2_1_fastq.txt','1_120829_AA001AAAXX_nophix_2_2_fastq.txt', '1_120829_AA001AAAXX_nophix_3_1_fastq.txt','1_120829_AA001AAAXX_nophix_3_2_fastq.txt', '1_120829_AA001AAAXX_nophix_4_1_fastq.txt','1_120829_AA001AAAXX_nophix_4_2_fastq.txt', '1_120829_AA001AAAXX_nophix_8_1_fastq.txt','1_120829_AA001AAAXX_nophix_8_2_fastq.txt'] self.eq(set(res_files), set(res[0].split())) ## Assert intermediate delivery output res = shell.exec_cmd(["ls", "-1", os.path.join(intermediate_delivery_dir, "120829_AA001AAAXX")]) self.eq(['1_120829_AA001AAAXX_nophix_1-sort-dup.align_metrics','1_120829_AA001AAAXX_nophix_1-sort-dup.bam'], res[0].split()[0:2]) self.eq(['1_120829_AA001AAAXX_nophix_8-sort-dup.insert_metrics','1_120829_AA001AAAXX_nophix_8-sort.bam', 'alignments'], res[0].split()[-3:]) ## Assert pruned yaml file contents with open(os.path.join(delivery_dir, "120829_AA001AAAXX", "project_run_info.yaml")) as fh: runinfo_yaml = yaml.load(fh) self.eq(runinfo_yaml['details'][0]['multiplex'][0]['name'], 'P1_101F_index1') self.eq(runinfo_yaml['details'][0]['multiplex'][0]['description'], 'J.Doe_00_01_P1_101F_index1') self.eq(set(runinfo_yaml['details'][0]['multiplex'][0]['files']), set([os.path.join(delivery_dir,"120829_AA001AAAXX", "1_120829_AA001AAAXX_barcode", os.path.basename(x)) for x in ['1_120829_AA001AAAXX_nophix_1_1_fastq.txt','1_120829_AA001AAAXX_nophix_1_2_fastq.txt']]))
def make_release(self): print('') print("Making Release for Version %s" % VERSION) print('-' * 77) if not self.app.pargs.noprompt: res = raw_input("Continue? [yN] ") if res not in ['Y', 'y', '1']: sys.exit(1) tmp = tempfile.mkdtemp() print("Destination: %s" % tmp) os.makedirs(os.path.join(tmp, 'source')) os.makedirs(os.path.join(tmp, 'doc')) self._do_pep8() self._do_tests() self._do_git() self._do_sphinx(os.path.join(tmp, 'doc')) tar_path = os.path.join(tmp, 'source', 'cement-%s.tar' % VERSION) gzip_path = "%s.gz" % tar_path print("Generating Release Files") cmd_args = ['git', 'archive', VERSION, '--prefix=cement-%s/' % VERSION, '--output=%s' % tar_path] out, err, res = shell.exec_cmd(cmd_args) cmd_args = ['gzip', tar_path] out, err, res = shell.exec_cmd(cmd_args) if res > 0: self._do_error("\n\nFailed generating git archive.\n\n" + "$ %s" % (' '.join(cmd_args), err)) print('')
def get_current_branch(self): try: stdout, stderr, exitcode = \ exec_cmd(['git', '--version']) if sys.version_info[0] >= 3: stdout = stdout.decode('utf8') LOG.debug('Git Version: ' + stdout) except: raise CommandError('Error getting "git --version".') stdout, stderr, exitcode = \ exec_cmd(['git', 'symbolic-ref', 'HEAD']) if sys.version_info[0] >= 3: stdout = stdout.decode('utf8') stderr = stderr.decode('utf8') stdout = stdout.rstrip() if exitcode != 0: io.log_warning( 'Git is in a detached head state. Using branch "default".') return 'default' self._handle_exitcode(exitcode, stderr) LOG.debug('git symbolic-ref result: ' + stdout) # Need to parse branch from ref manually because "--short" is # not supported on git < 1.8 return stdout.split('/')[-1]
def make_release(self): print('') print("Making Release for Version %s" % VERSION) print('-' * 77) if not self.app.pargs.noprompt: res = input("Continue? [yN] ") if res not in ['Y', 'y', '1']: sys.exit(1) tmp = tempfile.mkdtemp() print("Destination: %s" % tmp) os.makedirs(os.path.join(tmp, 'source')) os.makedirs(os.path.join(tmp, 'doc')) self._do_flake8() self._do_tests() self._do_git() self._do_sphinx(os.path.join(tmp, 'doc')) tar_path = os.path.join(tmp, 'source', 'cement-%s.tar' % VERSION) gzip_path = "%s.gz" % tar_path print("Generating Release Files") cmd_args = ['git', 'archive', VERSION, '--prefix=cement-%s/' % VERSION, '--output=%s' % tar_path] out, err, res = shell.exec_cmd(cmd_args) cmd_args = ['gzip', tar_path] out, err, res = shell.exec_cmd(cmd_args) if res > 0: self._do_error("\n\nFailed generating git archive.\n\n" + "$ %s" % (' '.join(cmd_args), err)) print('')
def get_current_branch(self): try: stdout, stderr, exitcode = \ exec_cmd(['git', '--version']) if sys.version_info[0] >= 3: stdout = stdout.decode('utf8') LOG.debug('Git Version: ' + stdout) except: raise CommandError('Error getting "git --version".') stdout, stderr, exitcode = \ exec_cmd(['git', 'symbolic-ref', 'HEAD']) if sys.version_info[0] >= 3: stdout = stdout.decode('utf8') stderr = stderr.decode('utf8') stdout = stdout.rstrip() if exitcode != 0: io.log_warning('Git is in a detached head state. Using branch "default".') return 'default' self._handle_exitcode(exitcode, stderr) LOG.debug('git symbolic-ref result: ' + stdout) # Need to parse branch from ref manually because "--short" is # not supported on git < 1.8 return stdout.split('/')[-1]
def test_wait(self): """Test that submitted shell jobs are run sequentially""" print "running first sleep" out = shell.exec_cmd(["sleep", "3"]) print "finishing first sleep" print "running second sleep" shell.exec_cmd(["sleep", "3"]) print "finishing second sleep"
def run(self): assert os.path.exists(self.app.pargs.run_config), \ "Run configuration file %s does not exist" % \ self.app.pargs.run_config config = yaml.load(open(self.app.pargs.run_config, 'r').read()) assert config, "Run configuration is un-readable (Invalid Yaml?)" assert 'commands' in config.keys(), \ "Run configuration has no commands defined" if 'sleep' not in config.keys(): config['sleep'] = 1 if 'prep' not in config.keys(): config['prep'] = [] if self.app.pargs.with_prep and 'prep' in config.keys(): for prep in config['prep']: shell.exec_cmd(prep, shell=True) items = [] index = 0 for command in config['commands']: item = { 'command' : command, 'id' : index, 'color' : COLORS[index], } item[self.app.pargs.mode] = None items.append(item) index += 1 while True: for i in items: mode = self.app.pargs.mode spawn = getattr(shell, 'spawn_%s' % mode) verbose = self.app.pargs.verbose if i[mode] is None or not i[mode].is_alive(): if self.app.pargs.no_color: msg = '{mode} #{id}: {command}'.format( mode=mode.capitalize(), id=i['id'], command=i['command'], ) else: msg = '{color}{mode} #{id}: {command}'.format( mode=mode.capitalize(), color=i['color'], id=i['id'], command=i['command'], ) print(msg) i[mode] = spawn(runner, args=(i['command'], verbose)) else: pass sleep(int(config['sleep']))
def _do_tests(self): print('Running Nose Tests') out, err, res = shell.exec_cmd(['which', 'nosetests']) nose = out.decode('utf-8').strip() if self.app.pargs.loud: cmd_args = ['coverage', 'run', nose, '--verbosity=3'] res = shell.exec_cmd2(cmd_args) else: cmd_args = ['coverage', 'run', nose, '--verbosity=0'] out, err, res = shell.exec_cmd(cmd_args) if res > 0: self._do_error("\n\nNose tests did not pass.\n\n" + "$ %s\n%s" % (' '.join(cmd_args), err))
def _do_tests(self): print('Running Nose Tests') out, err, res = shell.exec_cmd(['which', 'nosetests']) if self.app.pargs.loud: cmd_args = ['coverage', 'run', out.strip(), '--verbosity=3'] res = shell.exec_cmd2(cmd_args) else: cmd_args = ['coverage', 'run', out.strip(), '--verbosity=0'] out, err, res = shell.exec_cmd(cmd_args) if res > 0: self._do_error("\n\nNose tests did not pass.\n\n" + "$ %s\n%s" % (' '.join(cmd_args), err))
def setUp(self): ## setup empty files for k,v in data_files().items(): if not os.path.exists(os.path.join(filedir, k)): if not os.path.exists(os.path.dirname(os.path.join(filedir, k))): os.makedirs(os.path.dirname(os.path.join(filedir, k))) with open(os.path.join(filedir, k), "w") as fh: print "Preparing test: writing to file {}".format(k) fh.write(v) for f in empty_files(): if not os.path.exists(os.path.join(filedir, f)): print "Preparing test: touching file {}".format(f) if not os.path.exists(os.path.dirname(os.path.join(filedir, f))): os.makedirs(os.path.dirname(os.path.join(filedir, f))) shell.exec_cmd(['touch', os.path.join(filedir, f)])
def _do_pep8(self): print("Checking PEP8 Compliance") cmd_args = ['pep8', '-r', 'cement/', '--exclude=*.pyc'] out, err, res = shell.exec_cmd(cmd_args) if res > 0: self._do_error("\n\nPEP8 checks did not pass.\n\n" + "$ %s\n%s" % (' '.join(cmd_args), out))
def execute(self, bundle_name, paths, targets): if len(paths) < 1 or len(targets) < 1: self.log.error('Invalid rule: paths or targets is empty') rt_out_dir = self.out_dir + '/' + bundle_name if Path(rt_out_dir).exists(): print('Output directory exists. Files may be overwritten.') shell.Prompt("Press Enter To Continue", default='ENTER') else: Path(rt_out_dir).mkdir(parents=True) for path in paths: if not path.endswith('/'): path += '/' for target in targets: target.rstrip('/') if target != '*': target_path = path + target else: target_path = path.rstrip('/') self.log.debug('extracting: %s' % target_path, __name__) self.log.debug('to: %s' % rt_out_dir, __name__) cmd = ['adb', 'pull', '-a', target_path, rt_out_dir] stdout, stderr, exitcode = shell.exec_cmd(cmd) if exitcode != 0: print("error: failed to extract %s" % target_path, file=sys.stderr) print(stderr, file=sys.stderr) else: print('extracted: %s to %s' % (target_path, rt_out_dir))
def test_3_from_pre_to_casava_transfer(self): """Test from pre-casava to casava transfer to project directory""" self.app = self.make_app(argv = ['production', 'transfer', 'J.Doe_00_01', '-f', '120829_SN0001_0001_AA001AAAXX', '--from_pre_casava']) handler.register(ProductionController) self._run_app() res = shell.exec_cmd(["ls", "-1", os.path.join(delivery_dir, "P1_101F_index1", "120829_AA001AAAXX")]) self.eq(['1_120829_AA001AAAXX_barcode', '1_120829_AA001AAAXX_nophix_1-sort-dup.align_metrics', '1_120829_AA001AAAXX_nophix_1-sort-dup.bam', '1_120829_AA001AAAXX_nophix_1-sort-dup.dup_metrics', '1_120829_AA001AAAXX_nophix_1-sort-dup.hs_metrics', '1_120829_AA001AAAXX_nophix_1-sort-dup.insert_metrics', '1_120829_AA001AAAXX_nophix_1-sort.bam', 'P1_101F_index1-bcbb-config.yaml', 'alignments'], res[0].split())
def _do_flake8(self): print("Checking Flake8 Compliance (pep8, pycodestyle, mccabe, etc)") cmd_args = ['flake8', 'cement/', 'tests/'] out, err, res = shell.exec_cmd(cmd_args) if res > 0: self._do_error("\n\nFlake8 checks did not pass.\n\n" + "$ %s\n%s" % (' '.join(cmd_args), str(out)))
def _do_sphinx(self, dest_path): print("Building Documentation") cmd_args = ['rm', '-rf', 'docs/build/*'] cmd_args = ['sphinx-build', 'doc/source', dest_path] out, err, res = shell.exec_cmd(cmd_args) if res > 0: self._do_error("\n\nFailed to build sphinx documentation\n\n" + "$ %s\n%s" % (' '.join(cmd_args), out))
def get_message(self): stdout, stderr, exitcode = \ exec_cmd(['git', 'log', '--oneline', '-1']) if sys.version_info[0] >= 3: stdout = stdout.decode('utf8') stderr = stderr.decode('utf8') self._handle_exitcode(exitcode, stderr) return stdout.rstrip().split(' ', 1)[1]
def runpipe(): (stdout, stderr, returncode) = shell.exec_cmd(cmd_args, kw.get('shell', False)) if returncode and not ignore_error: if capture: self.app.log.error(stderr) raise Exception("Subprocess return code: {}".format(returncode)) if capture: return stdout
def test_wrapper(self): """Test basic wrapper functionality""" lsw = LsWrapper() self.assertEqual(repr(lsw), "<class 'tests.wrappers.test_wrappers.LsWrapper'>") print str(lsw) print lsw.cmd_args() out= shell.exec_cmd(lsw.cmd_args()) print out
def runpipe(): (stdout, stderr, returncode) = shell.exec_cmd(cmd, **{'shell': True}) if returncode and not ignore_error: if capture: LOG.error(stderr) raise Exception("Subprocess return code: {}".format(returncode)) if capture: return stdout
def runpipe(): (stdout, stderr, returncode) = shell.exec_cmd(cmd, **{'shell':True}) if returncode and not ignore_error: if capture: LOG.error(stderr) raise Exception("Subprocess return code: {}".format(returncode)) if capture: return stdout
def setUp(self): ## setup empty files for k, v in data_files().items(): if not os.path.exists(os.path.join(filedir, k)): if not os.path.exists(os.path.dirname(os.path.join(filedir, k))): os.makedirs(os.path.dirname(os.path.join(filedir, k))) with open(os.path.join(filedir, k), "w") as fh: print "Preparing test: writing to file {}".format(k) fh.write(v) for f in empty_files(): if not os.path.exists(os.path.join(filedir, f)): print "Preparing test: touching file {}".format(f) if not os.path.exists(os.path.dirname(os.path.join(filedir, f))): os.makedirs(os.path.dirname(os.path.join(filedir, f))) shell.exec_cmd(['touch', os.path.join(filedir, f)])
def _do_sphinx(self, dest_path): print("Building Documentation") cmd_args = ['rm', '-rf', 'docs/build/*'] cmd_args = ['sphinx-build', 'doc/source', dest_path] out, err, res = shell.exec_cmd(cmd_args) if res > 0: self._do_error("\n\nFailed to build sphinx documentation\n\n" + "$ %s\n%s" % (' '.join(cmd_args), str(out)))
def do_zip(self, location): io.log_info('creating zip using git archive HEAD') stdout, stderr, exitcode = \ exec_cmd(['git', 'archive', '-v', '--format=zip', '-o', location, 'HEAD']) if sys.version_info[0] >= 3: stderr = stderr.decode('utf8') self._handle_exitcode(exitcode, stderr) io.log_info('git archive output: ' + stderr)
def runpipe(): (stdout, stderr, returncode) = shell.exec_cmd(cmd_args) if returncode and not ignore_error: if capture: self.app.log.error(stderr) raise Exception( "Subprocess return code: {}".format(returncode)) if capture: return stdout
def _run_cmd(self, cmd, handle_exitcode=True): stdout, stderr, exitcode = exec_cmd(cmd) stdout = utils.decode_bytes(stdout).strip() stderr = utils.decode_bytes(stderr).strip() if handle_exitcode: self._handle_exitcode(exitcode, stderr) return stdout, stderr, exitcode
def _run_cmd(self, cmd, handle_exitcode=True): stdout, stderr, exitcode = exec_cmd(cmd) if sys.version_info[0] >= 3: stdout = stdout.decode('utf8') stderr = stderr.decode('utf8') stdout = stdout.strip() stderr = stderr.strip() if handle_exitcode: self._handle_exitcode(exitcode, stderr) return stdout, stderr, exitcode
def test_from_casava_to_casava_custom_transfer(self): """Test from casava to casava transfer to custom project directory""" self.app = self.make_app(argv = ['production', 'transfer', 'J.Doe_00_04', '--transfer_dir', 'j_doe_00_04_custom']) handler.register(ProductionController) self._run_app() delivery_dir = os.path.abspath(os.path.join(filedir, "data", "projects", "j_doe_00_04_custom", "data")) with open(os.path.join(delivery_dir, "P001_101_index3", "120924_CC003CCCXX", "P001_101_index3-bcbb-config.yaml")) as fh: runinfo_yaml = yaml.load(fh) res = shell.exec_cmd(["ls", "-1", os.path.join(delivery_dir, "P001_101_index3", "120924_CC003CCCXX")]) self.eq(len(set(res[0].split())), 22)
def get_current_branch(self): stdout, stderr, exitcode = \ exec_cmd(['git', 'rev-parse', '--abbrev-ref', 'HEAD']) if sys.version_info[0] >= 3: stdout = stdout.decode('utf8') stderr = stderr.decode('utf8') stdout = stdout.rstrip() self._handle_exitcode(exitcode, stderr) LOG.debug('git current-branch result: ' + stdout) return stdout
def test_7_from_casava_to_casava_transfer(self): """Test from casava to casava transfer to custom project directory""" self.app = self.make_app(argv = ['production', 'transfer', 'J.Doe_00_04', '--transfer_dir', 'j_doe_00_04_custom']) handler.register(ProductionController) self._run_app() delivery_dir = os.path.abspath(os.path.join(filedir, "data", "projects", "j_doe_00_04_custom", "data")) with open(os.path.join(delivery_dir, "P001_101_index3", "120924_CC003CCCXX", "P001_101_index3-bcbb-config.yaml")) as fh: runinfo_yaml = yaml.load(fh) self.eq(runinfo_yaml['details'][0]['multiplex'][0]['files'], [os.path.join(delivery_dir, "P001_101_index3", "120924_CC003CCCXX", "nophix", x) for x in ['1_120924_CC003CCCXX_7_nophix_1_fastq.txt.gz', '1_120924_CC003CCCXX_7_nophix_2_fastq.txt.gz']]) res = shell.exec_cmd(["ls", "-1", os.path.join(delivery_dir, "P001_101_index3", "120924_CC003CCCXX")]) self.eq(set(res[0].split()), set(['1_120924_CC003CCCXX_7_nophix-sort-dup-insert.pdf', '1_120924_CC003CCCXX_7_nophix-sort-dup-summary.aux', '1_120924_CC003CCCXX_7_nophix-sort-dup-summary.log', '1_120924_CC003CCCXX_7_nophix-sort-dup-summary.pdf', '1_120924_CC003CCCXX_7_nophix-sort-dup-summary.tex', '1_120924_CC003CCCXX_7_nophix-sort-dup.align_metrics', '1_120924_CC003CCCXX_7_nophix-sort-dup.bam', '1_120924_CC003CCCXX_7_nophix-sort.bam', '1_120924_CC003CCCXX_7_nophix-sort-dup.dup_metrics', '1_120924_CC003CCCXX_7_nophix-sort-dup.hs_metrics', '1_120924_CC003CCCXX_7_nophix-sort-dup.insert_metrics', 'P001_101_index3-bcbb-config.yaml', 'alignments', 'fastq_screen', 'nophix']))
def get_version_label(self): io.log_info('Getting version label from git with git-describe') stdout, stderr, exitcode = \ exec_cmd(['git', 'describe', '--always', '--abbrev=4']) if sys.version_info[0] >= 3: stdout = stdout.decode('utf8') stderr = stderr.decode('utf8') self._handle_exitcode(exitcode, stderr) #Replace dots with underscores return stdout[:-1].replace('.', '_')
def run(self): config = yaml.load(open(self.app.pargs.run_config, 'r').read()) if self.app.pargs.with_prep and 'prep' in config.keys(): for prep in config['prep']: shell.exec_cmd(prep, shell=True) items = [] index = 0 for command in config['commands']: item = { 'command' : command, 'id' : index, 'color' : COLORS[index], } item[self.app.pargs.mode] = None items.append(item) index += 1 while True: for i in items: mode = self.app.pargs.mode spawn = getattr(shell, 'spawn_%s' % mode) verbose = self.app.pargs.verbose if i[mode] is None or not i[mode].is_alive(): msg = '{color}{mode} #{tid}: {command}'.format( mode=mode.capitalize(), color=i['color'], tid=i['id'], command=i['command'], ) print(msg) i[mode] = spawn(runner, args=(i['command'], verbose)) else: pass sleep(int(config['sleep']))
def usage(self): cmd = ['adb', 'shell', 'dumpsys', 'usagestats'] stdout, stderr, exitcode = shell.exec_cmd(cmd) if exitcode == 0: if self.keyword: for line in stdout.decode('utf-8').split('\n'): if self.keyword in line: print(line) else: print(stdout.decode('utf-8')) else: self.log.warning('package.usage module exits unexpectedly: %d.' % exitcode)
def untracked_changes_exist(self): stdout, stderr, exitcode = \ exec_cmd(['git', 'diff', '--numstat']) if sys.version_info[0] >= 3: stdout = stdout.decode('utf8') stderr = stderr.decode('utf8') stdout = stdout.strip() LOG.debug('git diff --numstat result: ' + stdout + ' with errors: ' + stderr) if stdout: return True return False
def run(self): config = yaml.load(open(self.app.pargs.run_config, 'r').read()) if self.app.pargs.with_prep and 'prep' in config.keys(): for prep in config['prep']: shell.exec_cmd(prep, shell=True) items = [] index = 0 for command in config['commands']: item = { 'command': command, 'id': index, 'color': COLORS[index], } item[self.app.pargs.mode] = None items.append(item) index += 1 while True: for i in items: mode = self.app.pargs.mode spawn = getattr(shell, 'spawn_%s' % mode) verbose = self.app.pargs.verbose if i[mode] is None or not i[mode].is_alive(): msg = '{color}{mode} #{tid}: {command}'.format( mode=mode.capitalize(), color=i['color'], tid=i['id'], command=i['command'], ) print(msg) i[mode] = spawn(runner, args=(i['command'], verbose)) else: pass sleep(int(config['sleep']))
def do_zip(self, location): cwd = os.getcwd() try: # must be in project root for git archive to work. fileoperations._traverse_to_project_root() io.log_info('creating zip using git archive HEAD') stdout, stderr, exitcode = \ exec_cmd(['git', 'archive', '-v', '--format=zip', '-o', location, 'HEAD']) if sys.version_info[0] >= 3: stderr = stderr.decode('utf8') self._handle_exitcode(exitcode, stderr) io.log_info('git archive output: ' + stderr) finally: os.chdir(cwd)
def test_7_from_casava_to_casava_transfer(self): """Test from casava to casava transfer to custom project directory""" self.app = self.make_app(argv=[ 'production', 'transfer', 'J.Doe_00_04', '--transfer_dir', 'j_doe_00_04_custom' ]) handler.register(ProductionController) self._run_app() delivery_dir = os.path.abspath( os.path.join(filedir, "data", "projects", "j_doe_00_04_custom", "data")) with open( os.path.join(delivery_dir, "P001_101_index3", "120924_CC003CCCXX", "P001_101_index3-bcbb-config.yaml")) as fh: runinfo_yaml = yaml.load(fh) self.eq(runinfo_yaml['details'][0]['multiplex'][0]['files'], [ os.path.join(delivery_dir, "P001_101_index3", "120924_CC003CCCXX", "nophix", x) for x in [ '1_120924_CC003CCCXX_7_nophix_1_fastq.txt.gz', '1_120924_CC003CCCXX_7_nophix_2_fastq.txt.gz' ] ]) res = shell.exec_cmd([ "ls", "-1", os.path.join(delivery_dir, "P001_101_index3", "120924_CC003CCCXX") ]) self.eq( set(res[0].split()), set([ '1_120924_CC003CCCXX_7_nophix-sort-dup-insert.pdf', '1_120924_CC003CCCXX_7_nophix-sort-dup-summary.aux', '1_120924_CC003CCCXX_7_nophix-sort-dup-summary.log', '1_120924_CC003CCCXX_7_nophix-sort-dup-summary.pdf', '1_120924_CC003CCCXX_7_nophix-sort-dup-summary.tex', '1_120924_CC003CCCXX_7_nophix-sort-dup.align_metrics', '1_120924_CC003CCCXX_7_nophix-sort-dup.bam', '1_120924_CC003CCCXX_7_nophix-sort.bam', '1_120924_CC003CCCXX_7_nophix-sort-dup.dup_metrics', '1_120924_CC003CCCXX_7_nophix-sort-dup.hs_metrics', '1_120924_CC003CCCXX_7_nophix-sort-dup.insert_metrics', 'P001_101_index3-bcbb-config.yaml', 'alignments', 'fastq_screen', 'nophix' ]))
def _get_public_ssh_key(keypair_name): key_file = fileoperations.get_ssh_folder() + keypair_name if os.path.exists(key_file): file_name = key_file elif os.path.exists(key_file + '.pem'): file_name = key_file + '.pem' else: raise NotSupportedError(strings['ssh.filenotfound'].replace( '{key-name}', keypair_name)) try: stdout, stderr, returncode = exec_cmd(['ssh-keygen', '-y', '-f', file_name]) if returncode != 0: raise CommandError('An error occurred while trying ' 'to get ssh public key') key_material = stdout return key_material except OSError: CommandError(strings['ssh.notpresent'])
def _get_public_ssh_key(keypair_name): key_file = fileoperations.get_ssh_folder() + keypair_name if os.path.exists(key_file): file_name = key_file elif os.path.exists(key_file + '.pem'): file_name = key_file + '.pem' else: raise NotSupportedError(strings['ssh.filenotfound'].replace( '{key-name}', keypair_name)) try: stdout, stderr, returncode = exec_cmd( ['ssh-keygen', '-y', '-f', file_name]) if returncode != 0: raise CommandError('An error occurred while trying ' 'to get ssh public key') key_material = stdout return key_material except OSError: CommandError(strings['ssh.notpresent'])
def test_3_from_pre_to_casava_transfer(self): """Test from pre-casava to casava transfer to project directory""" self.app = self.make_app(argv=[ 'production', 'transfer', 'J.Doe_00_01', '-f', '120829_SN0001_0001_AA001AAAXX', '--from_pre_casava' ]) handler.register(ProductionController) self._run_app() res = shell.exec_cmd([ "ls", "-1", os.path.join(delivery_dir, "P1_101F_index1", "120829_AA001AAAXX") ]) self.eq([ '1_120829_AA001AAAXX_barcode', '1_120829_AA001AAAXX_nophix_1-sort-dup.align_metrics', '1_120829_AA001AAAXX_nophix_1-sort-dup.bam', '1_120829_AA001AAAXX_nophix_1-sort-dup.dup_metrics', '1_120829_AA001AAAXX_nophix_1-sort-dup.hs_metrics', '1_120829_AA001AAAXX_nophix_1-sort-dup.insert_metrics', '1_120829_AA001AAAXX_nophix_1-sort.bam', 'P1_101F_index1-bcbb-config.yaml', 'alignments' ], res[0].split())
def test_exec_cmd_shell_true(): out, err, ret = shell.exec_cmd(['echo KAPLA!'], shell=True) assert ret == 0 assert out == b'KAPLA!\n'
def test_exec_cmd_bad_command(): out, err, ret = shell.exec_cmd(['false']) assert ret == 1
def test_exec_cmd(self): out, err, ret = shell.exec_cmd(['echo', 'KAPLA!']) self.eq(ret, 0) self.eq(out, b'KAPLA!\n')
def test_exec_cmd_shell_true(self): out, err, ret = shell.exec_cmd(['echo KAPLA!'], shell=True) self.eq(ret, 0) self.eq(out, b'KAPLA!\n')
def test_exec_cmd_bad_command(self): out, err, ret = shell.exec_cmd(['false']) self.eq(ret, 1)