def _do_git(self):
# make sure we don't have any uncommitted changes
print('Checking for Uncommitted Changes')
out, err, res = shell.exec_cmd(['git', '--no-pager', 'diff'])
if len(out) > 0:
self._do_error('There are uncommitted changes. See `git status`.')
# make sure we don't have any un-added files
print('Checking for Untracked Files')
out, err, res = shell.exec_cmd(['git', 'status'])
if re.match('Untracked files', str(out)):
self._do_error('There are untracked files. See `git status`.')
# make sure there isn't an existing tag
print("Checking for Duplicate Git Tag")
out, err, res = shell.exec_cmd(['git', 'tag'])
for ver in str(out).split('\n'):
if ver == VERSION:
self._do_error("Tag %s already exists" % VERSION)
print("Tagging Git Release")
out, err, res = shell.exec_cmd(['git', 'tag', '-a', '-m', VERSION,
VERSION])
if res > 0:
self._do_error("Unable to tag release with git.")
def _do_git(self):
# make sure we don't have any uncommitted changes
print('Checking for Uncommitted Changes')
out, err, res = shell.exec_cmd(['git', '--no-pager', 'diff'])
if len(out) > 0:
self._do_error('There are uncommitted changes. See `git status`.')
# make sure we don't have any un-added files
print('Checking for Untracked Files')
out, err, res = shell.exec_cmd(['git', 'status'])
if re.match('Untracked files', out):
self._do_error('There are untracked files. See `git status`.')
# make sure there isn't an existing tag
print("Checking for Duplicate Git Tag")
out, err, res = shell.exec_cmd(['git', 'tag'])
for ver in out.split('\n'):
if ver == VERSION:
self._do_error("Tag %s already exists" % VERSION)
print("Tagging Git Release")
out, err, res = shell.exec_cmd(['git', 'tag', '-a', '-m', VERSION,
VERSION])
if res > 0:
self._do_error("Unable to tag release with git.")
def test_4_from_pre_to_pre_casava_transfer(self):
"""Test pre_casava transfer to project directory"""
self.app = self.make_app(argv = ['production', 'transfer', 'J.Doe_00_01', '-f', '120829_SN0001_0001_AA001AAAXX', '--from_pre_casava', '--to_pre_casava', '--quiet'])
handler.register(ProductionController)
self._run_app()
## Assert data output
res = shell.exec_cmd(["ls", "-1", os.path.join(delivery_dir, "120829_AA001AAAXX", "1_120829_AA001AAAXX_barcode")])
res_files = ['1_120829_AA001AAAXX_nophix_10_1_fastq.txt', '1_120829_AA001AAAXX_nophix_10_2_fastq.txt',
'1_120829_AA001AAAXX_nophix_12_1_fastq.txt','1_120829_AA001AAAXX_nophix_12_2_fastq.txt',
'1_120829_AA001AAAXX_nophix_1_1_fastq.txt','1_120829_AA001AAAXX_nophix_1_2_fastq.txt',
'1_120829_AA001AAAXX_nophix_2_1_fastq.txt','1_120829_AA001AAAXX_nophix_2_2_fastq.txt',
'1_120829_AA001AAAXX_nophix_3_1_fastq.txt','1_120829_AA001AAAXX_nophix_3_2_fastq.txt',
'1_120829_AA001AAAXX_nophix_4_1_fastq.txt','1_120829_AA001AAAXX_nophix_4_2_fastq.txt',
'1_120829_AA001AAAXX_nophix_8_1_fastq.txt','1_120829_AA001AAAXX_nophix_8_2_fastq.txt']
self.eq(set(res_files), set(res[0].split()))
## Assert intermediate delivery output
res = shell.exec_cmd(["ls", "-1", os.path.join(intermediate_delivery_dir, "120829_AA001AAAXX")])
self.eq(['1_120829_AA001AAAXX_nophix_1-sort-dup.align_metrics','1_120829_AA001AAAXX_nophix_1-sort-dup.bam'], res[0].split()[0:2])
self.eq(['1_120829_AA001AAAXX_nophix_8-sort-dup.insert_metrics','1_120829_AA001AAAXX_nophix_8-sort.bam', 'alignments'], res[0].split()[-3:])
## Assert pruned yaml file contents
with open(os.path.join(delivery_dir, "120829_AA001AAAXX", "project_run_info.yaml")) as fh:
runinfo_yaml = yaml.load(fh)
self.eq(runinfo_yaml['details'][0]['multiplex'][0]['name'], 'P1_101F_index1')
self.eq(runinfo_yaml['details'][0]['multiplex'][0]['description'], 'J.Doe_00_01_P1_101F_index1')
self.eq(set(runinfo_yaml['details'][0]['multiplex'][0]['files']), set([os.path.join(delivery_dir,"120829_AA001AAAXX", "1_120829_AA001AAAXX_barcode", os.path.basename(x)) for x in ['1_120829_AA001AAAXX_nophix_1_1_fastq.txt','1_120829_AA001AAAXX_nophix_1_2_fastq.txt']]))
def make_release(self):
print('')
print("Making Release for Version %s" % VERSION)
print('-' * 77)
if not self.app.pargs.noprompt:
res = raw_input("Continue? [yN] ")
if res not in ['Y', 'y', '1']:
sys.exit(1)
tmp = tempfile.mkdtemp()
print("Destination: %s" % tmp)
os.makedirs(os.path.join(tmp, 'source'))
os.makedirs(os.path.join(tmp, 'doc'))
self._do_pep8()
self._do_tests()
self._do_git()
self._do_sphinx(os.path.join(tmp, 'doc'))
tar_path = os.path.join(tmp, 'source', 'cement-%s.tar' % VERSION)
gzip_path = "%s.gz" % tar_path
print("Generating Release Files")
cmd_args = ['git', 'archive', VERSION,
'--prefix=cement-%s/' % VERSION,
'--output=%s' % tar_path]
out, err, res = shell.exec_cmd(cmd_args)
cmd_args = ['gzip', tar_path]
out, err, res = shell.exec_cmd(cmd_args)
if res > 0:
self._do_error("\n\nFailed generating git archive.\n\n" +
"$ %s" % (' '.join(cmd_args), err))
print('')
def get_current_branch(self):
try:
stdout, stderr, exitcode = \
exec_cmd(['git', '--version'])
if sys.version_info[0] >= 3:
stdout = stdout.decode('utf8')
LOG.debug('Git Version: ' + stdout)
except:
raise CommandError('Error getting "git --version".')
stdout, stderr, exitcode = \
exec_cmd(['git', 'symbolic-ref', 'HEAD'])
if sys.version_info[0] >= 3:
stdout = stdout.decode('utf8')
stderr = stderr.decode('utf8')
stdout = stdout.rstrip()
if exitcode != 0:
io.log_warning(
'Git is in a detached head state. Using branch "default".')
return 'default'
self._handle_exitcode(exitcode, stderr)
LOG.debug('git symbolic-ref result: ' + stdout)
# Need to parse branch from ref manually because "--short" is
# not supported on git < 1.8
return stdout.split('/')[-1]
def make_release(self):
print('')
print("Making Release for Version %s" % VERSION)
print('-' * 77)
if not self.app.pargs.noprompt:
res = input("Continue? [yN] ")
if res not in ['Y', 'y', '1']:
sys.exit(1)
tmp = tempfile.mkdtemp()
print("Destination: %s" % tmp)
os.makedirs(os.path.join(tmp, 'source'))
os.makedirs(os.path.join(tmp, 'doc'))
self._do_flake8()
self._do_tests()
self._do_git()
self._do_sphinx(os.path.join(tmp, 'doc'))
tar_path = os.path.join(tmp, 'source', 'cement-%s.tar' % VERSION)
gzip_path = "%s.gz" % tar_path
print("Generating Release Files")
cmd_args = ['git', 'archive', VERSION,
'--prefix=cement-%s/' % VERSION,
'--output=%s' % tar_path]
out, err, res = shell.exec_cmd(cmd_args)
cmd_args = ['gzip', tar_path]
out, err, res = shell.exec_cmd(cmd_args)
if res > 0:
self._do_error("\n\nFailed generating git archive.\n\n" +
"$ %s" % (' '.join(cmd_args), err))
print('')
def get_current_branch(self):
try:
stdout, stderr, exitcode = \
exec_cmd(['git', '--version'])
if sys.version_info[0] >= 3:
stdout = stdout.decode('utf8')
LOG.debug('Git Version: ' + stdout)
except:
raise CommandError('Error getting "git --version".')
stdout, stderr, exitcode = \
exec_cmd(['git', 'symbolic-ref', 'HEAD'])
if sys.version_info[0] >= 3:
stdout = stdout.decode('utf8')
stderr = stderr.decode('utf8')
stdout = stdout.rstrip()
if exitcode != 0:
io.log_warning('Git is in a detached head state. Using branch "default".')
return 'default'
self._handle_exitcode(exitcode, stderr)
LOG.debug('git symbolic-ref result: ' + stdout)
# Need to parse branch from ref manually because "--short" is
# not supported on git < 1.8
return stdout.split('/')[-1]
def test_wait(self):
"""Test that submitted shell jobs are run sequentially"""
print "running first sleep"
out = shell.exec_cmd(["sleep", "3"])
print "finishing first sleep"
print "running second sleep"
shell.exec_cmd(["sleep", "3"])
print "finishing second sleep"
def run(self):
assert os.path.exists(self.app.pargs.run_config), \
"Run configuration file %s does not exist" % \
self.app.pargs.run_config
config = yaml.load(open(self.app.pargs.run_config, 'r').read())
assert config, "Run configuration is un-readable (Invalid Yaml?)"
assert 'commands' in config.keys(), \
"Run configuration has no commands defined"
if 'sleep' not in config.keys():
config['sleep'] = 1
if 'prep' not in config.keys():
config['prep'] = []
if self.app.pargs.with_prep and 'prep' in config.keys():
for prep in config['prep']:
shell.exec_cmd(prep, shell=True)
items = []
index = 0
for command in config['commands']:
item = {
'command' : command,
'id' : index,
'color' : COLORS[index],
}
item[self.app.pargs.mode] = None
items.append(item)
index += 1
while True:
for i in items:
mode = self.app.pargs.mode
spawn = getattr(shell, 'spawn_%s' % mode)
verbose = self.app.pargs.verbose
if i[mode] is None or not i[mode].is_alive():
if self.app.pargs.no_color:
msg = '{mode} #{id}: {command}'.format(
mode=mode.capitalize(),
id=i['id'],
command=i['command'],
)
else:
msg = '{color}{mode} #{id}: {command}'.format(
mode=mode.capitalize(),
color=i['color'],
id=i['id'],
command=i['command'],
)
print(msg)
i[mode] = spawn(runner, args=(i['command'], verbose))
else:
pass
sleep(int(config['sleep']))
def _do_tests(self):
print('Running Nose Tests')
out, err, res = shell.exec_cmd(['which', 'nosetests'])
nose = out.decode('utf-8').strip()
if self.app.pargs.loud:
cmd_args = ['coverage', 'run', nose, '--verbosity=3']
res = shell.exec_cmd2(cmd_args)
else:
cmd_args = ['coverage', 'run', nose, '--verbosity=0']
out, err, res = shell.exec_cmd(cmd_args)
if res > 0:
self._do_error("\n\nNose tests did not pass.\n\n" +
"$ %s\n%s" % (' '.join(cmd_args), err))
def _do_tests(self):
print('Running Nose Tests')
out, err, res = shell.exec_cmd(['which', 'nosetests'])
if self.app.pargs.loud:
cmd_args = ['coverage', 'run', out.strip(), '--verbosity=3']
res = shell.exec_cmd2(cmd_args)
else:
cmd_args = ['coverage', 'run', out.strip(), '--verbosity=0']
out, err, res = shell.exec_cmd(cmd_args)
if res > 0:
self._do_error("\n\nNose tests did not pass.\n\n" +
"$ %s\n%s" % (' '.join(cmd_args), err))
def setUp(self):
## setup empty files
for k,v in data_files().items():
if not os.path.exists(os.path.join(filedir, k)):
if not os.path.exists(os.path.dirname(os.path.join(filedir, k))):
os.makedirs(os.path.dirname(os.path.join(filedir, k)))
with open(os.path.join(filedir, k), "w") as fh:
print "Preparing test: writing to file {}".format(k)
fh.write(v)
for f in empty_files():
if not os.path.exists(os.path.join(filedir, f)):
print "Preparing test: touching file {}".format(f)
if not os.path.exists(os.path.dirname(os.path.join(filedir, f))):
os.makedirs(os.path.dirname(os.path.join(filedir, f)))
shell.exec_cmd(['touch', os.path.join(filedir, f)])
def _do_pep8(self):
print("Checking PEP8 Compliance")
cmd_args = ['pep8', '-r', 'cement/', '--exclude=*.pyc']
out, err, res = shell.exec_cmd(cmd_args)
if res > 0:
self._do_error("\n\nPEP8 checks did not pass.\n\n" + "$ %s\n%s" %
(' '.join(cmd_args), out))
def execute(self, bundle_name, paths, targets):
if len(paths) < 1 or len(targets) < 1:
self.log.error('Invalid rule: paths or targets is empty')
rt_out_dir = self.out_dir + '/' + bundle_name
if Path(rt_out_dir).exists():
print('Output directory exists. Files may be overwritten.')
shell.Prompt("Press Enter To Continue", default='ENTER')
else:
Path(rt_out_dir).mkdir(parents=True)
for path in paths:
if not path.endswith('/'):
path += '/'
for target in targets:
target.rstrip('/')
if target != '*':
target_path = path + target
else:
target_path = path.rstrip('/')
self.log.debug('extracting: %s' % target_path, __name__)
self.log.debug('to: %s' % rt_out_dir, __name__)
cmd = ['adb', 'pull', '-a', target_path, rt_out_dir]
stdout, stderr, exitcode = shell.exec_cmd(cmd)
if exitcode != 0:
print("error: failed to extract %s" % target_path, file=sys.stderr)
print(stderr, file=sys.stderr)
else:
print('extracted: %s to %s' % (target_path, rt_out_dir))
def test_3_from_pre_to_casava_transfer(self):
"""Test from pre-casava to casava transfer to project directory"""
self.app = self.make_app(argv = ['production', 'transfer', 'J.Doe_00_01', '-f', '120829_SN0001_0001_AA001AAAXX', '--from_pre_casava'])
handler.register(ProductionController)
self._run_app()
res = shell.exec_cmd(["ls", "-1", os.path.join(delivery_dir, "P1_101F_index1", "120829_AA001AAAXX")])
self.eq(['1_120829_AA001AAAXX_barcode', '1_120829_AA001AAAXX_nophix_1-sort-dup.align_metrics', '1_120829_AA001AAAXX_nophix_1-sort-dup.bam', '1_120829_AA001AAAXX_nophix_1-sort-dup.dup_metrics', '1_120829_AA001AAAXX_nophix_1-sort-dup.hs_metrics', '1_120829_AA001AAAXX_nophix_1-sort-dup.insert_metrics', '1_120829_AA001AAAXX_nophix_1-sort.bam', 'P1_101F_index1-bcbb-config.yaml', 'alignments'], res[0].split())
def _do_pep8(self):
print("Checking PEP8 Compliance")
cmd_args = ['pep8', '-r', 'cement/', '--exclude=*.pyc']
out, err, res = shell.exec_cmd(cmd_args)
if res > 0:
self._do_error("\n\nPEP8 checks did not pass.\n\n" +
"$ %s\n%s" % (' '.join(cmd_args), out))
def _do_flake8(self):
print("Checking Flake8 Compliance (pep8, pycodestyle, mccabe, etc)")
cmd_args = ['flake8', 'cement/', 'tests/']
out, err, res = shell.exec_cmd(cmd_args)
if res > 0:
self._do_error("\n\nFlake8 checks did not pass.\n\n" +
"$ %s\n%s" % (' '.join(cmd_args), str(out)))
def _do_sphinx(self, dest_path):
print("Building Documentation")
cmd_args = ['rm', '-rf', 'docs/build/*']
cmd_args = ['sphinx-build', 'doc/source', dest_path]
out, err, res = shell.exec_cmd(cmd_args)
if res > 0:
self._do_error("\n\nFailed to build sphinx documentation\n\n" +
"$ %s\n%s" % (' '.join(cmd_args), out))
def get_message(self):
stdout, stderr, exitcode = \
exec_cmd(['git', 'log', '--oneline', '-1'])
if sys.version_info[0] >= 3:
stdout = stdout.decode('utf8')
stderr = stderr.decode('utf8')
self._handle_exitcode(exitcode, stderr)
return stdout.rstrip().split(' ', 1)[1]
def runpipe():
(stdout, stderr, returncode) = shell.exec_cmd(cmd_args, kw.get('shell', False))
if returncode and not ignore_error:
if capture:
self.app.log.error(stderr)
raise Exception("Subprocess return code: {}".format(returncode))
if capture:
return stdout
def test_wrapper(self):
"""Test basic wrapper functionality"""
lsw = LsWrapper()
self.assertEqual(repr(lsw), "<class 'tests.wrappers.test_wrappers.LsWrapper'>")
print str(lsw)
print lsw.cmd_args()
out= shell.exec_cmd(lsw.cmd_args())
print out
def runpipe():
(stdout, stderr, returncode) = shell.exec_cmd(cmd, **{'shell': True})
if returncode and not ignore_error:
if capture:
LOG.error(stderr)
raise Exception("Subprocess return code: {}".format(returncode))
if capture:
return stdout
def get_message(self):
stdout, stderr, exitcode = \
exec_cmd(['git', 'log', '--oneline', '-1'])
if sys.version_info[0] >= 3:
stdout = stdout.decode('utf8')
stderr = stderr.decode('utf8')
self._handle_exitcode(exitcode, stderr)
return stdout.rstrip().split(' ', 1)[1]
def runpipe():
(stdout, stderr, returncode) = shell.exec_cmd(cmd, **{'shell':True})
if returncode and not ignore_error:
if capture:
LOG.error(stderr)
raise Exception("Subprocess return code: {}".format(returncode))
if capture:
return stdout
def setUp(self):
## setup empty files
for k, v in data_files().items():
if not os.path.exists(os.path.join(filedir, k)):
if not os.path.exists(os.path.dirname(os.path.join(filedir,
k))):
os.makedirs(os.path.dirname(os.path.join(filedir, k)))
with open(os.path.join(filedir, k), "w") as fh:
print "Preparing test: writing to file {}".format(k)
fh.write(v)
for f in empty_files():
if not os.path.exists(os.path.join(filedir, f)):
print "Preparing test: touching file {}".format(f)
if not os.path.exists(os.path.dirname(os.path.join(filedir,
f))):
os.makedirs(os.path.dirname(os.path.join(filedir, f)))
shell.exec_cmd(['touch', os.path.join(filedir, f)])
def _do_sphinx(self, dest_path):
print("Building Documentation")
cmd_args = ['rm', '-rf', 'docs/build/*']
cmd_args = ['sphinx-build', 'doc/source', dest_path]
out, err, res = shell.exec_cmd(cmd_args)
if res > 0:
self._do_error("\n\nFailed to build sphinx documentation\n\n" +
"$ %s\n%s" % (' '.join(cmd_args), str(out)))
def do_zip(self, location):
io.log_info('creating zip using git archive HEAD')
stdout, stderr, exitcode = \
exec_cmd(['git', 'archive', '-v', '--format=zip',
'-o', location, 'HEAD'])
if sys.version_info[0] >= 3:
stderr = stderr.decode('utf8')
self._handle_exitcode(exitcode, stderr)
io.log_info('git archive output: ' + stderr)
def runpipe():
(stdout, stderr, returncode) = shell.exec_cmd(cmd_args)
if returncode and not ignore_error:
if capture:
self.app.log.error(stderr)
raise Exception(
"Subprocess return code: {}".format(returncode))
if capture:
return stdout
def _run_cmd(self, cmd, handle_exitcode=True):
stdout, stderr, exitcode = exec_cmd(cmd)
stdout = utils.decode_bytes(stdout).strip()
stderr = utils.decode_bytes(stderr).strip()
if handle_exitcode:
self._handle_exitcode(exitcode, stderr)
return stdout, stderr, exitcode
def _run_cmd(self, cmd, handle_exitcode=True):
stdout, stderr, exitcode = exec_cmd(cmd)
stdout = utils.decode_bytes(stdout).strip()
stderr = utils.decode_bytes(stderr).strip()
if handle_exitcode:
self._handle_exitcode(exitcode, stderr)
return stdout, stderr, exitcode
def do_zip(self, location):
io.log_info('creating zip using git archive HEAD')
stdout, stderr, exitcode = \
exec_cmd(['git', 'archive', '-v', '--format=zip',
'-o', location, 'HEAD'])
if sys.version_info[0] >= 3:
stderr = stderr.decode('utf8')
self._handle_exitcode(exitcode, stderr)
io.log_info('git archive output: ' + stderr)
def _run_cmd(self, cmd, handle_exitcode=True):
stdout, stderr, exitcode = exec_cmd(cmd)
if sys.version_info[0] >= 3:
stdout = stdout.decode('utf8')
stderr = stderr.decode('utf8')
stdout = stdout.strip()
stderr = stderr.strip()
if handle_exitcode:
self._handle_exitcode(exitcode, stderr)
return stdout, stderr, exitcode
def test_from_casava_to_casava_custom_transfer(self):
"""Test from casava to casava transfer to custom project directory"""
self.app = self.make_app(argv = ['production', 'transfer', 'J.Doe_00_04', '--transfer_dir', 'j_doe_00_04_custom'])
handler.register(ProductionController)
self._run_app()
delivery_dir = os.path.abspath(os.path.join(filedir, "data", "projects", "j_doe_00_04_custom", "data"))
with open(os.path.join(delivery_dir, "P001_101_index3", "120924_CC003CCCXX", "P001_101_index3-bcbb-config.yaml")) as fh:
runinfo_yaml = yaml.load(fh)
res = shell.exec_cmd(["ls", "-1", os.path.join(delivery_dir, "P001_101_index3", "120924_CC003CCCXX")])
self.eq(len(set(res[0].split())), 22)
def _run_cmd(self, cmd, handle_exitcode=True):
stdout, stderr, exitcode = exec_cmd(cmd)
if sys.version_info[0] >= 3:
stdout = stdout.decode('utf8')
stderr = stderr.decode('utf8')
stdout = stdout.strip()
stderr = stderr.strip()
if handle_exitcode:
self._handle_exitcode(exitcode, stderr)
return stdout, stderr, exitcode
def get_current_branch(self):
stdout, stderr, exitcode = \
exec_cmd(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])
if sys.version_info[0] >= 3:
stdout = stdout.decode('utf8')
stderr = stderr.decode('utf8')
stdout = stdout.rstrip()
self._handle_exitcode(exitcode, stderr)
LOG.debug('git current-branch result: ' + stdout)
return stdout
def test_7_from_casava_to_casava_transfer(self):
"""Test from casava to casava transfer to custom project directory"""
self.app = self.make_app(argv = ['production', 'transfer', 'J.Doe_00_04', '--transfer_dir', 'j_doe_00_04_custom'])
handler.register(ProductionController)
self._run_app()
delivery_dir = os.path.abspath(os.path.join(filedir, "data", "projects", "j_doe_00_04_custom", "data"))
with open(os.path.join(delivery_dir, "P001_101_index3", "120924_CC003CCCXX", "P001_101_index3-bcbb-config.yaml")) as fh:
runinfo_yaml = yaml.load(fh)
self.eq(runinfo_yaml['details'][0]['multiplex'][0]['files'], [os.path.join(delivery_dir, "P001_101_index3", "120924_CC003CCCXX", "nophix", x) for x in ['1_120924_CC003CCCXX_7_nophix_1_fastq.txt.gz', '1_120924_CC003CCCXX_7_nophix_2_fastq.txt.gz']])
res = shell.exec_cmd(["ls", "-1", os.path.join(delivery_dir, "P001_101_index3", "120924_CC003CCCXX")])
self.eq(set(res[0].split()), set(['1_120924_CC003CCCXX_7_nophix-sort-dup-insert.pdf', '1_120924_CC003CCCXX_7_nophix-sort-dup-summary.aux', '1_120924_CC003CCCXX_7_nophix-sort-dup-summary.log', '1_120924_CC003CCCXX_7_nophix-sort-dup-summary.pdf', '1_120924_CC003CCCXX_7_nophix-sort-dup-summary.tex', '1_120924_CC003CCCXX_7_nophix-sort-dup.align_metrics', '1_120924_CC003CCCXX_7_nophix-sort-dup.bam', '1_120924_CC003CCCXX_7_nophix-sort.bam', '1_120924_CC003CCCXX_7_nophix-sort-dup.dup_metrics', '1_120924_CC003CCCXX_7_nophix-sort-dup.hs_metrics', '1_120924_CC003CCCXX_7_nophix-sort-dup.insert_metrics', 'P001_101_index3-bcbb-config.yaml', 'alignments', 'fastq_screen', 'nophix']))
def get_current_branch(self):
stdout, stderr, exitcode = \
exec_cmd(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])
if sys.version_info[0] >= 3:
stdout = stdout.decode('utf8')
stderr = stderr.decode('utf8')
stdout = stdout.rstrip()
self._handle_exitcode(exitcode, stderr)
LOG.debug('git current-branch result: ' + stdout)
return stdout
def get_version_label(self):
io.log_info('Getting version label from git with git-describe')
stdout, stderr, exitcode = \
exec_cmd(['git', 'describe', '--always', '--abbrev=4'])
if sys.version_info[0] >= 3:
stdout = stdout.decode('utf8')
stderr = stderr.decode('utf8')
self._handle_exitcode(exitcode, stderr)
#Replace dots with underscores
return stdout[:-1].replace('.', '_')
def get_version_label(self):
io.log_info('Getting version label from git with git-describe')
stdout, stderr, exitcode = \
exec_cmd(['git', 'describe', '--always', '--abbrev=4'])
if sys.version_info[0] >= 3:
stdout = stdout.decode('utf8')
stderr = stderr.decode('utf8')
self._handle_exitcode(exitcode, stderr)
#Replace dots with underscores
return stdout[:-1].replace('.', '_')
def run(self):
config = yaml.load(open(self.app.pargs.run_config, 'r').read())
if self.app.pargs.with_prep and 'prep' in config.keys():
for prep in config['prep']:
shell.exec_cmd(prep, shell=True)
items = []
index = 0
for command in config['commands']:
item = {
'command' : command,
'id' : index,
'color' : COLORS[index],
}
item[self.app.pargs.mode] = None
items.append(item)
index += 1
while True:
for i in items:
mode = self.app.pargs.mode
spawn = getattr(shell, 'spawn_%s' % mode)
verbose = self.app.pargs.verbose
if i[mode] is None or not i[mode].is_alive():
msg = '{color}{mode} #{tid}: {command}'.format(
mode=mode.capitalize(),
color=i['color'],
tid=i['id'],
command=i['command'],
)
print(msg)
i[mode] = spawn(runner, args=(i['command'], verbose))
else:
pass
sleep(int(config['sleep']))
def usage(self):
cmd = ['adb', 'shell', 'dumpsys', 'usagestats']
stdout, stderr, exitcode = shell.exec_cmd(cmd)
if exitcode == 0:
if self.keyword:
for line in stdout.decode('utf-8').split('\n'):
if self.keyword in line:
print(line)
else:
print(stdout.decode('utf-8'))
else:
self.log.warning('package.usage module exits unexpectedly: %d.' %
exitcode)
def untracked_changes_exist(self):
stdout, stderr, exitcode = \
exec_cmd(['git', 'diff', '--numstat'])
if sys.version_info[0] >= 3:
stdout = stdout.decode('utf8')
stderr = stderr.decode('utf8')
stdout = stdout.strip()
LOG.debug('git diff --numstat result: ' + stdout + ' with errors: ' +
stderr)
if stdout:
return True
return False
def untracked_changes_exist(self):
stdout, stderr, exitcode = \
exec_cmd(['git', 'diff', '--numstat'])
if sys.version_info[0] >= 3:
stdout = stdout.decode('utf8')
stderr = stderr.decode('utf8')
stdout = stdout.strip()
LOG.debug('git diff --numstat result: ' + stdout +
' with errors: ' + stderr)
if stdout:
return True
return False
def run(self):
config = yaml.load(open(self.app.pargs.run_config, 'r').read())
if self.app.pargs.with_prep and 'prep' in config.keys():
for prep in config['prep']:
shell.exec_cmd(prep, shell=True)
items = []
index = 0
for command in config['commands']:
item = {
'command': command,
'id': index,
'color': COLORS[index],
}
item[self.app.pargs.mode] = None
items.append(item)
index += 1
while True:
for i in items:
mode = self.app.pargs.mode
spawn = getattr(shell, 'spawn_%s' % mode)
verbose = self.app.pargs.verbose
if i[mode] is None or not i[mode].is_alive():
msg = '{color}{mode} #{tid}: {command}'.format(
mode=mode.capitalize(),
color=i['color'],
tid=i['id'],
command=i['command'],
)
print(msg)
i[mode] = spawn(runner, args=(i['command'], verbose))
else:
pass
sleep(int(config['sleep']))
def do_zip(self, location):
cwd = os.getcwd()
try:
# must be in project root for git archive to work.
fileoperations._traverse_to_project_root()
io.log_info('creating zip using git archive HEAD')
stdout, stderr, exitcode = \
exec_cmd(['git', 'archive', '-v', '--format=zip',
'-o', location, 'HEAD'])
if sys.version_info[0] >= 3:
stderr = stderr.decode('utf8')
self._handle_exitcode(exitcode, stderr)
io.log_info('git archive output: ' + stderr)
finally:
os.chdir(cwd)
def do_zip(self, location):
cwd = os.getcwd()
try:
# must be in project root for git archive to work.
fileoperations._traverse_to_project_root()
io.log_info('creating zip using git archive HEAD')
stdout, stderr, exitcode = \
exec_cmd(['git', 'archive', '-v', '--format=zip',
'-o', location, 'HEAD'])
if sys.version_info[0] >= 3:
stderr = stderr.decode('utf8')
self._handle_exitcode(exitcode, stderr)
io.log_info('git archive output: ' + stderr)
finally:
os.chdir(cwd)
def test_7_from_casava_to_casava_transfer(self):
"""Test from casava to casava transfer to custom project directory"""
self.app = self.make_app(argv=[
'production', 'transfer', 'J.Doe_00_04', '--transfer_dir',
'j_doe_00_04_custom'
])
handler.register(ProductionController)
self._run_app()
delivery_dir = os.path.abspath(
os.path.join(filedir, "data", "projects", "j_doe_00_04_custom",
"data"))
with open(
os.path.join(delivery_dir, "P001_101_index3",
"120924_CC003CCCXX",
"P001_101_index3-bcbb-config.yaml")) as fh:
runinfo_yaml = yaml.load(fh)
self.eq(runinfo_yaml['details'][0]['multiplex'][0]['files'], [
os.path.join(delivery_dir, "P001_101_index3", "120924_CC003CCCXX",
"nophix", x) for x in [
'1_120924_CC003CCCXX_7_nophix_1_fastq.txt.gz',
'1_120924_CC003CCCXX_7_nophix_2_fastq.txt.gz'
]
])
res = shell.exec_cmd([
"ls", "-1",
os.path.join(delivery_dir, "P001_101_index3", "120924_CC003CCCXX")
])
self.eq(
set(res[0].split()),
set([
'1_120924_CC003CCCXX_7_nophix-sort-dup-insert.pdf',
'1_120924_CC003CCCXX_7_nophix-sort-dup-summary.aux',
'1_120924_CC003CCCXX_7_nophix-sort-dup-summary.log',
'1_120924_CC003CCCXX_7_nophix-sort-dup-summary.pdf',
'1_120924_CC003CCCXX_7_nophix-sort-dup-summary.tex',
'1_120924_CC003CCCXX_7_nophix-sort-dup.align_metrics',
'1_120924_CC003CCCXX_7_nophix-sort-dup.bam',
'1_120924_CC003CCCXX_7_nophix-sort.bam',
'1_120924_CC003CCCXX_7_nophix-sort-dup.dup_metrics',
'1_120924_CC003CCCXX_7_nophix-sort-dup.hs_metrics',
'1_120924_CC003CCCXX_7_nophix-sort-dup.insert_metrics',
'P001_101_index3-bcbb-config.yaml', 'alignments',
'fastq_screen', 'nophix'
]))
def _get_public_ssh_key(keypair_name):
key_file = fileoperations.get_ssh_folder() + keypair_name
if os.path.exists(key_file):
file_name = key_file
elif os.path.exists(key_file + '.pem'):
file_name = key_file + '.pem'
else:
raise NotSupportedError(strings['ssh.filenotfound'].replace(
'{key-name}', keypair_name))
try:
stdout, stderr, returncode = exec_cmd(['ssh-keygen', '-y', '-f',
file_name])
if returncode != 0:
raise CommandError('An error occurred while trying '
'to get ssh public key')
key_material = stdout
return key_material
except OSError:
CommandError(strings['ssh.notpresent'])
def _get_public_ssh_key(keypair_name):
key_file = fileoperations.get_ssh_folder() + keypair_name
if os.path.exists(key_file):
file_name = key_file
elif os.path.exists(key_file + '.pem'):
file_name = key_file + '.pem'
else:
raise NotSupportedError(strings['ssh.filenotfound'].replace(
'{key-name}', keypair_name))
try:
stdout, stderr, returncode = exec_cmd(
['ssh-keygen', '-y', '-f', file_name])
if returncode != 0:
raise CommandError('An error occurred while trying '
'to get ssh public key')
key_material = stdout
return key_material
except OSError:
CommandError(strings['ssh.notpresent'])
def test_3_from_pre_to_casava_transfer(self):
"""Test from pre-casava to casava transfer to project directory"""
self.app = self.make_app(argv=[
'production', 'transfer', 'J.Doe_00_01', '-f',
'120829_SN0001_0001_AA001AAAXX', '--from_pre_casava'
])
handler.register(ProductionController)
self._run_app()
res = shell.exec_cmd([
"ls", "-1",
os.path.join(delivery_dir, "P1_101F_index1", "120829_AA001AAAXX")
])
self.eq([
'1_120829_AA001AAAXX_barcode',
'1_120829_AA001AAAXX_nophix_1-sort-dup.align_metrics',
'1_120829_AA001AAAXX_nophix_1-sort-dup.bam',
'1_120829_AA001AAAXX_nophix_1-sort-dup.dup_metrics',
'1_120829_AA001AAAXX_nophix_1-sort-dup.hs_metrics',
'1_120829_AA001AAAXX_nophix_1-sort-dup.insert_metrics',
'1_120829_AA001AAAXX_nophix_1-sort.bam',
'P1_101F_index1-bcbb-config.yaml', 'alignments'
], res[0].split())
def test_exec_cmd_shell_true():
out, err, ret = shell.exec_cmd(['echo KAPLA!'], shell=True)
assert ret == 0
assert out == b'KAPLA!\n'
def test_exec_cmd_bad_command():
out, err, ret = shell.exec_cmd(['false'])
assert ret == 1
def test_exec_cmd(self):
out, err, ret = shell.exec_cmd(['echo', 'KAPLA!'])
self.eq(ret, 0)
self.eq(out, b'KAPLA!\n')
def test_exec_cmd_shell_true(self):
out, err, ret = shell.exec_cmd(['echo KAPLA!'], shell=True)
self.eq(ret, 0)
self.eq(out, b'KAPLA!\n')
def test_exec_cmd_bad_command(self):
out, err, ret = shell.exec_cmd(['false'])
self.eq(ret, 1)
