def create(self, part, opath, ipath):
try:
if not opath.endswith('.ubi'):
opath = '%s.ubi' % opath
if not os.path.isdir(ipath):
self.error('Input path is not a directory.')
if part.lower() == 'erootfs':
ini_file = os.path.join(self._app_path, 'firmware/ubi_cfg/%serootfs.ini' % self._ubi_version)
elif part.lower() == 'bulk':
ini_file = os.path.join(self._app_path, 'firmware/ubi_cfg/%sbulk.ini' % self._ubi_version)
else:
self.error('Partion must be erootfs, or bulk.')
mkfs = '-c %s -m %s -e %s -r %s' % (self._params['max_leb_cnt'],
self._params['min_io_size'],
self._params['leb_size'],
ipath)
ubinz = '-o %s -p %s -m %s -s %s -O %s %s' % (opath,
self._params['peb_size'],
self._params['min_io_size'],
self._params['sub_page_size'],
self._params['vid_hdr_offset'],
ini_file)
cmd_mkubi = shlex_split('sudo /usr/sbin/mkfs.ubifs %s temp.ubifs' % (mkfs))
cmd_ubinize = shlex_split('sudo /usr/sbin/ubinize %s' % (ubinz))
cmd_rmimg = shlex_split('sudo rm temp.ubifs')
cmd_chmod = shlex_split('sudo chmod 777 %s' % opath)
cmds = [cmd_mkubi, cmd_ubinize, cmd_rmimg, cmd_chmod]
self.popen_arr(cmds)
except Exception, e:
self.error(e)
def parseString(self, string):
if len(shlex_split(string)) < 1: return ()
name = shlex_split(string)[0] #.lower()
args = shlex_split(string)[1:]
return (name, args)
def test_functional(self):
"""Init new repository, create test file and save it in vcs"""
mygit.main(self.cwd_path, shlex_split("init")) # init new repository
test_file_name = "readme.md"
test_file_path = self.cwd_path / test_file_name
with Path.open(test_file_path, "w") as test_file: # create test file
test_file.write("hello world")
with Path.open(test_file_path, "rb") as test_file:
content = compress(test_file.read(), -1)
test_file_checksum = sha1(content).hexdigest()
state = get_current_state(self.constants)
assert state.status_not_indexed_paths == ['modified: readme.md']
mygit.main(self.cwd_path, shlex_split("index readme.md")) # index test file
state = get_current_state(self.constants)
assert state.status_not_indexed_paths == []
assert test_file_path in state.current_indexed_paths
assert state.current_indexed_paths.get(test_file_path) == test_file_checksum
mygit.main(self.cwd_path, shlex_split("commit created_readme")) # save test file in vcs
state = get_current_state(self.constants)
assert state.current_indexed_paths == {}
assert (self.constants.mygit_objects_path / test_file_checksum).exists() # was file really compressed & saved?
def runString(self, string, *argsv):
if len(shlex_split(string)) < 1: return
cmd = shlex_split(string)[0].lower()
args = shlex_split(string)[1:]
return self.commands[cmd](cmd, args, *argsv)
def on_text(self, buf, handler):
try:
cmd = buf.rstrip("\x00\n\r")
if cmd == "position":
handler.send_text(self.__class__.__name__)
else:
# NOTE: PY27
if isinstance(buf, unicode):
params = [
p.decode("utf8")
for p in shlex_split(cmd.encode("utf8"))
]
else:
params = shlex_split(cmd)
response = self.dispatch_cmd(handler, *params)
if response is not None:
logger.error("Shoud not response anything")
handler.send_text(response)
except RuntimeError as e:
handler.send_text(("error " + " ".join(e.args)).encode())
except IOError as e:
logger.debug("Connection close: %s" % e)
handler.close()
except Exception as e:
if DEBUG:
handler.send_text("error %s %s" % (UNKNOWN_ERROR, e))
else:
handler.send_text("error %s" % UNKNOWN_ERROR)
logger.exception(UNKNOWN_ERROR)
def test_index_delete_file(self):
mygit.main(self.cwd_path, shlex_split("init")) # init new repository
test_file_name = "readme.md"
test_file_path = self.cwd_path / test_file_name
with Path.open(test_file_path, "w") as test_file: # create test file
test_file.write("hello world")
mygit.main(self.cwd_path,
shlex_split("index readme.md")) # index test file
mygit.main(self.cwd_path,
shlex_split("commit upd_readme")) # fix file content
Path.unlink(test_file_path) # delete test file
state = get_current_state(self.constants)
assert state.status_not_indexed_paths == [
'deleted: readme.md'
] # yep mygit knows that you deleted it
assert state.current_indexed_paths == {} # but not indexed that change
mygit.main(self.cwd_path,
shlex_split("index readme.md")) # index removal
state = get_current_state(self.constants)
assert state.status_not_indexed_paths == []
assert state.current_indexed_paths.get(test_file_path) == 'deleted'
assert state.status_indexed_paths == ['deleted: readme.md']
mygit.main(self.cwd_path,
shlex_split("commit delete_readme")) # commit changes
state = get_current_state(self.constants)
assert state.status_not_indexed_paths == []
assert state.current_indexed_paths == {}
assert state.status_indexed_paths == [] # we saved all changes
def changeMusicVolume(change, steps=0):
"Changes volume according to string; can be either absolute ('40') or change ('2%-')."
if steps == 0:
call(shlex_split("amixer set Master " + str(change))) # CHANGEME somehow, if amixer doesn't work
else:
cur = get_state()
interval = 1. / steps
for a in drange(cur, int(change), float(int(change) - cur) / steps):
call(shlex_split("amixer set Master " + str(a)))
sleep(interval)
def _get_terminal_size_tput():
# get terminal width
# src: http://stackoverflow.com/questions/263890/how-do-i-find-the-width-height-of-a-terminal-window
try:
cols = int(subprocess_check_call(shlex_split('tput cols')))
rows = int(subprocess_check_call(shlex_split('tput lines')))
return (cols, rows)
except:
pass
def __GetTerminalSizeWithTPut():
from shlex import split as shlex_split
from subprocess import check_output
try:
width = int(check_output(shlex_split('tput cols')))
height = int(check_output(shlex_split('tput lines')))
return (width, height)
except:
pass
def umount(self):
try:
cmd_umount = shlex_split('sudo /bin/umount %s' % self._mount)
cmd_rmmod = shlex_split('sudo /sbin/rmmod ubifs ubi nandsim')
cmds = [cmd_umount, cmd_rmmod]
self.popen_arr(cmds)
if os.path.exists(self._mount):
self.popen(shlex_split('sudo rmdir %s' % self._mount))
except Exception, e:
self.error(e)
def umount(self):
try:
cmd_umount = shlex_split('sudo /bin/umount %s' % self._mount)
cmd_rmmod = shlex_split("sudo /sbin/rmmod jffs2 mtdblock mtdram")
cmds = [cmd_umount, cmd_rmmod]
self.popen_arr(cmds)
if os.path.exists(self._mount):
self.popen(shlex_split('sudo rmdir %s' % self._mount))
except Exception, e:
self.error(e)
def umount(self):
try:
cmd_umount = shlex_split('sudo /bin/umount %s' % self._mount)
cmd_rmmod = shlex_split("sudo /sbin/rmmod jffs2 mtdblock mtdram")
cmds = [cmd_umount, cmd_rmmod]
self.popen_arr(cmds)
if os.path.exists(self._mount):
self.popen(shlex_split('sudo rmdir %s' % self._mount))
except Exception, e:
self.error(e)
def umount(self):
try:
cmd_umount = shlex_split('sudo /bin/umount %s' % self._mount)
cmd_rmmod = shlex_split('sudo /sbin/rmmod ubifs ubi nandsim')
cmds = [cmd_umount, cmd_rmmod]
self.popen_arr(cmds)
if os.path.exists(self._mount):
self.popen(shlex_split('sudo rmdir %s' % self._mount))
except Exception, e:
self.error(e)
def changeMusicVolume(change, steps=0):
"Changes volume according to string; can be either absolute ('40') or change ('2%-')."
if steps == 0:
call(shlex_split(
"amixer set Master " +
str(change))) # CHANGEME somehow, if amixer doesn't work
else:
cur = get_state()
interval = 1. / steps
for a in drange(cur, int(change), float(int(change) - cur) / steps):
call(shlex_split("amixer set Master " + str(a)))
sleep(interval)
def create(self, opath, ipath):
try:
if not os.path.isdir(ipath):
self.error('Input path is not a directory.')
if not opath.endswith('.jffs2'):
opath = '%s.jffs2' % opath
cmd_mkjffs2 = shlex_split('sudo mkfs.jffs2 -p -r %s -e 128 -o %s' % (ipath, opath))
cmd_chmod = shlex_split('sudo chmod 777 %s' % opath)
cmds = [cmd_mkjffs2, cmd_chmod]
self.popen_arr(cmds)
except Exception, e:
self.error(e)
def create(self, opath, ipath):
try:
if not os.path.isdir(ipath):
self.error('Input path is not a directory.')
if not opath.endswith('.jffs2'):
opath = '%s.jffs2' % opath
cmd_mkjffs2 = shlex_split('sudo mkfs.jffs2 -p -r %s -e 128 -o %s' %
(ipath, opath))
cmd_chmod = shlex_split('sudo chmod 777 %s' % opath)
cmds = [cmd_mkjffs2, cmd_chmod]
self.popen_arr(cmds)
except Exception, e:
self.error(e)
def run_wikipedia(args: Namespace, tools_versions: dict) -> None:
if args.without_wikipedia:
logging.info("SKIPPING to create CZ WIKIPEDIA KB from its resource...")
return
if args.lang != "cs":
logging.info(
"SKIPPING to create CZ WIKIPEDIA KB from its resource due to different language was selected..."
)
return
logging.info("STARTING to create CZ WIKIPEDIA KB from its resources...")
git_root = os.path.join(BASEDIR, "kb_resources", "kb_cs_wikipedia")
add_tool_version(repo_dir=git_root, tools_versions=tools_versions)
script = os.path.join(git_root, "start.sh")
cmd = f"{script} -l {args.lang} -m {args.m} -d {args.wikipedia_dump} -I {args.wikipedia_indir}"
if args.log:
cmd += " --log"
ps = subprocess.Popen(shlex_split(cmd))
retcode = ps.wait()
if retcode != 0:
sys.exit(
"WIKIPEDIA KB creation failed (was terminated with code: {retcode})."
)
def logs(self):
call(shlex_split(f"./screen.sh logs {self.session_name}"))
self._wait()
f = open(self.log_filename, 'r+')
s = f.readlines()
f.close()
return s
def get_state():
# exp = r".*?(\d+) \[(\d+)%\]"
exp = r".*?(\d+) "
cmd = shlex_split("amixer get Master")
return int(
search(exp,
Popen(cmd, stdout=PIPE).stdout.readlines()[4]).groups()[0])
def receive(self, type='small', dblchk=0):
try:
if type == 'small':
request_len = self._request_small
trans_len = '01'
lba = '20'
else:
request_len = self._request_large
trans_len = '14'
lba = '40'
scsi_cmd = '%s %s -b -r %s -n 28 00 00 00 00 %s 00 00 %s 00' % (self._sg_raw, self._conn_iface.device_id, request_len, lba, trans_len)
scsi_cmd = shlex_split(scsi_cmd)
p = Popen(scsi_cmd, bufsize=0, stdout=PIPE, stderr=PIPE)
line = p.stdout.read()
if line == '':
if dblchk < 10:
sleep(1)
dblchk += 1
line = self.receive(type, dblchk)
if '102 BUSY' in line:
sleep(1)
line = self.receive(type, dblchk)
if line:
return line
else:
return False
except Exception, e:
self.error('Receiving error: %s' % e)
def cli(target):
from .panda import Panda
q = Qcows.get_qcow_info(target)
qcow = Qcows.get_qcow(target)
arch = q.arch
build_dir = Panda._find_build_dir(arch)
panda_args = [build_dir + f"/{arch}-softmmu/panda-system-{arch}"]
biospath = path.realpath(path.join(build_dir, "pc-bios"))
panda_args.extend(["-L", biospath])
if arch == 'mips64':
panda_args.extend(["-drive", f"file={qcow},if=virtio"])
else:
panda_args.append(qcow)
panda_args.extend(['-m', q.default_mem])
if q.extra_args:
extra_args = shlex_split(q.extra_args)
for x in extra_args:
if " " in x:
panda_args.append(repr(x))
else:
panda_args.append(x)
panda_args.extend(['-loadvm', q.snapshot])
ret = " ".join(panda_args)
if "-display none" in ret:
ret = ret.replace("-display none", "-nographic")
return ret
def tags(self, tag_names):
'''Get tags details from repository
To know how to parse, to see the output of following command
To get which branches a commit belongs to.
git --git-dir=/path/to/repo/.git branch --contains commit_hash
:return: sequence of GitTag
'''
git_cat_file = 'git --git-dir={0} cat-file -p `git --git-dir={0} rev-parse {1}`'
git_branch = 'git --git-dir={0} branch --contains {1}'
for tag_name in tag_names:
git_cmd = git_cat_file.format(self.git_dir, tag_name)
proc, stdout, stderr = git(git_cmd, shell=True)
try:
tag = GitTag.parse(stdout)
except InvalidTagError as error:
error.data = tag_name
yield error
else:
git_cmd = git_branch.format(self.git_dir, tag.commit_hash)
proc, stdout, stderr = git(shlex_split(git_cmd))
for line in sbuffer_iterator(stdout):
tag.add_branch_name(line.strip('* \r\n'))
yield tag
def _blind(dir_path):
# First strip additional arguments
subset_cmd = ('./subset.py -s core -m '
'-a Site,CSite,Sidechain,Contextgene {}').format(' '.join(
_find(dir_path, '.*\.a2')))
subset_main(shlex_split(subset_cmd))
# We need to rename the new core files
for a2_core_file_path in _find(dir_path, '.*\.a2.core'):
a2_file_path = a2_core_file_path[:-5]
assert isfile(a2_file_path)
move(a2_core_file_path, a2_file_path)
# Then negation and speculations
for a2_file_path in _find(dir_path, '.*\.a2'):
with open(a2_file_path, 'r') as a1_file:
a1_data = a1_file.read().split('\n')
# Write the annotations again without modifiers
with open(a2_file_path, 'w') as a1_file:
lines = []
for line in a1_data:
line_m = NESP_REGEX.match(line)
if line_m is None:
lines.append(line)
a1_file.write('\n'.join(lines))
def process(self, line):
""" Handle input.
"""
# Lex
try:
tokens = shlex_split(line)
except ValueError as error:
self.error("Syntax error (%s)", error.args[0])
return 2
# Parse
try:
args = self.__parser.parse_args(tokens)
except _ConsoleCommandError as error:
if tokens[0] in self.__commands:
# misused
self.error(error)
return 1
else:
# unknown
self.error(error)
return 127
# Dispatch
kwargs = vars(args)
f = kwargs.pop("f")
return f(**kwargs) or 0
def receive(self, type='small', dblchk=0):
try:
if type == 'small':
request_len = self._request_small
trans_len = '01'
lba = '20'
else:
request_len = self._request_large
trans_len = '14'
lba = '40'
scsi_cmd = '%s %s -b -r %s -n 28 00 00 00 00 %s 00 00 %s 00' % (
self._sg_raw, self._conn_iface.device_id, request_len, lba,
trans_len)
scsi_cmd = shlex_split(scsi_cmd)
p = Popen(scsi_cmd, bufsize=0, stdout=PIPE, stderr=PIPE)
line = p.stdout.read()
if line == '':
if dblchk < 10:
sleep(1)
dblchk += 1
line = self.receive(type, dblchk)
if '102 BUSY' in line:
sleep(1)
line = self.receive(type, dblchk)
if line:
return line
else:
return False
except Exception, e:
self.error('Receiving error: %s' % e)
def registry_information(self):
"""
actually list all available databases
"name" filed in retrieved information could be used to retrieve available datasets.
Example:
<MartRegistry>
<MartURLLocation database="ensembl_mart_65" default="1" displayName="ENSEMBL GENES 65 (SANGER UK)" host="www.biomart.org" includeDatasets="" martUser="" name="ensembl" path="/biomart/martservice" port="80" serverVirtualSchema="default" visible="1" />
<MartURLLocation database="snp_mart_65" default="0" displayName="ENSEMBL VARIATION 65 (SANGER UK)" host="www.biomart.org" includeDatasets="" martUser="" name="snp" path="/biomart/martservice" port="80" serverVirtualSchema="default" visible="1" />
<MartURLLocation database="functional_genomics_mart_65" default="0" displayName="ENSEMBL REGULATION 65 (SANGER UK)" host="www.biomart.org" includeDatasets="" martUser="" name="functional_genomics" path="/biomart/martservice" port="80" serverVirtualSchema="default" visible="1" />
...
</MartRegistry>
"""
params_dict = {"type":"registry"}
status, reason, data = self.easy_response(params_dict, echo=False)
# parse the output to make it more readable
databases = []
for line in data.strip().split('\n'):
ln = shlex_split(line)
ln_dict = dict(item.split('=') for item in ln if '=' in item)
if 'name' in ln_dict and 'displayName' in ln_dict:
databases.append(OrderedDict(biomart=ln_dict['name'], version=ln_dict['displayName']))
name_max_len = max(len(d['biomart']) for d in databases)
version_max_len = max(len(d['version']) for d in databases)
print "".ljust(5), 'biomart'.rjust(name_max_len+1), 'version'.rjust(version_max_len+1)
for i, d in enumerate(databases):
print str(i+1).ljust(5), d['biomart'].rjust(name_max_len+1), d['version'].rjust(version_max_len+1)
return databases
def dump(self):
try:
buf = b''
packets = int(FLASH_LEN / PACKET_SIZE)
total = 0
last_total = 0
for i in range(0, packets):
bts = '{0:04x}'.format(((i + 1) * 0x10))
byte1 = bts[0:2]
byte2 = bts[2:4]
cmdl = '%s %s -b -r %s -n FD 28 00 %s %s 00 06 00 08 00 00 00 00 00 47 50' % (
self._sg_raw, self._device_id, PACKET_SIZE, byte1, byte2)
cmd = shlex_split(cmdl)
if self.debug:
print("Write Cmd = %s" % cmdl)
p = Popen(cmd, stdout=PIPE, stderr=PIPE)
err = p.stderr.read()
data = p.stdout.read()
if not b'Good' in err:
self.error('SCSI error: {}'.format(err))
else:
buf += data
end = bytearray()
end.extend(buf[0:0x2eff0])
end.extend(b'\xFF' * 20)
end.extend(buf[0x2f004:-4])
end.extend(b'\xFF' * 4)
return end
except Exception as e:
self.error(e)
return None
def test_nonexisting_file(tmpdir):
tmpdir.chdir()
with pytest.raises(IOError):
main(
shlex_split(
"patch --current-version 1.2.0 --new-version 1.2.1 doesnotexist.txt"
))
def test_imcpack(tmpdir, capsys):
"""imcpack -d tmpdir/actual_output tmpdir/pack*"""
datapkg = f"{testdatapkg_parent}.pack"
contents2tmpdir = make_contents2destdir(datapkg, tmpdir)
# 1. prepare data files & paths
contents2tmpdir(recursive=True)
input_wildcard_arg = tmpdir.join("pack*")
output_dir_arg = tmpdir.join("actual_output")
# 2. run the actual imcunpack command
args = f'-v -d "{output_dir_arg!s}" "{input_wildcard_arg}"'
run_imcpack(shlex_split(args))
# 3. check verbose stdout
stdout_line1, stdout_rest = capsys.readouterr().out.split(sep="\n", maxsplit=1)
assert stdout_line1.startswith("packing ")
assert (
stdout_rest
== """\
'Piano' ->
'BrassLeft' ->
'BrassRight' ->
'BrassLPianoR' ->
"""
)
# 4. prepare paths for file comparison
actual_output_path = tmpdir.join("actual_output", "pack.IMC")
expected_ouput_path = tmpdir.join("expected_output.IMC")
# 5. check that the expected and actual output files are identical
assert filecmp.cmp(actual_output_path, expected_ouput_path, shallow=False)
def pipeopen(command,
important=True,
logger=log,
allowfork=False,
quiet=True,
close_fds=True,
env=None,
stdout=PIPE,
stderr=PIPE,
start_new_session=False):
if not quiet:
logger.log(logging.NOTICE if important else logging.DEBUG,
"Popen()ing: " + command)
args = shlex_split(str(command))
preexec_fn = None
if allowfork:
preexec_fn = unblock_sigchld
return Popen(
args,
stdin=PIPE,
stdout=stdout,
stderr=stderr,
close_fds=close_fds,
preexec_fn=preexec_fn,
env=env,
encoding='utf8',
start_new_session=start_new_session,
)
def parse(cls, argstr):
tokens = shlex_split(argstr)
executable = 'ocrd-%s' % tokens.pop(0)
input_file_grps = []
output_file_grps = []
parameters = {}
while tokens:
if tokens[0] == '-I':
for grp in tokens[1].split(','):
input_file_grps.append(grp)
tokens = tokens[2:]
elif tokens[0] == '-O':
for grp in tokens[1].split(','):
output_file_grps.append(grp)
tokens = tokens[2:]
elif tokens[0] == '-p':
parameters = {
**parameters,
**parse_json_string_or_file(tokens[1])
}
tokens = tokens[2:]
elif tokens[0] == '-P':
set_json_key_value_overrides(parameters, tokens[1:3])
tokens = tokens[3:]
else:
raise Exception(
"Failed parsing task description '%s' with tokens remaining: '%s'"
% (argstr, tokens))
return ProcessorTask(executable, input_file_grps, output_file_grps,
parameters)
def call_sg_raw(self, cmd, arg='None', buf_len=0):
try:
if len(self._settings[arg]) == 0:
self.error('Bad settings value')
elif len(self._cdb_cmds[cmd]) == 0:
self.error('Bad command')
if buf_len:
buf_len = '-r%s -b' % buf_len
else:
buf_len = ''
cmdl = '%s %s %s %s %s 00 00 00 00 00 00 00 00' % (self._sg_raw, buf_len, self._mount_config.device_id, self._cdb_cmds[cmd], self._settings[arg])
cmd = shlex_split(cmdl)
p = Popen(cmd, stdout=PIPE, stderr=PIPE)
err = p.stderr.read()
if not 'Good' in err:
if arg != 'None':
return False
else:
self.error('SCSI error.')
if arg != 'None':
sleep(1)
return p.stdout.read()
except Exception, e:
self.error(e)
def _blind(dir_path):
# First strip additional arguments
subset_cmd = ('./subset.py -s core -m '
'-a Site,CSite,Sidechain,Contextgene {}'
).format(' '.join(_find(dir_path, '.*\.a2')))
subset_main(shlex_split(subset_cmd))
# We need to rename the new core files
for a2_core_file_path in _find(dir_path, '.*\.a2.core'):
a2_file_path = a2_core_file_path[:-5]
assert isfile(a2_file_path)
move(a2_core_file_path, a2_file_path)
# Then negation and speculations
for a2_file_path in _find(dir_path, '.*\.a2'):
with open(a2_file_path, 'r') as a1_file:
a1_data = a1_file.read().split('\n')
# Write the annotations again without modifiers
with open(a2_file_path, 'w') as a1_file:
lines = []
for line in a1_data:
line_m = NESP_REGEX.match(line)
if line_m is None:
lines.append(line)
a1_file.write('\n'.join(lines))
def test_simple_replacement(tmpdir):
tmpdir.join("VERSION").write("1.2.0")
tmpdir.chdir()
main(
shlex_split(
"patch --current-version 1.2.0 --new-version 1.2.1 VERSION"))
assert "1.2.1" == tmpdir.join("VERSION").read()
def _extract(arch_path, dir_path):
tar_cmd = 'tar -x -z -f {} -C {}'.format(arch_path, dir_path)
tar_p = Popen(shlex_split(tar_cmd), stderr=PIPE)
tar_p.wait()
tar_p_stderr = tar_p.stderr.read()
if tar_p_stderr:
print >> stderr, tar_p_stderr
assert False, 'tar exited with an error'
def test_simple_replacement_in_utf8_file(tmpdir):
tmpdir.join("VERSION").write("Kröt1.3.0".encode('utf-8'), 'wb')
tmpdir.chdir()
main(
shlex_split(
"patch --current-version 1.3.0 --new-version 1.3.1 VERSION"))
out = tmpdir.join("VERSION").read('rb')
assert "'Kr\\xc3\\xb6t1.3.1'" in repr(out)
def test_nonexisting_file(tmpdir):
tmpdir.chdir()
tmpdir.join("mysourcecode.txt").write("1.2.3")
with pytest.raises(IOError):
main(shlex_split("patch --current-version 1.2.3 mysourcecode.txt doesnotexist2.txt"))
# first file is unchanged because second didn't exist
assert '1.2.3' == tmpdir.join("mysourcecode.txt").read()
def from_terminal(subject):
from subprocess import run as run_bash
from shlex import split as shlex_split
for command in bash_commands[subject]:
print('> EXECUTING:', command)
run_bash(shlex_split(command), timeout=None, check=True)
return True
def _parse_import(self, token):
parts = shlex_split(token.body)
fn = parts[0]
if len(parts) > 1:
assert parts[1] == 'as'
return ir.ImportNode(fn, parts[2])
else:
return ir.ImportNode(fn)
def _parse_import(self, token):
parts = shlex_split(token.body)
fn = parts[0]
if len(parts) > 1:
assert parts[1] == 'as'
return ir.ImportNode(fn, parts[2])
else:
return ir.ImportNode(fn)
def test_nonexisting_file(tmpdir):
tmpdir.chdir()
tmpdir.join("mysourcecode.txt").write("1.2.3")
with pytest.raises(IOError):
main(shlex_split("patch --current-version 1.2.3 mysourcecode.txt doesnotexist2.txt"))
# first file is unchanged because second didn't exist
assert '1.2.3' == tmpdir.join("mysourcecode.txt").read()
def _extract(arch_path, dir_path):
tar_cmd = 'tar -x -z -f {} -C {}'.format(arch_path, dir_path)
tar_p = Popen(shlex_split(tar_cmd), stderr=PIPE)
tar_p.wait()
tar_p_stderr = tar_p.stderr.read()
if tar_p_stderr:
print >> stderr, tar_p_stderr
assert False, 'tar exited with an error'
def pipeopen(command, allowfork=False):
args = shlex_split(str(command))
preexec_fn = fastclose
if allowfork:
preexec_fn = lambda : (fastclose(), unblock_sigchld())
return Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE,
close_fds=False, preexec_fn=preexec_fn)
def clone(self, repo_url, quiet=True):
'''Clone remote repository to local working directory'''
if os.path.exists(self.repo_dir):
raise OSError('Repo {0} already exists.'.format(self.repo_dir))
git_cmd = 'git clone {0} {1} {2}'.format('-q' if quiet else '',
repo_url,
self.repo_dir)
git(shlex_split(git_cmd))
def _train_model(c, train_path, model_path):
train_cmd = ("'{0}/ext/liblinear/train"
" -q -c {1} {2} {3}'").format(dirname(__file__), c, train_path,
model_path)
#print train_cmd
train_p = Popen(
shlex_split(train_cmd),
shell=True, executable='/bin/bash') # XXX: Nasty bash path
train_p.wait()
def _repack(tmp_dir, arch_path):
tar_cmd = 'tar cfz enriched.tar.gz {}'.format(
' '.join(basename(p) for p in _find(tmp_dir, '.*\.a2')))
tar_p = Popen(shlex_split(tar_cmd), stderr=PIPE, cwd=tmp_dir)
tar_p.wait()
tar_p_stderr = tar_p.stderr.read()
if tar_p_stderr:
print >> stderr, tar_p_stderr
assert False, 'tar exited with an error'
move(path_join(tmp_dir, 'enriched.tar.gz'), arch_path)
def _train_model(c, train_path, model_path):
# XXX: BAD BAD BAD PATH!
train_cmd = ("'/home/pontus/git/eepura/ext/liblinear/train"
" -q -c {} {} {}'").format(c, train_path,
model_path)
#print train_cmd
train_p = Popen(
shlex_split(train_cmd),
shell=True, executable='/bin/bash') # XXX: Nasty bash path
train_p.wait()
def pipeopen(command, important=True, logger=log, allowfork=False, quiet=False, close_fds=False):
if not quiet:
logger.log(logging.NOTICE if important else logging.DEBUG, "Popen()ing: " + command)
args = shlex_split(str(command))
preexec_fn = None
if allowfork:
preexec_fn = unblock_sigchld
return Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=close_fds, preexec_fn=preexec_fn)
def send(self, data, type='small'):
try:
data_len = len(data)
bytes_sent = 0
if type == 'small':
request_len = self._request_small
trans_len = '01'
lba = '20'
else:
request_len = self._request_large
trans_len = '14'
lba = '40'
if type == 'upload':
f = open(data, 'rb')
f.seek(0, 2)
data_len = f.tell()
f.seek(0)
i = 0
while bytes_sent != data_len:
if type == 'upload':
data_p = f.read(request_len)
else:
data_p = data[i:request_len+i]
data_p_len = len(data_p)
data_p += '\x00'*(request_len-data_p_len)
scsi_cmd = '%s %s -b -s %s -n 2A 80 00 00 00 %s 00 00 %s 00' % (self._sg_raw, self._conn_iface.device_id, request_len, lba, trans_len)
scsi_cmd = shlex_split(scsi_cmd)
p = Popen(scsi_cmd, bufsize=0, stdin=PIPE, stderr=PIPE)
p.stdin.write(data_p)
i += request_len
bytes_sent = bytes_sent + data_p_len
if type == 'upload' or type == 'script':
sys.stdout.write('\r Bytes sent: %s' % bytes_sent)
sys.stdout.flush()
ret = self.receive()
if not '103 CONT' in ret:
break
if type == 'upload':
f.close()
if type == 'upload' or type == 'script':
self.sendrtn('101 EOF:%s\x00' % (request_len - data_p_len))
return bytes_sent
except Exception, e:
self.error('Send error: %s' % e)
def test_config_file(tmpdir):
tmpdir.join("file1").write("0.9.34")
tmpdir.join("mybumpconfig.cfg").write("""[bumpversion]
current_version: 0.9.34
new_version: 0.9.35
files: file1""")
tmpdir.chdir()
main(shlex_split("patch --config-file mybumpconfig.cfg"))
assert "0.9.35" == tmpdir.join("file1").read()
def __init__(self, line):
r"""
Parses the line for the input key string and value
"""
# inital values
self.line_arr = []
self.keyword = ''
self.value_index = -1
self.unit = ''
self.comment_msg = ''
self.commented_out = False
#
# removing semi-colon if whole line was commented out
line = line.strip()
mat = re.match(r'^;(.*)', line)
if mat:
line = mat.group(1)
self.commented_out = True
#
# removing any comments after occurring the value
mat = re.search(r';.*', line)
if mat:
self.comment_msg = line[mat.start():].strip()
line = line[:mat.start()]
#
self.line = line
self.value = line
try:
self.line_arr = shlex_split(line) or ['']
except ValueError:
# TODO: Add debug message here saying shlex failed
self.line_arr = re.split(r'\s+', line)
self.line_arr = [v for v in self.line_arr if v]
self.keyword = re.match(r'[a-zA-z_-]*', self.line).group()
#
# if line has a colon the field after it will be used as the value
# otherwise the whole line is considered the value
if not re.search(r':(:?\s|$)', self.line):
return
#
for ifld, field in enumerate(self.line_arr):
if re.search(r':$', field):
try:
self.value = self.line_arr[ifld+1]
self.value_index = ifld+1
if len(self.line_arr) > ifld + 2:
self.unit = self.line_arr[ifld+2]
except IndexError:
self.value = 'NONE'
self.value_index = ifld+1
self.line_arr.append(self.value)
self.line = ' '.join(self.line_arr)
def remote_branch_names(self):
'''Get branch names from output of git-remote show'''
git_cmd = 'git --git-dir={0} branch -a'.format(self.git_dir)
proc, stdout, stderr = git(shlex_split(git_cmd))
ore = self.BRANCH_LINE_RE
for line in sbuffer_iterator(stdout):
branch_name = line.strip()
omatch = ore.match(branch_name)
if omatch is not None:
# ignore such line 'remotes/origin/HEAD -> origin/master'
if not omatch.group(1).startswith('HEAD ->'):
yield branch_name
def play_tts(self, tts_filepath):
""" Play a sound file using the command line specified by 'cmd_line_play
:param tts_filepath: filename path
:raises FileNotFoundError if the cmd to be executed is not found
"""
if self.__cmd_line_play is None:
raise RuntimeError("Unable to play '{}' since 'cmd_line_play' param was not specified".format(tts_filepath))
else:
args = shlex_split(self.__cmd_line_play.replace('[filename]', tts_filepath))
try:
call(args)
except FileNotFoundError as e:
raise FileNotFoundError("Executable specified in cmd_line_play not found ({})".format(str(e)))
def pipeopen(command, important=True, logger=log, allowfork=False, quiet=True, close_fds=True, env=None, stdout=PIPE, stderr=PIPE, start_new_session=False):
if not quiet:
logger.log(
logging.NOTICE if important else logging.DEBUG,
"Popen()ing: " + command
)
args = shlex_split(str(command))
preexec_fn = None
if allowfork:
preexec_fn = unblock_sigchld
return Popen(
args, stdin=PIPE, stdout=stdout, stderr=stderr,
close_fds=close_fds, preexec_fn=preexec_fn,
env=env, encoding='utf8', start_new_session=start_new_session,
)
def safe_copy(infile, outfile, **kwargs):
kwargs.setdefault('sleep', 10)
kwargs.setdefault('attempts', 10)
kwargs.setdefault('debug', False)
kwargs.setdefault('checksum', False)
kwargs.setdefault("checksum_blocksize", 4096)
kwargs.setdefault('mkdir',False)
sleep = parse_sleep(kwargs['sleep'])
xrootd = False
if kwargs['mkdir']: mkdir(dirname(outfile))
if kwargs['debug']:
print 'cp %s -> %s' % (infile, outfile)
# Try not to step on any toes....
infile = expandvars(infile)
outfile = expandvars(outfile)
cmnd = "cp %s %s" % (infile, outfile)
if infile.startswith("root:"):
if kwargs['debug']: print 'input file is on xrootd - switching to XRD library'
xrootd = True
if outfile.startswith("root:"):
if kwargs['debug']: print 'output file is on xrootd - switching to XRD library'
xrootd = True
if xrootd:
cmnd = "xrdcp -f %s %s" % (infile, outfile)
md5in = md5out = None
if kwargs['checksum'] and not xrootd:
md5in = md5sum(infile, blocksize=kwargs['checksum_blocksize'])
i = 1
while i < kwargs['attempts']:
if kwargs['debug'] and i > 0:
print "Attempting to copy file..."
status = sub.call(shlex_split(cmnd))
if status == 0:
if kwargs['checksum'] and not xrootd:
md5out = md5sum(outfile, blocksize=kwargs['checksum_blocksize'])
if md5in == md5out:
return status
else:
print '%i - copy successful but checksum does not match, try again in 5s'
time_sleep(5)
else:
print "%i - Copy failed; sleep %ss" % (i, sleep)
time_sleep(sleep)
i += 1
raise IOError("Failed to copy file")
def upload(self, path):
try:
if not os.path.exists(path) or os.path.isdir(path):
self.error('Surgeon not found.')
cbf.check(path)
buf = ''
with open(path, 'rb') as f:
buf = f.read()
f.close()
buf_len = len(buf)
packet_leftovers = buf_len % cbf.PACKET_SIZE
if packet_leftovers > 0:
padding_size = cbf.PACKET_SIZE - packet_leftovers
buf += struct.pack('%ss' % padding_size, '\xFF'*padding_size)
buf_len = len(buf)
packets = buf_len/cbf.PACKET_SIZE
byte1 = '00'
total = 0
last_total = 0
for i in range(0, packets):
cmdl = '%s %s -b -s %s -n 2A 00 00 00 00 %s 00 00 20 00' % (self._sg_raw, self._mount_config.device_id, cbf.PACKET_SIZE, byte1)
cmd = shlex_split(cmdl)
byte1 = '01'
p = Popen(cmd, stdin=PIPE, stderr=PIPE)
p.stdin.write(buf[last_total:last_total+cbf.PACKET_SIZE])
err = p.stderr.read()
if not 'Good' in err:
self.error('SCSI error.')
last_total += cbf.PACKET_SIZE
p = Popen([self._sg_verify, self._mount_config.device_id], stderr=PIPE)
err = p.stderr.read()
if len(err) != 0:
self.error('SCSI error.')
except Exception, e:
self.rerror(e)
def logs(self, rev_range=None, branch_name=None, no_merges=True):
'''Retreive logs from specific branch
You can specify a specific branch to retreive logs against it, or from
current branch by omitting the branch_name argument.
IMPORTANT: currently, no_merges is only a placeholder.
Logs are retreived by using this command,
git --git-dir=/path/to/repo/.git logs \
--pretty="format:%H,%at,%aN,%aE%n%s" --no-merges \
rev1...rev2 [branch_name]
'''
git_cmd = 'git --git-dir={0} log ' \
'--pretty="format:%n%H,%at,%aN,%aE%n%s" ' \
'--shortstat {1} {2} {3}'
git_cmd = git_cmd.format(self.git_dir,
# FIXME: to support no_merges argument
'--no-merges',
rev_range if rev_range else 'HEAD',
branch_name if branch_name else '')
proc, stdout, stderr = git(shlex_split(git_cmd))
log_lines = []
buffer = StringIO(stdout)
# If you want to understand this while-loop, issue git-log above is a
# good idea.
while True:
line = buffer.readline()
if not line:
if log_lines:
yield GitLog.parse(log_lines)
break
s = line.strip()
if s == u'':
if log_lines:
yield GitLog.parse(log_lines)
# Prepare for holding lines of next log
del log_lines[:]
# Here, skip any blank lines before each set of log lines
else:
log_lines.append(s)
buffer.close()
