def getspec( infile, region='relpix,box(-2,-2,0,0)', vsource=5., hann=5, tmpfile="junk" ):
'''dump out spectrum of selected region with imspec, return [chan, freqLSR, flux] arrays'''
# step 1: use imlist to retrieve velocity and freq information from the header
p= subprocess.Popen( ( shlex.split('imlist in=%s' % infile) ), \
stdout=subprocess.PIPE,stdin=subprocess.PIPE,stderr=subprocess.STDOUT)
result = p.communicate()[0]
lines = result.split("\n")
for line in lines :
if len(line) > 1 :
a = line.split()
n = string.find( line, "restfreq:" )
if n >= 0 :
restfreq = float( line[n+9:].split()[0] )
n = string.find( line, "crval3 :" )
if n >= 0 :
v1 = float( line[n+9:].split()[0] )
n = string.find( line, "cdelt3 :" )
if n >= 0 :
dv = float( line[n+9:].split()[0] )
print "restfreq = %.5f GHz; v1 = %.3f km/sec; dv = %.3f km/sec" % (restfreq,v1,dv)
# step 2: use imspec to dump out the spectrum for the selected region to tmpfile
chan = []
freq = []
flux = []
p= subprocess.Popen( ( shlex.split("imspec in=%s region=%s options=list,eformat,noheader,hanning,%d log=%s" % \
(infile,region,hann,tmpfile) )), stdout=subprocess.PIPE,stdin=subprocess.PIPE,stderr=subprocess.STDOUT)
time.sleep(1)
result = p.communicate()[0]
print result
if "Fatal Error" in result :
print " --- fatal --- "
return
# step 3: read velocities and flux densities from tmpfile, create arrays
fin = open( tmpfile, "r" )
for line in fin :
a = line.split()
if len(a) > 2 :
chan.append( int(a[0]) )
nchan = int( a[0] )
vlsr = float( a[1] )
flux.append( float( a[2] ) )
vlsrcalc = v1 + (nchan - 1) * dv
if abs(vlsrcalc-vlsr) > 0.05 :
print "WARNING: channel %d listed vlsr = %.2f, calculated = %.2f" % (nchan,vlsr,vlsrcalc)
fqLSR = restfreq * (1. - vlsrcalc/clight)
freq.append( fqLSR/(1.-vsource/clight) )
#print nchan, vlsrcalc, fqLSR, freq[-1]
# freq in rest frame of source
fin.close()
print "read in %d lines" % len(freq)
# step 4: sort in frequency order, return arrays
spectrum = numpy.array(sorted(zip(freq,chan,flux)))
# this sorts the chan,freq,flux triplets in freq order
a,b,c = numpy.split( spectrum, 3, axis=1 )
# this returns separate freq and flux arrays
return numpy.reshape(b,len(a)), numpy.reshape(a,len(b)), numpy.reshape(c,len(c))
def create_delete_dirlinks(filelink, operation):
if operation in ["-c", "--copy"]:
path = filelink.replace("Desktop.ini", "");
path = path.replace("\\", "\\\\")
source = path +"Desktop.ini";
target = path +"myDesktop.ini";
elif operation in ["-r", "--restore"]:
path = filelink.replace("myDesktop.ini", "");
path = path.replace("\\", "\\\\")
path1 = filelink.replace("\\myDesktop.ini", "");
path1 = path1.replace("\\", "\\\\")
source = path +"myDesktop.ini";
target = path +"Desktop.ini";
else:
raise sys.exit("Unknown operation. Available operations: -c | --copy | -r | --restore")
print("copying target to source file.")
args = shlex.split("cp "+source+" "+target);
p = subprocess.call(args);
print("Set hidden and system attributes of .ini file.")
args = shlex.split("attrib +H +S "+target);
p = subprocess.call(args);
if operation in ["-r", "--restore"]:
print("Set read-only attribute of directory.")
print(path1)
args = shlex.split("attrib +R "+path1);
p = subprocess.call(args);
def install_ohmyzsh():
print green('Installing: {0}'.format(red('oh-my-zsh', True)), True)
home = os.getenv('HOME')
current = os.path.abspath(os.path.dirname(__file__))
if not is_program_installed('zsh'):
install_program('zsh')
zsh_dir = os.path.join(home, '.oh-my-zsh')
backup = os.path.join(home, '.oh-my-zsh-save')
if os.path.exists(zsh_dir):
print green('\tBacking up folder: {0} --> {1}'.format(cyan(zsh_dir), cyan(backup)), True)
if os.path.exists(backup):
shutil.rmtree(backup)
shutil.move(zsh_dir, backup)
try:
# clone zsh repo
cmd = 'git clone --depth=1 https://github.com/robbyrussell/oh-my-zsh.git {0}'.format(zsh_dir)
check_output(shlex.split(cmd), stderr=subprocess.STDOUT, preexec_fn=init_child_process)
target = os.path.join(current, '.zshrc')
zshrc = os.path.join(home, '.zshrc')
print green('\tSymlinking file: {0} --> {1}'.format(cyan(zshrc), cyan(target)), True)
check_output(shlex.split('ln -sf {0} {1}'.format(target, zshrc)))
# change the shell
if 'zsh' not in os.environ['SHELL']:
os.system('chsh -s $(grep /zsh$ /etc/shells | tail -1)')
except CalledProcessError as e:
print '{0}\n{1}'.format(red(e, True), green(e.output, True))
finally:
print
def _getini(self, name):
try:
description, type, default = self._parser._inidict[name]
except KeyError:
raise ValueError("unknown configuration value: %r" %(name,))
try:
value = self.inicfg[name]
except KeyError:
if default is not None:
return default
if type is None:
return ''
return []
if type == "pathlist":
dp = py.path.local(self.inicfg.config.path).dirpath()
l = []
for relpath in shlex.split(value):
l.append(dp.join(relpath, abs=True))
return l
elif type == "args":
return shlex.split(value)
elif type == "linelist":
return [t for t in map(lambda x: x.strip(), value.split("\n")) if t]
elif type == "bool":
return bool(_strtobool(value.strip()))
else:
assert type is None
return value
def run_checks(hooks_all, hooks_modified, modified, path):
""" Run selected checks on the current git index """
retcode = 0
for command in hooks_all:
if not isinstance(command, list):
command = shlex.split(command)
retcode |= subprocess.call(command, env={'PATH': path})
for pattern, command in hooks_modified:
if not isinstance(command, list):
command = shlex.split(command)
for filename in modified:
if not fnmatch.fnmatch(filename, pattern):
continue
printed_filename = False
proc = subprocess.Popen(command + [filename],
env={'PATH': path},
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output, _ = proc.communicate()
if proc.returncode != 0:
if not printed_filename:
print(filename)
print('=' * len(filename))
printed_filename = True
print(command[0])
print('-' * len(command[0]))
print(output)
retcode |= proc.returncode
return retcode
def ask_ptxdist(pkg):
ptxdist = os.environ.get("PTXDIST", os.environ.get("ptxdist", "ptxdist"))
p = subprocess.Popen([ ptxdist, "-k", "make",
"/print-%s_DIR" % pkg,
"/print-%s_SUBDIR" % pkg,
"/print-%s_CONF_OPT" % pkg,
"/print-%s_AUTOCONF" % pkg,
"/print-%s_CONF_TOOL" %pkg,
"/print-CROSS_MESON_USR",
"/print-CROSS_AUTOCONF_USR",
"/print-PTXDIST_SYSROOT_HOST"],
stdout=subprocess.PIPE,
universal_newlines=True)
d = p.stdout.readline().strip()
subdir = p.stdout.readline().strip()
opt = shlex.split(p.stdout.readline().strip()) + shlex.split(p.stdout.readline().strip())
tool = p.stdout.readline().strip()
if not tool and opt:
tool = "autoconf"
meson_default = shlex.split(p.stdout.readline().strip())
autoconf_default = shlex.split(p.stdout.readline().strip())
sysroot_host = p.stdout.readline().strip()
if tool == "meson" and not opt:
opt = meson_default
if tool == "autoconf" and not opt:
opt = autoconf_default
if not d:
abort("'%s' is not a valid package: %s_DIR is undefined" % (pkg, pkg))
if not opt:
abort("'%s' is not a autoconf/meson package: %s_CONF_OPT and %s_AUTOCONF are undefined" % (pkg, pkg, pkg))
return (tool, d, subdir, opt, sysroot_host)
def replace_line_in_file(fname, startswith, replacement):
"""Will overwrite tempfile.txt."""
subprocess.call(shlex.split("cp %s tempfile.txt" % fname))
infile = open('tempfile.txt')
outfile = open(fname, 'w')
i = 0
for line in infile:
i += 1
written = False
for entry, repl in zip(startswith, replacement):
if isinstance(entry, int):
if i == entry:
outfile.write(repl)
written = True
break
elif isinstance(entry, str):
if line.startswith(entry):
outfile.write(repl)
written = True
break
if not written:
outfile.write(line)
subprocess.call(shlex.split("rm tempfile.txt"))
infile.close()
outfile.close()
async def handle_event(self, event):
if event['type'] == 'command' and event['command'] == self.name:
try:
result = self.argument_parser.parse_args(event['arguments'])
except (ValueError, argparse.ArgumentTypeError) as error:
await self.client.send_message(event['message'].channel, str(error))
return
if hasattr(result, 'set'):
self.settings.data['prefix'] = getattr(result, 'value')
self.settings.save()
await self.client.send_message(event['message'].channel, 'Set command prefix to: {}'.format(getattr(result, 'value')))
if event['type'] == 'message':
message = event['message']
if message.content.startswith(self.settings.data['prefix']):
split_text = shlex.split(message.content[len(self.settings.data['prefix']):])
clean_split_text = shlex.split(message.clean_content[len(self.settings.data['prefix']):])
await self.client.event_queue.put({
'type': 'command',
'message': message,
'command': split_text[0],
'arguments': split_text[1:],
'clean_arguments': clean_split_text[1:]
})
def shlex_split(string, env_replace=True, envs=None):
"""
Split the string, applying environment variable substitutions
Python2's shlex doesn't like unicode, so we have to convert the string
into bytes, run shlex.split() and convert it back to unicode.
This applies environment variable substitutions on the string when
quoting allows, and splits it into tokens at whitespace
delimiters.
Environment variable substitution is applied to the entire string,
even the part before any '=', which is not technically correct but
will only fail for invalid Dockerfile content.
:param string: str, string to split
:param env_replace: bool, whether to perform substitution
:param envs: dict, environment variables for substitution
"""
if env_replace:
string = EnvSubst(string, envs or {}).substitute()
if PY2 and isinstance(string, unicode):
string = u2b(string)
# this takes care of quotes
splits = shlex.split(string)
return map(b2u, splits)
else:
return shlex.split(string)
def find_last_svn_sha1():
'''
find sha1 of last svn revision.
'''
# git_svn_info = shlex.split("git svn info")
git_svn_url = shlex.split("git svn info --url")
# git_svn_log_commit = shlex.split("git svn log --show-commit --oneline --limit=1")
# find last svn commit in current history
try:
svn_url_result = subprocess.check_output(git_svn_url, stderr=subprocess.STDOUT)
logging.debug("Result:\n%s", svn_url_result)
svn_url = re.sub("\n", "", svn_url_result)
logging.debug("svn url: ", svn_url)
git_svn_log_commit_str = str.format('git log --grep="{0}" --oneline --max-count=1', svn_url)
logging.debug("Shell for finding last svn commits: %s.", git_svn_log_commit_str)
git_svn_log_commit = shlex.split(git_svn_log_commit_str)
logging.debug("Finding last svn commits: %s", git_svn_log_commit)
svn_info = subprocess.check_output(git_svn_log_commit)
logging.debug("svn log info: %s", svn_info)
m = re.findall(regex_sha1, svn_info)
logging.info("SVN SHA1: %s", m)
if m:
if len(m) == 1:
svn_sha1 = m[0]
logging.info("Found SHA1:%s", svn_sha1)
return svn_sha1
else:
logging.error("Please report error: Found more than one svn revision: %s", m)
else:
logging.error("SVN information is not found., Please make sure you are running this command under git-svn enabled repository.")
except subprocess.CalledProcessError, e:
logging.error("No git-svn information. Please make sure you are running this command under git-svn enabled repository.", e)
def launch_workers(self):
absenv, addenv = self.get_env()
p = sb.Popen(
shlex.split('ssh {sshname} cd {workdir}; sh -c "cat > job.sh"'.format(**self.params)), stdin=sb.PIPE
)
p.stdin.write("#!/bin/sh\n")
if self.params["type"] == "PBS":
spec = "ssh {sshname} cd {workdir}; qsub -N dmon -o logs/logs -t 1-{cores} -j oe job.sh"
else:
p.stdin.write("source ENV/bin/activate\n")
spec = "ssh {sshname} cd {workdir}; qsub -q normal.q -N dmon -e logs -o logs -t 1:{cores} -j y job.sh"
p.stdin.write("\n".join(("export {0}={1}".format(x, y) for x, y in absenv.iteritems())))
p.stdin.write("\n")
p.stdin.write("\n".join(("export {0}=${0}:{1}".format(x, y) for x, y in addenv.iteritems())))
p.stdin.write("\ncd {workdir}\n".format(**self.params))
p.stdin.write("\n{pythonloc} -m jobmon.mon\n".format(**self.params))
p.stdin.close()
p.wait()
p = sb.Popen(shlex.split(spec.format(**self.params)))
ret = p.wait()
if ret != "0":
print("Success!")
else:
raise Exception("Error launching monitor daemons with return code %s" % str(ret))
def writeArgs(game, args):
cfile = Variables.playonlinux_rep+"shortcuts/"+game
fichier = open(cfile,"r").readlines()
i = 0
line = []
while(i < len(fichier)): # On retire l'eventuel
fichier[i] = fichier[i].replace("\n","")
if("POL_Wine " not in fichier[i]):
line.append(fichier[i])
else:
try:
old_string = shlex.split(fichier[i])
new_string = shlex.split(str(args))
new_string = old_string[0:2] + new_string
new_string = " ".join([ pipes.quote(x) for x in new_string])
new_string = new_string+' "$@"'
line.append(new_string)
except:
line.append(fichier[i])
i += 1
fichier_write = open(cfile,"w")
i = 0
while(i < len(line)): # On ecrit
fichier_write.write(line[i]+"\n")
i+=1
def RunTiming(self, options):
Log.Info("Perform ALLKNN.", self.verbose)
# If the dataset contains two files then the second file is the query file.
# In this case we add this to the command line.
if len(self.dataset) == 2:
cmd = shlex.split(self.path + "mlpack_allknn -r " + self.dataset[0] +
" -q " + self.dataset[1] + " -v -n neighbors.csv -d distances.csv " +
options)
else:
cmd = shlex.split(self.path + "mlpack_allknn -r " + self.dataset +
" -v -n neighbors.csv -d distances.csv " + options)
# Run command with the nessecary arguments and return its output as a byte
# string. We have untrusted input so we disable all shell based features.
try:
s = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=False,
timeout=self.timeout)
except subprocess.TimeoutExpired as e:
Log.Warn(str(e))
return -2
except Exception as e:
Log.Fatal("Could not execute command: " + str(cmd))
return -1
# Return the elapsed time.
timer = self.parseTimer(s)
if not timer:
Log.Fatal("Can't parse the timer")
return -1
else:
time = self.GetTime(timer)
Log.Info(("total time: %fs" % (time)), self.verbose)
return time
def exec_command(self, cmd, args=None, printflag=True):
if printflag:
if args:
LOG.debug('cmd: %(cmd)s, args: %(args)s' %
{'cmd': cmd, 'args': args})
else:
LOG.debug('cmd: %s' % cmd)
cmd = [cmd]
if args:
if isinstance(args, six.text_type):
cmd += shlex.split(args.encode())
else:
cmd += shlex.split(args)
try:
stdout, stderr = utils.execute(*cmd, run_as_root=True)
ret = 0
except putils.ProcessExecutionError as e:
ret = e.exit_code
stdout = e.stdout
stderr = e.stderr
LOG.debug('cmd: %s' % six.text_type(cmd))
LOG.debug('from: %s' % six.text_type(inspect.stack()[2]))
LOG.debug('ret: %d' % ret)
LOG.debug('stdout: %s' % stdout.replace(os.linesep, ' '))
LOG.debug('stderr: %s' % stderr.replace(os.linesep, ' '))
return ret, stdout, stderr
def post(self, request):
try:
data = received_json_data=json.loads(request.body)
except Exception as e:
return JsonResponse(BAD_JSON_RESPONSE, status=400)
if 'cmd' not in data:
return JsonResponse(NO_CMD_RESPONSE, status=400)
cmd = data['cmd']
# Random key for the output
outputKey = generate_key()
outfile = open('/tmp/' + outputKey + '.out','w')
errfile = open('/tmp/' + outputKey + '.err','w')
try:
if '|' in cmd:
cmd_parts = cmd.split('|')
# Run the first part of the pipe
p = psutil.Popen(shlex.split(cmd_parts[0]),stdin=None, stdout=PIPE, stderr=errfile)
# Do the rest in background
t = threading.Thread(target=runPipe, args=(cmd_parts[1:], p, outfile, errfile))
t.start()
else:
# Run the command
p = psutil.Popen(shlex.split(cmd), stdout=outfile, stderr=errfile)
# Wait in background
t = threading.Thread(target=runCmd, args=(p, outfile, errfile))
t.start()
except OSError as e:
outfile.close()
errfile.close()
return JsonResponse(BAD_CMD_RESPONSE, status=400)
return JsonResponse({'result': 'ok', 'process': p.pid, 'output': outputKey })
def start(self):
"""Start the router: Configure the daemons, set the relevant sysctls,
and fire up all needed processes"""
self.cmd('ip', 'link', 'set', 'dev', 'lo', 'up')
# Build the config
self.config.build()
# Check them
err_code = False
for d in self.config.daemons:
out, err, code = self._processes.pexec(shlex.split(d.dry_run))
err_code = err_code or code
if code:
lg.error(d.NAME, 'configuration check failed ['
'rcode:', str(code), ']\n'
'stdout:', str(out), '\n'
'stderr:', str(err))
if err_code:
lg.error('Config checks failed, aborting!')
mininet.clean.cleanup()
sys.exit(1)
# Set relevant sysctls
for opt, val in self.config.sysctl:
self._old_sysctl[opt] = self._set_sysctl(opt, val)
# Fire up all daemons
for d in self.config.daemons:
self._processes.popen(shlex.split(d.startup_line))
# Busy-wait if the daemon needs some time before being started
while not d.has_started():
time.sleep(.001)
def run(self):
if platform.mac_ver()[0] < '10.7.':
cc = [get_config_var('CC')]
env = dict(os.environ)
env['MACOSX_DEPLOYMENT_TARGET'] = get_config_var('MACOSX_DEPLOYMENT_TARGET')
else:
cc = ['xcrun', 'clang']
env = dict(os.environ)
env['MACOSX_DEPLOYMENT_TARGET'] = get_config_var('MACOSX_DEPLOYMENT_TARGET')
if not os.path.exists('lib'):
os.mkdir('lib')
cflags = get_config_var('CFLAGS')
arch_flags = sum([shlex.split(x) for x in re.findall('-arch\s+\S+', cflags)], [])
root_flags = sum([shlex.split(x) for x in re.findall('-isysroot\s+\S+', cflags)], [])
cmd = cc + arch_flags + root_flags + ['-dynamiclib', '-o', os.path.abspath('lib/libshared.1.dylib'), 'src/sharedlib.c']
subprocess.check_call(cmd, env=env)
if os.path.exists('lib/libshared.dylib'):
os.unlink('lib/libshared.dylib')
os.symlink('libshared.1.dylib', 'lib/libshared.dylib')
if not os.path.exists('lib/stash'):
os.makedirs('lib/stash')
if os.path.exists('lib/libhalf.dylib'):
os.unlink('lib/libhalf.dylib')
cmd = cc + arch_flags + root_flags + ['-dynamiclib', '-o', os.path.abspath('lib/libhalf.dylib'), 'src/sharedlib.c']
subprocess.check_call(cmd, env=env)
os.rename('lib/libhalf.dylib', 'lib/stash/libhalf.dylib')
os.symlink('stash/libhalf.dylib', 'lib/libhalf.dylib')
def updateJobInformation(self, workflow, task, **kwargs):
"""
_updateJobInformation_
Update job information for all jobs in the workflow and task,
the change will take effect if the job is Idle or becomes idle.
The currently supported changes are only priority for which both the task (taskPriority)
and workflow priority (requestPriority) must be provided.
"""
if 'taskPriority' in kwargs and 'requestPriority' in kwargs:
# Do a priority update
priority = (int(kwargs['requestPriority']) + int(kwargs['taskPriority'] * self.maxTaskPriority))
command = 'condor_qedit -constraint \'WMAgent_SubTaskName == "%s" && WMAgent_RequestName == "%s" ' %(task, workflow)
command += '&& (JobPrio != %d)\' JobPrio %d' % (priority, priority)
command = shlex.split(command)
proc = subprocess.Popen(command, stderr = subprocess.PIPE,
stdout = subprocess.PIPE)
_, stderr = proc.communicate()
if proc.returncode != 0:
# Check if there are actually jobs to update
command = 'condor_q -constraint \'WMAgent_SubTaskName == "%s" && WMAgent_RequestName == "%s"' %(task, workflow)
command += ' && (JobPrio != %d)\'' % priority
command += ' -format \'WMAgentID:\%d:::\' WMAgent_JobID'
command = shlex.split(command)
proc = subprocess.Popen(command, stderr = subprocess.PIPE,
stdout = subprocess.PIPE)
stdout, _ = proc.communicate()
if stdout != '':
msg = 'HTCondor edit failed with exit code %d\n'% proc.returncode
msg += 'Error was: %s' % stderr
raise BossAirPluginException(msg)
return
async def _validate(self, verrors, name, data):
if data["encryption"]:
if not data["encryption_password"]:
verrors.add(f"{name}.encryption_password", "This field is required when encryption is enabled")
credentials = await self._get_credentials(data["credentials"])
if not credentials:
verrors.add(f"{name}.credentials", "Invalid credentials")
if verrors:
raise verrors
provider = REMOTES[credentials["provider"]]
schema = []
if provider.buckets:
schema.append(Str("bucket", required=True, empty=False))
schema.append(Str("folder", required=True))
schema.extend(provider.task_schema)
attributes_verrors = validate_attributes(schema, data, additional_attrs=True)
if not attributes_verrors:
await provider.pre_save_task(data, credentials, verrors)
verrors.add_child(f"{name}.attributes", attributes_verrors)
try:
shlex.split(data["args"])
except ValueError as e:
verrors.add(f"{name}.args", f"Parse error: {e.args[0]}")
def reposync(self, repoid, repoconfig, newest, urlonly, quiet):
cmd = 'reposync --norepopath -r %s -p %s' % (repoid, repoid)
if repoconfig:
cmd += ' -c %s' % repoconfig
if self.str2bool(newest) is True:
cmd += ' -n'
if self.str2bool(urlonly) is True:
cmd += ' -u'
proc = subprocess.Popen(shlex.split(cmd), stdin=None,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
o, e = proc.communicate()
return o, e
if self.str2bool(quiet) is False:
proc = subprocess.Popen(shlex.split(cmd), stdin=None,
stdout=None, stderr=subprocess.PIPE)
o, e = proc.communicate()
else:
proc = subprocess.Popen(shlex.split(cmd), stdin=None,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
o, e = proc.communicate()
if repoid and (self.str2bool(urlonly) is False):
cwd = os.getcwd()
os.chdir(repoid)
# Check if RPMS dir already exists
if not os.path.lexists('RPMS'):
os.symlink('.', 'RPMS')
os.chdir(cwd)
def _is_command_malformed(self, original_command, modified_command):
"""
Checks if the command to be executed is safe and it's not in the block list
defined by the user. Returns False if the modified command is ok, True if
otherwise.
TODO: Use global command block list.
TODO: complete block idioms
"""
block_chars = set(["|", "$", "#"])
if original_command == modified_command:
return False
orig_cmd_args = shlex.split(original_command)
if not isinstance(modified_command, basestring):
modified_command = ""
mod_cmd_args = shlex.split(modified_command)
block_flag = False
orig_args_len = len(orig_cmd_args)
for index in xrange(0, len(mod_cmd_args)):
if index < orig_args_len and orig_cmd_args[index] == mod_cmd_args[index]:
continue
for char in block_chars:
if char in mod_cmd_args[index]:
block_flag = True
break
return block_flag
def start(self):
sys.stderr.flush()
call(['toilet', '-f', 'smbraille', 'Starting QEMU'])
sys.stdout.flush()
self.log.info("Starting VM, ssh port redirected to localhost:%s (inside docker, not exposed by default)", self.ssh_port)
if self.is_running():
raise VMError("VM is running, shutdown first")
if self._interactive:
self.qemu_process = Popen(shlex.split(QEMU_RUN_INTERACTIVE.format(ssh_port=self.ssh_port, ram=self.ram)))
return
else:
self.log.info("Starting in non-interactive mode. Terminal output is disabled.")
self.qemu_process = Popen(shlex.split(QEMU_RUN.format(ssh_port=self.ssh_port, ram=self.ram)), stdout=DEVNULL, stdin=DEVNULL, stderr=PIPE)
def keep_waiting():
return self.is_running()
logging.info("waiting for ssh to be open in the VM (timeout {}s)".format(self.timeout_s))
ssh_working = wait_ssh_open('127.0.0.1', self.ssh_port, keep_waiting, self.timeout_s)
if not self.is_running():
(_, stderr) = self.qemu_process.communicate()
raise VMError("VM failed to start, retcode: {}, stderr: {}".format( self.retcode(), stderr.decode()))
if not ssh_working:
if self.is_running():
self.log.error("VM running but SSH is not working")
self.terminate()
raise VMError("SSH is not working after {} seconds".format(self.timeout_s))
self.log.info("VM is online and SSH is up")
def start_server(socketname, compid, side, container, command, copyfrom, copyto):
comparison_home = "comparisons/{}".format(compid)
donefile = "{}/{}.done".format(comparison_home, side)
if os.path.exists(donefile):
return
context = zmq.Context()
socket = context.socket(zmq.PUSH)
socket.connect(socketname)
cmd, cidfile = docker_command(container, command)
time.sleep(2)
logfile = "{}/{}.log".format(comparison_home, side)
msgstub = {"to": "update_logs", "compid": compid, "side": side}
start_docker(cmd, cidfile, socket, logfile=logfile, msgstub=msgstub)
open(donefile, "a").close()
print "docker done... copying result"
container_id = open(cidfile).read()
cmd = "docker cp {}:{} {}/{}.{}".format(container_id, copyfrom, comparison_home, side, copyto)
subprocess.call(shlex.split(cmd))
cmd = "docker rm {}".format(container_id)
subprocess.call(shlex.split(cmd))
print "exit, runserver"
def main():
# Path to this script
my_path = os.path.dirname(os.path.realpath(__file__))
parser = argparse.ArgumentParser()
parser.add_argument("exe", help="""Path the MOM executable.""")
parser.add_argument("--git_repo", default=my_path, help="""
The path to the git repo.""")
parser.add_argument("--verbose", default=False, action='store_true',
help="""Verbose output, will print the hashes.""")
args = parser.parse_args()
readelf_out = sp.check_output(shlex.split('readelf -p .rodata {}'.format(args.exe)))
m = re.search('MOM_VERSION=(\w{40})', readelf_out)
exe_hash = m.group(1)
if args.verbose:
print('Exe hash {}'.format(exe_hash), file=sys.stderr)
curr_dir = os.getcwd()
os.chdir(args.git_repo)
git_out = sp.check_output(shlex.split('git rev-parse HEAD'))
os.chdir(curr_dir)
if args.verbose:
print('Repo hash {}'.format(git_out.strip()), file=sys.stderr)
if exe_hash == git_out.strip():
return 0
else:
return 1
def setup_module(module):
print ("")
print ("start applications backend")
global pid
global backend_address
if backend_address == "http://127.0.0.1:5000/":
# Set test mode for applications backend
os.environ['TEST_ALIGNAK_BACKEND'] = '1'
os.environ['TEST_ALIGNAK_BACKEND_DB'] = 'test_alignak_webui-datatable'
# Delete used mongo DBs
exit_code = subprocess.call(
shlex.split('mongo %s --eval "db.dropDatabase()"' % os.environ['TEST_ALIGNAK_BACKEND_DB'])
)
assert exit_code == 0
# No console output for the applications backend ...
FNULL = open(os.devnull, 'w')
pid = subprocess.Popen(
shlex.split('alignak_backend --hostname 127.0.0.1 --port 5000'), stdout=FNULL, stderr=FNULL
)
print ("PID: %s" % pid)
time.sleep(1)
def startPacketCaptureReader(self, source_node_name):
"""
Starts the packet capture reader.
"""
if not os.path.isfile(self._capture_file_path):
raise FileNotFoundError("the {} capture file does not exist on this host".format(self._capture_file_path))
if self._tail_process and self._tail_process.poll() is None:
self._tail_process.kill()
self._tail_process = None
if self._capture_reader_process and self._capture_reader_process.poll() is None:
self._capture_reader_process.kill()
self._capture_reader_process = None
command = self._settings["packet_capture_reader_command"]
# PCAP capture file path
command = command.replace("%c", '"' + self._capture_file_path + '"')
# Add description
description = "{} {} to {} {}".format(source_node_name, self.name(),
self.destinationNode().name(), self.destinationPort().name())
command = command.replace("%d", description)
if "|" in command:
# live traffic capture (using tail)
command1, command2 = command.split("|", 1)
info = None
if sys.platform.startswith("win"):
# hide tail window on Windows
info = subprocess.STARTUPINFO()
info.dwFlags |= subprocess.STARTF_USESHOWWINDOW
info.wShowWindow = subprocess.SW_HIDE
if hasattr(sys, "frozen"):
tail_path = os.path.dirname(os.path.abspath(sys.executable)) # for Popen to find tail.exe
else:
# We suppose a developer will have tail the standard GNS3 location
tail_path = "C:\\Program Files\\GNS3"
command1 = command1.replace("tail.exe", os.path.join(tail_path, "tail.exe"))
command1 = command1.strip()
command2 = command2.strip()
else:
try:
command1 = shlex.split(command1)
command2 = shlex.split(command2)
except ValueError as e:
msg = "Invalid packet capture command {}: {}".format(command, str(e))
print(msg)
log.error(msg)
return
self._tail_process = subprocess.Popen(command1, startupinfo=info, stdout=subprocess.PIPE)
self._capture_reader_process = subprocess.Popen(command2, stdin=self._tail_process.stdout, stdout=subprocess.PIPE)
self._tail_process.stdout.close()
else:
# normal traffic capture
if not sys.platform.startswith("win"):
command = shlex.split(command)
self._capture_reader_process = subprocess.Popen(command)
def run_tool(path):
if os.path.getsize(path) < 100:
print("pcap file too small, not splitting")
return
# need to make directories to store results from pcapsplitter
base_dir = path.rsplit('/', 1)[0]
timestamp = ""
try:
timestamp = '-'.join(str(datetime.datetime.now()).split(' ')) + '-UTC'
timestamp = timestamp.replace(':', '_')
except Exception as e: # pragma: no cover
print("couldn't create output directory with unique timestamp")
# make directory for tool name recognition of piping to other tools
output_dir = os.path.join(base_dir, 'pcap-node-splitter' + '-' + timestamp)
try:
os.mkdir(output_dir)
os.mkdir(output_dir + '/clients')
os.mkdir(output_dir + '/servers')
except OSError: # pragma: no cover
print("couldn't make directories for output of this tool")
try:
subprocess.check_call(shlex.split("./PcapSplitter -f " +
path + " -o " + output_dir + '/clients' + " -m client-ip"))
subprocess.check_call(shlex.split("./PcapSplitter -f " +
path + " -o " + output_dir + '/servers' + " -m server-ip"))
except Exception as e:
print(str(e))
def run_command(command):
if "|" in command:
command_parts = command.split('|')
elif ">" in command or ">>" in command or "<" in command or "<<" in command:
p = subprocess.Popen(command,stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
return ''.join(list(iter(p.stdout.readline, b'')))
else:
command_parts = []
command_parts.append(command)
i = 0
p = {}
for command_part in command_parts:
command_part = command_part.strip()
if i == 0:
p[i]=subprocess.Popen(shlex.split(command_part),stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
else:
p[i]=subprocess.Popen(shlex.split(command_part),stdin=p[i-1].stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
i = i +1
(output, err) = p[i-1].communicate()
exit_code = p[0].wait()
if exit_code != 0:
errorStr = "Shell Output: " + str(output) + '\n'
errorStr += "Shell Error: " + str(err) + '\n'
return errorStr
else:
return str(output)
def main():
parser = argparse.ArgumentParser(description="decrypt bzipped tarred sets",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--rootdir", "-r", help="directory where sets are")
parser.add_argument("--keys", "-k", nargs='+', default=[], help="decrypt keys")
parser.add_argument("--sets", "-s", nargs='+', default=[], help="decrypt sets")
parser.add_argument("--template", default=".tar.bz2.openssl", help="suffix of input files")
parser.add_argument("--opensslargstr", default="enc -d -aes-256-cbc -salt", help="openssl options")
parser.add_argument("--tarargstr", default="-jxf", help="tar options")
try:
args = parser.parse_args()
except IOError as msg:
parser.error(str(msg))
if len(args.keys) != len(args.sets):
sys.stderr.write("Must have same number of keys as sets\n")
sys.exit(1)
for k, s in zip(args.keys, args.sets):
infile = os.path.join(args.rootdir, "%s%s" % (s, args.template))
opensslcmd = "openssl %s -k %s" % (args.opensslargstr, k)
tarcmd = "tar -C %s %s -" % (args.rootdir, args.tarargstr)
op = subprocess.Popen(shlex.split(opensslcmd), stdin=open(infile, 'r'), stdout=subprocess.PIPE)
tp = subprocess.check_call(shlex.split(tarcmd), stdin=op.stdout)
def __init__(self, config, file_name, file_path, info, stop_event, exceptions):
threading.Thread.__init__(self)
self.config = config
self.seconds = int(info['duration'])
self.title = ''
self.author = ''
self.album = ''
self.track = ''
self.genre = ''
self.comment = 'Made by RadioCo www.RadioCo.org'
if 'title' in info and info['title']:
self.title = unicode(info['title'])
if 'author' in info and info['author']:
self.author = unicode(info['author'])
if 'album' in info and info['album']:
self.album = unicode(info['album'])
if 'track' in info and info['track']:
self.track = str(info['track'])
if 'genre' in info and info['genre']:
self.genre = unicode(info['genre'])
self.file_name = file_name
self.file_path = file_path
self.stop_event = stop_event
self.exceptions = exceptions
self.command_1 = []
for row in shlex.split(str(config.get('SETTINGS', 'recorder_command'))):
self.command_1.append(self.replace(row))
self.command_2 = []
for row in shlex.split(str(config.get('SETTINGS', 'recorder_command_2'))):
self.command_2.append(self.replace(row))
def subprompt(entity, changed=False, ask_all=False, parent_prompt=None):
global history
if history is None:
history = get_history(entity_name, InMemoryHistory())
if ask_all:
for k, v in form.items():
if k not in entity:
try:
if prompt_parameter(k, entity, entity_name, form,
parent_prompt):
changed = True
except KeyboardInterrupt:
entity[k] = None
except EOFError:
exit_dshell()
while True:
try:
text = prompt(
f"{parent_prompt}{entity_name}('{entity['name']}')> ",
history=history,
auto_suggest=AutoSuggestFromHistory())
except KeyboardInterrupt:
continue
except EOFError:
exit_dshell(rc=1)
try:
namespace = parser.parse_args(shlex.split(text))
except (ValueError, argparse.ArgumentError) as e:
print(e)
continue
except SystemExit:
continue
if namespace.cmd == 'preview':
dprint(entity)
elif namespace.cmd == 'set':
if namespace.parameter not in form.keys():
dprint("Not a valid parameter. Available: " +
', '.join(form.keys()))
else:
try:
if prompt_parameter(
namespace.parameter, entity, form,
f"{parent_prompt}{entity_name}('{entity['name']}')"
):
changed = True
except KeyboardInterrupt:
continue # Control-C pressed. Try again.
except EOFError:
exit_dshell(rc=1)
elif namespace.cmd == 'submit':
resp = ntwrk.post(f'api_1_0.{entity_name.replace("_", "")}list',
json=entity)
dprint(resp)
if resp.ok:
return
elif namespace.cmd == 'delete':
if namespace.parameter not in form.keys():
dprint("Not a valid parameter. Available: " +
', '.join(form.keys()))
else:
entity.pop(namespace.parameter)
changed = True
elif namespace.cmd == 'dump':
with open(namespace.file, 'w') as dumpfile:
json.dump(entity, dumpfile, indent=4)
changed = False
elif namespace.cmd == 'exit':
if changed:
text = ''
while text.lower().strip() not in ('y', 'n'):
text = prompt(
'If you exit you will lose the changes. Do you want to continue? (y/n): '
)
if text.lower().strip() == 'y':
break
else:
break
#Code by Tejaswini
#Date 10 May,2020
import numpy as np
from matplotlib import pyplot as plt
#if using termux
import subprocess
import shlex
#end if
data=np.loadtxt('ee18btech11047.dat')
plt.plot(data[:,0],data[:,1])
plt.grid()
plt.xlabel("time")
plt.ylabel("oscillating response")
plt.title("Output from spice simulation")
#if using termux
plt.savefig('./figs/ee18btech11047/ee18btech11047_spice.pdf')
plt.savefig('./figs/ee18btech11047/ee18btech11047_spice.eps')
subprocess.run(shlex.split("termux-open ./figs/ee18btech11047/ee18btech11047_spice.pdf"))
#else
#plt.show()
def create_options(self, arg, from_tty):
usage = "usage: %prog [options]"
description = (
"Command that can load EFI PE/COFF and TE image symbols. "
"If you are having trouble in PEI try adding --pei. "
"Given any address search backward for the PE/COFF (or TE header) "
"and then parse the PE/COFF image to get debug info. "
"The address can come from the current pc, pc values in the "
"frame, or an address provided to the command"
"")
self.parser = optparse.OptionParser(description=description,
prog='efi symbols',
usage=usage,
add_help_option=False)
self.parser.add_option(
'-a',
'--address',
type="str",
dest='address',
help='Load symbols for image that contains address',
default=None)
self.parser.add_option('-c',
'--clear',
action='store_true',
dest='clear',
help='Clear the cache of loaded images',
default=False)
self.parser.add_option('-f',
'--frame',
action='store_true',
dest='frame',
help='Load symbols for current stack frame',
default=False)
self.parser.add_option('-p',
'--pc',
action='store_true',
dest='pc',
help='Load symbols for pc',
default=False)
self.parser.add_option(
'--pei',
action='store_true',
dest='pei',
help='Load symbols for PEI (searches every 4 bytes)',
default=False)
self.parser.add_option(
'-e',
'--extended',
action='store_true',
dest='extended',
help='Try to load all symbols based on config tables',
default=False)
self.parser.add_option(
'-r',
'--range',
type="long",
dest='range',
help='How far to search backward for start of PE/COFF Image',
default=None)
self.parser.add_option('-s',
'--stride',
type="long",
dest='stride',
help='Boundary to search for PE/COFF header',
default=None)
self.parser.add_option(
'-t',
'--thread',
action='store_true',
dest='thread',
help='Load symbols for the frames of all threads',
default=False)
self.parser.add_option('-v',
'--verbose',
action='store_true',
dest='verbose',
help='Show more info on symbols loading in gdb',
default=False)
self.parser.add_option('-h',
'--help',
action='store_true',
dest='help',
help='Show help for the command',
default=False)
return self.parser.parse_args(shlex.split(arg))
def get_binaryinprolist(self):
'''
return binary proc_dict
if fail to find binary, return {}
if error, return None
proc_dict = {
pid(str): proc(str),
...
}
'''
self.logger.info("get_binaryinprolist start")
proc_dict = {}
if sys.platform == 'win32':
# Only Support Win7 or newer (Don't support cygwin)
cmdlist = shlex.split('wmic process get processid,executablepath')
try:
self.logger.info("Windows cmd: %s", ' '.join(cmdlist))
_wmilist = subprocess.check_output(cmdlist)
wmilist = _to_unicode(_wmilist)
except subprocess.CalledProcessError:
self.logger.error(
"wmic run error, please check manually in host")
return None
except WindowsError as err:
self.logger.error("wmic run exception: {}".format(err))
self.logger.exception("Stack: ")
return None
pattern = re.compile(r'(.*?) *(\d{1,8})')
for proline in wmilist.split(u'\r\n'):
_ = pattern.search(proline)
if not _:
continue
proc, pid = _.groups()
if os.path.basename(proc).lower() == u'{}.exe'.format(
self._binaryname):
proc_dict.update({pid: proc})
for thirdbinary_p in self.thirdbinary_p:
if os.path.basename(proc) == thirdbinary_p:
self.logger.critical(
"Find 3rd %s, please uninstall it to prevent unexpected error",
self._binaryname)
self.logger.critical("3rd %s: %s", self._binaryname,
proc)
elif sys.platform in ('linux2', 'linux'):
pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]
for pid in pids:
try:
proc = open(os.path.join('/proc', pid,
'exe'), 'rb').read().decode(
'UTF-8',
OUT_ERROR_HANDLING).strip()
except IOError:
continue
if os.path.basename(proc) == self._binaryname:
proc_dict.update({pid: proc})
else:
raise Exception("Don't support your system: {system}".format(
system=sys.platform))
for pid, proc in proc_dict.items():
self.logger.info("%s process: pid(%s)|proc(%s)", self._binaryname,
pid, proc)
self.logger.info("get_binaryinprolist complete")
return proc_dict
def _process_options(self, options):
global VARIANTS
if options.sancov_dir:
self.sancov_dir = options.sancov_dir
if not os.path.exists(self.sancov_dir):
print("sancov-dir %s doesn't exist" % self.sancov_dir)
raise base_runner.TestRunnerError()
options.command_prefix = shlex.split(options.command_prefix)
options.extra_flags = sum(map(shlex.split, options.extra_flags), [])
if options.gc_stress:
options.extra_flags += GC_STRESS_FLAGS
if self.build_config.asan:
options.extra_flags.append("--invoke-weak-callbacks")
options.extra_flags.append("--omit-quit")
if options.novfp3:
options.extra_flags.append("--noenable-vfp3")
if options.no_variants:
print(
"Option --no-variants is deprecated. "
"Pass --variants=default instead.")
assert not options.variants
options.variants = "default"
if options.exhaustive_variants:
# TODO(machenbach): Switch infra to --variants=exhaustive after M65.
print(
"Option --exhaustive-variants is deprecated. "
"Pass --variants=exhaustive instead.")
# This is used on many bots. It includes a larger set of default
# variants.
# Other options for manipulating variants still apply afterwards.
assert not options.variants
options.variants = "exhaustive"
if options.quickcheck:
assert not options.variants
options.variants = "stress,default"
options.slow_tests = "skip"
options.pass_fail_tests = "skip"
if self.build_config.predictable:
options.variants = "default"
options.extra_flags.append("--predictable")
options.extra_flags.append("--verify_predictable")
options.extra_flags.append("--no-inline-new")
# TODO(machenbach): Figure out how to test a bigger subset of variants on
# msan.
if self.build_config.msan:
options.variants = "default"
if options.j == 0:
options.j = multiprocessing.cpu_count()
if options.random_seed_stress_count <= 1 and options.random_seed == 0:
options.random_seed = self._random_seed()
# Use developer defaults if no variant was specified.
options.variants = options.variants or "dev"
# Resolve variant aliases and dedupe.
# TODO(machenbach): Don't mutate global variable. Rather pass mutated
# version as local variable.
VARIANTS = list(
set(
reduce(
list.__add__,
(VARIANT_ALIASES.get(v, [v])
for v in options.variants.split(",")),
[],
)))
if not set(VARIANTS).issubset(ALL_VARIANTS):
print "All variants must be in %s" % str(ALL_VARIANTS)
raise base_runner.TestRunnerError()
def CheckTestMode(name, option):
if not option in ["run", "skip", "dontcare"]:
print "Unknown %s mode %s" % (name, option)
raise base_runner.TestRunnerError()
CheckTestMode("slow test", options.slow_tests)
CheckTestMode("pass|fail test", options.pass_fail_tests)
if self.build_config.no_i18n:
base_runner.TEST_MAP["bot_default"].remove("intl")
base_runner.TEST_MAP["default"].remove("intl")
import os
import pwd
import shlex
import sys
from collections import namedtuple
from .fast_data_types import set_boss as set_c_boss
appname = 'kitty'
version = (0, 8, 3)
str_version = '.'.join(map(str, version))
_plat = sys.platform.lower()
is_macos = 'darwin' in _plat
base = os.path.dirname(os.path.abspath(__file__))
editor = shlex.split(os.environ.get('EDITOR', 'vim'))
ScreenGeometry = namedtuple('ScreenGeometry', 'xstart ystart xnum ynum dx dy')
WindowGeometry = namedtuple('WindowGeometry', 'left top right bottom xnum ynum')
def _get_config_dir():
if 'KITTY_CONFIG_DIRECTORY' in os.environ:
return os.path.abspath(os.path.expanduser(os.environ['KITTY_CONFIG_DIRECTORY']))
locations = []
if 'XDG_CONFIG_HOME' in os.environ:
locations.append(os.path.abspath(os.path.expanduser(os.environ['XDG_CONFIG_HOME'])))
locations.append(os.path.expanduser('~/.config'))
if is_macos:
with open(tmp_spk_vad_scp, "w") as writer:
writer.write(spk_vad_scp_content)
add_deltas = ("add-deltas " + delta_opts + " scp:" + tmp_spk_feats_scp +
" ark:- |")
apply_cmvn = "apply-cmvn-sliding --norm-vars=false --center=true --cmn-window=300 ark:- ark:- |"
select_voiced_frame = ("select-voiced-frames ark:- scp,s,cs:" +
tmp_spk_vad_scp + " ark:- |")
feats = ("ark,s,cs:" + add_deltas + " " + apply_cmvn + " " +
select_voiced_frame)
acc_stats_command = (
"gmm-global-acc-stats --binary=false --update-flags=" +
update_flags_str + " " + dubm + " " + shlex.quote(feats) + " " +
tmp_spk_acc_file)
args = shlex.split(acc_stats_command)
p = subprocess.Popen(args,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
p.wait()
output_model = model_dir + "/" + spk_id + "-identity.gmm"
map_command = ("gmm-global-est-map --update-flags=" + update_flags_str +
" " + dubm + " " + tmp_spk_acc_file + " " + output_model)
args = shlex.split(map_command)
p = subprocess.Popen(args,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
p.wait()
# delete all the tmp file
def cueExtractArc(fileName, fileLen, justChecking):
if fileLen > MAX_REASONABLE_CUE_SIZE: #sanity check
return 0
curTrackBinPath = None
curTrackFullBinPath = None
curTrackIndex = -1
cueTracks = []
try:
with open(fileName, "r") as f:
for line in f:
values = shlex.split(line)
if len(values) > 0:
cmd = values[0].lower()
if cmd == "file" and len(values) >= 2:
curTrackBinPath = None
curTrackIndex = -1
path = values[1]
type = values[2].lower()
if type == "binary" or type == "motorola":
fullPath = os.path.join(
rapi.getDirForFilePath(fileName), path)
if os.path.exists(fullPath):
curTrackBinPath = path
curTrackFullBinPath = fullPath
else:
print("Warning: Can't find track data:",
fullPath)
else:
print("Not searching for volume in track type:",
type)
elif cmd == "track" and len(values) >= 2:
if curTrackBinPath:
trackTypeDict = {
#haven't tested most of these, let me know if one of them falls over
"audio": (2352, 2352, 0),
"cdg": (2448, 2448, 0),
"mode1/2048": (2048, 2048, 0),
"mode1/2352": (2352, 2048, 16), #mode 1
"mode2/2048": (2352, 2048, 24), #xa form 1
"mode2/2324": (2352, 2324, 24), #xa form 2
"mode2/2336": (2352, 2336, 16), #mode 2
"mode2/2352": (
2352, 2048, 24
), #xa "raw" (mmv, but typically want to discard error data as used by psx rips and such)
"cdi/2336": (2352, 2048, 24),
"cdi/2352": (2352, 2324, 24)
}
trackType = values[2].lower()
if trackType in trackTypeDict:
curTrackIndex = len(cueTracks)
sectorSize, userSize, headerSize = trackTypeDict[
trackType]
cueTracks.append(
CueTrack(curTrackBinPath,
curTrackFullBinPath,
int(values[1]), sectorSize,
userSize, headerSize))
else:
print("Warning: Unsupported track type:",
trackType)
curTrackIndex = -1
elif cmd == "pregap" and curTrackIndex >= 0 and len(
values) >= 2:
cueTracks[curTrackIndex].preGap += msfToLba(values[1])
elif cmd == "postgap" and curTrackIndex >= 0 and len(
values) >= 2:
cueTracks[curTrackIndex].postGap += msfToLba(values[1])
elif cmd == "index" and curTrackIndex >= 0 and len(
values) >= 3:
i = int(values[1])
if i == 0:
cueTracks[curTrackIndex].preGap += msfToLba(
values[2])
elif i == 1:
#supposedly this can also be absolute instead of relative to the last track, but don't have any samples to test that.
cueTracks[curTrackIndex].startLba += msfToLba(
values[2])
#otherwise discard the line (this includes rem)
except:
return 0
if len(cueTracks) == 0:
return 0
if justChecking:
return 1
cueTracks = sorted(
cueTracks, key=noeCmpToKey(lambda a, b: a.trackIndex - b.trackIndex))
currentLba = 0
tracks = []
for cueTrack in cueTracks:
cueTrack.calculateSizeInBlocks()
currentLba += cueTrack.startLba + cueTrack.preGap
track = NoeImgTrack(open(cueTrack.fullBinPath,
"rb"), cueTrack.trackIndex, currentLba,
cueTrack.sectorSize, cueTrack.userSize,
cueTrack.headerSize)
track.binPath = cueTrack.binPath
track.fullBinPath = cueTrack.fullBinPath
tracks.append(track)
currentLba += cueTrack.sizeInBlocks + cueTrack.postGap
img = NoeImgReader(tracks)
for track in tracks:
imgVol = img.readFileSystemVolume(track.index)
if imgVol:
print("Read volume information from", track.fullBinPath)
for imgFile in imgVol.files:
writePath = track.binPath + "_track%02i/" % track.index + imgFile.path
print("Writing", writePath)
rapi.exportArchiveFile(writePath, img.readFileData(imgFile))
img.close()
return 1
def set_time(date):
subprocess.call(shlex.split("timedatectl set-ntp false"))
subprocess.call(shlex.split("sudo date -s '" + time.ctime(date) + "'"))
subprocess.call(shlex.split("sudo hwclock -w"))
def process_line(line):
'''process one line of pin definition file'''
global allpins
a = shlex.split(line)
# keep all config lines for later use
alllines.append(line)
if a[0].startswith('P') and a[0][1] in ports and a[0] in config:
print("WARNING: Pin %s redefined" % a[0])
config[a[0]] = a[1:]
if a[0] == 'MCU':
global mcu_type
mcu_type = a[2]
if a[0].startswith('P') and a[0][1] in ports:
# it is a port/pin definition
try:
port = a[0][1]
pin = int(a[0][2:])
label = a[1]
type = a[2]
extra = a[3:]
except Exception:
error("Bad pin line: %s" % a)
return
p = generic_pin(port, pin, label, type, extra)
portmap[port][pin] = p
allpins.append(p)
if not type in bytype:
bytype[type] = []
bytype[type].append(p)
bylabel[label] = p
af = get_alt_function(mcu_type, a[0], label)
if af is not None:
p.af = af
if a[0] == 'SPIDEV':
spidev.append(a[1:])
if a[0] == 'ROMFS':
romfs.append((a[1],a[2]))
if a[0] == 'undef':
print("Removing %s" % a[1])
config.pop(a[1], '')
bytype.pop(a[1],'')
bylabel.pop(a[1],'')
#also remove all occurences of defines in previous lines if any
for line in alllines[:]:
if line.startswith('define') and a[1] in line:
alllines.remove(line)
newpins = []
for pin in allpins:
if pin.type == a[1]:
continue
if pin.label == a[1]:
continue
if pin.portpin == a[1]:
continue
newpins.append(pin)
allpins = newpins
if a[0] == 'env':
print("Adding environment %s" % ' '.join(a[1:]))
if len(a[1:]) < 2:
error("Bad env line for %s" % a[0])
env_vars[a[1]] = ' '.join(a[2:])
def _packaging(
self
, name
, binaries=[]
, datas=[]
, hiddenimports=[]
, exclude_modules=[]
, upx_enable=False
# , working_dir="tmp_pyinstaller"
, mode_debug=True
, python_ver=3
, is_onefile=True
, do_clean=True
, is_gui_app=False
, app_debug=False
, app_dist="dist"
):
"""
１つのファイル/ディレクトリにパッケージング
:type hiddenimports: list
:param do_clean: Clean intermediate product or not
:return: なし
"""
""" 頻出参照変数生成 """
self.app_dist = app_dist # TODO: 任意に変更できるように要修正
# obsolute_working_path = self._ppath_wcd.as_posix()
# 空フォルダ生成
# os.makedirs(obsolute_working_path)
# os.chdir(obsolute_working_path)
# self.copy_dest = obsolute_working_path
# self.copy_dest = self._ppath_wcd.as_posix()
# フォルダTreeの削除
# if os.path.exists(obsolute_working_path) and do_clean:
# shutil.rmtree(obsolute_working_path)
self.filename_entry_point = "entry_point.py"
# self._ppath_entry_point = os.path.join(self._pl_packing_data, self.filename_entry_point)
self._ppath_entry_point = Path(self._pl_packing_data).joinpath(self.filename_entry_point)
""" making package dir """
if not self._pl_packing_data.exists():
os.mkdir(self._pl_packing_data.as_posix())
""" making entry_point.py """
self._codec_main_python_file = self.check_encoding(
file_path=self._ppath_main_python.as_posix())
with open(self._ppath_main_python.as_posix(), 'r', encoding=self._codec_main_python_file) as f: # Caution) as_posix() func. requires for python3.5,3.6 or later?
lines = f.readlines()
lines_strip = [line.strip() for line in lines]
dat_from = [line for line in lines_strip if
line.startswith('from ') or line.startswith('import ')] # or '_MEIPASS' in line
# TODO: コード抽出は行単位であるため，スコープを考慮した抽出に変える必要を要検討
# TODO: 行冒頭の空白が除去されて書き出されている
with open(self._ppath_entry_point.as_posix(), mode='w') as to_f: # Caution) as_posix() func. requires for python3.5,3.6 or later?
if self._pl_header_file and self._pl_header_file.name != "":
if not self._pl_header_file.exists():
raise Exception("Not found header file")
with open(self._pl_header_file, "r") as f_header:
lines = f_header.readlines()
if lines is not None and lines != "":
to_f.write(self._newline_for_output.join(lines) + self._newline_for_output)
to_f.write(self._newline_for_output.join(dat_from) + self._newline_for_output)
""" add entry point """
# to_f.write("if hasattr(sys, \"_MEIPASS\"):")
# to_f.write(" sys._MEIPASS")
if self._start_method:
to_f.write("from %s import %s%s" % (
self._ppath_main_python.stem, self._start_method, self._newline_for_output))
to_f.write("%s()%s" % (self._start_method, self._newline_for_output))
else:
# No indicate entry_point
to_f.write("import %s%s" %
(self._ppath_main_python.stem, self._newline_for_output)
)
# main_py = """
# from {main_python_filename_without_ext} import timeline_headquater
# timeline_headquater()
# """.strip().format(
# # encoding=self.enc_on_os()
# main_python_filename_without_ext=self.main_python_filename_without_ext
# )
#
# f = open(self.filename_app_main, 'w', encoding=self.enc_on_os()) # 書き込みモードで開く
# # f = open(filename_app_main, 'w') # 書き込みモードで開く
# f.write(main_py) # 引数の文字列をファイルに書き込む
# f.close() # ファイルを閉じる
"""
Make parameter for cui command of PyInstaller
"""
cmd_build = "pyinstaller"
""" Add binaries """
for x in self.binaries_inc_param:
cmd_build += " " + x
# """ add binaries """
# if binaries is None:
# binaries = []
#
# """ add generated libs """
# paths_gen_lib = self._get_filepath_list(self.path_store_gen_libs, ext=self._ext_lib())
# FIXME: 下記フラット構造しか対応してない
#
# for a_path in paths_gen_lib:
# binaries.append([a_path, "."])
# for x in binaries: # TODO: functionize
# cmd_build += " --add-binary '{path_include}{data_separator}{dir_place}'".format(path_include=x[0],
# data_separator=_sep4data_bin,
# dir_place=x[1])
for x in hiddenimports:
# cmd_build += " --hidden-import=%s%s" % (x, self._newline_for_output)
cmd_build += " --hidden-import=%s" % x
for x in exclude_modules:
cmd_build += " --exclude-module %s" % x
def get_operator_data_or_binary(cmd_building, ope, binaries_or_datas):
"""
parameter of data and binary for CUI of pyInstaller
:param ope:
:type ope:str
:param binaries_or_datas:[[path of lib,dir_to_set_lib],[...]]
:type binaries_or_datas:[[]]
:return:
"""
# res = ""
if binaries_or_datas is None or len(binaries_or_datas) == 0:
return cmd_building
for x in binaries_or_datas:
cmd_building += " {operator} '{path_include}{data_separator}{dir_place}' ".format(
operator=ope,
path_include=x[0],
data_separator=self._sep4data_bin,
dir_place=x[1])
return cmd_building
""" add 'data's and 'binaries' from indicated parameter """
cmd_build = get_operator_data_or_binary(cmd_build, "--add-data", datas)
cmd_build = get_operator_data_or_binary(cmd_build, "--add-binary", binaries)
if is_onefile:
cmd_build += " --onefile"
if is_gui_app:
cmd_build += " --windowed" # -w, --windowed, --noconsoleは同義．別途，コマンドプロンプトが現れない、エラーを除く。
if app_debug:
cmd_build += " --debug"
if upx_enable:
cmd_build += " --upx"
else:
cmd_build += " --noupx"
# if do_clean:
# cmd_build += " --clean -y" # -y: outputフォルダ削除確認にyesを引き渡す？
cmd_build += " -n %s" % name
# if self.os_cat1== OSType.windows:
# cmd_build += " '%s'" % self._ppath_entry_point
# elif self.os_cat1== OSType.unix:
# cmd_build += " '%s'" % self._ppath_entry_point
# else:
# raise Exception("Out of expected OSType")
cmd_build += " '%s'" % self._ppath_entry_point # ''でくくった方が安定してpyinstallerが起動する
print("src CMD:", cmd_build)
""" write out log to confirm """
with open("pyinstaller_parameter.log", "w", encoding=self.enc_on_os) as f:
f.write(cmd_build)
args = shlex.split(cmd_build)
# print(":", args)
p = subprocess.Popen(args, shell=False) # Success!
p.wait()
def tokenize(string):
return shlex.split(string)
def splitOptions(options): # Use python's shell command lexer to correctly split the list of options in # accordance with the POSIX standard return shlex.split(options)
def run_tests(self):
import shlex
# import here, cause outside the eggs aren't loaded
import pytest
errno = pytest.main(shlex.split(self.pytest_args))
sys.exit(errno)
def invoke(self, cli, args=None, input=None, env=None,
catch_exceptions=True, color=False, mix_stderr=False, **extra):
"""Invokes a command in an isolated environment. The arguments are
forwarded directly to the command line script, the `extra` keyword
arguments are passed to the :meth:`~clickpkg.Command.main` function of
the command.
This returns a :class:`Result` object.
.. versionadded:: 3.0
The ``catch_exceptions`` parameter was added.
.. versionchanged:: 3.0
The result object now has an `exc_info` attribute with the
traceback if available.
.. versionadded:: 4.0
The ``color`` parameter was added.
:param cli: the command to invoke
:param args: the arguments to invoke. It may be given as an iterable
or a string. When given as string it will be interpreted
as a Unix shell command. More details at
:func:`shlex.split`.
:param input: the input data for `sys.stdin`.
:param env: the environment overrides.
:param catch_exceptions: Whether to catch any other exceptions than
``SystemExit``.
:param extra: the keyword arguments to pass to :meth:`main`.
:param color: whether the output should contain color codes. The
application can still override this explicitly.
"""
exc_info = None
with self.isolation(input=input, env=env, color=color) as outstreams:
exception = None
exit_code = 0
if isinstance(args, string_types):
args = shlex.split(args)
try:
prog_name = extra.pop("prog_name")
except KeyError:
prog_name = self.get_default_prog_name(cli)
try:
cli.main(args=args or (), prog_name=prog_name, **extra)
except SystemExit as e:
exc_info = sys.exc_info()
exit_code = e.code
if exit_code is None:
exit_code = 0
if exit_code != 0:
exception = e
if not isinstance(exit_code, int):
sys.stdout.write(str(exit_code))
sys.stdout.write('\n')
exit_code = 1
except Exception as e:
if not catch_exceptions:
raise
exception = e
exit_code = 1
exc_info = sys.exc_info()
finally:
sys.stdout.flush()
stdout = outstreams[0].getvalue() if not outstreams[0].closed else b""
stderr = outstreams[1] and outstreams[1].getvalue()
return Result(runner=self,
stdout_bytes=stdout,
stderr_bytes=stderr,
exit_code=exit_code,
exception=exception,
exc_info=exc_info)
def _gen_shared_lib(
self
# , is_build_so_lib
, working_dir="tmp_cython" # FIXME: obsolute and to self.
, python_ver=3
, clean_c_files=False
, build_dir="build_lib"
, cython_one_liblize=False
, rpath_for_attach=""
):
"""
CythonでSharedLibsを作る
:param do_clean: SharedLibs生成フォルダ，ファイルを全て消去
:param build_dir: ShareLibsを生成するフォルダ
:param cython_one_liblize: SharedLibを１つにまとめる. 将来，DefaultをTrueへ
:param rpath_for_attach: The relative path to store lib, data, files
:return:
"""
# if is_build_so_lib:
# ターゲットのpathへ遷移
# self.cython_wcd_path = os.path.join(self.pathdir_include_main_python_file, working_dir)
# self.cython_working_path = self.pathdir_include_main_python_file
if not self._cython_pwcd.exists():
self._cython_pwcd.mkdir()
# if do_clean:
# shutil.rmtree(self._cython_pwcd.as_posix())
os.chdir(self._cython_pwcd.as_posix())
print("[Debug] chdir to :", self._cython_pwcd.as_posix())
py_src_paths = self._get_filepath_list(self._ppath_main_python.parent.as_posix(),
ext="py") # FIXME: pyxも含められる用に
# os.chdir(self.base_script_abs_path)
""" Remove header python """
if self._pl_header_file and self._pl_header_file.name in py_src_paths:
py_src_paths.remove(self._pl_header_file.name)
for file in py_src_paths:
if os.path.exists(file):
print("[Debug] File is exists:", file)
else:
print("[Debug] File is not exists:", file)
if cython_one_liblize:
# １つの.soへ集約できる．Extentionsクラスを使うから
setup_py = '''
#cython: language_level={python_ver}, boundscheck=False
from distutils.core import setup
from Cython.Build import cythonize
from distutils.extension import Extension
extensions = [Extension("{app_name}", {py_src_paths})]
setup(
name="{app_name}",
ext_modules=cythonize(
extensions
# build according to python3
,compiler_directives={compiler_directives}
#, working="{dir_dist_in_cython}"
)
)
'''.strip().format(
python_ver=python_ver
, app_name=self._app_name
, py_src_paths=py_src_paths
, compiler_directives='{"language_level":%s}' % python_ver
, dir_dist_in_cython=self.dir_dist_in_cython
)
else:
setup_py = '''
#cython: language_level={python_ver}, boundscheck=False
from distutils.core import setup,Extension
from Cython.Build import cythonize
setup(
name="{app_name}",
ext_modules=cythonize(
{py_src_paths}
# build according to python3
, compiler_directives={compiler_directives}
#, working="{dir_dist_in_cython}"
)
)
'''.strip().format(
python_ver=python_ver
, app_name=self._app_name
, py_src_paths=py_src_paths
# , compiler_directives="{\"language_level\":%s}" % python_ver
, compiler_directives='{"language_level":%s}' % python_ver
, dir_dist_in_cython=self.dir_gen_shared_libs_obsolute
)
# print("[Debug] setup_test_code2.py---------------------------")
# print(setup_py)
with open(self.setup_file_name, 'w') as f: # 書き込みモードで開く
f.write(setup_py) # 引数の文字列をファイルに書き込む
""" Detect OS and Build """
# --inplace: ビルド対象の相対フォルダ(例:src以下)の構造を再現する．本プログラムでは，--inplaceパラはON/OFF両方対応
# cmd_build_base = "python {0} build_ext --build-lib {1}".format(self.setup_file_name, self.dir_dist_in_cython)
cmd_build = "python {setup_file_name} build_ext --build-lib {build_lib}".format(
setup_file_name=self.setup_file_name
, build_lib=self.dir_gen_shared_libs_obsolute # cannot abs path.
)
if self.os_cat1 == OSCategory1.windows:
cmd_build = cmd_build + " --compiler=mingw32"
if not skip_throw_cython:
print("Building Libraries with Command:", cmd_build)
args = shlex.split(cmd_build)
# print("分割コマンド:", args)
p = subprocess.Popen(args, shell=False) # Success!
p.wait()
""" Clean C files """
if clean_c_files:
for pysrc_path in py_src_paths:
tmp = Path(pysrc_path).with_suffix(".c")
if tmp.exists():
tmp.unlink()
"""
生成したLibのPATHからPyinstaller用binaries_includeリストを作る
"""
# if self.os_cat1()==OSType.windows:
# common_prefix = os.path.commonprefix(self.path_store_gen_libs) # 前のver.ではこちらを適用
# elif self.os_cat1()==OSType.unix:
# common_prefix = self.path_store_gen_libs
# else:
# raise Exception("PATH落ち")
# common_prefix = self.path_store_gen_libs # TODO: common_prefix削除
# １つ上位のPATHを取得
# common_prefix = os.path.abspath(os.path.join(common_prefix, os.pardir))
# common_prefix += "/"
# common_prefix = common_prefix_childdir
self.paths_each_gen_libs = self._get_filepath_list(path=self._cython_pwcd.as_posix(), ext=self._ext_lib)
# gen_lib_filename = [os.path.basename(x) for x in self.paths_each_gen_libs] # type: list[str]
# app_dirs_place_libs = [libdir.replace(filename, "") for (libdir, filename) in
# zip(self.paths_each_gen_libs, gen_lib_filename)]
# 書式： binaries = [("取り込みたいlibPATH","配置したいlibのdir"),...]
# self.binaries_inc = [
# (os.path.join(self.base_script_abs_path, x), os.path.join(".", y.replace(common_prefix, ""))) for (x, y) in
# zip(self.path_each_gen_libs, app_dirs_place_libs)]
# def get_dir_to_set_libs(dir):
# """
# 下記でhiddenimportするライブラリへのPATHを返す
# :param dir:
# :return:
# """
# if dir == "":
# return "."
# else:
# return "." + dir
""" write out parameters '--add-binary'command to plyinstaller """
# self.binaries_inc_param = ["--add-binary '%s:%s'%s" % (
# (os.path.join(self.base_script_abs_path, x)
# , get_dir_to_set_libs(y.replace(common_prefix, ""))
# ,self._newline_for_output)
# ) for (x, y) in zip(self.paths_each_gen_libs, app_dirs_place_libs)]
self.binaries_inc_param = []
""" write out parameters '--add-binary'command to plyinstaller """
for a_path_gen_lib in self.paths_each_gen_libs:
self.binaries_inc_param.append("--add-binary \'%s%s%s\'" % (
a_path_gen_lib
, self._sep4data_bin
, rpath_for_attach))
# , a_path_gen_lib.replace(common_prefix + self.path_sep, "")))
logging.debug("parameter:%s" % self.binaries_inc_param)
def get_cc_version(conf, cc, gcc=False, icc=False, clang=False):
"""
Run the preprocessor to determine the compiler version
The variables CC_VERSION, DEST_OS, DEST_BINFMT and DEST_CPU will be set in *conf.env*
"""
cmd = cc + ['-dM', '-E', '-']
env = conf.env.env or None
try:
p = Utils.subprocess.Popen(cmd, stdin=Utils.subprocess.PIPE, stdout=Utils.subprocess.PIPE, stderr=Utils.subprocess.PIPE, env=env)
p.stdin.write('\n'.encode())
out = p.communicate()[0]
except Exception:
conf.fatal('Could not determine the compiler version %r' % cmd)
if not isinstance(out, str):
out = out.decode(sys.stdout.encoding or 'iso8859-1')
if gcc:
if out.find('__INTEL_COMPILER') >= 0:
conf.fatal('The intel compiler pretends to be gcc')
if out.find('__GNUC__') < 0 and out.find('__clang__') < 0:
conf.fatal('Could not determine the compiler type')
if icc and out.find('__INTEL_COMPILER') < 0:
conf.fatal('Not icc/icpc')
if clang and out.find('__clang__') < 0:
conf.fatal('Not clang/clang++')
k = {}
if icc or gcc or clang:
out = out.splitlines()
for line in out:
lst = shlex.split(line)
if len(lst)>2:
key = lst[1]
val = lst[2]
k[key] = val
def isD(var):
return var in k
def isT(var):
return var in k and k[var] != '0'
# Some documentation is available at http://predef.sourceforge.net
# The names given to DEST_OS must match what Utils.unversioned_sys_platform() returns.
if not conf.env.DEST_OS:
conf.env.DEST_OS = ''
for i in MACRO_TO_DESTOS:
if isD(i):
conf.env.DEST_OS = MACRO_TO_DESTOS[i]
break
else:
if isD('__APPLE__') and isD('__MACH__'):
conf.env.DEST_OS = 'darwin'
elif isD('__unix__'): # unix must be tested last as it's a generic fallback
conf.env.DEST_OS = 'generic'
if isD('__ELF__'):
conf.env.DEST_BINFMT = 'elf'
elif isD('__WINNT__') or isD('__CYGWIN__') or isD('_WIN32'):
conf.env.DEST_BINFMT = 'pe'
conf.env.LIBDIR = conf.env.BINDIR
elif isD('__APPLE__'):
conf.env.DEST_BINFMT = 'mac-o'
if not conf.env.DEST_BINFMT:
# Infer the binary format from the os name.
conf.env.DEST_BINFMT = Utils.destos_to_binfmt(conf.env.DEST_OS)
for i in MACRO_TO_DEST_CPU:
if isD(i):
conf.env.DEST_CPU = MACRO_TO_DEST_CPU[i]
break
Logs.debug('ccroot: dest platform: ' + ' '.join([conf.env[x] or '?' for x in ('DEST_OS', 'DEST_BINFMT', 'DEST_CPU')]))
if icc:
ver = k['__INTEL_COMPILER']
conf.env['CC_VERSION'] = (ver[:-2], ver[-2], ver[-1])
else:
if isD('__clang__'):
conf.env['CC_VERSION'] = (k['__clang_major__'], k['__clang_minor__'], k['__clang_patchlevel__'])
else:
try:
conf.env['CC_VERSION'] = (k['__GNUC__'], k['__GNUC_MINOR__'], k['__GNUC_PATCHLEVEL__'])
except KeyError:
conf.env['CC_VERSION'] = (k['__GNUC__'], k['__GNUC_MINOR__'], 0)
return k
def run(self):
while self._running:
command = input('cli> ')
args = self._parser.parse_args(shlex.split(command))
args = vars(args)
args['func'](args)
def _read_non_linear_iraf_wcs(header, wcsdim):
"""Read non-linear wavelength solutions written by IRAF
Extracts the appropriate information and organize it in a dictionary for
calling the method _set_math_model which decides what is the appropriate
mathematical model to be used according the the type of wavelength solution
it is dealing with.
Parameters
----------
header : :class:`~astropy.io.fits.header.Header`
Full header of file being loaded
wcsdim : int
Number of the wcs dimension to be read.
Returns
-------
spectral_axis : :class:`~numpy.ndarray`
Mathematical model of wavelength solution evaluated for each pixel
position
"""
wat_wcs_dict = {}
wcs_parser = {'aperture': int, 'beam': int, 'dtype': int,
'dstart': float, 'avdelt': float,
'pnum': lambda x: int(float(x)), 'z': float,
'alow': lambda x: int(float(x)), 'ahigh': lambda x: int(float(x)),
'weight': float, 'zeropoint': float,
'ftype': int, 'order': lambda x: int(float(x)),
'pmin': lambda x: int(float(x)), 'pmax': lambda x: int(float(x))}
ctypen = header['CTYPE{:d}'.format(wcsdim)]
log.info('Attempting to read CTYPE{:d}: {:s}'.format(wcsdim, ctypen))
if ctypen == 'MULTISPE':
# This is extracting all header cards for f'WAT{wcsdim}_*' into a list
wat_head = header['WAT{:d}*'.format(wcsdim)]
if len(wat_head) == 1:
log.debug('Get units')
wat_array = wat_head[0].split(' ')
for pair in wat_array:
split_pair = pair.split('=')
wat_wcs_dict[split_pair[0]] = split_pair[1]
elif len(wat_head) > 1:
wat_string = ''
for key in wat_head:
wat_string += f'{header[key]:68s}' # Keep header from stripping trailing blanks!
wat_array = shlex.split(wat_string.replace('=', ' '))
if len(wat_array) % 2 == 0:
for i in range(0, len(wat_array), 2):
# if wat_array[i] not in wcs_dict.keys():
wat_wcs_dict[wat_array[i]] = wat_array[i + 1]
# print(wat_array[i], wat_array[i + 1])
for key in wat_wcs_dict.keys():
log.debug("{:d} -{:s}- {:s}".format(wcsdim, key, wat_wcs_dict[key]))
specn = [k for k in wat_wcs_dict.keys() if k.startswith('spec')]
spectral_axis = np.empty((len(specn), header['NAXIS1']))
for n, sp in enumerate(specn):
spec = wat_wcs_dict[sp].split()
wcs_dict = dict((k, wcs_parser[k](spec[i])) for i, k in enumerate(wcs_parser.keys()))
wcs_dict['fpar'] = [float(i) for i in spec[15:]]
log.debug(f'Retrieving model for {sp}: {wcs_dict["dtype"]} {wcs_dict["ftype"]}')
math_model = _set_math_model(wcs_dict=wcs_dict)
spectral_axis[n] = math_model(range(1, wcs_dict['pnum'] + 1)) / (1. + wcs_dict['z'])
log.info(f'Constructed spectral axis of shape {spectral_axis.shape}')
return spectral_axis
def save_file(filename, common_info, trans_opts, dryrun):
"""Transfers a single file along with file information to the
Data Backbone"s staging area
Parameters
----------
filename : `str`
Name of file to save to Data Backbone. Includes any local path.
common_info : `dict`
File information common to all files (like LSST user name).
trans_opts : `dict`
Options for the transfer command (e.g., dest http url prefix).
dryrun : `bool`
Controls whether file is actually transferred.
"""
logging.info("Saving %s", filename)
all_files = []
digest = {}
# add actual file to save to the collection of files to put in the tarball
afile = {"filename": filename,
"arcname": os.path.basename(filename),
"chksum": calc_chksum(name=filename)}
all_files.append(afile)
digest[afile["arcname"]] = afile["chksum"]
afile = create_physical_data(filename, afile["chksum"], "md5", common_info)
all_files.append(afile)
digest[afile["filename"]] = afile["chksum"]
all_files.append(create_digest_data(filename, digest))
transcmd = create_transfer_cmd(filename, trans_opts, common_info['uuid'])
if not dryrun:
for i in range(1, trans_opts['num_tries'] + 1):
try:
logging.info("Transfer attempt %d of %d", i, trans_opts['num_tries'])
# send output of tar as stdin to curl
process_trans = subprocess.Popen(shlex.split(transcmd),
shell=False,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
with tarfile.open(fileobj=process_trans.stdin, mode="w|") as tar:
for afile in all_files:
logging.debug("%s afile keys = %s", afile["filename"], afile.keys())
if "data" in afile:
tar.addfile(afile["tarinfo"], afile["data"])
else:
tar.add(afile["filename"], arcname=afile["arcname"])
# Explicit close to address problems in https://jira.lsstcorp.org/browse/DM-16181
process_trans.stdin.close()
logging.debug("Tar done, waiting for curl to end")
process_trans.wait()
logging.debug("process_trans.returncode = %s" % process_trans.returncode)
if process_trans.returncode != 0:
raise RuntimeError("Non-zero transfer returncode (%s)" % process_trans.returncode)
except:
out = process_trans.communicate()[0].decode("utf-8")
if process_trans.returncode != 0:
msg = "; ".join(out.split('\n')[:2])
logging.warning(msg)
if i < trans_opts['num_tries']:
logging.warning("Transfer problem (Try: %d of %d). Trying again.",
i, trans_opts['num_tries'])
else:
logging.error("Aborting transfer due to problems.")
if process_trans.returncode != 0:
logging.info("Transfer command = %s", transcmd)
# Skip printing broken pipe traceback as problem happened in curl
raise RuntimeError(msg) from None
else:
raise
else:
break
logging.info("Completed transfer of %s", filename)
else:
logging.info("Dryrun. Skipped transfer of %s", filename)
def index(self, input_bam):
cmd = self.samtools_cmd + ' index {}.bam'.format(input_bam)
return (shlex.split(cmd))
for j in lengths:
char = "".join(random.choice(test_chars) for k in range(j))
j = str(j).zfill(3)
len_capture_string = "%s/other/data_read/%d.pcap" % (OSX_RECV_DATA,i)
try:
os.remove(len_capture_string)
except:
pass
filter_string = "tcp port 5223 and net 17.0.0.0/8"
#Run applescript and finish
applescript_cmd = "ssh -i /Users/coulls/.ssh/id_rsa [email protected] \"osascript %s/type_letter.scpt %s\"" % (OSX_RECV_CODE,char)
subprocess.call(shlex.split(applescript_cmd))
time.sleep(1)
applescript_cmd = "ssh -i /Users/coulls/.ssh/id_rsa [email protected] \"osascript %s/enter_letter.scpt\"" % (OSX_RECV_CODE)
subprocess.call(shlex.split(applescript_cmd))
time.sleep(4)
#Open tcpdump and sleep
tcpdump_cmd = "tcpdump -i en1 -w %s %s" % (len_capture_string, filter_string)
tcpdump_proc = subprocess.Popen(shlex.split(tcpdump_cmd))
time.sleep(1)
applescript_cmd = "osascript change_focus.scpt"
subprocess.call(shlex.split(applescript_cmd))
time.sleep(4)
def runCmdReturnJson(cmd, trace=False, silentErrors=False):
cmdArr = shlex.split(cmd)
return Utils.runCmdArrReturnJson(cmdArr,
trace=trace,
silentErrors=silentErrors)
def translate(cmd):
import shlex
"""with a def you can easily change your subprocess call"""
args = shlex.split(cmd)
p = subprocess.call(args)
def getUserArguments(self):
try:
return shlex.split(self.textEntry.get())
except ValueError:
# shlex syntax error: ignore and return the entire string as a single argument
return [self.textEntry.get()]
def SplitArgs(value):
if isinstance(value, list):
return value
return shlex.split(value)
def main():
""" This section is for arguments parsing """
module = AnsibleModule(
argument_spec=dict(
pn_cliusername=dict(required=True, type='str'),
pn_clipassword=dict(required=True, type='str', no_log=True),
pn_cliswitch=dict(required=False, type='str'),
state=dict(required=True, type='str',
choices=['present', 'absent', 'update']),
pn_vrouter_name=dict(required=True, type='str'),
pn_ospf_area=dict(required=True, type='str'),
pn_stub_type=dict(type='str', choices=['none', 'stub', 'nssa',
'stub-no-summary',
'nssa-no-summary']),
pn_prefix_listin=dict(type='str'),
pn_prefix_listout=dict(type='str'),
pn_quiet=dict(type='bool', default='True')
)
)
# Accessing the arguments
cliusername = module.params['pn_cliusername']
clipassword = module.params['pn_clipassword']
cliswitch = module.params['pn_cliswitch']
state = module.params['state']
vrouter_name = module.params['pn_vrouter_name']
ospf_area = module.params['pn_ospf_area']
stub_type = module.params['pn_stub_type']
prefix_listin = module.params['pn_prefix_listin']
prefix_listout = module.params['pn_prefix_listout']
quiet = module.params['pn_quiet']
command = get_command_from_state(state)
# Building the CLI command string
cli = '/usr/bin/cli'
if quiet is True:
cli += ' --quiet '
cli += ' --user %s:%s ' % (cliusername, clipassword)
if cliswitch:
if cliswitch == 'local':
cli += ' switch-local '
else:
cli += ' switch ' + cliswitch
cli += ' %s vrouter-name %s area %s ' % (command, vrouter_name, ospf_area)
if stub_type:
cli += ' stub-type ' + stub_type
if prefix_listin:
cli += ' prefix-list-in ' + prefix_listin
if prefix_listout:
cli += ' prefix-list-out ' + prefix_listout
# Run the CLI command
ospfcommand = shlex.split(cli)
# 'out' contains the output
# 'err' contains the error messages
result, out, err = module.run_command(ospfcommand)
# Response in JSON format
if result != 0:
module.exit_json(
command=cli,
stderr=err.rstrip("\r\n"),
changed=False
)
else:
module.exit_json(
command=cli,
stdout=out.rstrip("\r\n"),
changed=True
)
def _parse(self, parser, confpath):
"""
Parse file in .wks format using provided parser.
"""
with open(confpath) as conf:
lineno = 0
for line in conf:
line = line.strip()
lineno += 1
if line and line[0] != '#':
try:
line_args = shlex.split(line)
parsed = parser.parse_args(line_args)
except ArgumentError as err:
raise KickStartError('%s:%d: %s' % \
(confpath, lineno, err))
if line.startswith('part'):
# SquashFS does not support filesystem UUID
if parsed.fstype == 'squashfs':
if parsed.fsuuid:
err = "%s:%d: SquashFS does not support UUID" \
% (confpath, lineno)
raise KickStartError(err)
if parsed.label:
err = "%s:%d: SquashFS does not support LABEL" \
% (confpath, lineno)
raise KickStartError(err)
if parsed.use_label and not parsed.label:
err = "%s:%d: Must set the label with --label" \
% (confpath, lineno)
raise KickStartError(err)
# using ArgumentParser one cannot easily tell if option
# was passed as argument, if said option has a default
# value; --overhead-factor/--extra-space cannot be used
# with --fixed-size, so at least detect when these were
# passed with non-0 values ...
if parsed.fixed_size:
if parsed.overhead_factor or parsed.extra_space:
err = "%s:%d: arguments --overhead-factor and --extra-space not "\
"allowed with argument --fixed-size" \
% (confpath, lineno)
raise KickStartError(err)
else:
# ... and provide defaults if not using
# --fixed-size iff given option was not used
# (again, one cannot tell if option was passed but
# with value equal to 0)
if '--overhead-factor' not in line_args:
parsed.overhead_factor = self.DEFAULT_OVERHEAD_FACTOR
if '--extra-space' not in line_args:
parsed.extra_space = self.DEFAULT_EXTRA_SPACE
self.partnum += 1
self.partitions.append(Partition(parsed, self.partnum))
elif line.startswith('include'):
self._parse(parser, parsed.path)
elif line.startswith('bootloader'):
if not self.bootloader:
self.bootloader = parsed
else:
err = "%s:%d: more than one bootloader specified" \
% (confpath, lineno)
raise KickStartError(err)
def get_commit_log():
output = subprocess.check_output(
shlex.split('git log --pretty=%s --color'), stderr=subprocess.STDOUT)
output = output.decode('ascii')
output = output.split('\n')
return output
def read_rc(path):
with open(path) as rc:
return dict(token.split("=") for token in shlex.split(rc.read()))
