def link_alive_tcp( link, remote_ip ):
"""
Returns status of the link.
If the link is alive, returns its rtt to the remote_ip.
Use this method to check if the link is alive:
$ nc -v -s 192.168.0.101 -w 1 1.1.1.1 35501
nc: connect to 1.1.1.1 port 35501 (tcp) timed out: Operation now in progress
$ nc -v -s 192.168.0.101 -w 1 5.9.243.189 35501
nc: connect to 5.9.243.189 port 35501 (tcp) failed: Connection refused
"""
if link[0] in 'el':
source_ip = ip_of_interface( link )
if source_ip is None:
log("Can't detect IP address of %s" % link)
return
cmd = ['nc', '-v', '-s', source_ip, '-w', '1', remote_ip, '35501' ]
p = Popen(cmd, stdout=PIPE, stderr=STDOUT)
stdout = p.communicate()[0]
if 'Connection refused' in stdout:
return '1'
return None
def test_drivenmatlabagent(volttron_instance1):
print("** Setting up test_drivenagent module **")
wrapper = volttron_instance1
#Write config files for master driver
process = Popen(['python', 'config_builder.py',
'--count=1',
'--publish-only-depth-all',
'--campus=fakecampus',
'--building=fakebuilding',
'--interval=5',
'--config-dir=../../applications/pnnl/FakeDrivenMatlabAgent/tests',
'fake',
'../../applications/pnnl/FakeDrivenMatlabAgent/tests/test_fake.csv',
'null'],
env=volttron_instance1.env, cwd='scripts/scalability-testing',
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
result = process.wait()
print result
assert result == 0
#Actuator Agent
agent_uuid = volttron_instance1.install_agent(
agent_dir="services/core/ActuatorAgent",
config_file="services/core/ActuatorAgent/actuator-deploy.service",
start=True)
print("agent id: ", agent_uuid)
assert agent_uuid
actuator_agent = wrapper.build_agent()
#Driven Matlab Agent
agent_uuid = volttron_instance1.install_agent(
agent_dir="applications/pnnl/FakeDrivenMatlabAgent",
config_file=config_wh,
start=True)
print("agent id: ", agent_uuid)
assert agent_uuid
driven_agent = wrapper.build_agent()
#Fake Master Driver
agent_uuid = volttron_instance1.install_agent(
agent_dir="services/core/MasterDriverAgent",
config_file="applications/pnnl/FakeDrivenMatlabAgent/tests/master-driver.agent",
start=True)
print("agent id: ", agent_uuid)
assert agent_uuid
driver_agent = wrapper.build_agent()
gevent.sleep(5)
path = 'fakecampus/fakebuilding/fakedriver0/HPWH_Phy0_PowerState'
value = driven_agent.vip.rpc.call('platform.actuator', 'get_point', path).get()
print('The set point value is '+str(value))
assert value == 1
path = 'fakecampus/fakebuilding/fakedriver0/ERWH_Phy0_ValveState'
value = driven_agent.vip.rpc.call('platform.actuator', 'get_point', path).get()
print('The set point value is '+str(value))
assert value == 1
def save_weather_data( location, filename ):
if location == NOT_FOUND_LOCATION:
location_not_found = True
location = DEFAULT_LOCATION
else:
location_not_found = False
p = Popen( [ WEGO, '--city=%s' % location ], stdout=PIPE, stderr=PIPE )
stdout, stderr = p.communicate()
if p.returncode != 0:
error( stdout + stderr )
dirname = os.path.dirname( filename )
if not os.path.exists( dirname ):
os.makedirs( dirname )
if location_not_found:
stdout += NOT_FOUND_MESSAGE
open( filename, 'w' ).write( stdout )
p = Popen( [ "bash", ANSI2HTML, "--palette=solarized", "--bg=dark" ], stdin=PIPE, stdout=PIPE, stderr=PIPE )
stdout, stderr = p.communicate( stdout )
if p.returncode != 0:
error( stdout + stderr )
open( filename+'.html', 'w' ).write( stdout )
def run_autobahn():
"""
Spawn the autobahn test suite in a subprocess
"""
import os.path
cmd = ['wstest -m fuzzingclient -s %s/autobahn.json' % (
os.path.dirname(__file__),)]
wstest = Popen(cmd, stderr=PIPE, stdout=PIPE, shell=True)
if wstest.wait():
# something went wrong, it's boom time.
stdout, stderr = wstest.communicate(None)
sys.stderr.write(stderr)
sys.stderr.flush()
sys.stdout.write(stdout)
sys.stderr.flush()
raise RuntimeError
# parse the generated report to see if we have failures
chk = Popen(
'fgrep gevent_websocket reports/clients/index.html | grep Fail',
stdout=PIPE, shell=True)
stdout, stderr = chk.communicate(None)
if stdout:
sys.stderr.write('Autobahn test failures:\n' + stdout)
raise SystemExit(1)
def call_subprocess(self, cmd):
times = datetime.datetime.now()
# latest 14.0.4 requires "HOME" env variable to be passed
# copy current environment variables and add "HOME" variable
# pass the newly created environment variable to Popen subprocess
env_home = os.environ.copy()
env_home['HOME'] = HOME_ENV_PATH
# stdout and stderr are redirected.
# stderr not used (stdout validation is done so stderr check is
# is not needed)
try:
p = Popen(cmd, stdout=PIPE, \
stderr=PIPE, shell=True, env=env_home)
while p.poll() is None:
gevent.sleep(0.1)
now = datetime.datetime.now()
diff = now - times
if diff.seconds > 5:
os.kill(p.pid, signal.SIGKILL)
os.waitpid(-1, os.WNOHANG)
message = "command:" + cmd + " ---> hanged"
ssdlog = StorageStatsDaemonLog(message = message)
self.call_send(ssdlog)
return None
except:
pass
return None
# stdout is used
return p.stdout.read()
def _auction_fucn(self, args):
process = None
try:
process = Popen(args)
self.processes[process.pid] = process
rc = process.wait()
if rc == 0:
self.logger.info(
"Finished {}".format(args[2]),
extra={
'MESSAGE_ID': 'CHRONOGRAPH_WORKER_COMPLETE_SUCCESSFUL'
}
)
else:
self.logger.error(
"Exit with error {}".format(args[2]),
extra={
'MESSAGE_ID': 'CHRONOGRAPH_WORKER_COMPLETE_EXCEPTION'
}
)
except Exception as error:
self.logger.critical(
"Exit with error {} params: {} error: {}".format(
args[2], repr(args), repr(error)),
extra={'MESSAGE_ID': 'CHRONOGRAPH_WORKER_COMPLETE_EXCEPTION'})
if process:
del self.processes[process.pid]
def logs_downloader(logs_queued, recipient):
# Build tar command with tar command and logs sent by client
archive_path = '%ssrc/rockstor/logs/' % settings.ROOT_DIR
archive_file = 'requested_logs.tgz'
# If log download requested by Log Reader serve a personalized tgz
# file with log file name
if (recipient == 'reader_response'):
archive_file = '%s.tgz' % logs_queued[0]
archive_path += archive_file
download_command = []
download_command.extend(self.tar_utility)
download_command.append(archive_path)
# Get every log location via logs dictionary
for log in logs_queued:
download_command.append(self.build_log_path(log))
# Build download archive
download_process = Popen(download_command, bufsize=1, stdout=PIPE)
download_process.communicate()
# Return ready state for logs archive download specifying recipient
# (logManager or LogDownloader)
self.emit('logsdownload',
{
'key': 'logManager:logsdownload', 'data': {
'archive_name': '/logs/%s' % archive_file,
'recipient': recipient
}
})
def try_get_video_name(self, video_url):
print "Getting name for %s"%video_url
cmd=['youtube-dl', '--get-title',video_url]
temp = Popen(cmd, stdout=PIPE)
out, err = temp.communicate()
print out
return out
def execute_cmd_with_output(cmd, stdin=None):
log.debug("Running command: %r" % cmd)
p = Popen(cmd, bufsize=-1, stdout=PIPE, stderr=PIPE, stdin=stdin)
(msgs, errs) = p.communicate()
if p.returncode != 0:
raise Exception('Failed to run command')
return (msgs, errs)
def test_ia_upload():
# upload from stdin.
cmd = ('echo "Hello World!" |'
'ia upload iacli-test-item - --remote-name="stdin.txt" --size-hint=8')
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert proc.returncode == 0
cmd = ('echo "Hello World!" |'
'ia upload iacli-test-item -')
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert proc.returncode == 1
assert stderr == '--remote-name is required when uploading from stdin.\n'
# upload file.
cmd = 'ia upload iacli-test-item setup.py'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert proc.returncode == 0
# upload debug.
cmd = 'ia upload iacli-test-item setup.py --debug'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert proc.returncode == 0
# upload non-200 status_code.
cmd = ('echo "Hello World!" |'
'ia upload iacli-test-item - --remote-name="iacli-test-item_meta.xml"')
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert stderr == 'error "AccessDenied" (403): Access Denied\n'
assert proc.returncode == 1
def ipv6_supported():
"""Checks whether we can support IPv6 on this host.
:returns tuple[bool,str]: supported, reason for lack of support or None.
"""
if not os.path.exists("/proc/sys/net/ipv6"):
return False, "/proc/sys/net/ipv6 is missing (IPv6 compiled out?)"
try:
check_call(["which", "ip6tables"])
except FailedSystemCall:
return False, ("ip6tables not installed; Calico IPv6 support requires "
"Linux kernel v3.3 or above and ip6tables v1.4.14 or "
"above.")
try:
# Use -C, which checks for a particular rule. We don't expect the rule
# to exist but iptables will give us a distinctive error if the
# rpfilter module is missing.
proc = Popen(["ip6tables", "-C", "FORWARD", "-m", "rpfilter"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = proc.communicate()
if "Couldn't load match" in err:
return False, (
"ip6tables is missing required rpfilter match module; "
"Calico IPv6 support requires Linux kernel v3.3 or "
"above and ip6tables v1.4.14 or above."
)
except OSError:
return False, "Failed to execute ip6tables"
return True, None
def start_agent(self, agent_uuid):
self.logit('Starting agent {}'.format(agent_uuid))
self.logit("VOLTTRON_HOME SETTING: {}".format(
self.env['VOLTTRON_HOME']))
cmd = ['volttron-ctl']
cmd.extend(['start', agent_uuid])
p = Popen(cmd, env=self.env,
stdout=sys.stdout, stderr=sys.stderr)
p.wait()
# Confirm agent running
cmd = ['volttron-ctl']
cmd.extend(['status', agent_uuid])
res = subprocess.check_output(cmd, env=self.env)
# 776 TODO: Timing issue where check fails
time.sleep(.1)
self.logit("Subprocess res is {}".format(res))
assert 'running' in res
pidpos = res.index('[') + 1
pidend = res.index(']')
pid = int(res[pidpos: pidend])
assert psutil.pid_exists(pid), \
"The pid associated with agent {} does not exist".format(pid)
self.started_agent_pids.append(pid)
return pid
def get_doc_index(document, filepath):
words = {}
command = 'tesseract ' + filepath + ' -l hrv stdout'
#text = subprocess.check_output(command, shell=True)
print '>>> ', command
sub = Popen([command], stdout=PIPE, shell=True)
text, err = sub.communicate()
print '>>> ', command, 'DONE'
# extract page num from file path
match = re.search('.*xl-(\d+).png', filepath)
if match:
page = int(match.groups()[0])
for word in text.strip().split():
# skip small words
if len(word) <= 2:
continue
if word in words:
document_dict = words[word]
if document['name'] in document_dict:
pages_set = words[word][document['name']]
pages_set.add(page)
words[word][document['name']] = pages_set
else:
words[word][document['name']] = set([page])
else:
# init word
words[word] = {document['name']: set([page])}
return len(words)
def worker():
print 'start'
sub = Popen(['sleep 10'], stdout=PIPE, shell=True)
# sub = Popen(['top'], stdout=PIPE, shell=True)
out, err = sub.communicate()
print 'end'
return out.rstrip()
def ping(ip, task_id):
loss = latency = None
p = Popen(['ping', '-c', '3', ip], stdout=PIPE, stderr=PIPE)
out, err = p.communicate()
out = out.split('\n')
for line in out:
line = line.rstrip()
match = re.match('.* ([0-9]+)% packet loss.*', line)
if match:
loss = match.group(1)
match = re.match('.*([0-9\.]+)/([0-9\.]+)/([0-9\.])+/([0-9\.]+) ms.*', line)
if match:
latency = match.group(2)
ping = Ping()
ping.ip = ip
ping.task_id = uuid.UUID(task_id)
if loss:
ping.loss = loss
if latency:
ping.latency = float(latency)
ping.save()
def do_job(self,jobid,props,*args,**kwargs):
log.info("do job: %s"%(job))
if props:
props = json.loads(props)
else:
props = {}
payload = {"id":jobid,"status":"building"}
r = requests.post('%s/api/job'%(self.options.master_base_url),data=payload)
log.info("update job %s status,result:%s"%(jobid,r))
if props.has_key("steps"):
steps = props["steps"]
for step in steps:
if step.has_key("shell"):
shell = step["shell"]
log.info("shell: %s"%(shell[:256]))
sub = Popen([shell], stdout=PIPE, stderr=PIPE, shell=True)
stdio_q = Queue()
def handle_stdout():
for l in sub.stdout:
stdio_q.put_nowait((0,l))
def handle_stderr():
for l in sub.stderr:
stdio_q.put_nowait((1,l))
def handle_stdio_q():
#stdout 0 stderr 1 extra 2 end 255
current_text_type = None
stdio_list = []
need_flush = False
timeout = None
while 1:
ttype,text = stdio_q.get()
if ttype!=current_text_type and len(stdio_list)>0:
need_flush = True
if len(stdio_list)>50:
need_flush = True
if need_flush:
text2flush = "".join(stdio_list)
payload = {"id":jobid,"text_type":current_text_type,"stdio_text":text2flush}
r = requests.post('%s/api/job'%(self.options.master_base_url),data=payload)
need_flush = False
if ttype==255:
break
current_text_type = ttype
stdio_list.append(text)
glet_stdout = gevent.spawn(handle_stdout)
glet_stderr = gevent.spawn(handle_stderr)
glet_stdio_q = gevent.spawn(handle_stdio_q)
sub.wait()
stdio_q.put_nowait((255,""))
glet_stdout.kill()
glet_stderr.kill()
payload = {"id":jobid,"status":3}#JOB_STATUS_FINISHED
r = requests.post('%s/api/job'%(self.options.master_base_url),data=payload)
log.info("update job %d status,result:%s"%(jobid,r))
def fetch(url):
sys.stdout.write(".")
sys.stdout.flush()
p = Popen(['curl', '-w', '@curl-format.txt', '-o', '/dev/null', '-s', url], stdout=PIPE)
out, err = p.communicate()
out = re.sub(r'(\d+),(\d+)', r'\1.\2', out)
return json.loads(out)
def test_invoke():
# Run a subprocess through Popen to make sure
# libev is handling SIGCHLD. This could *probably* be simplified to use
# just hub.loop.install_sigchld
p = Popen("true", stdout=PIPE, stderr=PIPE)
gevent.sleep(0)
p.communicate()
gevent.sleep(0)
def _download_by_curl(self, method, url, headers, file_path):
header_str = ' '.join([ "%s: %s" % (k, v) for k,v in headers.items() ])
request_cmd = 'curl -J -k -b sid=%s -H "%s" -X %s -o %s %s -s' % \
(self.cookies['sid'], header_str, method, file_path, url)
sub = Popen(request_cmd, stdout=PIPE, shell=True)
response, err = sub.communicate()
return response
def test_ia_metadata_exists():
cmd = 'ia metadata --exists iacli_test-doesnotexist'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert proc.returncode == 1
cmd = 'ia metadata --exists nasa'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert proc.returncode == 0
def test_ia_metadata_formats():
cmd = 'ia metadata --formats iacli_test_item'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
test_output_set = set([
"Text",
"Archive BitTorrent",
"Metadata",
])
assert set(stdout[:-1].split('\n')) == test_output_set
def block_and_check_process_output(process_args, fail_on_error=True, retry_count=0):
process = Popen(process_args)
process.wait()
if process.returncode != 0:
if retry_count > 0:
print '{} failed. Retrying...'.format(' '.join(process_args))
block_and_check_process_output(process_args, fail_on_error=fail_on_error, retry_count=retry_count-1)
else:
print process.stderr
if fail_on_error:
exit(1)
def _cmd(cmdargs):
"""Executes the passed command.
:param cmdargs: A list of arguments that should be passed to Popen.
:type cmdargs: [str]
"""
if verbose:
print(cmdargs)
process = Popen(cmdargs, env=os.environ, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
process.wait()
def exec_command(*args, **kwargs):
shell = kwargs.get("shell", False)
process = Popen(args, stdout=PIPE, stderr=PIPE, close_fds=True, shell=shell)
retcode = process.wait()
output = process.stdout.read()
unused_err = process.stderr.read()
if retcode:
_logger.debug("Command '%s' returned non-zero exit status %d", args, retcode)
return retcode, output.strip()
def html_wrapper(data):
"""
Convert ANSI text `data` to HTML
"""
proc = Popen(
["bash", ANSI2HTML, "--palette=solarized", "--bg=dark"],
stdin=PIPE, stdout=PIPE, stderr=PIPE)
data = data.encode('utf-8')
stdout, stderr = proc.communicate(data)
if proc.returncode != 0:
error(stdout + stderr)
return stdout.decode('utf-8')
def worker(code):
m = judgeMarket(code)
print 'start process %s.%s' % (code, m)
target_file = os.path.join(DATA_DIR, '%s.csv' % code)
if os.path.exists(target_file):
print '%s quote file already exists' % code
else:
cmd = "wget -t 10 %s%s.%s -O %s" % (WGET_URL, code, m, target_file)
print 'fetch data : %s' % cmd
sub = Popen([cmd], stdout=PIPE, shell=True)
out, err = sub.communicate()
print out.rstrip()
def static_reader(reader, log_path):
if (valid_log(log_path)):
# Log file exist and greater than 0, perform data collecting
# Build reader command
read_command = build_reader_command(reader)
# Define popen process and once completed split stdout by lines
reader_process = Popen(read_command, bufsize=1, stdout=PIPE)
log_content = reader_process.communicate()[0]
log_contentsize = getsizeof(log_content)
log_content = log_content.splitlines(True)
# Starting from content num of lines decide if serve it 1
# line/time or in 200 lines chunks
reader_type = 'fast' if (len(log_content) <= 200) else 'slow'
chunk_size = logs_loader[reader_type]['lines']
reader_sleep = logs_loader[reader_type]['sleep']
log_content_chunks = [log_content[x:x+chunk_size] for x in
xrange(0, len(log_content), chunk_size)] # noqa F821
total_rows = len(log_content)
else: # Log file missing or size 0, gently inform user
# Log not exist or empty so we send fake values for rows,
# chunks, etc to uniform data on existing functions and avoid
# client side extra checks
log_content = 'Selected log file is empty or doesn\'t exist'
log_content = log_content.splitlines(True)
total_rows = 1
log_contentsize = getsizeof(log_content)
log_content_chunks = []
log_content_chunks.append(log_content)
reader_sleep = 0
# Serve each chunk with emit and sleep before next one to avoid
# client side browser overload
current_rows = 0
for data_chunks in log_content_chunks:
chunk_content = ''.join(data_chunks)
current_rows += len(data_chunks)
self.emit('logcontent',
{
'key': 'logManager:logcontent',
'data': {
'current_rows': current_rows,
'total_rows': total_rows,
'chunk_content': chunk_content,
'content_size': log_contentsize
}
})
gevent.sleep(reader_sleep)
def process_video(self, video_url, timestamp=None):
print "Playing %s"%video_url
cmd=['youtube-dl', '-g','-f best',video_url]
temp = Popen(cmd, stdout=PIPE)
out, err = temp.communicate()
video_direct = out
cmd=['omxplayer', '-o', 'hdmi', '--vol', '-2000', video_direct[:-1]]
print " ".join(cmd)
self.sub = Popen(cmd, stdin=PIPE)
omxdbus = OmxDbus()
self.sub.communicate()
self.sub = None
self.task.put({'play_complete':timestamp})
def block_until_sshable(config, instance_name, tries=1, wait_time_seconds=10):
if tries > 10:
print "Tried {} times to ssh to {}, something bad happened".format(tries, instance_name)
exit(1)
process_args = zdgcutil.ssh(config, instance_name)
process = Popen(process_args)
process.wait()
ret_code = process.returncode
if ret_code != 0:
print "{} not yet sshable, waiting {} seconds".format(instance_name, wait_time_seconds)
time.sleep(wait_time_seconds)
block_until_sshable(config, instance_name, tries + 1)
def test_ia_mine():
with open('testlist.txt', 'w') as fp:
fp.write('\n'.join(['nasa', 'iacli-test-item']))
cmd = 'ia mine testlist.txt --cache'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert proc.returncode == 0
cmd = 'ia mine testlist.txt --output=d.json'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert proc.returncode == 0
cmd = 'ia mine testlist.txt'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert proc.returncode == 0
# Test ids from stdin.
cmd = 'echo "nasa" | ia mine -'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert proc.returncode == 0
try:
os.remove('testlist.txt')
os.remove('d.json')
os.remove('nasa_meta.json')
os.remove('iacli-test-item_meta.json')
except OSError:
pass
def detect_ipv6_supported():
"""Checks whether we can support IPv6 on this host.
:returns tuple[bool,str]: supported, reason for lack of support or None.
"""
if not os.path.exists("/proc/sys/net/ipv6"):
return False, "/proc/sys/net/ipv6 is missing (IPv6 compiled out?)"
try:
check_call(["which", "ip6tables"])
except FailedSystemCall:
return False, ("ip6tables not installed; Calico IPv6 support requires "
"Linux kernel v3.3 or above and ip6tables v1.4.14 or "
"above.")
# Check for the existence of the IPv6 NAT table.
try:
check_call(["ip6tables-save", "--table", "nat"])
except FailedSystemCall:
return False, "Failed to load IPv6 NAT table"
try:
# Use -C, which checks for a particular rule. We don't expect the rule
# to exist but iptables will give us a distinctive error if the
# rpfilter module is missing.
proc = Popen(["ip6tables", "-C", "FORWARD", "-m", "rpfilter"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate()
if "Couldn't load match" in err:
return False, (
"ip6tables is missing required rpfilter match module; "
"Calico IPv6 support requires Linux kernel v3.3 or "
"above and ip6tables v1.4.14 or above.")
except OSError:
return False, "Failed to execute ip6tables"
return True, None
def link_alive_icmp( link, remote_ip ):
"""
Returns if the link is alive, and if it is, returns its ttl.
If the link is not alive, returns None.
"""
if link[0] in 'el':
source_ip = ip_of_interface( link )
if source_ip is None:
log("Can't detect IP address of %s" % link)
return
cmd = ['/bin/ping', '-I', source_ip, '-c', '1', '-W', '1', remote_ip ]
p = Popen(cmd, stdout=PIPE, stderr=STDOUT)
stdout = p.communicate()[0]
if p.returncode:
return None
for line in stdout.splitlines():
if 'bytes from' in line:
try:
return line.strip().split()[6][5:]
except:
return None
def run_server(execs, cwd, server, logfile, retval, test_id):
os.putenv("LISTEN", server.iproto)
server.process = Popen(execs, stdout=PIPE, stderr=PIPE, cwd=cwd)
sampler.register_process(server.process.pid, test_id, server.name)
test_timeout = Options().args.test_timeout
timer = Timer(test_timeout, timeout_handler,
(server.process, test_timeout))
timer.start()
stdout, stderr = server.process.communicate()
timer.cancel()
sys.stdout.write_bytes(stdout)
with open(logfile, 'ab') as f:
f.write(stderr)
retval['returncode'] = server.process.wait()
server.process = None
def _htmlize(ansi_output, title, parsed_query):
"""Return HTML representation of `ansi_output`.
Use `title` as the title of the page.
Format page according to query parameters from `parsed_query`."""
cmd = ["bash", ANSI2HTML, "--palette=solarized"]
if not parsed_query.get('inverted_colors'):
cmd += ["--bg=dark"]
proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate(ansi_output.encode("utf-8"))
stdout = stdout.decode("utf-8")
stderr = stderr.decode("utf-8")
if proc.returncode != 0:
error(stdout + stderr)
if parsed_query.get('inverted_colors'):
stdout = stdout.replace(
'<body class="">', '<body class="" style="background:white;color:#777777">')
title = "<title>%s</title>" % title
opengraph = _get_opengraph(parsed_query)
stdout = re.sub("<head>", "<head>" + title + opengraph, stdout)
return stdout
def test_ia_metadata_exists():
cmd = 'ia metadata --exists iacli_test-doesnotexist'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert proc.returncode == 1
cmd = 'ia metadata --exists nasa'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert proc.returncode == 0
def filtered_call(logger, source, program, arguments):
"""
Generic filter call.
Calls PROGRAM with OPTIONS given as keyword arguments to this function
which are supplied as command line options to the PROGRAM.
The function assumes the PROGRAM takes the SOURCE as input on stdin
and returns its output on stdout.
"""
filter_call = [utils.resolve_executable(program)]
args = [str(key) + str(option) for key, option in arguments.iteritems()]
filter_call.extend(args)
logger.debug("{} call: {}".format(program, ' '.join(filter_call)))
result = Popen(filter_call, stdin=source.stdout, stdout=PIPE, stderr=PIPE)
return result
def test_ia_search():
cmd = 'ia search iacli-test-item --sort=date:asc'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert proc.returncode == 0
cmd = 'ia search "identifier:iacli-test-item" --number-found'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert stdout == '1\n'
assert proc.returncode == 0
def start_driver(self):
while True:
try:
try:
call(['rm', '/var/run/pigpio.pid'])
except Exception, e:
pass
self.cleanup()
self.driver_name = 'swave2'
swave_path = os.path.join(self.file_dir, self.driver_name)
print "swave executable: ", swave_path
self.swave = Popen([swave_path])
sleep(1) # change this line with "as soon as started"
self.send_signal(self.HB)
self.run_heartbeat = True
gevent.spawn(self.send_heartbeat)
break
except:
def mac_lookup(self, ip_address):
if AceConfig.osplatform != 'Windows':
Popen(['ping', '-c 1', ip_address], stdout=PIPE, shell=False)
try:
pid = Popen(['arp', '-n', ip_address],
stdout=PIPE,
shell=False)
except:
Stat.logger.error("Can't execute arp! Install net-tools")
return "Local IP address "
else:
popen_params = {'stdout': PIPE, 'shell': False}
CREATE_NO_WINDOW = 0x08000000 # CREATE_NO_WINDOW
CREATE_NEW_PROCESS_GROUP = 0x00000200 # note: could get it from subprocess
DETACHED_PROCESS = 0x00000008 # 0x8 | 0x200 == 0x208
popen_params.update(creationflags=CREATE_NO_WINDOW
| CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS)
Popen(["ping", "-n 1", ip_address], **popen_params)
pid = Popen(["arp", "-a", ip_address], **popen_params)
s = pid.communicate()[0]
mac_address = re.search(r"(([a-f\d]{1,2}(\:|\-)){5}[a-f\d]{1,2})", s)
response = None
if mac_address != None:
mac_address = mac_address.groups()[0]
lookup_url = 'http://macvendors.co/api/vendorname/%s'
try:
headers = {
'User-Agent': 'API Browser',
'Accept-Encoding': 'gzip, deflate',
'Connection': 'close'
}
response = requests.get(lookup_url % mac_address,
headers=headers,
timeout=5).text
except:
Stat.logger.error("Can't obtain vendor for MAC address %s" %
mac_address)
else:
Stat.logger.error("Can't obtain MAC address for Local IP:%s" %
ip_address)
return "Local IP address " if not response else response
def test_ia_metadata_modify():
# Modify test item.
valid_key = "foo-{k}".format(k=int(time()))
cmd = 'ia metadata --modify="{k}:test_value" iacli_test_item'.format(
k=valid_key)
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert proc.returncode == 0
# Submit illegal modification.
cmd = 'ia metadata --modify="-foo:test_value" iacli_test_item'
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
assert proc.returncode == 1
assert "Illegal tag name" in stderr
def launch_commands(self, commands):
all_commands = commands #[self.parsed_args.slurmSetupCommands] + commands
self.script = self.create_sbatch_script(all_commands, self.stdio_dir,
self.parsed_args)
self.write_sbatch_workerscript_to_file("create_additional_workers_" +
self.parsed_args.slurmJobName +
".sh")
call_pars = shlex.split(self.parsed_args.slurmBinary) #pylint: disable=E1103
self.logger.debug("Launching '{}' with input:\n{}".format(
" ".join(call_pars), self.script))
#print("launching '" + " ".join(call_pars) + "' with input:\n" + self.script)
#Make the sbatch call
(out, err) = Popen(call_pars, stdin=PIPE, stdout=PIPE,
stderr=PIPE).communicate(self.script)
self.logger.debug("Launch result: {}".format(str((out, err))))
#print("launch result: " + str((out,err)))
if err:
self.logger.error("Slurm batch launch error: {}".format(err))
raise Exception("Slurm batch launch error: {}".format(err))
#TODO: assert that correct results were received
return (out, err)
def launch_python_function(
self,
f,
join_all_greenlets_at_end=True): #TODO: add file_paths to config
main_f = create_main_function_wrapper(f)
cmd = create_python_function_command(main_f)
call_pars = shlex.split(cmd)
rand_id = random.randint(0, sys.maxint)
stdout_file_path = os.path.join(self.stdio_dir,
hex(rand_id) + "_stdout.log")
stderr_file_path = os.path.join(self.stdio_dir,
hex(rand_id) + "_stderr.log")
stdout_file = open(stdout_file_path, "w")
stderr_file = open(stderr_file_path, "w")
#TODO: Close files on failure. Use: scope.on_exception
g = gevent.spawn(
Popen(call_pars, stdout=stdout_file,
stderr=stderr_file).communicate)
g.link(lambda _: stdout_file.close())
g.link(lambda _: stderr_file.close())
return g
def copy_untar_ref_db(source_file, destination, logger):
"""
Copies and uncompresses gzipped tar file containing reference database to destination.
"""
workdir = os.path.dirname(destination)
tar_call = [utils.resolve_executable("tar"), "-xf", source_file]
if source_file.lower().endswith((".tar.gz", ".tar", ".tgz")):
logger.info("It appears reference DB '%s' is in tar/gz format",
source_file)
logger.info("Extracting database tarball...")
shutil.copy(source_file, destination)
tar = Popen(tar_call, stdout=PIPE, stderr=PIPE, cwd=workdir)
logger.debug("tar call:{}".format(tar_call))
tar_stream_data = tar.communicate()
if tar.returncode is not 0:
logger.error("tar returncode {}".format(tar.returncode))
logger.error(
"tar stdout: {}\ntar stderr: {}".format(tar_stream_data))
raise PipelineError("tar exited with return code {}".format(
tar.returncode))
else:
logger.info("Untar of reference DB successful.")
elif source_file.lower().endswith((".gz")):
logger.info("It appears reference DB '%s' is in gz format",
source_file)
logger.info("Gunzipping database file...")
shutil.copy(source_file, destination)
gunzip_call = [utils.resolve_executable("gunzip"), source_file]
gunzip = Popen(gunzip_call, stdout=PIPE, stderr=PIPE, cwd=workdir)
logger.debug("gunzip call:{}".format(tar_call))
gunzip_stream_data = gunzip.communicate()
if gunzip.returncode is not 0:
logger.error("gunzip returncode {}".format(gunzip.returncode))
logger.error("gunzip stdout: {}\ngunzip stderr: {}".format(
gunzip_stream_data))
raise PipelineError("gunzip exited with return code {}".format(
gunzip.returncode))
else:
logger.info("Gunzip of reference DB successful.")
else:
logger.error(
"Don't know what to do with {}, it does not look like a (gzipped) tar file"
.format(source_file))
raise FileFormatError(source_file)
return destination
def get_moon(parsed_query):
location = parsed_query['orig_location']
html = parsed_query['html_output']
lang = parsed_query['lang']
hemisphere = parsed_query['hemisphere']
date = None
if '@' in location:
date = location[location.index('@') + 1:]
location = location[:location.index('@')]
cmd = [globals.PYPHOON]
if lang:
cmd += ["-l", lang]
if not hemisphere:
cmd += ["-s", "south"]
if date:
try:
dateutil.parser.parse(date)
except Exception as e:
print("ERROR: %s" % e)
else:
cmd += [date]
p = Popen(cmd, stdout=PIPE, stderr=PIPE)
stdout = p.communicate()[0]
stdout = stdout.decode("utf-8")
if parsed_query.get('no-terminal', False):
stdout = globals.remove_ansi(stdout)
if html:
p = Popen(
["bash", globals.ANSI2HTML, "--palette=solarized", "--bg=dark"],
stdin=PIPE,
stdout=PIPE,
stderr=PIPE)
stdout, stderr = p.communicate(stdout.encode("utf-8"))
stdout = stdout.decode("utf-8")
stderr = stderr.decode("utf-8")
if p.returncode != 0:
globals.error(stdout + stderr)
return stdout
def run_profiler(nursery: Nursery, running_nodes: List[RunningNode],
profiler_data_directory: str) -> List[Popen]:
os.makedirs(os.path.expanduser(profiler_data_directory), exist_ok=True)
profiler_processes: List[Popen] = list()
for node in running_nodes:
args = [
"py-spy",
"record",
"--pid",
str(node.process.pid),
"--output",
os.path.join(
profiler_data_directory,
f"{node.config.address}-{datetime.utcnow().isoformat()}.data",
),
]
profiler = Popen(args, stdout=DEVNULL, stderr=DEVNULL)
nursery.exec_under_watch(profiler)
return profiler_processes
def do_start(self, cmd=None):
self.send_success('start ok')
DEBUG('------------task start------------')
task_ids = [
int(task_id) for task_id in self.task_ids.split(',')
if int(task_id) not in TASK
]
for task_id in task_ids:
try:
if cmd is None:
cmd = 'python topscan.py -t %s' % task_id
else:
cmd = 'python topscan.py -t %s %s' % (task_id, cmd)
self.sub = Popen(cmd, shell=True, cwd=CWD)
pid = int(self.sub.pid)
TASK[task_id] = pid
INFO('%s start a new task,task_id:%s,pid:%s' %
(self.MODULE_NAME, task_id, pid))
except Exception:
ERROR('%s start a new task,task_id:%s failed' %
(self.MODULE_NAME, task_id))
def live_reader(log_path):
# Switch live reader state to True
self.livereading = True
# Build reader command from readers dict
read_command = build_reader_command("tailf")
self.livereader_process = Popen(read_command, bufsize=1, stdout=PIPE)
while self.livereading:
live_out = self.livereader_process.stdout.readline()
self.emit(
"logcontent",
{
"key": "logManager:logcontent",
"data": {
"current_rows": 1,
"total_rows": 1,
"chunk_content": live_out,
"content_size": 1,
},
},
)
def _run_vim_script(script_lines, text_lines):
"""
Apply `script_lines` to `lines_classes`
and returns the result
"""
script_vim = NamedTemporaryFile(delete=True)
textfile = NamedTemporaryFile(delete=True)
open(script_vim.name, "w").write("\n".join(script_lines))
open(textfile.name, "w").write("\n".join(text_lines))
script_vim.file.close()
textfile.file.close()
my_env = os.environ.copy()
my_env['HOME'] = PATH_VIM_ENVIRONMENT
cmd = ["script", "-q", "-c",
"vim -S %s %s" % (script_vim.name, textfile.name)]
Popen(cmd, shell=False, stdout=FNULL, stderr=FNULL, env=my_env).communicate()
return open(textfile.name, "r").read()
def live_reader(log_path):
# Switch live reader state to True
self.livereading = True
# Build reader command from readers dict
read_command = build_reader_command('tailf')
self.livereader_process = Popen(read_command,
bufsize=1,
stdout=PIPE)
while self.livereading:
live_out = self.livereader_process.stdout.readline()
self.emit(
'logcontent', {
'key': 'logManager:logcontent',
'data': {
'current_rows': 1,
'total_rows': 1,
'chunk_content': live_out,
'content_size': 1
}
})
def mediainfo(filepath):
"""Return dictionary with media info(codec, duration, size, bitrate...) from filepath
"""
prober = get_prober_name()
command_args = ["-v", "quiet", "-show_format", "-show_streams", filepath]
command = [prober, '-of', 'old'] + command_args
res = Popen(command, stdout=PIPE)
output = res.communicate()[0].decode("utf-8")
if res.returncode != 0:
command = [prober] + command_args
output = Popen(command, stdout=PIPE).communicate()[0].decode("utf-8")
rgx = re.compile(r"(?:(?P<inner_dict>.*?):)?(?P<key>.*?)\=(?P<value>.*?)$")
info = {}
if sys.platform == 'win32':
output = output.replace("\r", "")
for line in output.split("\n"):
# print(line)
mobj = rgx.match(line)
if mobj:
# print(mobj.groups())
inner_dict, key, value = mobj.groups()
if inner_dict:
try:
info[inner_dict]
except KeyError:
info[inner_dict] = {}
info[inner_dict][key] = value
else:
info[key] = value
return info
def get_moon(location, html=False, lang=None, query=None):
if query is None:
query = {}
date = None
if '@' in location:
date = location[location.index('@') + 1:]
location = location[:location.index('@')]
cmd = [PYPHOON]
if date:
try:
dateutil.parser.parse(date)
except Exception as e:
print("ERROR: %s" % e)
else:
cmd += [date]
env = os.environ.copy()
if lang:
env['LANG'] = lang
p = Popen(cmd, stdout=PIPE, stderr=PIPE, env=env)
stdout = p.communicate()[0]
if query.get('no-terminal', False):
stdout = remove_ansi(stdout)
if html:
p = Popen(["bash", ANSI2HTML, "--palette=solarized", "--bg=dark"],
stdin=PIPE,
stdout=PIPE,
stderr=PIPE)
stdout, stderr = p.communicate(stdout)
if p.returncode != 0:
error(stdout + stderr)
return stdout
#!/usr/bin/env python
# -*- coding: utf8 -*-
import gevent
from gevent.subprocess import Popen, PIPE
def cron():
while True:
print("cron")
gevent.sleep(0.2)
g = gevent.spawn(cron)
sub = Popen(['sleep 1; uname'], stdout=PIPE, shell=True)
out, err = sub.communicate()
g.kill()
print(out.rstrip())
def publish_agent(request, volttron_instance1):
"""
Fixture used for setting up the environment.
1. Creates fake driver configs
2. Starts the master driver agent with the created fake driver agents
3. Starts the actuator agent
4. Creates an instance Agent class for publishing and returns it
:param request: pytest request object
:param volttron_instance1: instance of volttron in which test cases are run
:return: an instance of fake agent used for publishing
"""
# Reset master driver config store
cmd = ['volttron-ctl', 'config', 'delete', PLATFORM_DRIVER, '--all']
process = Popen(cmd,
env=volttron_instance1.env,
cwd='scripts/scalability-testing',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
result = process.wait()
print(result)
assert result == 0
# Add master driver configuration files to config store.
cmd = [
'volttron-ctl', 'config', 'store', PLATFORM_DRIVER, 'fake.csv',
'fake_unit_testing.csv', '--csv'
]
process = Popen(cmd,
env=volttron_instance1.env,
cwd='scripts/scalability-testing',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
result = process.wait()
print(result)
assert result == 0
config_name = "devices/fakedriver"
cmd = [
'volttron-ctl', 'config', 'store', PLATFORM_DRIVER, config_name,
'fake_unit_testing.config', '--json'
]
process = Popen(cmd,
env=volttron_instance1.env,
cwd='scripts/scalability-testing',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
result = process.wait()
print(result)
assert result == 0
# Start the master driver agent which would intern start the fake driver
# using the configs created above
master_uuid = volttron_instance1.install_agent(
agent_dir=get_services_core("MasterDriverAgent"),
config_file={},
start=True)
print("agent id: ", master_uuid)
gevent.sleep(2) # wait for the agent to start and start the devices
# Start the actuator agent through which publish agent should communicate
# to fake device. Start the master driver agent which would intern start
# the fake driver using the configs created above
actuator_uuid = volttron_instance1.install_agent(
agent_dir=get_services_core("ActuatorAgent"),
config_file=get_services_core("ActuatorAgent/tests/actuator.config"),
start=True)
print("agent id: ", actuator_uuid)
gevent.sleep(2)
example_uuid = volttron_instance1.install_agent(
agent_dir=get_examples("ConfigActuation"),
config_file={},
vip_identity="config_actuation")
gevent.sleep(2)
# 3: Start a fake agent to publish to message bus
publish_agent = volttron_instance1.build_agent(identity=TEST_AGENT)
# 4: add a tear down method to stop sqlhistorian agent and the fake agent
# \that published to message bus
def stop_agent():
print("In teardown method of module")
volttron_instance1.stop_agent(actuator_uuid)
volttron_instance1.stop_agent(master_uuid)
volttron_instance1.stop_agent(example_uuid)
volttron_instance1.remove_agent(actuator_uuid)
volttron_instance1.remove_agent(master_uuid)
volttron_instance1.remove_agent(example_uuid)
publish_agent.core.stop()
request.addfinalizer(stop_agent)
return publish_agent
import sys
from gevent.subprocess import Popen
from util import alarm
alarm(3)
popen = Popen([sys.executable, '-c', 'pass'])
while popen.poll() is None:
pass
def test_cov_update_published(volttron_instance, test_agent):
"""Tests the functionality of BACnet change of value forwarding in the
Master Driver and driver.py"""
# Reset master driver config store
cmd = ['volttron-ctl', 'config', 'delete', PLATFORM_DRIVER, '--all']
process = Popen(cmd,
env=volttron_instance.env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
result = process.wait()
assert result == 0
# Add fake device configuration
cmd = [
'volttron-ctl', 'config', 'store', PLATFORM_DRIVER, 'fake.csv',
'examples/configurations/drivers/fake.csv', '--csv'
]
process = Popen(cmd,
env=volttron_instance.env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
result = process.wait()
assert result == 0
cmd = [
'volttron-ctl', 'config', 'store', PLATFORM_DRIVER,
"devices/fakedriver", 'examples/configurations/drivers/fake.config',
'--json'
]
process = Popen(cmd,
env=volttron_instance.env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
result = process.wait()
assert result == 0
# install master driver, start the master driver, which starts the device
master_uuid = volttron_instance.install_agent(
agent_dir=get_services_core("MasterDriverAgent"),
config_file={},
start=True)
print("agent id: ", master_uuid)
# tell the master driver to forward the value
point_name = "PowerState"
device_path = "fakedriver"
result_dict = {"fake1": "test", "fake2": "test", "fake3": "test"}
test_agent.vip.rpc.call(PLATFORM_DRIVER, 'forward_bacnet_cov_value',
device_path, point_name, result_dict)
# wait for the publishes to make it to the bus
gevent.sleep(2)
# Mock checks
# Should have one "PowerState" publish for each item in the result dict
# Total all publishes likely will include regular scrapes
assert test_agent.cov_callback.call_count >= 3
test_count = 0
for call_arg in test_agent.cov_callback.call_args_list:
if call_arg[0][5][0].get("PowerState", False):
test_count += 1
assert test_count == 3
def startStreamReader(self, url, cid, counter, req_headers=None):
logger = logging.getLogger('StreamReader')
logger.debug('Open video stream: %s' % url)
self._streamReaderState = 1
transcoder = None
if 'range' in req_headers: del req_headers['range']
logger.debug('Get headers from client: %s' % req_headers)
with requests.get(url,
headers=req_headers,
stream=True,
timeout=(5, None)) as self._streamReaderConnection:
try:
if self._streamReaderConnection.status_code not in (200, 206):
logger.error('Failed to open video stream %s' % url)
return None
if url.endswith('.m3u8'):
self._streamReaderConnection.headers = {
'Content-Type': 'application/octet-stream',
'Connection': 'Keep-Alive',
'Keep-Alive': 'timeout=15, max=100'
}
popen_params = {
"bufsize": AceConfig.readchunksize,
"stdout": PIPE,
"stderr": None,
"shell": False
}
if AceConfig.osplatform == 'Windows':
ffmpeg_cmd = 'ffmpeg.exe '
CREATE_NO_WINDOW = 0x08000000
CREATE_NEW_PROCESS_GROUP = 0x00000200
DETACHED_PROCESS = 0x00000008
popen_params.update(creationflags=CREATE_NO_WINDOW
| DETACHED_PROCESS
| CREATE_NEW_PROCESS_GROUP)
else:
ffmpeg_cmd = 'ffmpeg '
ffmpeg_cmd += '-hwaccel auto -hide_banner -loglevel fatal -re -i %s -c copy -f mpegts -' % url
transcoder = Popen(ffmpeg_cmd.split(), **popen_params)
out = transcoder.stdout
logger.warning(
'HLS stream detected. Ffmpeg transcoding started')
else:
out = self._streamReaderConnection.raw
except requests.exceptions.RequestException:
logger.error('Failed to open video stream %s' % url)
logger.error(traceback.format_exc())
except:
logger.error(traceback.format_exc())
else:
with self._lock:
self._streamReaderState = 2
self._lock.notifyAll()
self.play_event()
while 1:
self.getPlayEvent(
) # Wait for PlayEvent (stop/resume sending data from AceEngine to streamReaderQueue)
clients = counter.getClients(cid)
try:
data = out.read(AceConfig.readchunksize)
except:
data = None
if data is not None and clients:
if self._streamReaderQueue.full():
self._streamReaderQueue.get()
self._streamReaderQueue.put(data)
for c in clients:
try:
c.queue.put(data, timeout=5)
except gevent.queue.Full: #Queue.Full client does not read data from buffer until 5sec - disconnect it
if len(clients) > 1:
logger.debug('Disconnecting client: %s' %
c.handler.clientip)
c.destroy()
elif counter.count(cid) == 0:
logger.debug(
'All clients disconnected - broadcast stoped')
break
else:
logger.warning('No data received - broadcast stoped')
counter.deleteAll(cid)
break
finally:
with self._lock:
self._streamReaderState = None
self._lock.notifyAll()
if transcoder:
try:
transcoder.kill()
logger.warning('Ffmpeg transcoding stoped')
except:
pass
def _get_cheat(topic):
cmd = ["cheat", topic]
proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
answer = proc.communicate()[0].decode('utf-8')
return answer
def run_command(
command=None,
arguments=None,
working_directory=None,
standard_output=None,
error_output=None,
show_output=None,
raise_exceptions=None,
environment_overlay=None):
# type: (str, Iterable[str], str, [], [], bool, bool, EnvironmentOverlay) -> int
if show_output is None:
show_output = True
if raise_exceptions is None:
raise_exceptions = True
if arguments is None:
arguments = list()
if command:
arguments.insert(0, command)
if not arguments:
raise ValueError('In order to run a command, a command, arguments, or both must be specified.')
log.debug(
"Invoking: {1}{0}"
"\tCapture stdout?: {2}\tCapture stderr?: {3}\tShow output?: {4}\tRaise exceptions: {5}{0}"
"\tWorking directory: {6}{0}"
"\tEnvironment overlay: {7}{0}".format(
linesep,
' '.join(arguments),
standard_output is not None,
error_output is not None,
show_output,
raise_exceptions,
working_directory,
repr(environment_overlay)))
environment = None if not environment_overlay else environment_overlay.overlay(environ)
process = Popen(
args=arguments,
cwd=working_directory,
stdout=PIPE,
stderr=PIPE,
env=environment)
stdout_routine = spawn(_synchronize_stream, process.stdout, stdout, standard_output, show_output)
stderr_routine = spawn(_synchronize_stream, process.stderr, stderr, error_output, show_output)
stdout_routine.join()
stderr_routine.join()
exit_code = process.wait()
log.debug('Exit code: ' + str(exit_code))
if exit_code != 0:
message = str(
"Non-zero exit detected from sub-process:{0}"
"\tArguments: {1}{0}"
"\tExit code: {2}{0}"
"\tWorking directory: {3}{0}".format(
linesep,
' '.join(arguments),
str(exit_code),
repr(working_directory)))
if raise_exceptions:
log.debug('raising exception: ' + message)
process.stderr.close()
process.stdout.close()
raise RuntimeError(message)
log.error(message)
return exit_code
def _execute(self, cmd):
p = Popen(cmd, stdin=PIPE, stderr=PIPE,
stdout=PIPE, shell=True, encoding='utf8')
return p.communicate()[0]
def save_weather_data(location,
filename,
lang=None,
query=None,
location_name=None,
full_address=None):
ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]')
def remove_ansi(sometext):
return ansi_escape.sub('', sometext)
if _is_invalid_location(location):
error("Invalid location: %s" % location)
NOT_FOUND_MESSAGE_HEADER = ""
while True:
location_not_found = False
if location in ["test-thunder"]:
test_name = location[5:]
test_file = TEST_FILE.replace('NAME', test_name)
stdout = open(test_file, 'r').read()
stderr = ""
break
print "LOCATION = ", location
if location == NOT_FOUND_LOCATION:
location_not_found = True
location = DEFAULT_LOCATION
cmd = [WEGO, '--city=%s' % location]
if query.get('inverted_colors'):
cmd += ['-inverse']
if query.get('use_ms_for_wind'):
cmd += ['-wind_in_ms']
if query.get('narrow'):
cmd += ['-narrow']
if lang and lang in SUPPORTED_LANGS:
cmd += ['-lang=%s' % lang]
if query.get('use_imperial', False):
cmd += ['-imperial']
if location_name:
cmd += ['-location_name', location_name]
p = Popen(cmd, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate()
if p.returncode != 0:
print "ERROR: location not found: %s" % location
if 'Unable to find any matching weather location to the query submitted' in stderr:
if location != NOT_FOUND_LOCATION:
NOT_FOUND_MESSAGE_HEADER = u"ERROR: %s: %s\n---\n\n" % (
get_message('UNKNOWN_LOCATION', lang), location)
location = NOT_FOUND_LOCATION
continue
error(stdout + stderr)
break
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
if location_not_found:
stdout += get_message('NOT_FOUND_MESSAGE', lang).encode('utf-8')
stdout = NOT_FOUND_MESSAGE_HEADER.encode('utf-8') + stdout
if 'days' in query:
if query['days'] == '0':
stdout = "\n".join(stdout.splitlines()[:7]) + "\n"
if query['days'] == '1':
stdout = "\n".join(stdout.splitlines()[:17]) + "\n"
if query['days'] == '2':
stdout = "\n".join(stdout.splitlines()[:27]) + "\n"
first = stdout.splitlines()[0].decode('utf-8')
rest = stdout.splitlines()[1:]
if query.get('no-caption', False):
separator = None
if ':' in first:
separator = ':'
if u'：' in first:
separator = u'：'
if separator:
first = first.split(separator, 1)[1]
stdout = "\n".join([first.strip().encode('utf-8')] +
rest) + "\n"
if query.get('no-terminal', False):
stdout = remove_ansi(stdout)
if query.get('no-city', False):
stdout = "\n".join(stdout.splitlines()[2:]) + "\n"
if full_address and query.get('format', 'txt') != 'png':
line = "%s: %s [%s]\n" % (get_message(
'LOCATION',
lang).encode('utf-8'), full_address.encode('utf-8'), location)
stdout += line
if query.get('padding', False):
lines = [x.rstrip() for x in stdout.splitlines()]
max_l = max(len(remove_ansi(x).decode('utf8')) for x in lines)
last_line = " " * max_l + " .\n"
stdout = " \n" + "\n".join(" %s " % x
for x in lines) + "\n" + last_line
open(filename, 'w').write(stdout)
cmd = ["bash", ANSI2HTML, "--palette=solarized"]
if not query.get('inverted_colors'):
cmd += ["--bg=dark"]
p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate(stdout)
if p.returncode != 0:
error(stdout + stderr)
if query.get('inverted_colors'):
stdout = stdout.replace(
'<body class="">',
'<body class="" style="background:white;color:#777777">')
title = "<title>%s</title>" % first.encode('utf-8')
opengraph = get_opengraph()
stdout = re.sub("<head>", "<head>" + title + opengraph, stdout)
open(filename + '.html', 'w').write(stdout)
