def doCompilerTest(self, vm_cmd, time_parse_info, stderr_redir=True):
start_time = time()
cmd_args = shlex.split(vm_cmd)
print(cmd_args)
if stderr_redir:
vm_process = subprocess.run(cmd_args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, timeout=300)
else:
vm_process = subprocess.run(cmd_args, stdout=subprocess.PIPE, shell=True, timeout=300)
#vm_process.wait(300)
end_time = time()
total_time = end_time - start_time
#stdoutlines = [str(line, 'utf8') for line in vm_process.stdout]
stdoutlines = [line for line in vm_process.stdout.decode('utf8').rstrip().split('\n')]
print('stdout_lines: ', ("\n").join(stdoutlines), end="\n")
try:
compile_line = stdoutlines[time_parse_info['compile_line_num']]
print('compile_lines: ', compile_line)
compile_match = re.search(time_parse_info['compile_regex'], compile_line)
print('compile_match: ', compile_match)
compile_time = durationpy.from_str(compile_match[1])
print('compile_time: ', compile_time)
exec_line = stdoutlines[time_parse_info['exec_line_num']]
print('exec_line: ', exec_line)
exec_match = re.search(time_parse_info['exec_regex'], exec_line)
print('exec_match: ', exec_match)
exec_time = durationpy.from_str(exec_match[1])
print('exec_time: ', exec_time)
except Exception as e:
error_msg = ["Error parsing engine output. exception: {}".format(e)] + \
["engine output:"] + stdoutlines
raise Exception("\n".join(error_msg))
return Record(time=total_time, compile_time=compile_time.total_seconds(), exec_time=exec_time.total_seconds())
def duration_is_valid(duration: Optional[str]) -> bool:
if duration is None:
return True
try:
durationpy.from_str(duration)
return True
# Really?! durationpy raises bare Exception??
except Exception:
return False
def test_formatter(self):
for [input, passes, expected] in cases:
if passes:
dt = durationpy.from_str(input)
ds = durationpy.to_str(dt)
actual = durationpy.from_str(ds).total_seconds() * 1000
self.assertEquals(
expected, actual,
"{}, expecting {}, got {}".format(input, expected, actual))
def test_parser(self):
for [input, passes, expected] in cases:
if passes:
actual = durationpy.from_str(input).total_seconds() * 1000
self.assertEquals(
expected, actual,
"{}, expecting {}, got {}".format(input, expected, actual))
else:
with self.assertRaises(Exception):
durationpy.from_str(input)
def is_delay_valid(self, delay):
if not delay:
return True
try:
if delay[-1].isdigit():
# delay can not be end with digit
return False
durationpy.from_str(delay)
except:
return False
return True
def saveData(url):
r = urllib2.Request(url)
rgxs = [
'Query\ \(id\=(.+?)\)', 'Session\ ID\:\ (.+?)$',
'Query\ submitted\:\ (.+?)\ \(', 'Planning\ finished\:\ (.+?)\ \(',
'Ready\ to\ start\ on\ \d+\ backends\:\ (.+?)\ \(',
'First\ row\ fetched\:\ (.+?)\ \(', 'ClientFetchWaitTimer\:\ (.+?)$',
'RowMaterializationTimer\:\ (.+?)$'
]
content = urllib2.urlopen(r).read()
s = ""
val = ""
for rgx in rgxs:
m = re.findall(rgx, content, re.MULTILINE)
if m:
try:
val = durationpy.from_str(str(m[-1])).total_seconds() * 1000
except:
val = str(m[-1])
s += "%s\t" % val
return s[:-1]
def _emit_mapping(self, obj: KubernetesObject, rule_count: int,
rule: Dict[str, Any]) -> None:
hosts = rule.get('hosts', [])
split_mapping_specs: List[Dict[str, Any]] = []
paths = rule.get('http', {}).get('paths', [])
for path in paths:
global_headers = path.get('appendHeaders', {})
splits = path.get('splits', [])
for split in splits:
service_name = split.get('serviceName')
if not service_name:
continue
service_namespace = split.get('serviceNamespace',
obj.namespace)
service_port = split.get('servicePort', 80)
headers = split.get('appendHeaders', {})
headers = {**global_headers, **headers}
split_mapping_specs.append({
'service':
f"{service_name}.{service_namespace}:{service_port}",
'add_request_headers':
headers,
'weight':
split.get('percent', 100),
'prefix':
path.get('path', '/'),
'timeout_ms':
int(
durationpy.from_str(path.get(
'timeout', '15s')).total_seconds() * 1000),
})
for split_count, (host, split_mapping_spec) in enumerate(
itertools.product(hosts, split_mapping_specs)):
mapping_identifier = f"{obj.name}-{rule_count}-{split_count}"
spec = {
'ambassador_id': obj.ambassador_id,
'host': host,
}
spec.update(split_mapping_spec)
mapping = NormalizedResource.from_data(
'Mapping',
mapping_identifier,
namespace=obj.namespace,
generation=obj.generation,
labels=obj.labels,
spec=spec,
)
self.logger.debug(
f"Generated mapping from Knative {obj.kind}: {mapping}")
self.manager.emit(mapping)
def __init__(self):
self.logger = logging.getLogger(App.__name__)
self.logger.info('launched Karakteraz')
try:
with open('app.yml', 'r') as stream:
self.configuration = yaml.load(stream)['karakteraz']
except yaml.YAMLError:
self.logger.exception('configuration format error')
sys.exit(1)
except FileNotFoundError:
self.logger.exception('configuration not found')
sys.exit(1)
self.configuration['notification']['email'] = None if sum(
[1 for _, v in self.configuration['notification']['email'].items() if v is None]) > 0 else \
self.configuration['notification']['email']
self.configuration['notification']['telegram'] = None if sum(
[1 for _, v in self.configuration['notification']['telegram'].items() if v is None]) > 0 else \
self.configuration['notification']['telegram']
self.configuration['watch-list'] = [x.upper() for x in self.configuration['watch-list']]
self.configuration['results-page'] = 'https://fsweb.no/studentweb/resultater.jsf'
self.logger.info(
'installed configuration:\n{}'.format(yaml.dump(self.configuration, default_flow_style=False)))
self.interval = durationpy.from_str(self.configuration['frequency']).total_seconds()
self.driver = webdriver.Firefox()
self.driver.wait = WebDriverWait(self.driver, 10)
self.scheduler = BlockingScheduler()
def set_duration(k, v):
v = str(v)
try:
# d is a datetime.timedelta
d = durationpy.from_str(v)
except Exception:
raise ValueError('%s service config has invalid duration: %s' % (name, v))
self.__dict__[k] = d
def _decode_value(typ, value):
if value is None:
return None
kind = typ['kind']
if kind == 'primitive':
name = typ['name']
if name in [
'uint8', 'uint16', 'uint32', 'uint64', 'int8', 'int16',
'int32', 'int64'
]:
return int(value)
if name == 'duration':
return durationpy.from_str(value)
if name == 'time':
return dateutil.parser.isoparse(value)
if name in ['float16', 'float32', 'float64']:
return float(value)
if name == 'decimal':
return decimal.Decimal(value)
if name == 'bool':
return value == 'T'
if name == 'bytes':
return binascii.a2b_hex(value[2:])
if name == 'string':
return value
if name == 'ip':
return ipaddress.ip_address(value)
if name == 'net':
return ipaddress.ip_network(value)
if name in 'type':
return value
if name == 'null':
return None
raise Exception(f'unknown primitive name {name}')
if kind == 'record':
return {
f['name']: _decode_value(f['type'], v)
for f, v in zip(typ['fields'], value)
}
if kind == 'array':
return [_decode_value(typ['type'], v) for v in value]
if kind == 'set':
return {_decode_value(typ['type'], v) for v in value}
if kind == 'map':
key_type, val_type = typ['key_type'], typ['val_type']
return {
_decode_value(key_type, v[0]): _decode_value(val_type, v[1])
for v in value
}
if kind == 'union':
type_index, val = value
return _decode_value(typ['types'][int(type_index)], val)
if kind == 'enum':
return typ['symbols'][int(value)]
if kind in ['error', 'named']:
return _decode_value(typ['type'], value)
raise Exception(f'unknown type kind {kind}')
def string_to_timestamp(timestring):
"""
Accepts a str, returns an int timestamp.
"""
ts = None
# Uses an extended version of Go's duration string.
try:
delta = durationpy.from_str(timestring);
past = datetime.datetime.utcnow() - delta
ts = calendar.timegm(past.timetuple())
return ts
except Exception as e:
pass
if ts:
return ts
# else:
# print("Unable to parse timestring.")
return 0
def string_to_timestamp(timestring):
"""
Accepts a str, returns an int timestamp.
"""
ts = None
# Uses an extended version of Go's duration string.
try:
delta = durationpy.from_str(timestring);
past = datetime.datetime.utcnow() - delta
ts = calendar.timegm(past.timetuple())
return ts
except Exception as e:
pass
if ts:
return ts
# else:
# print("Unable to parse timestring.")
return 0
async def main() -> None:
args = parse_args()
assert are_args_valid(args)
resolver: Callable = aiohttp.resolver.DefaultResolver
if args.resolve is not None:
host, address = args.resolve.split(":")
if args.url.host == host:
custom_resolution = {host: address}
def resolver():
return CustomResolver(custom_mappings=custom_resolution)
results: Deque[Result] = deque(maxlen=args.request_history)
shutdown_event = asyncio.Event()
def shutdown_signal_handler(_, __):
shutdown_event.set()
for shutdown_signal in (signal.SIGINT, signal.SIGTERM):
signal.signal(shutdown_signal, shutdown_signal_handler)
tasks = []
if args.duration is not None:
duration = durationpy.from_str(args.duration)
async def stop_test():
await asyncio.wait([shutdown_event.wait()], timeout=duration)
shutdown_event.set()
tasks.append(stop_test())
async def renderer() -> None:
while True:
if shutdown_event.is_set():
return
stats = build_stats(url=args.url, method=args.method, results=results)
output = render_stats(stats, _format=args.output_format)
os.system("clear")
print(output, flush=True)
await asyncio.sleep(0.1)
tasks.append(renderer())
timeout = aiohttp.ClientTimeout(connect=args.timeout)
connector = aiohttp.TCPConnector(
force_close=True, limit=0, resolver=resolver(), verify_ssl=_str_to_bool(args.verify_tls, default=True)
)
async with aiohttp.ClientSession(timeout=timeout, connector=connector) as session:
async def worker() -> None:
while not shutdown_event.is_set():
result = await request(
url=args.url,
method=args.method,
session=session,
follow_redirects=_str_to_bool(args.follow_redirects, default=True),
)
results.append(result)
for _ in range(args.workers):
tasks.append(worker())
await asyncio.gather(*tasks)
