def store_in_database(basepath: str, package_info_filepath: str,
binary_path: str, parameter: str, crash_dir_path: str,
crash_db_path: str, afl_dir_path: str,
configuration_dir: str):
print("Package info for package {0}".format(package_info_filepath))
package_info_dict = {}
with open(basepath + "/" +
package_info_filepath) as package_info_filepointer:
text = package_info_filepointer.read()
tmp_list = [item.strip().split(":", 1) for item in text.split("\n")]
for item in tmp_list:
if len(item) == 2:
package_info_dict[item[0].strip()] = item[1].strip()
package, created = Package.objects.get_or_create(
name=package_info_dict["Name"],
version=ArchCrawler.get_package_version(package_info_dict["Name"]))
binary, created = Binary.objects.get_or_create(path=binary_path,
package=package)
p = pathlib.Path(afl_dir_path)
p = p.relative_to(*p.parts[:2]) # type: pathlib.PosixPath
binary.afl_dir = os.path.abspath(configuration_dir + "/" +
str(p)) # First path is the volume path
# binary.afl_dir= configuration_dir+"/"+afl_dir_path
binary.save()
crash_db_full_path = basepath + "/" + crash_db_path
if not os.path.exists(crash_db_full_path):
print("Error: The database {0} does not exist".format(
crash_db_full_path))
print(crash_db_path)
return
print("Opening database {0}".format(crash_db_full_path))
connect = sqlite3.connect(basepath + "/" + crash_db_path)
c = connect.cursor()
c.execute(
"select count(*) from sqlite_master where type='table' and name='Data';"
)
if c.fetchone()[0] != 1:
print("Error: The table Data does not exist")
return
c.execute("select count(*) from Data;")
conv = Ansi2HTMLConverter()
if c.fetchone()[0] > 0:
print("Crashes for package {0} binary {1}".format(
package, binary_path))
for row in c.execute('SELECT * FROM Data'):
print(basepath + "/" + crash_dir_path + "/" + row[0])
if not os.path.exists(basepath + "/" + crash_dir_path + "/" +
row[0]):
continue
with open(basepath + "/" + crash_dir_path + "/" + row[0],
"rb") as crash_filepointer:
crash_data = crash_filepointer.read()
try:
rendered_text = conv.convert(row[5].decode("utf-8"))
except Exception as e:
rendered_text = ""
crash, created = Crash.objects.get_or_create(
binary=binary,
parameter=parameter,
exploitability=row[1],
file_blob=crash_data,
description=row[2],
execution_output=rendered_text)
crash.name = row[0]
else:
print("No crashes for package {0} binary {1}".format(
package, binary_path))
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
from ansi2html import Ansi2HTMLConverter
REF_TYPE = {"fig": "Figure", "exm": "Example"}
ADM_TYPE = {
"comment": "NOTE",
"note": "NOTE",
"caution": "WARNING",
"tip": "TIP"
}
conv = Ansi2HTMLConverter(inline=True, scheme="solarized", linkify=False)
callout_code_re = re.compile(r'#? ?<([0-9]{1,2})>')
callout_text_re = re.compile(r'<([0-9]{1,2})>')
def pygments(key, value, format, _):
if format == "asciidoc":
# Fix references to figures
if (key == "Str") and value.startswith("@ref"):
# stderr.write(f"{key}\t{value}\n")
_, ref_type, ref_id, _ = re.split("\(|:|\)", value)
return Str(f"<<{ref_type}:{ref_id}>>")
def ansi2html(s):
return mark_safe(
Ansi2HTMLConverter(inline=True, escaped=False).convert(s, full=False))
def emulate(request):
context = {"success": False}
projectName = request.GET["projectName"]
project = getProjectByName(projectName)
if not project:
return redirect("/")
projectPath = getProjectPath(projectName)
if request.method == 'POST':
form = emulateForm(request.POST)
if form.is_valid():
tracepoints = form.cleaned_data["tracepoints"]
if len(tracepoints) > 2:
for w in " \r\n\t":
tracepoints = tracepoints.replace(w, ",")
tracepoints = filter(lambda x: len(x) > 0,
tracepoints.split(","))
tracepoints = list(map(lambda x: int(x, 16), tracepoints))
else:
tracepoints = []
try:
stdin = form.cleaned_data["stdin"]
stdin = stdin.replace(" ", "").replace("\n",
"").replace("\r", "")
print(stdin)
stdin = unhexlify(stdin)
except:
import traceback
traceback.print_exc()
stdin = ""
binaryPath = os.path.join(projectPath, "gen",
request.GET["emulatorName"])
emulator = uc.emu(
binaryPath,
stdin,
tracepoints,
emulator_base=project.cfg["config"]["EMULATION_CODE_BASE"])
emulator.run()
#prepare results for html
results = emulator.results
from ansi2html import Ansi2HTMLConverter
conv = Ansi2HTMLConverter()
for r in results:
del r["stdout"]
r["pc_symbolized"] = project.symbolize(r["regs"]["pc"])
r["stderr"] = base64.b64encode(
r["stderr"].encode("utf-8")).decode("utf-8")
r["memdif_rendered"] = base64.b64encode(
r["memdif_rendered"].encode("utf-8")).decode("utf-8")
r["memdif_html"] = base64.b64encode(
conv.convert(
r["memdif_rendered"]).encode("utf-8")).decode("utf-8")
emulator.coverage_activity_json = json.dumps(
emulator.coverage_activity)
emulator.read_activity_json = json.dumps(emulator.read_activity)
emulator.write_activity_json = json.dumps(emulator.write_activity)
#context["segments"] = sorted(project.cfg["segments"].items(), key=lambda x: x[1]["addr"])
context["symbols_json"] = json.dumps(project.get_symbols())
context["emulator"] = emulator
context["tracefile_b64"] = base64.b64encode(
emulator.get_tracefile()).decode()
context["project"] = project
context["success"] = True
else:
form = emulateForm()
context["emulateForm"] = form
context["projectName"] = projectName
context["emulatorName"] = request.GET["emulatorName"]
return render_wrapper(request, 'emulate.html', context)
def _jkmagic(kernel_name, **kwargs):
r"""
Called when user issues `my_kernel = jupyter("kernel_name")` from a cell, not intended to be called directly by user.
Start a jupyter kernel and create a sagews function for it. See docstring for class JUPYTER above.
Based on http://jupyter-client.readthedocs.io/en/latest/api/index.html
INPUT:
- ``kernel_name`` -- name of kernel as it appears in output of `jupyter kernelspec list`
"""
# CRITICAL: We import these here rather than at module scope, since they can take nearly a second
# i CPU time to import.
import jupyter_client # TIMING: takes a bit of time
from ansi2html import Ansi2HTMLConverter # TIMING: this is surprisingly bad.
from Queue import Empty # TIMING: cheap
import base64, tempfile, sys, re # TIMING: cheap
import warnings
import sage.misc.latex
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
km, kc = jupyter_client.manager.start_new_kernel(
kernel_name=kernel_name)
import sage.interfaces.cleaner
sage.interfaces.cleaner.cleaner(km.kernel.pid, "km.kernel.pid")
import atexit
atexit.register(km.shutdown_kernel)
atexit.register(kc.hb_channel.close)
# inline: no header or style tags, useful for full == False
# linkify: little gimmik, translates URLs to anchor tags
conv = Ansi2HTMLConverter(inline=True, linkify=True)
def hout(s, block=True, scroll=False, error=False):
r"""
wrapper for ansi conversion before displaying output
INPUT:
- ``s`` - string to display in output of sagews cell
- ``block`` - set false to prevent newlines between output segments
- ``scroll`` - set true to put output into scrolling div
- ``error`` - set true to send text output to stderr
"""
# `full = False` or else cell output is huge
if "\x1b[" in s:
# use html output if ansi control code found in string
h = conv.convert(s, full=False)
if block:
h2 = '<pre style="font-family:monospace;">' + h + '</pre>'
else:
h2 = '<pre style="display:inline-block;margin-right:-1ch;font-family:monospace;">' + h + '</pre>'
if scroll:
h2 = '<div style="max-height:320px;width:80%;overflow:auto;">' + h2 + '</div>'
salvus.html(h2)
else:
if error:
sys.stderr.write(s)
sys.stderr.flush()
else:
sys.stdout.write(s)
sys.stdout.flush()
def run_code(code=None, **kwargs):
def p(*args):
from smc_sagews.sage_server import log
if run_code.debug:
log("kernel {}: {}".format(kernel_name,
' '.join(str(a) for a in args)))
if kwargs.get('get_kernel_client', False):
return kc
if kwargs.get('get_kernel_manager', False):
return km
if kwargs.get('get_kernel_name', False):
return kernel_name
if code is None:
return
# execute the code
msg_id = kc.execute(code)
# get responses
shell = kc.shell_channel
iopub = kc.iopub_channel
stdinj = kc.stdin_channel
# buffering for %capture because we don't know whether output is stdout or stderr
# until shell execute_reply message is received with status 'ok' or 'error'
capture_mode = not hasattr(sys.stdout._f, 'im_func')
# handle iopub messages
while True:
try:
msg = iopub.get_msg()
msg_type = msg['msg_type']
content = msg['content']
except Empty:
# shouldn't happen
p("iopub channel empty")
break
p('iopub', msg_type, str(content)[:300])
if msg['parent_header'].get('msg_id') != msg_id:
p('*** non-matching parent header')
continue
if msg_type == 'status' and content['execution_state'] == 'idle':
break
def display_mime(msg_data):
'''
jupyter server does send data dictionaries, that do contain mime-type:data mappings
depending on the type, handle them in the salvus API
'''
# sometimes output is sent in several formats
# 1. if there is an image format, prefer that
# 2. elif default text or image mode is available, prefer that
# 3. else choose first matching format in modes list
from smc_sagews.sage_salvus import show
def show_plot(data, suffix):
r"""
If an html style is defined for this kernel, use it.
Otherwise use salvus.file().
"""
suffix = '.' + suffix
fname = tempfile.mkstemp(suffix=suffix)[1]
with open(fname, 'w') as fo:
fo.write(data)
if run_code.smc_image_scaling is None:
salvus.file(fname)
else:
img_src = salvus.file(fname, show=False)
htms = '<img src="{0}" smc-image-scaling="{1}" />'.format(
img_src, run_code.smc_image_scaling)
salvus.html(htms)
os.unlink(fname)
mkeys = msg_data.keys()
imgmodes = ['image/svg+xml', 'image/png', 'image/jpeg']
txtmodes = [
'text/html', 'text/plain', 'text/latex', 'text/markdown'
]
if any('image' in k for k in mkeys):
dfim = run_code.default_image_fmt
#print('default_image_fmt %s'%dfim)
dispmode = next((m for m in mkeys if dfim in m), None)
if dispmode is None:
dispmode = next(m for m in imgmodes if m in mkeys)
#print('dispmode is %s'%dispmode)
# https://en.wikipedia.org/wiki/Data_scheme#Examples
# <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEU
# <img src='data:image/svg+xml;utf8,<svg ... > ... </svg>'>
if dispmode == 'image/svg+xml':
data = msg_data[dispmode]
show_plot(data, 'svg')
elif dispmode == 'image/png':
data = base64.standard_b64decode(msg_data[dispmode])
show_plot(data, 'png')
elif dispmode == 'image/jpeg':
data = base64.standard_b64decode(msg_data[dispmode])
show_plot(data, 'jpg')
return
elif any('text' in k for k in mkeys):
dftm = run_code.default_text_fmt
if capture_mode:
dftm = 'plain'
dispmode = next((m for m in mkeys if dftm in m), None)
if dispmode is None:
dispmode = next(m for m in txtmodes if m in mkeys)
if dispmode == 'text/plain':
p('text/plain', msg_data[dispmode])
# override if plain text is object marker for latex output
if re.match('<IPython.core.display.\w+ object>',
msg_data[dispmode]):
p("overriding plain -> latex")
show(msg_data['text/latex'])
else:
txt = re.sub(r"^\[\d+\] ", "", msg_data[dispmode])
hout(txt)
elif dispmode == 'text/html':
salvus.html(msg_data[dispmode])
elif dispmode == 'text/latex':
p('text/latex', msg_data[dispmode])
sage.misc.latex.latex.eval(msg_data[dispmode])
elif dispmode == 'text/markdown':
salvus.md(msg_data[dispmode])
return
# reminder of iopub loop is switch on value of msg_type
if msg_type == 'execute_input':
# the following is a cheat to avoid forking a separate thread to listen on stdin channel
# most of the time, ignore "execute_input" message type
# but if code calls python3 input(), wait for message on stdin channel
if 'code' in content and kernel_name in [
'python3', 'anaconda3'
]:
ccode = content['code']
if (re.match('^[^#]*\W?input\(', ccode)):
# FIXME input() will be ignored if it's aliased to another name
p('iopub input call: ', ccode)
try:
# do nothing if no messsage on stdin channel within 0.5 sec
imsg = stdinj.get_msg(timeout=0.5)
imsg_type = imsg['msg_type']
icontent = imsg['content']
p('stdin', imsg_type, str(icontent)[:300])
# kernel is now blocked waiting for input
if imsg_type == 'input_request':
prompt = '' if icontent[
'password'] else icontent['prompt']
value = salvus.raw_input(prompt=prompt)
xcontent = dict(value=value)
xmsg = kc.session.msg('input_reply', xcontent)
p('sending input_reply', xcontent)
stdinj.send(xmsg)
except:
pass
elif msg_type == 'execute_result':
if not 'data' in content:
continue
p('execute_result data keys: ', content['data'].keys())
display_mime(content['data'])
elif msg_type == 'display_data':
if 'data' in content:
display_mime(content['data'])
elif msg_type == 'status':
if content['execution_state'] == 'idle':
# when idle, kernel has executed all input
break
elif msg_type == 'clear_output':
salvus.clear()
elif msg_type == 'stream':
if 'text' in content:
# bash kernel uses stream messages with output in 'text' field
# might be ANSI color-coded
if 'name' in content and content['name'] == 'stderr':
hout(content['text'], error=True)
else:
hout(content['text'], block=False)
elif msg_type == 'error':
# XXX look for ename and evalue too?
if 'traceback' in content:
tr = content['traceback']
if isinstance(tr, list):
for tr in content['traceback']:
hout(tr + '\n', error=True)
else:
hout(tr, error=True)
# handle shell messages
while True:
try:
msg = shell.get_msg(timeout=0.2)
msg_type = msg['msg_type']
content = msg['content']
except Empty:
# shouldn't happen
p("shell channel empty")
break
if msg['parent_header'].get('msg_id') == msg_id:
p('shell', msg_type, len(str(content)), str(content)[:300])
if msg_type == 'execute_reply':
if content['status'] == 'ok':
if 'payload' in content:
payload = content['payload']
if len(payload) > 0:
if 'data' in payload[0]:
data = payload[0]['data']
if 'text/plain' in data:
text = data['text/plain']
hout(text, scroll=True)
break
else:
# not our reply
continue
return
# 'html', 'plain', 'latex', 'markdown' - support depends on jupyter kernel
run_code.default_text_fmt = 'html'
# 'svg', 'png', 'jpeg' - support depends on jupyter kernel
run_code.default_image_fmt = 'png'
# set to floating point fraction e.g. 0.5
run_code.smc_image_scaling = None
# set True to record jupyter messages to sage_server log
run_code.debug = False
return run_code
def ansi_to_html(text):
from ansi2html import Ansi2HTMLConverter
conv = Ansi2HTMLConverter(inline=True, linkify=True)
return conv.convert(text, full=False)
import re
import subprocess
from standardweb.lib import cache
from ansi2html import Ansi2HTMLConverter
from django.contrib import messages
ansi_converter = Ansi2HTMLConverter()
ansi_pat = re.compile(r'\x1b[^m]*m')
try:
git_revision = subprocess.check_output('git log -n 1 --pretty=format:"%H"',
shell=True)
except Exception, e:
print 'Could not get git revision, ignoring: %s', e
git_revision = ''
def iso_date(date):
return date.strftime("%Y-%m-%d %H:%M:%SZ")
def elapsed_time_string(total_minutes):
hours = int(total_minutes / 60)
if not hours:
return '%d %s' % (total_minutes,
'minute' if total_minutes == 1 else 'minutes')
def format_log(filename, msg):
with open(os.path.join(location, filename), 'r') as f:
conv = Ansi2HTMLConverter()
html = conv.convert(f.read(), full=False)
msg = f"{html}"
return msg
def main(api_url):
cont = True
while (cont):
log_buffer = StringIO()
conv = Ansi2HTMLConverter()
try:
print("Worker starting")
print("api_url: " + str(api_url))
r = requests.get(f"{api_url}/api/ag/v1/fetch_job")
print("Request response: " + r.text)
fetched = r.json()
print("fetched " + str(fetched))
if fetched["queue_empty"]:
print("Queue empty 1")
cont = False
break
skeleton_name = fetched["skeleton"]
skeleton_zip = requests.get(
f"{api_url}/api/ag/v1/skeleton/{skeleton_name}")
os.makedirs(GRADING_DIR, exist_ok=True)
with open(f"{GRADING_DIR}/{skeleton_name}.zip", "wb") as f:
f.write(skeleton_zip.content)
proc = subprocess.Popen(["unzip", "-o", skeleton_name],
cwd=GRADING_DIR)
proc.wait()
assert proc.returncode == 0, "Unzip failed"
access_token = fetched["access_token"]
backup_id = fetched["backup_id"]
job_id = fetched["job_id"]
backup_assignment_url = f"https://okpy.org/api/v3/backups/{backup_id}?access_token={access_token}"
backup_response = requests.get(backup_assignment_url)
backup_json = backup_response.json()
try:
file_dict = backup_json["data"]["messages"][0]["contents"]
except Exception as e:
print("Something went wrong with getting the file dict")
print("Backup Assignment URL")
print(backup_assignment_url)
print("Backup response:")
print(backup_response.text)
print("Backup json:")
print(backup_json)
print(e, flush=True)
cont = False
break
files_to_grade = []
for name, content in file_dict.items():
if name == "submit":
continue
files_to_grade.append(name)
with open(f"{GRADING_DIR}/{name}", "w") as f:
f.write(content)
assert len(
files_to_grade) == 1, "Only support grading 1 notebook file"
os.chdir(GRADING_DIR)
# Do the grading in another context to avoid name space collisions and the like
# Should really add change the autograder to do this.
print("MAKE NEW CONTEXT", flush=True)
ctx = mp.get_context('spawn')
q = ctx.Queue()
p = ctx.Process(target=get_gofer_grade,
args=(
files_to_grade[0],
q,
))
p.start()
p.join() # this blocks until the process terminates
result = q.get()
print("DONE USING NEW CONTEXT", flush=True)
print(result)
log_buffer.write(str(result))
okpy_result, path_to_score = gofer_wrangle(result)
# print(res)
path_to_score["bid"] = backup_id
path_to_score["assignment"] = skeleton_name
#report_breakdown_url = f"{api_url}/api/ag/v1/report_result"
#requests.post(report_breakdown_url, json=path_to_score)
score_content = {
"bid": backup_id,
"score": okpy_result["total"],
"kind": "Autograder",
"message": okpy_result["msg"],
}
score_endpoint = (
f"https://okpy.org/api/v3/score/?access_token={access_token}")
resp = requests.post(score_endpoint, json=score_content)
assert resp.status_code == 200, resp.json()
except Exception as e:
print("Things went wrong")
print("Exception: " + str(e))
print("Stack trace")
print(traceback.format_exc())
print(e)
cont = False
break
#print(log_buffer.getvalue())
print("Pandas Version: " + str(pd.__version__), file=sys.stderr)
print("Sending log to api/ag/v1/report_done")
report_done_endpoint = f"{api_url}/api/ag/v1/report_done/{job_id}"
resp = requests.post(report_done_endpoint,
data=conv.convert(log_buffer.getvalue()))
if resp.status_code == 200:
print("Everything Successful")
else:
print("Response error code: " + str(resp.status_code))
cont = False
break
import gym
from ansi2html import Ansi2HTMLConverter
import imgkit
env = gym.make('Taxi-v3')
conv = Ansi2HTMLConverter(font_size='xx-large', dark_bg=False, scheme='xterm')
for i in range(env.observation_space.n):
out = env.render(i)
html = conv.convert(out)
options = {
'format' : 'png',
'crop-w' : 208,
'crop-h' : 244,
}
imgkit.from_string(html, "{}.png".format(str(i).zfill(3)), options=options)
import os
import pathlib
import re
import shlex
from datetime import timedelta
import tempfile
from typing import Any, Callable, cast
from ansi2html import Ansi2HTMLConverter
# global converter
ansi2HTMLconverter = Ansi2HTMLConverter(inline=True, linkify=True)
TEMP_DIR = pathlib.Path(tempfile.gettempdir() + os.sep +
'security-scans-wrapper-temp')
def error(msg: str) -> None:
raise RuntimeError(f"security-scans-wrapper: {msg}")
def _check_substitution_leftovers(cmd: str) -> None:
placeholder_regex = re.compile(r'{{((?:}(?!})|[^}])*)}}')
all_leftovers_placeholder = placeholder_regex.findall(cmd)
if all_leftovers_placeholder:
error(
"the following place holders could not get interpolated: "
f"{', '.join(all_leftovers_placeholder)}. Use --arg KEY=VALUE to pass values."
)
def __init__(self, parent=None):
QtGui.QWidget.__init__(self, parent)
self.showMaximized()
self.ui = Ui_MainWindow()
self.scrollLock = False
self.ui.setupUi(self)
self.updateTimer = QtCore.QTimer()
self.logTimer = QtCore.QTimer()
self.processHandler = ProcessHandler(self)
# commands
self.commands = {}
self.allCommands = {}
self.progressTopics = {}
self.progress = {}
self.rosoutNames = {}
self.statusTopics = {}
self.readCommandConfig()
self.readProgressAndStatusConfig()
self.readRosOutConfig()
self.setupCommandAllGUI()
self.setupCommandGUI(self.commands)
# latex=False,
# inline=False,
# dark_bg=True,
# font_size='normal',
# linkify=False,
# escaped=True,
# markup_lines=False,
# output_encoding='utf-8',
# scheme='ansi2html',
# title=''
self.ansiConv = Ansi2HTMLConverter(False, False, False, 'normal',
False, True, False, 'utf-8',
'ansi2html')
# ping
self.ping = Ping()
self.ping.start()
# machines
self.machines = {}
self.stateLabels = {}
self.readMachineConfig()
self.setupMachinesGUI()
self.ui.textEditConsole.setReadOnly(True)
self.lastConsole = ""
self.updateTimer.timeout.connect(self.updateWindow)
self.logTimer.timeout.connect(self.updateConsoleOutput)
# self.ui.actionStopAll.triggered.connect(self.onActionStopAll)
#########################################
#CALLBACKS DEFINITIONS ##################
self.ui.toolButtonClear.clicked.connect(self.onToolButtonClear)
self.ui.toolButtonClose.clicked.connect(self.onToolButtonClose)
self.ui.toolButtonScrollLock.clicked.connect(
self.onToolButtonScrollLock)
# self.ui.pushButtonCoreStop.clicked.connect(self.onCoreStop)
# self.ui.pushButtonPerceptionStart.clicked.connect(self.onPerceptionStart)
#########################################
self.logTimer.start(100)
self.updateTimer.start(100)
def test_latex_linkify(self):
ansi = 'http://python.org/'
target = '\\url{%s}' % ansi
latex = Ansi2HTMLConverter(latex=True, inline=True,
linkify=True).convert(ansi)
assert (target in latex)
def test_latex_inline(self):
ansi = '\x1b[33mYELLOW/BROWN'
target = '\\textcolor[HTML]{aa5500}{YELLOW/BROWN}'
latex = Ansi2HTMLConverter(latex=True, inline=True).convert(ansi)
assert (target in latex)
def run(request):
if request.POST.get("raw"):
command = RawCommand(request.POST.get("command"))
parsed_command = command.parse()
# Schedules.
if request.POST.get("schedule_type"):
schedule_type = request.POST.get("schedule_type")
schedule_name = request.POST.get(
"schedule_name",
datetime.datetime.now().strftime("%Y%m%d%H%M%S"))
schedule_parsed = [{
"client":
"local",
"batch":
None,
"tgt_type":
parsed_command[0]["tgt_type"],
"tgt":
parsed_command[0]["tgt"],
"fun":
"schedule.add",
"arg": [
schedule_name,
"function={}".format(parsed_command[0]["fun"]),
"job_args={}".format(parsed_command[0]["arg"]),
],
}]
if schedule_type == "once":
schedule_date = request.POST.get("schedule")
schedule_parsed[0]["arg"].append(
"once={}".format(schedule_date))
schedule_parsed[0]["arg"].append("once_fmt=%Y-%m-%d %H:%M:%S")
else:
cron = request.POST.get("cron")
schedule_parsed[0]["arg"].append("cron={}".format(cron))
ret = run_raw(schedule_parsed)
if "error" in ret:
return Response(ret["error"], status=401)
formatted = nested_output.output(ret)
conv = Ansi2HTMLConverter(inline=False, scheme="xterm")
html = conv.convert(formatted, ensure_trailing_newline=True)
return HttpResponse(html)
cli_ret = request.POST.get("cli")
conv = Ansi2HTMLConverter(inline=False, scheme="xterm")
ret = run_raw(parsed_command)
if "error" in ret:
return Response(ret["error"], status=401)
formatted = "\n"
# Error.
if isinstance(ret, str):
item_ret = nested_output.output(ret)
formatted += item_ret + "\n\n"
# runner or wheel client.
elif isinstance(ret, list):
for item in ret:
item_ret = nested_output.output(item)
formatted += item_ret + "\n\n"
# Highstate.
elif (parsed_command[0]["fun"] in ["state.apply", "state.highstate"]
and parsed_command[0]["client"] != "local_async"):
for state, out in ret.items():
minion_ret = highstate_output.output({state: out})
formatted += minion_ret + "\n\n"
# Everything else.
else:
for state, out in ret.items():
minion_ret = nested_output.output({state: out})
formatted += minion_ret + "\n\n"
if cli_ret:
return JsonResponse({"results": formatted})
html = conv.convert(formatted, ensure_trailing_newline=True)
return HttpResponse(html)
def translated_call(**kwargs):
def html_newlines(text):
r = '<br />\n'
text = text.replace(' ', ' ')
text = text.replace('\r\n',
r).replace('\n\r', r).replace(
'\r', r).replace('\n', r)
return text
__format__ = False # Default __format__ value
ignore_cookie_set = False
try:
#print('Processing: %r with args: %r and kwargs: %r' % (func, args, kwargs, ))
# Pipe web input into Python web call
kwargs2 = _process_input(flask.request.args)
kwargs3 = _process_input(flask.request.form)
try:
# kwargs4 = _process_input(flask.request.get_json())
kwargs4 = ut.from_json(flask.request.data)
except:
kwargs4 = {}
kwargs.update(kwargs2)
kwargs.update(kwargs3)
kwargs.update(kwargs4)
# Update the request object to include the final rectified inputs for possible future reference
flask.request.processed = ut.to_json(kwargs)
jQuery_callback = None
if 'callback' in kwargs and 'jQuery' in kwargs[
'callback']:
jQuery_callback = str(kwargs.pop('callback', None))
kwargs.pop('_', None)
#print('KWARGS: %s' % (kwargs, ))
#print('COOKIES: %s' % (request.cookies, ))
__format__ = request.cookies.get('__format__', None)
__format__ = kwargs.pop('__format__', __format__)
if __format__ is not None:
__format__ = str(__format__).lower()
ignore_cookie_set = __format__ in [
'onetime', 'true'
]
__format__ = __format__ in [
'true', 'enabled', 'enable'
]
resp_tup = translate_ibeis_webcall(func, **kwargs)
rawreturn, success, code, message = resp_tup
except WebException as webex:
# ut.printex(webex)
print('CAUGHT2: %r' % (webex, ))
rawreturn = webex.get_rawreturn(
DEBUG_PYTHON_STACK_TRACE_JSON_RESPONSE)
success = False
code = webex.code
message = webex.message
jQuery_callback = None
except Exception as ex:
print('CAUGHT2: %r' % (ex, ))
# ut.printex(ex)
rawreturn = None
if DEBUG_PYTHON_STACK_TRACE_JSON_RESPONSE:
rawreturn = str(traceback.format_exc())
success = False
code = 500
message = str(ex)
# errmsg = str(ex)
# message = 'API error, Python Exception thrown: %s' % (errmsg)
if "'int' object is not iterable" in message:
rawreturn = '''
HINT: the input for this call is most likely expected to be a list.
Try adding a comma at the end of the input (to cast the conversion into a list) or encapsulate the input with [].
'''
jQuery_callback = None
#print('RECEIVED FORMAT: %r' % (__format__, ))
if __format__:
# Hack for readable error messages
webreturn = translate_ibeis_webreturn(
rawreturn, success, code, message, jQuery_callback)
webreturn = ut.repr3(ut.from_json(webreturn),
strvals=True)
try:
from ansi2html import Ansi2HTMLConverter
conv = Ansi2HTMLConverter()
webreturn = conv.convert(webreturn)
except ImportError as ex:
ut.printex(ex,
'pip install ansi2html',
iswarning=True)
webreturn = ut.strip_ansi(webreturn)
webreturn = '<p><samp>\n' + html_newlines(
webreturn) + '\n</samp></p>'
webreturn = '<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-8">\n' + webreturn
def get_func_href(funcname):
url = 'http://' + request.environ[
'HTTP_HOST'] + flask.url_for(
funcname) + '?__format__=True'
return '<a href="{url}">{url}</a>'.format(url=url)
if not success:
webreturn += '<pre>See logs for details: %s</pre>' % get_func_href(
'get_current_log_text')
webreturn += '<pre>Might also look into db_info: %s</pre>' % get_func_href(
'get_dbinfo')
else:
webreturn = translate_ibeis_webreturn(
rawreturn, success, code, message, jQuery_callback)
webreturn = ut.strip_ansi(webreturn)
resp = flask.make_response(webreturn, code)
resp.status_code = code
if not __format__:
resp.headers[
'Content-Type'] = 'application/json; charset=utf-8'
resp.headers['mimetype'] = 'application/json'
if not ignore_cookie_set:
if __format__:
resp.set_cookie('__format__', 'enabled')
else:
resp.set_cookie('__format__', '', expires=0)
return resp
def get_log_html(self):
conv = Ansi2HTMLConverter()
return conv.convert(self.log)
def __init__(self): self.logs = '' self.conv = Ansi2HTMLConverter()
- make sure columns are correctly designated for both formats
- make sure abbreviations are unified:
- either different rules for different abbreviations
OR
- better:
- apply your own abbr. unifier (lemmatization data scripts)
to make the data unified.
- then insure that the abbr. in SPARQL match
- Find a solution for rendering words in SPARQL.
Perhaps, FLASK templates would be the best solution also to corpus-specific
placeholders' rendering.
'''
#
#---/ ANSI 2 HTML /------------------------------------------------------------
#
a2h_conv = Ansi2HTMLConverter()
#
#---/ Variables /--------------------------------------------------------------
#
_path = os.path.dirname(os.path.abspath(__file__))
sp = subprocesses()
#
#---/ CDLI-CoNLL > CONLL-U /---------------------------------------------------
#
class CC2CU(common_functions, CdliCoNLLtoCoNLLUConverter):
'''
Wrapper around CDLI-CoNLL-to-CoNLLU-Converter:
https://github.com/cdli-gh/CDLI-CoNLL-to-CoNLLU-Converter
def test_escaped_explicit(self):
test = "<p>awesome</p>"
expected = "<p>awesome</p>"
html = Ansi2HTMLConverter(escaped=True).convert(test, full=False)
self.assertEqual(expected, html)
def output_html(self, obj):
if obj and obj.output is not None:
conv = Ansi2HTMLConverter()
output = mark_safe(conv.convert(obj.output, full=False))
return display_as_code(output)
return None
def test_unescaped(self):
test = "<p>awesome</p>"
expected = "<p>awesome</p>"
html = Ansi2HTMLConverter(escaped=False).convert(test, full=False)
self.assertEqual(expected, html)
def format_log(log):
conv = Ansi2HTMLConverter(dark_bg=False, scheme="solarized", markup_lines=True)
headers = conv.produce_headers()
content = conv.convert(log, full=False)
content = '<pre class="ansi2html-content">{}</pre>'.format(content)
return headers + content
def test_markup_lines(self):
test = " wat \n "
expected = '<span id="line-0"> wat </span>\n<span id="line-1"> </span>'
html = Ansi2HTMLConverter(markup_lines=True).convert(test, full=False)
self.assertEqual(expected, html)
# -*- coding: utf-8 -*-
import xml.etree.ElementTree as ET
import outer_packages
import argparse
import glob
from pprint import pprint
import sys, os
from collections import OrderedDict
import copy
import datetime, time
import traceback
import yaml
import subprocess, shlex
from ansi2html import Ansi2HTMLConverter
converter = Ansi2HTMLConverter(inline=True)
convert = converter.convert
def ansi2html(text):
return convert(text, full=False)
FUNCTIONAL_CATEGORY = 'Functional' # how to display those categories
ERROR_CATEGORY = 'Error'
def try_write(file, text):
try:
file.write(text)
except:
def test_no_markup_lines(self):
test = " wat \n "
expected = test
html = Ansi2HTMLConverter().convert(test, full=False)
self.assertEqual(expected, html)
'#00c200',
'#c7c400',
'#0037da',
'#c930c7',
'#00c5c7',
'#c7c7c7',
'#676767',
'#ff6d67',
'#5ff967',
'#fefb67',
'#6871ff',
'#ff76ff',
'#5ffdff',
'#feffff',
) * 2
conv = Ansi2HTMLConverter(inline=True, scheme='iterm')
def ansi_to_html(output):
if output is None:
return None
try:
return conv.convert(output, full=False)
except Exception as e:
return f'[Error converting ANSI to HTML: {e}]\n\n{output}'
def prepare(element_html, data):
element = lxml.html.fragment_fromstring(element_html)
required_attribs = []
optional_attribs = []
def test_not_linkify(self):
ansi = "http://threebean.org"
target = '<a href="http://threebean.org">http://threebean.org</a>'
html = Ansi2HTMLConverter().convert(ansi)
assert (target not in html)
def htmlize(ansi):
conv = Ansi2HTMLConverter(inline=True, dark_bg=True)
return conv.convert(ansi, full=False)
NOT_FOUND_LOCATION = "NOT_FOUND"
DEFAULT_LOCATION = "Oymyakon"
NOT_FOUND_MESSAGE = """
We were unable to find your location,
so we have brought you to Oymyakon,
one of the coldest permanently inhabited locales on the planet.
"""
if not os.path.exists(os.path.dirname(LOG_FILE)):
os.makedirs(os.path.dirname(LOG_FILE))
logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG)
reader = geoip2.database.Reader(GEOLITE)
geolocator = Nominatim()
ansiconv = Ansi2HTMLConverter()
reHeaderLine = re.compile(r"^Weather for (\-?[\d\.]+,\-?[\d\.]+)$",
re.IGNORECASE | re.MULTILINE)
my_loader = jinja2.ChoiceLoader([
app.jinja_loader,
jinja2.FileSystemLoader(TEMPLATES),
])
app.jinja_loader = my_loader
class Limits:
def __init__(self):
self.intervals = ['min', 'hour', 'day']
self.divisor = {
