def index():
"""Renders and returns the index-page HTML template."""
sw = Stopwatch()
try:
return render_template("index.html")
except Exception as ex:
error_handler.log_error(ex)
return str(ex)
finally:
di.service_stats.log_command(name="index", elapsed_ms=sw.elapsed_ms)
def set_task_iterations(self, id_: UUID, total_iterations: int) -> None:
"""Sets the total number of iterations after a task has been started, when the total
isn't known until later."""
try:
with self.__task_lock:
if id_ in self.__running_tasks:
task_stats = self.__running_tasks[id_]
task_stats.set_total_iterations(total_iterations)
except Exception as ex:
error_handler.log_error(ex)
def update_task(self, id_: UUID, completed_iterations: int, is_total: bool) -> None:
"""Updates running task with new number of completed iterations. This number can be
just the number of new iterations since the last update, or the total number of iterations
since the task was started."""
try:
with self.__task_lock:
if id_ in self.__running_tasks:
task_stats = self.__running_tasks[id_]
task_stats.update_task(completed_iterations, is_total)
except Exception as ex:
error_handler.log_error(ex)
def begin_task(self, name: str, total_iterations: int = 0) -> UUID:
"""Starts a long running task and returns its UUID. Task must be ended on completion."""
try:
task_stats = TaskStats(name, total_iterations)
with self.__task_lock:
if task_stats.id not in self.__running_tasks:
self.__running_tasks[task_stats.id] = task_stats
return task_stats.id
except Exception as ex:
error_handler.log_error(ex)
return uuid4()
def ping() -> Response:
"""Returns the plain text "Up", "LoadingData", or "Down" depending on service state."""
sw = Stopwatch()
try:
response = di.service_state.state.name
except Exception as ex:
error_handler.log_error(ex)
response = str(ex)
finally:
di.service_stats.log_command(name="ping", elapsed_ms=sw.elapsed_ms)
return Response(response=response, mimetype="text/plain")
def log_command(self, name: str, elapsed_ms: int) -> None:
"""Logs a command."""
try:
if name is None:
name = ""
cmd_stats = OperationStats(name, elapsed_ms, datetime.now(), False)
with self.__queue_lock:
if name not in self.__command_queue:
self.__command_queue[name] = []
self.__command_queue[name].append(cmd_stats)
except Exception as ex:
error_handler.log_error(ex)
def error(errors: List[Exception], command: str = "unknown") -> str:
"""Writes the standard JSON response for fatal error, if a more specific command handler was not reached."""
try:
w = JsonWriter()
w.write_start_object()
__write_info(w, command, 0)
__write_errors(w, errors)
w.write_end_object()
json = w.to_string()
return json
except Exception as ex:
error_handler.log_error(ex)
return ""
def end_task(self, id_: UUID) -> None:
"""Ends the specified task."""
try:
with self.__task_lock:
if id_ in self.__running_tasks:
task_stats = self.__running_tasks.pop(id_)
task_stats.end_task()
self.__completed_tasks.append(task_stats)
six_hours_ago = datetime.now() - timedelta(hours=6)
while len(self.__completed_tasks) > 0 and self.__completed_tasks[0].end_time < six_hours_ago:
self.__completed_tasks.pop(0)
except Exception as ex:
error_handler.log_error(ex)
def get_stats() -> Response:
"""Generates service statistics and returns JSON response."""
errors = []
sw = Stopwatch()
try:
response = command_writer.get_stats()
except Exception as ex:
errors.append(ex)
error_handler.log_error(ex)
response = command_writer.error(errors, "getstats")
finally:
di.service_stats.log_command(name="getstats", elapsed_ms=sw.elapsed_ms)
return Response(response, mimetype="application/json")
def __data_process_thread(self) -> None:
"""Internal data processing thread."""
try:
self.__signal.set()
while True:
try:
now = datetime.now()
six_hours_ago = now - timedelta(hours=6)
if now >= self.__last_swap + timedelta(seconds=6):
self.swap_and_rollup(self.__operation_queue, self.__operation_rollups, now, six_hours_ago)
self.swap_and_rollup(self.__command_queue, self.__command_rollups, now, six_hours_ago)
self.__last_swap = now
except Exception as ex:
error_handler.log_error(ex)
time.sleep(0.25)
except Exception as ex:
error_handler.log_error(ex)
def word_split() -> Response:
"""Performs word split operation, returns JSON response with metadata OR plain text."""
errors: List[Exception] = []
output = "json"
sw = Stopwatch()
try:
# parse params
inputs: List[str] = (request.args.get("input")
or "").replace("|", ",").split(",")
pass_display = int(request.args.get("passdisplay") or "5")
exhaustive = (request.args.get("exhaustive") or "0") == "1"
verbosity = VerbosityLevel(int(request.args.get("verbosity") or "0"))
output = (request.args.get("output") or "json").lower()
cache = (request.args.get("cache") or "1") == "1"
# parse restrictions
if not exhaustive:
max_input_chars = config.default_max_input_chars
max_terms = config.default_max_terms
max_passes = config.default_max_passes
else:
max_input_chars = config.exhaustive_max_input_chars
max_terms = config.exhaustive_max_terms
max_passes = config.exhaustive_max_passes
# limit input
for i in range(len(inputs)):
if len(inputs[i]) > max_input_chars:
inputs[i] = remove(inputs[i], max_input_chars)
if len(inputs) > 1000:
del inputs[1000:]
# perform splits
results = []
for s in inputs:
if (verbosity < VerbosityLevel.High) and (not exhaustive):
result = di.word_splitter.simple_split(s, cache, max_terms,
max_passes, errors)
else:
result = di.word_splitter.full_split(s, cache, pass_display,
max_terms, max_passes,
errors)
results.append(result)
# write response
response = ""
if output == "json":
response = command_writer.word_split(verbosity, inputs,
pass_display, exhaustive,
results, sw.elapsed_ms,
errors)
elif output == "text":
for r in results:
response += r.output + "\n"
except Exception as ex:
errors.append(ex)
error_handler.log_error(ex)
response = command_writer.error(errors, "wordsplit")
finally:
di.service_stats.log_command(name="wordsplit",
elapsed_ms=sw.elapsed_ms)
return Response(
response,
mimetype="application/json" if output == "json" else "text/plain")