def test_executors_sqlite(tmp_path):
"""test each executor type with an sqlite database
"""
from qme.main import Queue
config_dir = os.path.join(str(tmp_path), ".qme")
queue = Queue(config_dir=config_dir, database="sqlite")
# The following commands should map to the following executors
commands = ["ls"]
executors = ["shell"]
for i, command in enumerate(commands):
# The executor name should be chosen based on the command
name = executors[i]
task = queue.run("ls")
assert task.executor.name == name
assert task.summary()
# Task.load includes the file dump, the upper level keys should be same
content = task.load()
for key in ["executor", "uid", "data", "command"]:
assert key in content
# Task.export includes the executor specific data
data = task.export()
for key in ["pwd", "user", "timestamp"]:
assert key in data
def main(args, extra):
# Create a queue object
queue = Queue(config_dir=args.config_dir)
query = " ".join(args.query).strip()
if not query:
sys.exit("Please provide a query to search for.")
results = queue.search(query)
bot.table(results)
def main(args, extra):
# Create a queue object, run the command to match to an executor
queue = Queue(config_dir=args.config_dir)
# Case 1: empty list indicates listing all
if not args.executor:
bot.table(queue.list())
else:
# Each in the list can be a full executor or a task id
for executor in args.executor:
bot.table(queue.list(executor))
def main(args, extra):
# Create a queue object
queue = Queue(config_dir=args.config_dir)
query = " ".join(args.query).strip()
if not query:
sys.exit("Please provide a query to search for.")
results = queue.search(query)
results = [[r.taskid, r.command,
str(r.timestamp),
str(r.executor)] for r in results]
bot.table(results)
def start(port=5000, debug=True, queue=None, host=None, level="DEBUG"):
"""Start can be invoked when this file is executed (see __main__ below)
or used as a function to programmatically start a server. If started
via qme view, we can add the queue to the server. If you want to change
the hostname, set the environment variable QME_HOSTNAME or set on command
line with qme start.
"""
host = host or QME_HOSTNAME
bot.info(f"QueueMe: running on http://{host}:{port}")
# If the user doesn't specify a queue, use default
if not queue:
from qme.main import Queue
queue = Queue()
# Customize the logger to log to the app folder
file_handler = logging.FileHandler(
os.path.join(queue.config_dir, "dashboard.log"))
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s")
file_handler.setFormatter(formatter)
app.logger.addHandler(file_handler)
app.logger.setLevel(getattr(logging, level))
# Add the queue to the server
app.queue = queue
socketio.run(app, port=port, debug=debug, host=host)
def run(args, extra):
# Create a queue object, run the command to match to an executor
queue = Queue(config_dir=args.config_dir)
# Don't allow an empty command!
if not args.cmd:
sys.exit("Please provide a command for QueueMe to execute.")
command = args.cmd
# --help needs to be quoted, make sure if provided, gets parsed into command
if any(["--help" in x for x in args.cmd]):
command = []
for item in args.cmd:
command += shlex.split(item)
# Extra might include unparsed arguments
queue.run(command=command + extra, message=args.message)
def main(args, extra):
result = None
if not args.actions:
sys.exit("Please provide an executor to list actions for, or an action.")
# User has provided an executor to list actions for
elif len(args.actions) == 1:
# Assume looking for executor first, fall back to action on last job
try:
executor = get_named_executor(args.actions[0])
actions = [[x] for x in executor.get_actions()]
bot.table(actions)
except:
queue = Queue(config_dir=args.config_dir)
task = queue.get()
result = task.run_action(args.actions[0])
# NEED TO DEBUG THIS FOR SLURM
# user provided exec <action> <taskid>
elif len(args.actions) == 2:
taskid, action = args.actions
queue = Queue(config_dir=args.config_dir)
task = queue.get(taskid)
result = task.run_action(action)
# Print result to terminal
if result:
if isinstance(result, list):
result = " ".join(result)
print(result)
def test_executor_shell(tmp_path):
"""Test the shell executor. A shell executor should have added a returncode,
output, and error to it's data export.
"""
from qme.main import Queue
config_dir = os.path.join(str(tmp_path), ".qme")
queue = Queue(config_dir=config_dir)
task = queue.run("ls")
assert task.executor.name == "shell"
assert task.filename == os.path.join(queue.config_dir, "database", "shell",
"%s.json" % task.taskid)
assert task.summary()
# Task.load includes the file dump, the upper level keys should be same
content = task.load()
for key in ["executor", "uid", "data"]:
assert key in content
# Task.export includes the executor specific data
data = task.export()
for key in ["cmd", "pwd", "output", "error", "returncode"]:
assert key in data
def main(args, extra):
# Create a queue object, run the command to match to an executor
queue = Queue(config_dir=args.config_dir)
# Pass the queue object to start a server
try:
from qme.app.server import start
start(port=args.port,
queue=queue,
debug=args.debug,
level=args.log_level)
except ModuleNotFoundError:
sys.exit(
"You must 'pip install qme[app]' 'pip install qme[all]' to use the dashboard."
)
def main(args, extra):
# Create a queue object
queue = Queue(config_dir=args.config_dir)
task = queue.get(args.taskid)
print(json.dumps(task.load(), indent=4))
def test_relational(tmp_path):
"""Test loading and using a queue with the filesystem database.
"""
from qme.main import Queue
config_dir = os.path.join(str(tmp_path), ".qme")
queue = Queue(config_dir=config_dir, database="sqlite")
assert os.path.exists(config_dir)
assert queue.config_dir == config_dir
assert queue.config.configfile == os.path.join(queue.config_dir,
"config.ini")
assert queue.database == "sqlite"
assert queue.db.database == "sqlite"
# Test list, empty without anything
assert not queue.list()
# Run a task
task = queue.run("ls")
assert task.executor.name == "shell"
assert task.taskid.startswith("shell")
assert len(queue.list()) == 1
# Rerun the task, should still only have one
rerun = queue.rerun()
assert rerun.taskid == task.taskid
assert len(queue.list()) == 1
# queue.get should return last task, given no id
lasttask = queue.get()
assert lasttask.taskid == task.taskid
# Run a new task
newtask = queue.run("whoami")
assert len(queue.list()) == 2
exports = newtask.export()
# Search for tasks
assert len(queue.search("ls")) > 0
# Check exports
for required in ["pwd", "output", "error", "cmd", "returncode"]:
assert required in exports
assert exports["pwd"] == os.getcwd()
assert exports["cmd"] == ["whoami"]
assert exports["returncode"] == 0
# Get a task id that isn't the last task
notlast = queue.get(task.taskid)
assert task.taskid == notlast.taskid
# Clean up a specific task (no prompt)
queue.clear(task.taskid, noprompt=True)
assert len(queue.list()) == 1
queue.clear(noprompt=True)
assert not queue.list()
def main(args, extra):
# Clear an executor, taskid, or target
queue = Queue(config_dir=args.config_dir)
queue.clear(args.target, noprompt=args.force)
def rerun(args, extra):
# Create a queue object, run the command to match to an executor
queue = Queue(config_dir=args.config_dir)
queue.rerun(taskid=args.taskid, message=args.message)