def get_constant(self,
const,
prefix=None,
missing_ok=False,
missing_default=None):
if "constants" not in self:
raise ConfigurationError(
"constants section not present in the device config.")
constants = self["constants"]
if prefix:
if prefix in constants:
if const in constants[prefix]:
return constants[prefix][const]
if missing_ok:
return missing_default
raise ConfigurationError(
"Constant %s,%s does not exist in the device config 'constants' section."
% (prefix, const))
if const in constants:
return constants[const]
if missing_ok:
return missing_default
raise ConfigurationError(
"Constant %s does not exist in the device config 'constants' section."
% const)
def boot_check(cls, device, parameters):
if not device:
raise JobError('job "device" was None')
if 'method' not in parameters:
raise ConfigurationError("method not specified in boot parameters")
if 'actions' not in device:
raise ConfigurationError('Invalid device configuration, no "actions" in device configuration')
if 'boot' not in device['actions']:
raise ConfigurationError('"boot" is not in the device configuration actions')
if 'methods' not in device['actions']['boot']:
raise ConfigurationError('Device misconfiguration, no "methods" in device configuration boot action')
def deploy_check(cls, device, parameters):
if not device:
raise JobError('job "device" was None')
if 'actions' not in device:
raise ConfigurationError('Invalid device configuration, no "actions" in device configuration')
if 'to' not in parameters:
raise ConfigurationError('"to" not specified in deploy parameters')
if 'deploy' not in device['actions']:
raise ConfigurationError('"deploy" is not in the device configuration actions')
if 'methods' not in device['actions']['deploy']:
raise ConfigurationError('Device misconfiguration, no "methods" in device configuration deploy actions')
def accepts(cls, device, parameters):
if "method" not in parameters:
raise ConfigurationError("method not specified in boot parameters")
if parameters["method"] != "new_connection":
return False, "new_connection not in method"
if "actions" not in device:
raise ConfigurationError("Invalid device configuration")
if "boot" not in device["actions"]:
return False, "boot not in device actions"
if "methods" not in device["actions"]["boot"]:
raise ConfigurationError("Device misconfiguration")
if "method" not in parameters:
return False, "no boot method"
return True, "accepted"
def accepts(cls, device, parameters):
if 'method' not in parameters:
raise ConfigurationError("method not specified in boot parameters")
if parameters['method'] != 'new_connection':
return False, 'new_connection not in method'
if 'actions' not in device:
raise ConfigurationError("Invalid device configuration")
if 'boot' not in device['actions']:
return False, 'boot not in device actions'
if 'methods' not in device['actions']['boot']:
raise ConfigurationError("Device misconfiguration")
if 'method' not in parameters:
return False, 'no boot method'
return True, 'accepted'
def create_custom_job(self,
template,
job_data,
job_ctx=None,
validate=True):
if validate:
validate_job(job_data, strict=False)
if job_ctx:
job_data["context"] = job_ctx
else:
job_ctx = job_data.get("context")
(data, device_dict) = self.create_device(template, job_ctx)
device = NewDevice(yaml_safe_load(data))
print("####### Device configuration #######")
print(data)
print("#######")
try:
parser = JobParser()
job = parser.parse(yaml_safe_dump(job_data), device, 4999, None,
"")
except (ConfigurationError, TypeError) as exc:
print("####### Parser exception ########")
print(device)
print("#######")
raise ConfigurationError("Invalid device: %s" % exc)
job.logger = DummyLogger()
return job
def run(self, connection, max_end_time):
connection = super().run(connection, max_end_time)
if not connection:
raise LAVABug("Cannot transfer overlay, no connection available.")
overlay_full_path = self.get_namespace_data(action="compress-overlay",
label="output",
key="file")
if not overlay_full_path:
raise JobError("No overlay file identified for the transfer.")
if not overlay_full_path.startswith(DISPATCHER_DOWNLOAD_DIR):
raise ConfigurationError(
"overlay should already be in DISPATCHER_DOWNLOAD_DIR")
overlay_path = overlay_full_path[len(DISPATCHER_DOWNLOAD_DIR) + 1:]
overlay = os.path.basename(overlay_path)
connection.sendline("rm %s" % overlay)
connection.wait()
cmd = self.parameters["transfer_overlay"]["download_command"]
ip_addr = dispatcher_ip(self.job.parameters["dispatcher"], "http")
connection.sendline("%s http://%s/tmp/%s" %
(cmd, ip_addr, overlay_path))
connection.wait()
unpack = self.parameters["transfer_overlay"]["unpack_command"]
connection.sendline(unpack + " " + overlay)
connection.wait()
return connection
def validate(self):
super().validate()
cmd_name = self.parameters["name"]
if cmd_name in self.builtin_commands:
if cmd_name in self.job.device["commands"]:
self.cmd = {"do": self.job.device["commands"][cmd_name]}
return True
else:
self.errors = "Command '%s' not defined for this device" % cmd_name
return False
user_commands = self.job.device.get("commands", {}).get("users")
if not user_commands:
self.errors = "Device has no configured user commands"
return False
try:
self.cmd = user_commands[cmd_name]
if not isinstance(self.cmd["do"], str) or not isinstance(
self.cmd.get("undo", ""), str):
raise ConfigurationError('User command "%s" is invalid: '
"'do' and 'undo' should be strings" %
cmd_name)
return True
except KeyError:
self.errors = "Unknown user command '%s'" % cmd_name
return False
def read_settings(self, filename):
"""
NodeDispatchers need to use the same port and blocksize as the Coordinator,
so read the same conffile.
The protocol header is hard-coded into the server & here.
"""
settings = {
"port": 3079,
"blocksize": 4 * 1024,
"poll_delay": 1,
"coordinator_hostname": "localhost",
}
self.logger = logging.getLogger("dispatcher")
json_default = {}
with open(filename) as stream:
jobdata = stream.read()
try:
json_default = json.loads(jobdata)
except ValueError as exc:
raise ConfigurationError(
"Invalid JSON settings for %s: %s" % (self.name, exc)
)
if "port" in json_default:
settings["port"] = json_default["port"]
if "blocksize" in json_default:
settings["blocksize"] = json_default["blocksize"]
if "poll_delay" in json_default:
settings["poll_delay"] = json_default["poll_delay"]
if "coordinator_hostname" in json_default:
settings["coordinator_hostname"] = json_default["coordinator_hostname"]
return settings
def accepts(cls, device, parameters):
if parameters["method"] != "barebox":
return False, '"method" was not "barebox"'
if "commands" not in parameters:
raise ConfigurationError("commands not specified in boot parameters")
if "barebox" in device["actions"]["boot"]["methods"]:
return True, "accepted"
return False, '"barebox" was not in the device configuration boot methods'
def accepts(cls, device, parameters):
if parameters["method"] != "uuu":
return False, '"method" was not "uuu"'
if "commands" not in parameters:
raise ConfigurationError("commands not specified in boot parameters")
params = device["actions"]["boot"]["methods"]["uuu"]["options"]
if not params["usb_otg_path"]:
raise ConfigurationError(
"uuu_usb_otg_path not defined in device definition"
)
if params["corrupt_boot_media_command"] is None:
raise ConfigurationError(
"uuu_corrupt_boot_media_command not defined in device definition"
)
if "u-boot" in device["actions"]["boot"]["methods"]:
return True, "accepted"
return False, '"uuu" was not in the device configuration boot methods'
def accepts(cls, device, parameters):
if parameters['method'] != 'u-boot':
return False, '"method" was not "u-boot"'
if 'commands' not in parameters:
raise ConfigurationError(
"commands not specified in boot parameters")
if 'u-boot' in device['actions']['boot']['methods']:
return True, 'accepted'
return False, '"u-boot" was not in the device configuration boot methods'
def accepts(cls, device, parameters):
if 'method' not in parameters:
raise ConfigurationError("method not specified in boot parameters")
if parameters["method"] not in ["grub", "grub-efi"]:
return False, '"method" was not "grub" or "grub-efi"'
if 'actions' not in device:
raise ConfigurationError("Invalid device configuration")
if 'boot' not in device['actions']:
return False, '"boot" was not in the device configuration actions'
if 'methods' not in device['actions']['boot']:
raise ConfigurationError("Device misconfiguration")
params = device['actions']['boot']['methods']
if 'grub' in params and 'sequence' in params['grub']:
return False, '"sequence" was in "grub" parameters'
if 'grub' in params or 'grub-efi' in params:
return True, 'accepted'
else:
return False, '"grub" or "grub-efi" was not in the device configuration boot methods'
def accepts(cls, device, parameters):
if "method" not in parameters:
raise ConfigurationError("method not specified in boot parameters")
if parameters["method"] not in ["grub", "grub-efi"]:
return False, '"method" was not "grub" or "grub-efi"'
if "actions" not in device:
raise ConfigurationError("Invalid device configuration")
if "boot" not in device["actions"]:
return False, '"boot" was not in the device configuration actions'
if "methods" not in device["actions"]["boot"]:
raise ConfigurationError("Device misconfiguration")
params = device["actions"]["boot"]["methods"]
if "grub" not in params:
return False, '"grub" was not in the device configuration boot methods'
if "grub-efi" in params:
return False, '"grub-efi" was not in the device configuration boot methods'
if "sequence" in params["grub"]:
return True, "accepted"
return False, '"sequence" not in device configuration boot methods'
def poll(self, message, timeout=None):
"""
Blocking, synchronous polling of the Coordinator on the configured port.
Single send operations greater than 0xFFFF are rejected to prevent truncation.
:param msg_str: The message to send to the Coordinator, as a JSON string.
:return: a JSON string of the response to the poll
"""
if not timeout:
timeout = self.poll_timeout.duration
if isinstance(timeout, float):
timeout = int(timeout)
elif not isinstance(timeout, int):
raise ConfigurationError("Invalid timeout duration type: %s %s" % (type(timeout), timeout))
msg_len = len(message)
if msg_len > 0xFFFE:
raise JobError("Message was too long to send!")
c_iter = 0
response = None
delay = self.settings['poll_delay']
self.logger.debug("Connecting to LAVA Coordinator on %s:%s timeout=%d seconds.",
self.settings['coordinator_hostname'],
self.settings['port'], timeout)
while True:
c_iter += self.settings['poll_delay']
if self._connect(delay):
delay = self.settings['poll_delay']
else:
delay += 2
continue
if not c_iter % int(10 * self.settings['poll_delay']):
self.logger.debug("sending message: %s waited %s of %s seconds",
json.loads(message)['request'], c_iter,
timeout)
# blocking synchronous call
if not self._send_message(message):
continue
self.sock.shutdown(socket.SHUT_WR)
response = self._recv_message()
self.sock.close()
try:
json_data = json.loads(response)
except ValueError:
self.logger.debug("response starting '%s' was not JSON", response[:42])
self.finalise_protocol()
break
if json_data['response'] != 'wait':
break
else:
time.sleep(delay)
# apply the default timeout to each poll operation.
if c_iter > timeout:
self.finalise_protocol()
raise MultinodeProtocolTimeoutError(
"protocol %s timed out" % self.name)
return response
def __init__(self, target):
super().__init__({})
# Parse the yaml configuration
try:
if isinstance(target, str):
with open(target) as f_in:
data = f_in.read()
data = yaml.safe_load(data)
elif isinstance(target, dict):
data = target
else:
data = target.read()
data = yaml.safe_load(data)
if data is None:
raise ConfigurationError("Missing device configuration")
self.update(data)
except yaml.parser.ParserError:
raise ConfigurationError("%s could not be parsed" % target)
self.setdefault('power_state', 'off') # assume power is off at start of job
def validate(self):
super().validate()
cmd_name = self.parameters['name']
try:
user_commands = self.job.device['commands']['users']
except KeyError:
raise ConfigurationError(
"Unable to get device.commands.users dictionary")
try:
self.cmd = user_commands[cmd_name]
if not isinstance(self.cmd['do'], str) or \
not isinstance(self.cmd.get('undo', ""), str):
raise ConfigurationError("User command \"%s\" is invalid: "
"'do' and 'undo' should be strings" %
cmd_name)
return True
except KeyError:
self.errors = "Unknown user command '%s'" % cmd_name
return False
def validate(self):
super().validate()
cmd_name = self.parameters["name"]
try:
user_commands = self.job.device["commands"]["users"]
except KeyError:
raise ConfigurationError(
"Unable to get device.commands.users dictionary")
try:
self.cmd = user_commands[cmd_name]
if not isinstance(self.cmd["do"], str) or not isinstance(
self.cmd.get("undo", ""), str):
raise ConfigurationError('User command "%s" is invalid: '
"'do' and 'undo' should be strings" %
cmd_name)
return True
except KeyError:
self.errors = "Unknown user command '%s'" % cmd_name
return False
def validate(self):
super().validate()
if 'prompts' not in self.parameters:
self.errors = "Unable to identify boot prompts from job definition."
try:
boot = self.job.device['actions']['boot']['methods']['qemu']
qemu_binary = which(boot['parameters']['command'])
self.sub_command = [qemu_binary]
self.sub_command.extend(boot['parameters'].get('options', []))
except AttributeError as exc:
raise ConfigurationError(exc)
except (KeyError, TypeError):
self.errors = "Invalid parameters for %s" % self.name
def validate(self):
super().validate()
if "prompts" not in self.parameters:
self.errors = "Unable to identify boot prompts from job definition."
try:
boot = self.job.device["actions"]["boot"]["methods"]["qemu"]
qemu_binary = which(boot["parameters"]["command"])
self.sub_command = [qemu_binary]
self.sub_command.extend(boot["parameters"].get("options", []))
except AttributeError as exc:
raise ConfigurationError(exc)
except (KeyError, TypeError):
self.errors = "Invalid parameters for %s" % self.name
def parse(cls, data):
"""
Parsed timeouts can be set in device configuration or device_type configuration
and can therefore exceed the clamp.
"""
if not isinstance(data, dict):
raise ConfigurationError("Invalid timeout data")
duration = datetime.timedelta(days=data.get('days', 0),
hours=data.get('hours', 0),
minutes=data.get('minutes', 0),
seconds=data.get('seconds', 0))
if not duration:
return Timeout.default_duration()
return int(duration.total_seconds())
def __init__(self):
import pymongo
self.client = pymongo.MongoClient(settings.MONGO_DB_URI)
try:
# Test connection.
# The ismaster command is cheap and does not require auth.
self.client.admin.command("ismaster")
except (pymongo.errors.ConnectionFailure, pymongo.errors.OperationFailure):
raise ConfigurationError(
"MongoDB URI is not configured properly. Unable to connect to MongoDB."
)
self.db = self.client[settings.MONGO_DB_DATABASE]
self.db.logs.create_index([("job_id", 1), ("dt", 1)])
super().__init__()
def validate(self):
super().validate()
try:
boot = self.job.device['actions']['boot']['methods']['dfu']
dfu_binary = which(boot['parameters']['command'])
self.base_command = [dfu_binary]
self.base_command.extend(boot['parameters'].get('options', []))
if self.job.device['board_id'] == '0000000000':
self.errors = "[FLASH_DFU] board_id unset"
if self.job.device['usb_vendor_id'] == '0000':
self.errors = '[FLASH_DFU] usb_vendor_id unset'
if self.job.device['usb_product_id'] == '0000':
self.errors = '[FLASH_DFU] usb_product_id unset'
self.usb_vendor_id = self.job.device['usb_vendor_id']
self.usb_product_id = self.job.device['usb_product_id']
self.board_id = self.job.device['board_id']
self.base_command.extend(['--serial', self.board_id])
self.base_command.extend([
'--device',
'%s:%s' % (self.usb_vendor_id, self.usb_product_id)
])
except AttributeError as exc:
raise ConfigurationError(exc)
except (KeyError, TypeError):
self.errors = "Invalid parameters for %s" % self.name
substitutions = {}
for action in self.get_namespace_keys('download-action'):
dfu_full_command = []
image_arg = self.get_namespace_data(action='download-action',
label=action,
key='image_arg')
action_arg = self.get_namespace_data(action='download-action',
label=action,
key='file')
if not image_arg or not action_arg:
self.errors = "Missing image_arg for %s. " % action
continue
if not isinstance(image_arg, str):
self.errors = "image_arg is not a string (try quoting it)"
continue
substitutions["{%s}" % action] = action_arg
dfu_full_command.extend(self.base_command)
dfu_full_command.extend(substitute([image_arg], substitutions))
self.exec_list.append(dfu_full_command)
if len(self.exec_list) < 1:
self.errors = "No DFU command to execute"
def validate(self):
super().validate()
try:
boot = self.job.device["actions"]["boot"]["methods"]["dfu"]
dfu_binary = which(boot["parameters"]["command"])
self.base_command = [dfu_binary]
self.base_command.extend(boot["parameters"].get("options", []))
if self.job.device["board_id"] == "0000000000":
self.errors = "[FLASH_DFU] board_id unset"
if self.job.device["usb_vendor_id"] == "0000":
self.errors = "[FLASH_DFU] usb_vendor_id unset"
if self.job.device["usb_product_id"] == "0000":
self.errors = "[FLASH_DFU] usb_product_id unset"
self.usb_vendor_id = self.job.device["usb_vendor_id"]
self.usb_product_id = self.job.device["usb_product_id"]
self.board_id = self.job.device["board_id"]
self.base_command.extend(["--serial", self.board_id])
self.base_command.extend([
"--device",
"%s:%s" % (self.usb_vendor_id, self.usb_product_id)
])
except AttributeError as exc:
raise ConfigurationError(exc)
except (KeyError, TypeError):
self.errors = "Invalid parameters for %s" % self.name
substitutions = {}
for action in self.get_namespace_keys("download-action"):
dfu_full_command = []
image_arg = self.get_namespace_data(action="download-action",
label=action,
key="image_arg")
action_arg = self.get_namespace_data(action="download-action",
label=action,
key="file")
if not image_arg or not action_arg:
self.errors = "Missing image_arg for %s. " % action
continue
if not isinstance(image_arg, str):
self.errors = "image_arg is not a string (try quoting it)"
continue
substitutions["{%s}" % action] = action_arg
dfu_full_command.extend(self.base_command)
dfu_full_command.extend(substitute([image_arg], substitutions))
self.exec_list.append(dfu_full_command)
if not self.exec_list:
self.errors = "No DFU command to execute"
def create_custom_job(self, template, job_data):
job_ctx = job_data.get('context')
(data, device_dict) = self.create_device(template, job_ctx)
device = NewDevice(yaml.safe_load(data))
if self.debug:
print('####### Device configuration #######')
print(data)
print('#######')
try:
parser = JobParser()
job = parser.parse(yaml.dump(job_data), device, 4999, None, "")
except (ConfigurationError, TypeError) as exc:
print('####### Parser exception ########')
print(device)
print('#######')
raise ConfigurationError("Invalid device: %s" % exc)
job.logger = DummyLogger()
return job
def run(self, connection, max_end_time):
connection = super().run(connection, max_end_time)
if not connection:
raise LAVABug("Cannot transfer overlay, no connection available.")
ip_addr = dispatcher_ip(self.job.parameters['dispatcher'])
overlay_full_path = self.get_namespace_data(action='compress-overlay',
label='output',
key='file')
if not overlay_full_path:
raise JobError("No overlay file identified for the transfer.")
if not overlay_full_path.startswith(DISPATCHER_DOWNLOAD_DIR):
raise ConfigurationError(
"overlay should already be in DISPATCHER_DOWNLOAD_DIR")
overlay_path = overlay_full_path[len(DISPATCHER_DOWNLOAD_DIR) + 1:]
overlay = os.path.basename(overlay_path)
dwnld = self.parameters['transfer_overlay']['download_command']
dwnld += " http://%s/tmp/%s" % (ip_addr, overlay_path)
unpack = self.parameters['transfer_overlay']['unpack_command']
unpack += ' ' + overlay
connection.sendline("rm %s; %s && %s" % (overlay, dwnld, unpack))
return connection
for doc in docs:
doc_dict = doc.to_dict()
result.append(
json.dumps({
"dt": doc.id,
"lvl": doc_dict["lvl"],
"msg": doc_dict["msg"]
}))
return "\n".join(["- %s" % x for x in result])
def size(self, job, start=0, end=None):
# TODO: should be implemented.
return None
def write(self, job, line, output=None, idx=None):
line = yaml_load(line)[0]
doc_ref = (self.db.collection(self.root_collection).document(
"%02d-%02d-%02d" % (job.submit_time.year, job.submit_time.month,
job.submit_time.day)).collection(str(
job.id)).document(line["dt"]))
doc_ref.set({"lvl": line["lvl"], "msg": line["msg"]})
logs_backend_str = settings.LAVA_LOG_BACKEND.rsplit(".", 1)
try:
logs_class = getattr(import_module(logs_backend_str[0]),
logs_backend_str[1])
except (AttributeError, ModuleNotFoundError) as exc:
raise ConfigurationError(str(exc))
logs_instance = logs_class()