def get_constant(self, const, prefix=None, missing_ok=False, missing_default=None): if "constants" not in self: raise ConfigurationError( "constants section not present in the device config.") constants = self["constants"] if prefix: if prefix in constants: if const in constants[prefix]: return constants[prefix][const] if missing_ok: return missing_default raise ConfigurationError( "Constant %s,%s does not exist in the device config 'constants' section." % (prefix, const)) if const in constants: return constants[const] if missing_ok: return missing_default raise ConfigurationError( "Constant %s does not exist in the device config 'constants' section." % const)
def boot_check(cls, device, parameters): if not device: raise JobError('job "device" was None') if 'method' not in parameters: raise ConfigurationError("method not specified in boot parameters") if 'actions' not in device: raise ConfigurationError('Invalid device configuration, no "actions" in device configuration') if 'boot' not in device['actions']: raise ConfigurationError('"boot" is not in the device configuration actions') if 'methods' not in device['actions']['boot']: raise ConfigurationError('Device misconfiguration, no "methods" in device configuration boot action')
def deploy_check(cls, device, parameters): if not device: raise JobError('job "device" was None') if 'actions' not in device: raise ConfigurationError('Invalid device configuration, no "actions" in device configuration') if 'to' not in parameters: raise ConfigurationError('"to" not specified in deploy parameters') if 'deploy' not in device['actions']: raise ConfigurationError('"deploy" is not in the device configuration actions') if 'methods' not in device['actions']['deploy']: raise ConfigurationError('Device misconfiguration, no "methods" in device configuration deploy actions')
def accepts(cls, device, parameters): if "method" not in parameters: raise ConfigurationError("method not specified in boot parameters") if parameters["method"] != "new_connection": return False, "new_connection not in method" if "actions" not in device: raise ConfigurationError("Invalid device configuration") if "boot" not in device["actions"]: return False, "boot not in device actions" if "methods" not in device["actions"]["boot"]: raise ConfigurationError("Device misconfiguration") if "method" not in parameters: return False, "no boot method" return True, "accepted"
def accepts(cls, device, parameters): if 'method' not in parameters: raise ConfigurationError("method not specified in boot parameters") if parameters['method'] != 'new_connection': return False, 'new_connection not in method' if 'actions' not in device: raise ConfigurationError("Invalid device configuration") if 'boot' not in device['actions']: return False, 'boot not in device actions' if 'methods' not in device['actions']['boot']: raise ConfigurationError("Device misconfiguration") if 'method' not in parameters: return False, 'no boot method' return True, 'accepted'
def create_custom_job(self, template, job_data, job_ctx=None, validate=True): if validate: validate_job(job_data, strict=False) if job_ctx: job_data["context"] = job_ctx else: job_ctx = job_data.get("context") (data, device_dict) = self.create_device(template, job_ctx) device = NewDevice(yaml_safe_load(data)) print("####### Device configuration #######") print(data) print("#######") try: parser = JobParser() job = parser.parse(yaml_safe_dump(job_data), device, 4999, None, "") except (ConfigurationError, TypeError) as exc: print("####### Parser exception ########") print(device) print("#######") raise ConfigurationError("Invalid device: %s" % exc) job.logger = DummyLogger() return job
def run(self, connection, max_end_time): connection = super().run(connection, max_end_time) if not connection: raise LAVABug("Cannot transfer overlay, no connection available.") overlay_full_path = self.get_namespace_data(action="compress-overlay", label="output", key="file") if not overlay_full_path: raise JobError("No overlay file identified for the transfer.") if not overlay_full_path.startswith(DISPATCHER_DOWNLOAD_DIR): raise ConfigurationError( "overlay should already be in DISPATCHER_DOWNLOAD_DIR") overlay_path = overlay_full_path[len(DISPATCHER_DOWNLOAD_DIR) + 1:] overlay = os.path.basename(overlay_path) connection.sendline("rm %s" % overlay) connection.wait() cmd = self.parameters["transfer_overlay"]["download_command"] ip_addr = dispatcher_ip(self.job.parameters["dispatcher"], "http") connection.sendline("%s http://%s/tmp/%s" % (cmd, ip_addr, overlay_path)) connection.wait() unpack = self.parameters["transfer_overlay"]["unpack_command"] connection.sendline(unpack + " " + overlay) connection.wait() return connection
def validate(self): super().validate() cmd_name = self.parameters["name"] if cmd_name in self.builtin_commands: if cmd_name in self.job.device["commands"]: self.cmd = {"do": self.job.device["commands"][cmd_name]} return True else: self.errors = "Command '%s' not defined for this device" % cmd_name return False user_commands = self.job.device.get("commands", {}).get("users") if not user_commands: self.errors = "Device has no configured user commands" return False try: self.cmd = user_commands[cmd_name] if not isinstance(self.cmd["do"], str) or not isinstance( self.cmd.get("undo", ""), str): raise ConfigurationError('User command "%s" is invalid: ' "'do' and 'undo' should be strings" % cmd_name) return True except KeyError: self.errors = "Unknown user command '%s'" % cmd_name return False
def read_settings(self, filename): """ NodeDispatchers need to use the same port and blocksize as the Coordinator, so read the same conffile. The protocol header is hard-coded into the server & here. """ settings = { "port": 3079, "blocksize": 4 * 1024, "poll_delay": 1, "coordinator_hostname": "localhost", } self.logger = logging.getLogger("dispatcher") json_default = {} with open(filename) as stream: jobdata = stream.read() try: json_default = json.loads(jobdata) except ValueError as exc: raise ConfigurationError( "Invalid JSON settings for %s: %s" % (self.name, exc) ) if "port" in json_default: settings["port"] = json_default["port"] if "blocksize" in json_default: settings["blocksize"] = json_default["blocksize"] if "poll_delay" in json_default: settings["poll_delay"] = json_default["poll_delay"] if "coordinator_hostname" in json_default: settings["coordinator_hostname"] = json_default["coordinator_hostname"] return settings
def accepts(cls, device, parameters): if parameters["method"] != "barebox": return False, '"method" was not "barebox"' if "commands" not in parameters: raise ConfigurationError("commands not specified in boot parameters") if "barebox" in device["actions"]["boot"]["methods"]: return True, "accepted" return False, '"barebox" was not in the device configuration boot methods'
def accepts(cls, device, parameters): if parameters["method"] != "uuu": return False, '"method" was not "uuu"' if "commands" not in parameters: raise ConfigurationError("commands not specified in boot parameters") params = device["actions"]["boot"]["methods"]["uuu"]["options"] if not params["usb_otg_path"]: raise ConfigurationError( "uuu_usb_otg_path not defined in device definition" ) if params["corrupt_boot_media_command"] is None: raise ConfigurationError( "uuu_corrupt_boot_media_command not defined in device definition" ) if "u-boot" in device["actions"]["boot"]["methods"]: return True, "accepted" return False, '"uuu" was not in the device configuration boot methods'
def accepts(cls, device, parameters): if parameters['method'] != 'u-boot': return False, '"method" was not "u-boot"' if 'commands' not in parameters: raise ConfigurationError( "commands not specified in boot parameters") if 'u-boot' in device['actions']['boot']['methods']: return True, 'accepted' return False, '"u-boot" was not in the device configuration boot methods'
def accepts(cls, device, parameters): if 'method' not in parameters: raise ConfigurationError("method not specified in boot parameters") if parameters["method"] not in ["grub", "grub-efi"]: return False, '"method" was not "grub" or "grub-efi"' if 'actions' not in device: raise ConfigurationError("Invalid device configuration") if 'boot' not in device['actions']: return False, '"boot" was not in the device configuration actions' if 'methods' not in device['actions']['boot']: raise ConfigurationError("Device misconfiguration") params = device['actions']['boot']['methods'] if 'grub' in params and 'sequence' in params['grub']: return False, '"sequence" was in "grub" parameters' if 'grub' in params or 'grub-efi' in params: return True, 'accepted' else: return False, '"grub" or "grub-efi" was not in the device configuration boot methods'
def accepts(cls, device, parameters): if "method" not in parameters: raise ConfigurationError("method not specified in boot parameters") if parameters["method"] not in ["grub", "grub-efi"]: return False, '"method" was not "grub" or "grub-efi"' if "actions" not in device: raise ConfigurationError("Invalid device configuration") if "boot" not in device["actions"]: return False, '"boot" was not in the device configuration actions' if "methods" not in device["actions"]["boot"]: raise ConfigurationError("Device misconfiguration") params = device["actions"]["boot"]["methods"] if "grub" not in params: return False, '"grub" was not in the device configuration boot methods' if "grub-efi" in params: return False, '"grub-efi" was not in the device configuration boot methods' if "sequence" in params["grub"]: return True, "accepted" return False, '"sequence" not in device configuration boot methods'
def poll(self, message, timeout=None): """ Blocking, synchronous polling of the Coordinator on the configured port. Single send operations greater than 0xFFFF are rejected to prevent truncation. :param msg_str: The message to send to the Coordinator, as a JSON string. :return: a JSON string of the response to the poll """ if not timeout: timeout = self.poll_timeout.duration if isinstance(timeout, float): timeout = int(timeout) elif not isinstance(timeout, int): raise ConfigurationError("Invalid timeout duration type: %s %s" % (type(timeout), timeout)) msg_len = len(message) if msg_len > 0xFFFE: raise JobError("Message was too long to send!") c_iter = 0 response = None delay = self.settings['poll_delay'] self.logger.debug("Connecting to LAVA Coordinator on %s:%s timeout=%d seconds.", self.settings['coordinator_hostname'], self.settings['port'], timeout) while True: c_iter += self.settings['poll_delay'] if self._connect(delay): delay = self.settings['poll_delay'] else: delay += 2 continue if not c_iter % int(10 * self.settings['poll_delay']): self.logger.debug("sending message: %s waited %s of %s seconds", json.loads(message)['request'], c_iter, timeout) # blocking synchronous call if not self._send_message(message): continue self.sock.shutdown(socket.SHUT_WR) response = self._recv_message() self.sock.close() try: json_data = json.loads(response) except ValueError: self.logger.debug("response starting '%s' was not JSON", response[:42]) self.finalise_protocol() break if json_data['response'] != 'wait': break else: time.sleep(delay) # apply the default timeout to each poll operation. if c_iter > timeout: self.finalise_protocol() raise MultinodeProtocolTimeoutError( "protocol %s timed out" % self.name) return response
def __init__(self, target): super().__init__({}) # Parse the yaml configuration try: if isinstance(target, str): with open(target) as f_in: data = f_in.read() data = yaml.safe_load(data) elif isinstance(target, dict): data = target else: data = target.read() data = yaml.safe_load(data) if data is None: raise ConfigurationError("Missing device configuration") self.update(data) except yaml.parser.ParserError: raise ConfigurationError("%s could not be parsed" % target) self.setdefault('power_state', 'off') # assume power is off at start of job
def validate(self): super().validate() cmd_name = self.parameters['name'] try: user_commands = self.job.device['commands']['users'] except KeyError: raise ConfigurationError( "Unable to get device.commands.users dictionary") try: self.cmd = user_commands[cmd_name] if not isinstance(self.cmd['do'], str) or \ not isinstance(self.cmd.get('undo', ""), str): raise ConfigurationError("User command \"%s\" is invalid: " "'do' and 'undo' should be strings" % cmd_name) return True except KeyError: self.errors = "Unknown user command '%s'" % cmd_name return False
def validate(self): super().validate() cmd_name = self.parameters["name"] try: user_commands = self.job.device["commands"]["users"] except KeyError: raise ConfigurationError( "Unable to get device.commands.users dictionary") try: self.cmd = user_commands[cmd_name] if not isinstance(self.cmd["do"], str) or not isinstance( self.cmd.get("undo", ""), str): raise ConfigurationError('User command "%s" is invalid: ' "'do' and 'undo' should be strings" % cmd_name) return True except KeyError: self.errors = "Unknown user command '%s'" % cmd_name return False
def validate(self): super().validate() if 'prompts' not in self.parameters: self.errors = "Unable to identify boot prompts from job definition." try: boot = self.job.device['actions']['boot']['methods']['qemu'] qemu_binary = which(boot['parameters']['command']) self.sub_command = [qemu_binary] self.sub_command.extend(boot['parameters'].get('options', [])) except AttributeError as exc: raise ConfigurationError(exc) except (KeyError, TypeError): self.errors = "Invalid parameters for %s" % self.name
def validate(self): super().validate() if "prompts" not in self.parameters: self.errors = "Unable to identify boot prompts from job definition." try: boot = self.job.device["actions"]["boot"]["methods"]["qemu"] qemu_binary = which(boot["parameters"]["command"]) self.sub_command = [qemu_binary] self.sub_command.extend(boot["parameters"].get("options", [])) except AttributeError as exc: raise ConfigurationError(exc) except (KeyError, TypeError): self.errors = "Invalid parameters for %s" % self.name
def parse(cls, data): """ Parsed timeouts can be set in device configuration or device_type configuration and can therefore exceed the clamp. """ if not isinstance(data, dict): raise ConfigurationError("Invalid timeout data") duration = datetime.timedelta(days=data.get('days', 0), hours=data.get('hours', 0), minutes=data.get('minutes', 0), seconds=data.get('seconds', 0)) if not duration: return Timeout.default_duration() return int(duration.total_seconds())
def __init__(self): import pymongo self.client = pymongo.MongoClient(settings.MONGO_DB_URI) try: # Test connection. # The ismaster command is cheap and does not require auth. self.client.admin.command("ismaster") except (pymongo.errors.ConnectionFailure, pymongo.errors.OperationFailure): raise ConfigurationError( "MongoDB URI is not configured properly. Unable to connect to MongoDB." ) self.db = self.client[settings.MONGO_DB_DATABASE] self.db.logs.create_index([("job_id", 1), ("dt", 1)]) super().__init__()
def validate(self): super().validate() try: boot = self.job.device['actions']['boot']['methods']['dfu'] dfu_binary = which(boot['parameters']['command']) self.base_command = [dfu_binary] self.base_command.extend(boot['parameters'].get('options', [])) if self.job.device['board_id'] == '0000000000': self.errors = "[FLASH_DFU] board_id unset" if self.job.device['usb_vendor_id'] == '0000': self.errors = '[FLASH_DFU] usb_vendor_id unset' if self.job.device['usb_product_id'] == '0000': self.errors = '[FLASH_DFU] usb_product_id unset' self.usb_vendor_id = self.job.device['usb_vendor_id'] self.usb_product_id = self.job.device['usb_product_id'] self.board_id = self.job.device['board_id'] self.base_command.extend(['--serial', self.board_id]) self.base_command.extend([ '--device', '%s:%s' % (self.usb_vendor_id, self.usb_product_id) ]) except AttributeError as exc: raise ConfigurationError(exc) except (KeyError, TypeError): self.errors = "Invalid parameters for %s" % self.name substitutions = {} for action in self.get_namespace_keys('download-action'): dfu_full_command = [] image_arg = self.get_namespace_data(action='download-action', label=action, key='image_arg') action_arg = self.get_namespace_data(action='download-action', label=action, key='file') if not image_arg or not action_arg: self.errors = "Missing image_arg for %s. " % action continue if not isinstance(image_arg, str): self.errors = "image_arg is not a string (try quoting it)" continue substitutions["{%s}" % action] = action_arg dfu_full_command.extend(self.base_command) dfu_full_command.extend(substitute([image_arg], substitutions)) self.exec_list.append(dfu_full_command) if len(self.exec_list) < 1: self.errors = "No DFU command to execute"
def validate(self): super().validate() try: boot = self.job.device["actions"]["boot"]["methods"]["dfu"] dfu_binary = which(boot["parameters"]["command"]) self.base_command = [dfu_binary] self.base_command.extend(boot["parameters"].get("options", [])) if self.job.device["board_id"] == "0000000000": self.errors = "[FLASH_DFU] board_id unset" if self.job.device["usb_vendor_id"] == "0000": self.errors = "[FLASH_DFU] usb_vendor_id unset" if self.job.device["usb_product_id"] == "0000": self.errors = "[FLASH_DFU] usb_product_id unset" self.usb_vendor_id = self.job.device["usb_vendor_id"] self.usb_product_id = self.job.device["usb_product_id"] self.board_id = self.job.device["board_id"] self.base_command.extend(["--serial", self.board_id]) self.base_command.extend([ "--device", "%s:%s" % (self.usb_vendor_id, self.usb_product_id) ]) except AttributeError as exc: raise ConfigurationError(exc) except (KeyError, TypeError): self.errors = "Invalid parameters for %s" % self.name substitutions = {} for action in self.get_namespace_keys("download-action"): dfu_full_command = [] image_arg = self.get_namespace_data(action="download-action", label=action, key="image_arg") action_arg = self.get_namespace_data(action="download-action", label=action, key="file") if not image_arg or not action_arg: self.errors = "Missing image_arg for %s. " % action continue if not isinstance(image_arg, str): self.errors = "image_arg is not a string (try quoting it)" continue substitutions["{%s}" % action] = action_arg dfu_full_command.extend(self.base_command) dfu_full_command.extend(substitute([image_arg], substitutions)) self.exec_list.append(dfu_full_command) if not self.exec_list: self.errors = "No DFU command to execute"
def create_custom_job(self, template, job_data): job_ctx = job_data.get('context') (data, device_dict) = self.create_device(template, job_ctx) device = NewDevice(yaml.safe_load(data)) if self.debug: print('####### Device configuration #######') print(data) print('#######') try: parser = JobParser() job = parser.parse(yaml.dump(job_data), device, 4999, None, "") except (ConfigurationError, TypeError) as exc: print('####### Parser exception ########') print(device) print('#######') raise ConfigurationError("Invalid device: %s" % exc) job.logger = DummyLogger() return job
def run(self, connection, max_end_time): connection = super().run(connection, max_end_time) if not connection: raise LAVABug("Cannot transfer overlay, no connection available.") ip_addr = dispatcher_ip(self.job.parameters['dispatcher']) overlay_full_path = self.get_namespace_data(action='compress-overlay', label='output', key='file') if not overlay_full_path: raise JobError("No overlay file identified for the transfer.") if not overlay_full_path.startswith(DISPATCHER_DOWNLOAD_DIR): raise ConfigurationError( "overlay should already be in DISPATCHER_DOWNLOAD_DIR") overlay_path = overlay_full_path[len(DISPATCHER_DOWNLOAD_DIR) + 1:] overlay = os.path.basename(overlay_path) dwnld = self.parameters['transfer_overlay']['download_command'] dwnld += " http://%s/tmp/%s" % (ip_addr, overlay_path) unpack = self.parameters['transfer_overlay']['unpack_command'] unpack += ' ' + overlay connection.sendline("rm %s; %s && %s" % (overlay, dwnld, unpack)) return connection
for doc in docs: doc_dict = doc.to_dict() result.append( json.dumps({ "dt": doc.id, "lvl": doc_dict["lvl"], "msg": doc_dict["msg"] })) return "\n".join(["- %s" % x for x in result]) def size(self, job, start=0, end=None): # TODO: should be implemented. return None def write(self, job, line, output=None, idx=None): line = yaml_load(line)[0] doc_ref = (self.db.collection(self.root_collection).document( "%02d-%02d-%02d" % (job.submit_time.year, job.submit_time.month, job.submit_time.day)).collection(str( job.id)).document(line["dt"])) doc_ref.set({"lvl": line["lvl"], "msg": line["msg"]}) logs_backend_str = settings.LAVA_LOG_BACKEND.rsplit(".", 1) try: logs_class = getattr(import_module(logs_backend_str[0]), logs_backend_str[1]) except (AttributeError, ModuleNotFoundError) as exc: raise ConfigurationError(str(exc)) logs_instance = logs_class()