def handle_command(self, data: bytes, stream: Stream) -> None: """ Handles an osp command in a string. """ try: tree = secET.fromstring(data) except secET.ParseError: logger.debug("Erroneous client input: %s", data) raise OspdCommandError('Invalid data') command_name = tree.tag logger.debug('Handling %s command request.', command_name) command = self.commands.get(command_name, None) if not command and command_name != "authenticate": raise OspdCommandError('Bogus command name') if not self.initialized and command.must_be_initialized: exception = OspdCommandError( '%s is still starting' % self.daemon_info['name'], 'error' ) response = exception.as_xml() stream.write(response) return response = command.handle_xml(tree) if isinstance(response, bytes): stream.write(response) else: for data in response: stream.write(data)
def preprocess_scan_params(self, xml_params): """ Processes the scan parameters. """ params = {} for param in xml_params: params[param.tag] = param.text or '' # Validate values. for key in params: param_type = self.get_scanner_param_type(key) if not param_type: continue if param_type in ['integer', 'boolean']: try: params[key] = int(params[key]) except ValueError: raise OspdCommandError('Invalid %s value' % key, 'start_scan') from None if param_type == 'boolean': if params[key] not in [0, 1]: raise OspdCommandError('Invalid %s value' % key, 'start_scan') elif param_type == 'selection': selection = self.get_scanner_param_default(key).split('|') if params[key] not in selection: raise OspdCommandError('Invalid %s value' % key, 'start_scan') if self.get_scanner_param_mandatory(key) and params[key] == '': raise OspdCommandError('Mandatory %s value is missing' % key, 'start_scan') return params
def parse_filters(self, vt_filter: str) -> List: """ Parse a string containing one or more filters and return a list of filters Arguments: vt_filter (string): String containing filters separated with semicolon. Return: List with filters. Each filters is a list with 3 elements e.g. [arg, operator, value] """ filter_list = vt_filter.split(';') filters = list() for single_filter in filter_list: filter_aux = re.split(r'(\W)', single_filter, 1) if len(filter_aux) < 3: raise OspdCommandError( "Invalid number of argument in the filter", "get_vts" ) _element, _oper, _val = filter_aux if _element not in self.allowed_filter: raise OspdCommandError("Invalid filter element", "get_vts") if _oper not in self.filter_operator: raise OspdCommandError("Invalid filter operator", "get_vts") filters.append(filter_aux) return filters
def store_data(self, filename: str, data_object: Any) -> str: """ Pickle a object and store it in a file named""" storage_file_path = Path(self._storage_path) / filename try: # create parent directories recursively parent_dir = storage_file_path.parent parent_dir.mkdir(parents=True, exist_ok=True) except Exception as e: # pylint: disable=broad-except raise OspdCommandError( 'Not possible to access dir for %s. %s' % (filename, e), 'start_scan', ) try: pickled_data = pickle.dumps(data_object) except pickle.PicklingError as e: raise OspdCommandError( 'Not possible to pickle scan info for %s. %s' % (filename, e), 'start_scan', ) try: with open(str(storage_file_path), 'wb', opener=self._fd_opener) as scan_info_f: scan_info_f.write(pickled_data) except Exception as e: # pylint: disable=broad-except self._fd_close() raise OspdCommandError( 'Not possible to store scan info for %s. %s' % (filename, e), 'start_scan', ) self._fd_close() return self._pickled_data_hash_generator(pickled_data)
def stop_scan(self, scan_id: str) -> None: scan_process = self.scan_processes.get(scan_id) if not scan_process: raise OspdCommandError('Scan not found {0}.'.format(scan_id), 'stop_scan') if not scan_process.is_alive(): raise OspdCommandError('Scan already stopped or finished.', 'stop_scan') self.set_scan_status(scan_id, ScanStatus.STOPPED) logger.info('%s: Scan stopping %s.', scan_id, scan_process.ident) self.stop_scan_cleanup(scan_id) try: scan_process.terminate() except AttributeError: logger.debug('%s: The scanner task stopped unexpectedly.', scan_id) try: _terminate_process_group(scan_process) except ProcessLookupError as e: logger.info('%s: Scan already stopped %s.', scan_id, scan_process.pid) if scan_process.ident != os.getpid(): scan_process.join(0) logger.info('%s: Scan stopped.', scan_id)
def handle_command(self, data: bytes, stream: Stream) -> None: """Handles an osp command in a string.""" try: tree = secET.fromstring(data) except secET.ParseError as e: logger.debug("Erroneous client input: %s", data) raise OspdCommandError('Invalid data') from e command_name = tree.tag logger.debug('Handling %s command request.', command_name) command = self.commands.get(command_name, None) if not command and command_name != "authenticate": raise OspdCommandError('Bogus command name') if not self.initialized and command.must_be_initialized: exception = OspdCommandError( f'{self.daemon_info["name"]} is still starting', 'error') response = exception.as_xml() stream.write(response) return response = command.handle_xml(tree) write_success = True if isinstance(response, bytes): write_success = stream.write(response) else: for data in response: write_success = stream.write(data) if not write_success: break scan_id = tree.get('scan_id') if self.scan_exists(scan_id) and command_name == "get_scans": if write_success: logger.debug( '%s: Results sent successfully to the client. Cleaning ' 'temporary result list.', scan_id, ) self.scan_collection.clean_temp_result_list(scan_id) else: logger.debug( '%s: Failed sending results to the client. Restoring ' 'result list into the cache.', scan_id, ) self.scan_collection.restore_temp_result_list(scan_id)
def handle_xml(self, xml: Element) -> bytes: """Handles <get_performance> command. @return: Response string for <get_performance> command. """ start = xml.attrib.get('start') end = xml.attrib.get('end') titles = xml.attrib.get('titles') cmd = ['gvmcg'] if start: try: int(start) except ValueError: raise OspdCommandError( 'Start argument must be integer.', 'get_performance' ) from None cmd.append(start) if end: try: int(end) except ValueError: raise OspdCommandError( 'End argument must be integer.', 'get_performance' ) from None cmd.append(end) if titles: combined = "(" + ")|(".join(GVMCG_TITLES) + ")" forbidden = "^[^|&;]+$" if re.match(combined, titles) and re.match(forbidden, titles): cmd.append(titles) else: raise OspdCommandError( 'Arguments not allowed', 'get_performance' ) try: output = subprocess.check_output(cmd) except (subprocess.CalledProcessError, OSError) as e: raise OspdCommandError( 'Bogus get_performance format. %s' % e, 'get_performance' ) from None return simple_response_str( 'get_performance', 200, 'OK', output.decode() )
def get_filtered_vts_list(self, vts, vt_filter): """ Gets a collection of vulnerability test from the vts dictionary, which match the filter. Arguments: vt_filter (string): Filter to apply to the vts collection. vts (dictionary): The complete vts collection. Returns: Dictionary with filtered vulnerability tests. """ if not vt_filter: raise OspdCommandError('vt_filter: A valid filter is required.') filters = self.parse_filters(vt_filter) if not filters: return None _vts_aux = vts.copy() for _element, _oper, _filter_val in filters: for vt_id in _vts_aux.copy(): if not _vts_aux[vt_id].get(_element): _vts_aux.pop(vt_id) continue _elem_val = _vts_aux[vt_id].get(_element) _val = self.format_filter_value(_element, _elem_val) if self.filter_operator[_oper](_val, _filter_val): continue else: _vts_aux.pop(vt_id) return _vts_aux
def start_scan(self, scan_id: str, target: Dict) -> None: """ Starts the scan with scan_id. """ os.setsid() if target is None or not target: raise OspdCommandError('Erroneous target', 'start_scan') logger.info("%s: Scan started.", scan_id) self.process_exclude_hosts(scan_id, target.get('exclude_hosts')) self.process_finished_hosts(scan_id, target.get('finished_hosts')) try: self.set_scan_status(scan_id, ScanStatus.RUNNING) ret = self.exec_scan(scan_id) except Exception as e: # pylint: disable=broad-except self.add_scan_error( scan_id, name='', host=self.get_scan_host(scan_id), value='Host process failure (%s).' % e, ) logger.exception('While scanning: %s', scan_id) else: logger.info("%s: Host scan finished.", scan_id) if self.get_scan_status(scan_id) != ScanStatus.STOPPED: self.finish_scan(scan_id)
def unpickle_scan_info(self, scan_id: str) -> None: """Unpickle a stored scan_inf corresponding to the scan_id and store it in the scan_table""" scan_info = self.scans_table.get(scan_id) scan_info_hash = scan_info.pop('scan_info_hash') pickler = DataPickler(self.file_storage_dir) unpickled_scan_info = pickler.load_data(scan_id, scan_info_hash) if not unpickled_scan_info: pickler.remove_file(scan_id) raise OspdCommandError( 'Not possible to unpickle stored scan info for %s' % scan_id, 'start_scan', ) scan_info['results'] = list() scan_info['temp_results'] = list() scan_info['progress'] = ScanProgress.INIT.value scan_info['target_progress'] = dict() scan_info['count_alive'] = 0 scan_info['count_dead'] = 0 scan_info['count_total'] = None scan_info['excluded_simplified'] = None scan_info['target'] = unpickled_scan_info.pop('target') scan_info['vts'] = unpickled_scan_info.pop('vts') scan_info['options'] = unpickled_scan_info.pop('options') scan_info['start_time'] = int(time.time()) scan_info['end_time'] = 0 self.scans_table[scan_id] = scan_info pickler.remove_file(scan_id)
def stop_scan(self, scan_id: str) -> None: if ( scan_id in self.scan_collection.ids_iterator() and self.get_scan_status(scan_id) == ScanStatus.QUEUED ): logger.info('Scan %s has been removed from the queue.', scan_id) self.scan_collection.remove_file_pickled_scan_info(scan_id) self.set_scan_status(scan_id, ScanStatus.STOPPED) return scan_process = self.scan_processes.get(scan_id) if not scan_process: raise OspdCommandError( 'Scan not found {0}.'.format(scan_id), 'stop_scan' ) if not scan_process.is_alive(): raise OspdCommandError( 'Scan already stopped or finished.', 'stop_scan' ) self.set_scan_status(scan_id, ScanStatus.STOPPED) logger.info( '%s: Stopping Scan with the PID %s.', scan_id, scan_process.ident ) self.stop_scan_cleanup(scan_id) try: scan_process.terminate() except AttributeError: logger.debug('%s: The scanner task stopped unexpectedly.', scan_id) try: _terminate_process_group(scan_process) except ProcessLookupError: logger.info( '%s: Scan with the PID %s is already stopped.', scan_id, scan_process.pid, ) if scan_process.ident != os.getpid(): scan_process.join(0) logger.info('%s: Scan stopped.', scan_id)
def handle_client_stream(self, stream: Stream) -> None: """ Handles stream of data received from client. """ data = b'' request_parser = RequestParser() while True: try: buf = stream.read() if not buf: break data += buf if request_parser.has_ended(buf): break except (AttributeError, ValueError) as message: logger.error(message) return except (ssl.SSLError) as exception: logger.debug('Error: %s', exception) break except (socket.timeout) as exception: logger.debug('Request timeout: %s', exception) break if len(data) <= 0: logger.debug("Empty client stream") return response = None try: self.handle_command(data, stream) except OspdCommandError as exception: response = exception.as_xml() logger.debug('Command error: %s', exception.message) except Exception: # pylint: disable=broad-except logger.exception('While handling client command:') exception = OspdCommandError('Fatal error', 'error') response = exception.as_xml() if response: stream.write(response) stream.close()
def set_feed_vendor(self, feed_vendor: str) -> None: """Set the feed vendor. Parameters: feed_home (str): Identifies the feed home. """ if not feed_vendor: raise OspdCommandError('A feed vendor parameter is required', 'set_feed_vendor') self.feed_vendor = feed_vendor
def set_feed_name(self, feed_name: str) -> None: """Set the feed name. Parameters: feed_name (str): Identifies the feed name. """ if not feed_name: raise OspdCommandError('A feed name parameter is required', 'set_feed_name') self.feed_name = feed_name
def set_vts_version(self, vts_version: str) -> None: """Add into the vts dictionary an entry to identify the vts version. Parameters: vts_version (str): Identifies a unique vts version. """ if not vts_version: raise OspdCommandError('A vts_version parameter is required', 'set_vts_version') self.vts_version = vts_version
def handle_xml(self, xml: Element) -> bytes: help_format = xml.get('format') if help_format is None or help_format == "text": # Default help format is text. return simple_response_str('help', 200, 'OK', self._daemon.get_help_text()) elif help_format == "xml": text = get_elements_from_dict( {k: v.as_dict() for k, v in self._daemon.commands.items()}) return simple_response_str('help', 200, 'OK', text) raise OspdCommandError('Bogus help format', 'help')
def handle_xml(self, xml: Element) -> Iterator[bytes]: """ Handles <get_vts> command. Writes the vt collection on the stream. The <get_vts> element accept two optional arguments. vt_id argument receives a single vt id. filter argument receives a filter selecting a sub set of vts. If both arguments are given, the vts which match with the filter are return. @return: Response string for <get_vts> command on fail. """ xml_helper = XmlStringHelper() vt_id = xml.get('vt_id') vt_filter = xml.get('filter') _details = xml.get('details') vt_details = False if _details == '0' else True if vt_id and vt_id not in self._daemon.vts: text = "Failed to find vulnerability test '{0}'".format(vt_id) raise OspdCommandError(text, 'get_vts', 404) filtered_vts = None if vt_filter: filtered_vts = self._daemon.vts_filter.get_filtered_vts_list( self._daemon.vts, vt_filter) vts_selection = self._daemon.get_vts_selection_list( vt_id, filtered_vts) # List of xml pieces with the generator to be iterated yield xml_helper.create_response('get_vts') begin_vts_tag = xml_helper.create_element('vts') val = len(self._daemon.vts) begin_vts_tag = xml_helper.add_attr(begin_vts_tag, "total", val) if filtered_vts: val = len(filtered_vts) begin_vts_tag = xml_helper.add_attr(begin_vts_tag, "sent", val) if self._daemon.vts.sha256_hash is not None: begin_vts_tag = xml_helper.add_attr(begin_vts_tag, "sha256_hash", self._daemon.vts.sha256_hash) yield begin_vts_tag for vt in self._daemon.get_vt_iterator(vts_selection, vt_details): yield xml_helper.add_element(self._daemon.get_vt_xml(vt)) yield xml_helper.create_element('vts', end=True) yield xml_helper.create_response('get_vts', end=True)
def handle_command(self, data: bytes, stream: Stream) -> None: """ Handles an osp command in a string. """ try: tree = secET.fromstring(data) except secET.ParseError: logger.debug("Erroneous client input: %s", data) raise OspdCommandError('Invalid data') command_name = tree.tag logger.debug('Handling %s command request.', command_name) command = self.commands.get(command_name, None) if not command and command_name != "authenticate": raise OspdCommandError('Bogus command name') response = command.handle_xml(tree) if isinstance(response, bytes): stream.write(response) else: for data in response: stream.write(data)
def handle_xml(self, xml: Element) -> bytes: """Handles <stop_scan> command. @return: Response string for <stop_scan> command. """ scan_id = xml.get('scan_id') if scan_id is None or scan_id == '': raise OspdCommandError('No scan_id attribute', 'stop_scan') self._daemon.stop_scan(scan_id) # Don't send response until the scan is stopped. try: self._daemon.scan_processes[scan_id].join() except KeyError: pass return simple_response_str('stop_scan', 200, 'OK')
def get_filtered_vts_list( self, vts: Vts, vt_filter: str ) -> Optional[List[str]]: """ Gets a collection of vulnerability test from the vts dictionary, which match the filter. Arguments: vt_filter: Filter to apply to the vts collection. vts: The complete vts collection. Returns: List with filtered vulnerability tests. The list can be empty. None in case of filter parse failure. """ if not vt_filter: raise OspdCommandError('vt_filter: A valid filter is required.') filters = self.parse_filters(vt_filter) if not filters: return None vt_oid_list = list(vts) for _element, _oper, _filter_val in filters: for vt_oid in vts: if vt_oid not in vt_oid_list: continue vt = vts.get(vt_oid) if vt is None or not vt.get(_element): vt_oid_list.remove(vt_oid) continue _elem_val = vt.get(_element) _val = self.format_filter_value(_element, _elem_val) if self.filter_operator[_oper](_val, _filter_val): continue else: vt_oid_list.remove(vt_oid) return vt_oid_list
def handle_xml(self, xml: Element) -> bytes: """ Handles <delete_scan> command. @return: Response string for <delete_scan> command. """ scan_id = xml.get('scan_id') if scan_id is None: return simple_response_str('delete_scan', 404, 'No scan_id attribute') if not self._daemon.scan_exists(scan_id): text = "Failed to find scan '{0}'".format(scan_id) return simple_response_str('delete_scan', 404, text) self._daemon.check_scan_process(scan_id) if self._daemon.delete_scan(scan_id): return simple_response_str('delete_scan', 200, 'OK') raise OspdCommandError('Scan in progress', 'delete_scan')
def handle_xml(self, xml: Element) -> bytes: """Handles <check_feed> command. Return: Response string for <check_feed> command. """ feed = Element('feed') feed_status = self._daemon.check_feed_self_test() if not feed_status or not isinstance(feed_status, dict): raise OspdCommandError('No feed status available', 'check_feed') for key, value in feed_status.items(): elem = SubElement(feed, key) elem.text = value content = [feed] return simple_response_str('check_feed', 200, 'OK', content)
def handle_xml(self, xml: Element) -> bytes: """Handles <get_scans> command. @return: Response string for <get_scans> command. """ scan_id = xml.get('scan_id') if scan_id is None or scan_id == '': raise OspdCommandError('No scan_id attribute', 'get_scans') details = xml.get('details') pop_res = xml.get('pop_results') max_res = xml.get('max_results') progress = xml.get('progress') if details and details == '0': details = False else: details = True pop_res = pop_res and pop_res == '1' if max_res: max_res = int(max_res) progress = progress and progress == '1' responses = [] if scan_id in self._daemon.scan_collection.ids_iterator(): self._daemon.check_scan_process(scan_id) scan = self._daemon.get_scan_xml( scan_id, details, pop_res, max_res, progress ) responses.append(scan) else: text = "Failed to find scan '{0}'".format(scan_id) return simple_response_str('get_scans', 404, text) return simple_response_str('get_scans', 200, 'OK', responses)
def test_default_params(self): e = OspdCommandError('message') self.assertEqual('message', e.message) self.assertEqual(400, e.status) self.assertEqual('osp', e.command)
def handle_xml(self, xml: Element) -> bytes: """Handles <start_scan> command. Return: Response string for <start_scan> command. """ current_queued_scans = self._daemon.get_count_queued_scans() if ( self._daemon.max_queued_scans and current_queued_scans >= self._daemon.max_queued_scans ): logger.info( 'Maximum number of queued scans set to %d reached.', self._daemon.max_queued_scans, ) raise OspdCommandError( 'Maximum number of queued scans set to %d reached.' % self._daemon.max_queued_scans, 'start_scan', ) target_str = xml.get('target') ports_str = xml.get('ports') # For backward compatibility, if target and ports attributes are set, # <targets> element is ignored. if target_str is None or ports_str is None: target_element = xml.find('targets/target') if target_element is None: raise OspdCommandError('No targets or ports', 'start_scan') else: scan_target = OspRequest.process_target_element(target_element) else: scan_target = { 'hosts': target_str, 'ports': ports_str, 'credentials': {}, 'exclude_hosts': '', 'finished_hosts': '', 'options': {}, } logger.warning( "Legacy start scan command format is being used, which " "is deprecated since 20.08. Please read the documentation " "for start scan command." ) scan_id = xml.get('scan_id') if scan_id is not None and scan_id != '' and not valid_uuid(scan_id): raise OspdCommandError('Invalid scan_id UUID', 'start_scan') if xml.get('parallel'): logger.warning( "parallel attribute of start_scan will be ignored, sice " "parallel scan is not supported by OSPd." ) scanner_params = xml.find('scanner_params') if scanner_params is None: raise OspdCommandError('No scanner_params element', 'start_scan') # params are the parameters we got from the <scanner_params> XML. params = self._daemon.preprocess_scan_params(scanner_params) # VTS is an optional element. If present should not be empty. vt_selection = {} # type: Dict scanner_vts = xml.find('vt_selection') if scanner_vts is not None: if len(scanner_vts) == 0: raise OspdCommandError('VTs list is empty', 'start_scan') else: vt_selection = OspRequest.process_vts_params(scanner_vts) scan_params = self._daemon.process_scan_params(params) scan_id_aux = scan_id scan_id = self._daemon.create_scan( scan_id, scan_target, scan_params, vt_selection ) if not scan_id: id_ = Element('id') id_.text = scan_id_aux return simple_response_str('start_scan', 100, 'Continue', id_) logger.info( 'Scan %s added to the queue in position %d.', scan_id, current_queued_scans + 1, ) id_ = Element('id') id_.text = scan_id return simple_response_str('start_scan', 200, 'OK', id_)
def test_string_conversion(self): e = OspdCommandError('message foo bar', 'command', '304') self.assertEqual('message foo bar', str(e))
def test_is_ospd_error(self): e = OspdCommandError('message') self.assertIsInstance(e, OspdError)
def test_as_xml(self): e = OspdCommandError('message') self.assertEqual( b'<osp_response status="400" status_text="message" />', e.as_xml())
def test_constructor(self): e = OspdCommandError('message', 'command', '304') self.assertEqual('message', e.message) self.assertEqual('command', e.command) self.assertEqual('304', e.status)
def handle_xml(self, xml: Element) -> Iterator[bytes]: """Handles <get_vts> command. Writes the vt collection on the stream. The <get_vts> element accept two optional arguments. vt_id argument receives a single vt id. filter argument receives a filter selecting a sub set of vts. If both arguments are given, the vts which match with the filter are return. @return: Response string for <get_vts> command on fail. """ self._daemon.vts.is_cache_available = False xml_helper = XmlStringHelper() vt_id = xml.get('vt_id') vt_filter = xml.get('filter') _details = xml.get('details') version_only = xml.get('version_only') vt_details = False if _details == '0' else True if self._daemon.vts and vt_id and vt_id not in self._daemon.vts: self._daemon.vts.is_cache_available = True text = f"Failed to find vulnerability test '{vt_id}'" raise OspdCommandError(text, 'get_vts', 404) filtered_vts = None if vt_filter and not version_only: try: filtered_vts = self._daemon.vts_filter.get_filtered_vts_list( self._daemon.vts, vt_filter) except OspdCommandError as filter_error: self._daemon.vts.is_cache_available = True raise filter_error if not version_only: vts_selection = self._daemon.get_vts_selection_list( vt_id, filtered_vts) # List of xml pieces with the generator to be iterated yield xml_helper.create_response('get_vts') begin_vts_tag = xml_helper.create_element('vts') # Add Feed information as attributes begin_vts_tag = xml_helper.add_attr(begin_vts_tag, "vts_version", self._daemon.get_vts_version()) begin_vts_tag = xml_helper.add_attr(begin_vts_tag, "feed_vendor", self._daemon.get_feed_vendor()) begin_vts_tag = xml_helper.add_attr(begin_vts_tag, "feed_home", self._daemon.get_feed_home()) begin_vts_tag = xml_helper.add_attr(begin_vts_tag, "feed_name", self._daemon.get_feed_name()) val = len(self._daemon.vts) begin_vts_tag = xml_helper.add_attr(begin_vts_tag, "total", val) if filtered_vts and not version_only: val = len(filtered_vts) begin_vts_tag = xml_helper.add_attr(begin_vts_tag, "sent", val) if self._daemon.vts.sha256_hash is not None: begin_vts_tag = xml_helper.add_attr(begin_vts_tag, "sha256_hash", self._daemon.vts.sha256_hash) yield begin_vts_tag if not version_only: for vt in self._daemon.get_vt_iterator(vts_selection, vt_details): yield xml_helper.add_element(self._daemon.get_vt_xml(vt)) yield xml_helper.create_element('vts', end=True) yield xml_helper.create_response('get_vts', end=True) self._daemon.vts.is_cache_available = True