def get_subpack_info(self, pkgname, database, pkgname_info): """ get subpacks info for source package Args: pkgname: package name database: database pkgname_info: package info which include subpacks Returns: subpacks: subpack list """ subpacks_bin_list = pkgname_info[pkgname].get('subpacks') if not subpacks_bin_list: _msg = "Error in getting subpack info." LOGGER.error(_msg) return [] subpacks = [] for subpack_name in subpacks_bin_list: subpack_dict = { "bin_name": subpack_name, "provides": self.get_provides(subpack_name, database), "requires": self.get_requires(subpack_name, database) } subpacks.append(subpack_dict) return subpacks
def _import_error(self, error, record=True): if record: LOGGER.error(error) self._fail.append(self.elastic_index) fails = self._delete_index() if fails: LOGGER.warning("Delete the failed ES database:%s ." % fails)
def get_build_info(self, pkgname, database): """ get build dependency info Args: pkgname: package name database: database Returns: build_depend_info: build dependency for source package Attributes: TypeError: object does not support this property or method """ try: build_obj = BuildRequires([database]) build_depend = build_obj.get_build_req([pkgname], database) if not build_depend: return [] build_depend_info = build_depend[0].get("requires") build_package_list = [] for requires_info in build_depend_info: build_package_list.append(requires_info.get("component")) return build_package_list except (TypeError, AttributeError) as e: LOGGER.error(e) return []
def load_config(self, path): """ Description: Read the contents of the configuration file load each node data in the yaml configuration file as a list to return Args: path: Initialize the repo source configuration file for the data Returns: Initialize the contents of the database configuration file Raises: FileNotFoundError: The specified file does not exist TypeError: Wrong type of data """ if not path: raise ValueError( "The configuration source for the data dependency initialization is not specified" ) if not os.path.exists(path): raise FileNotFoundError("system initialization configuration file " "does not exist: %s" % path) # load yaml configuration file with open(path, "r", encoding="utf-8") as file_context: try: self._repo = yaml.load(file_context.read(), Loader=yaml.FullLoader) except yaml.YAMLError as yaml_error: LOGGER.error(yaml_error) raise ValueError( "The format of the yaml configuration " "file is wrong please check and try again:{0}".format( yaml_error)) from yaml_error
def __init__(self, db_engine=None, host=None, port=None): self.db_engine = db_engine or "elastic" if self.db_engine not in self.__DATABASE_ENGINE_TYPE: LOGGER.error("DataBase %s is not support" % self.db_engine) raise DatabaseConfigException() self._host = host or configuration.DATABASE_HOST self._port = port or configuration.DATABASE_PORT self.session = None
def src_package_info(self, src_name_list, database_list=None): """ get a source package info (provides, requires, etc) Args: src_name_list: source package list database_list: database list Returns: src_package_info_res Attributes: AttributeError: Cannot find the attribute of the corresponding object IndexError: list index out of range TypeError: object does not support this property or method Raises: ElasticSearchQueryException: dataBase connect failed DatabaseConfigException: dataBase config error """ try: src_package_info_res = {} query_package = QueryPackage() for database in database_list: single_db_src_info = query_package.get_src_info( src_name_list, database, 1, 20).get("data") if not single_db_src_info: return {} database_src_info_list = [] for pkg_info in single_db_src_info: pkgname = list(pkg_info.keys())[0] build_package_info = self.get_build_info(pkgname, database) subpacks = self.get_subpack_info(pkgname, database, pkg_info) database_src_info_list.append({ "src_name": pkgname, "license": pkg_info[pkgname].get("license"), "version": pkg_info[pkgname].get("version"), "url": pkg_info[pkgname].get("url"), "summary": pkg_info[pkgname].get("summary"), "description": pkg_info[pkgname].get("description"), "build_dep": build_package_info, "subpacks": subpacks }) src_package_info_res[database] = database_src_info_list return src_package_info_res except (AttributeError, IndexError, TypeError) as e: LOGGER.error(e) return {}
def _redis(): try: if REDIS_CONN.keys("pkgship_*"): REDIS_CONN.delete(*REDIS_CONN.keys("pkgship_*")) except redis.RedisError: LOGGER.error( "There is an exception in Redis service. Please check it later." " After restart , please check the key value with pkgship_ prefix" )
def __init__(self, host=None, port=None): self._host = host self._port = port try: self.client = Elasticsearch( [{"host": self._host, "port": self._port}], timeout=60) except LocationValueError: LOGGER.error("The host of database in package.ini is empty") raise DatabaseConfigException()
def update_setting(self): """ Update the ES configuration, currently the maximum number of modified queries :return: None """ try: self.client.indices.put_settings(index='_all', body={'index': { 'max_result_window': MAX_ES_QUERY_NUM}}) except ElasticsearchException: LOGGER.error('Set max_result_window of all indies failed')
def inner(*args, **kwargs): try: return func(*args, **kwargs) except (KeyError, TypeError, ValueError, AttributeError, IndexError, TypeError) as error: LOGGER.error(error) raise ValueError("input json_data is error! please check") except (IOError, OSError) as error: LOGGER.error(error) raise ValueError("input json_data is error! please check")
def all_bin_packages(self, database, page_num=1, page_size=20, package_list=None, command_line=False): """ get all binary package info Args: database: database page_num: paging query index page_size: paging query size package_list: package list name command_line: command_line or UI Returns: all_bin_dict: all binary package information dict for example:: { "total": "", "data": "" } Attributes: AttributeError: Cannot find the attribute of the corresponding object KeyError: key not exists TypeError: object does not support this property or method Raises: ElasticSearchQueryException: dataBase connect failed DatabaseConfigException: dataBase config error """ try: # query all binary package info from database query_package = QueryPackage() all_bin_info = query_package.get_bin_info(package_list, database, page_num, page_size, command_line) if not all_bin_info["data"]: _msg = "An error occurred when getting bianry package info." raise PackageInfoGettingError(_msg) parsed_all_bin_info = [] total_num = all_bin_info.get("total") for pkg_info in all_bin_info.get("data"): single_pkg = [] for pkgname, info_values in pkg_info.items(): single_pkg = self.__parse_pkg_info(info_values, pkgname, database) single_pkg["source_name"] = info_values["src_name"] parsed_all_bin_info.append(single_pkg) all_bin_dict = {"total": total_num, "data": parsed_all_bin_info} return all_bin_dict except (AttributeError, KeyError, TypeError) as e: LOGGER.error(e) return {}
def create_engine(db_engine, **kwargs): """ Get database class by engines name """ try: module = importlib.import_module("packageship.application.database.engines.%s" % db_engine) except ModuleNotFoundError: # Log here, the caller throws an exception LOGGER.error("DataBase engine module not exist") return None cls = getattr(module, db_engine) return cls(**kwargs)
def _repo_files(self): xml = True if "db_file" in self._repo else False try: self._find_files(xml=xml) except (FileNotFoundError, ValueError) as error: LOGGER.error(error) return False if xml: exists = all([self._repo["file_list"], self._repo["db_file"]]) else: exists = self._sqlite() return exists
def get_provides(self, pkgname, database): """ get provides info from bedepend query result Args: pkgname: package name database: database Returns: provides_info_res: which include component, required_by_bin, required_by_src for example: [{'component': 'comp1', 'required_by_bin': ['A1', 'B1'], 'required_by_src': ['T', 'R']}] """ try: # getting bedend query res be_depend_obj = BeDependRequires() be_depend_info = be_depend_obj.get_be_req([pkgname], database) if not be_depend_info: return [] provides_info_res = [] depend_info = be_depend_info[0].get("provides") for component_info in depend_info: # getting required_by_bin install_require_info = component_info.get("install_require") required_by_bin = [] for req_info in install_require_info: if req_info.get("req_bin_name"): required_by_bin.append(req_info.get("req_bin_name")) # getting required_by_src build_require_info = component_info.get("build_require") required_by_src = [] for req_info in build_require_info: if req_info.get("req_src_name"): required_by_src.append(req_info.get("req_src_name")) component_dict = { "component": component_info.get("component"), "required_by_bin": required_by_bin, "required_by_src": required_by_src } provides_info_res.append(component_dict) return provides_info_res except (TypeError, IndexError) as e: LOGGER.error(e) return []
def _delete_file_path(self, folder_path): """ Delete folder Args: folder_path: Folder path Returns: None """ try: if folder_path and os.path.exists(folder_path): shutil.rmtree(folder_path) except (IOError, OSError) as error: LOGGER.error(error) raise IOError("Failed to delete folder")
def connection(self): """ Returns: specify database connection Raises: DatabaseConfigException, database class not found """ self.session = engine.create_engine(db_engine=self.db_engine, host=self._host, port=self._port) if self.session is None: LOGGER.error( "Failed to create database engine %s, please check database configuration" % self.db_engine) raise DatabaseConfigException() return self.session
def count(self, index, body): """ Obtain data volume of specify index Args: index: index of elasticsearch body: query body of elasticsearch Returns: data volume Raises: ElasticSearchQueryException,including connection timeout, server unreachable, index does not exist, etc. """ try: data_count = self.client.count(index=index, body=body) return data_count except ElasticsearchException as e: LOGGER.error(str(e)) raise ElasticSearchQueryException()
def query(self, index, body): """ Elasticsearch query function Args: index: index of elasticsearch body: query body of elasticsearch Returns: elasticsearch data Raises: ElasticSearchQueryException,including connection timeout, server unreachable, index does not exist, etc. """ try: result = self.client.search(index=index, body=body) return result except ElasticsearchException as e: LOGGER.error(str(e)) raise ElasticSearchQueryException()
def insert(self, index, body, doc_type="_doc"): """ Single insert ES data Args: index: Index to ES database body: inserted data set doc_type: document type Returns: Raises: ElasticSearchQueryException,including connection timeout, server unreachable, index does not exist, etc. """ try: self.client.index(index=index, body=body, doc_type=doc_type) except ElasticsearchException as e: LOGGER.error(str(e)) raise ElasticSearchQueryException()
def del_temporary_file(path, folder=False): """ Description: Delete temporary files or folders Args: path: temporary files or folders folder: file or folder, fsiles are deleted by default """ if not os.path.exists(path): return try: if folder: shutil.rmtree(path) else: os.remove(path) except IOError as error: LOGGER.error(error)
def import_depend(self, path=None): """ Description: Initializes import dependency data Args: path: repo source file """ # Initialize the judgment of the process self._process() if not path: path = configuration.INIT_CONF_PATH try: self._config.load_config(path) except (ValueError, FileNotFoundError) as error: raise InitializeError(str(error)) from error if not self._config.validate: raise InitializeError(self._config.message) self._clear_all_index() # Clear the cached value for a particular key self._redis() for repo in self._config: self._data = ESJson() self._repo = repo try: if not self._repo_files(): raise RepoError("Repo source data error: %s" % self.elastic_index) xml_files = self._xml() if xml_files["xml"] and xml_files["filelist"]: self._xml_parse(**xml_files) self._save() self._session.update_setting() except (RepoError, ElasticsearchException) as error: LOGGER.error(error) self._fail.append(self.elastic_index) if isinstance(error, ElasticsearchException): self._delete_index() finally: # delete temporary directory del_temporary_file(configuration.TEMPORARY_DIRECTORY, folder=True) self._repo = None
def scan(self, index, body): """ Elasticsearch scan function, obtain all data Args: index: index of elasticsearch body: query body of elasticsearch Returns: elasticsearch all data Raises: ElasticSearchQueryException,including connection timeout, server unreachable, index does not exist, etc. """ try: result = helpers.scan( client=self.client, index=index, query=body, scroll='3m', timeout='1m') result_list = [res for res in result] return result_list except ElasticsearchException as e: LOGGER.error(str(e)) raise ElasticSearchQueryException()
def send_memory_file(self, folder_path): """ Main method of sending files Args: folder_path: The file path Returns: zip_file: File stream Raises: IOError: IOError """ try: memory_file = self._send_bytes_io(folder_path) return memory_file except (OSError, IOError) as error: LOGGER.error(error) raise IOError("File sending failed") finally: self._delete_file_path(folder_path)
def get_requires(self, pkgname, database): """ get requires info from install dependency Args: pkgname: package name database: database Returns: required_by_bin: binary package information that depends on this component for example: [{'component': 'lib.so', 'provided_by': ['libJudy']}, {'component': 'lib4.so', 'provided_by': ['libJudy']}, {'component': 'lib2.so', 'provided_by': ['libJudy2']}] Attributes: TypeError: object does not support this property or method """ try: install_depend_obj = InstallRequires([database]) install_depend_info = install_depend_obj.get_install_req([pkgname], database) if not install_depend_info: return [] depend_info = install_depend_info[0].get("requires") required_by_bin = [] for component_info in depend_info: component_dict = { 'component': component_info.get("component"), 'provided_by': [component_info.get("com_bin_name", "")] } if component_dict not in required_by_bin: required_by_bin.append(component_dict) return required_by_bin except TypeError as e: LOGGER.error(e) return []
def dbs_compare(self, dbs_depend_info): """ Compare the dependent information in different databases, write it into a csv file and convert it to excel。 :param dbs_depend_info: the dependent information in different databases :return: True/False """ LOGGER.info( 'Start to compare the dependent information in different databases' ) if not dbs_depend_info: print('[ERROR] No data in all databases') return False try: # The first database is recorded separately as the benchmark database base_database = dbs_depend_info[0] base_depend_dict = defaultdict(list) self._write_data_to_csv(base_database, base_depend_dict) # The latter database is used as a comparison database record compare_depend_list = list() for compare_database in dbs_depend_info[1:]: compare_depend_dict = defaultdict(list) self._write_data_to_csv(compare_database, compare_depend_dict) compare_depend_list.append(compare_depend_dict) # Compare the dependency difference between base database and other databases self._compare_data(base_depend_dict, compare_depend_list) # Set file permissions self._set_file_permissions() return True except (ValueError, AttributeError) as e: print( '[ERROR] Failed to save the data by comparing the difference, please check the log location' ) LOGGER.error( f'Failed to save the data by comparing the difference, message is {str(e)}' ) return False
def parse_filelist_info(self, filelists): """ Get filelist info include dir, file, ghost for package Args: filelists: Returns: filelist_dict for example: {'dir': ['/usr/share/ext', '/usr/share/int'], 'file': ['/usr/lib64/libJudy.so'], 'ghost': []} Raises: KeyError: key that doesn't exist in the dictionary IndexError: index is out of range """ if not filelists: _msg = "Error in getting filelist info." LOGGER.error(_msg) return {} def is_append(lst, data): if data not in lst: lst.append(data) return lst filelist_dict = {"dir": [], "file": [], "ghost": []} try: for info in filelists: all_list = list( map(lambda x: info["dirname"] + '/' + x, info["filenames"].split("/"))) for index, type_ in enumerate(info["filetypes"]): if type_ == "d": is_append(filelist_dict["dir"], all_list[index]) elif type_ == "f": is_append(filelist_dict["file"], all_list[index]) elif type_ == "g": is_append(filelist_dict["ghost"], all_list[index]) else: _msg = "The filetype of filelist is not in ['d', 'f', 'g']" LOGGER.error(_msg) return {} return filelist_dict except (KeyError, IndexError) as e: LOGGER.error(e) return {}
def bin_package_info(self, bin_name_list, database_list=None): """ Query for binary package details Args: bin_name_list: binary package name list database_list: database list Returns: binary package detail info Attributes: AttributeError: Cannot find the attribute of the corresponding object IndexError: list index out of range TypeError: object does not support this property or method Raises: ElasticSearchQueryException: dataBase connect failed DatabaseConfigException: dataBase config error """ try: query_package = QueryPackage() bin_package_info_res = {} for database in database_list: single_db_bin_info = query_package.get_bin_info( bin_name_list, database, 1, 20).get("data") if not single_db_bin_info: return {} database_bin_info_list = [] for pkg_info in single_db_bin_info: pkgname = list(pkg_info.keys())[0] # get provides from bedepend info provides_info = self.get_provides(pkgname, database) # get requires from install info requires_info = self.get_requires(pkgname, database) database_bin_info_list.append({ "bin_name": pkgname, "version": pkg_info[pkgname].get("version"), "url": pkg_info[pkgname].get("url"), "license": pkg_info[pkgname].get("license"), "release": pkg_info[pkgname].get("release"), "summary": pkg_info[pkgname].get("summary"), "description": pkg_info[pkgname].get("description"), "src_name": pkg_info[pkgname].get("src_name"), "provides": provides_info, "requires": requires_info, "filelist": self.parse_filelist_info( pkg_info[pkgname].get("file_list")) }) bin_package_info_res[database] = database_bin_info_list return bin_package_info_res except (AttributeError, IndexError, TypeError) as e: LOGGER.error(e) return {}