def load_config(self, path):
"""
Description: Read the contents of the configuration file load each
node data in the yaml configuration file as a list to return
Args:
path: Initialize the repo source configuration file for the data
Returns:
Initialize the contents of the database configuration file
Raises:
FileNotFoundError: The specified file does not exist
TypeError: Wrong type of data
"""
if not path:
raise ValueError(
"The configuration source for the data dependency initialization is not specified"
)
if not os.path.exists(path):
raise FileNotFoundError("system initialization configuration file "
"does not exist: %s" % path)
# load yaml configuration file
with open(path, "r", encoding="utf-8") as file_context:
try:
self._repo = yaml.load(file_context.read(),
Loader=yaml.FullLoader)
except yaml.YAMLError as yaml_error:
LOGGER.error(yaml_error)
raise ValueError(
"The format of the yaml configuration "
"file is wrong please check and try again:{0}".format(
yaml_error)) from yaml_error
def be_depend(self):
"""
get source(binary) rpm package(s) bedepend relation
"""
searched_pkgs = set()
update_meth = self.__get_update_data_method()
to_search = self.__init_to_serach_pkgs()
is_init = True
while to_search:
resp = self.provide.get_be_req(to_search, self.database)
if not resp:
break
next_search = set()
for bedep_info in resp:
searched_pkgs.add(bedep_info.get("binary_name"))
next_pkgs = update_meth(bedep_info)
next_search.update(next_pkgs)
if is_init and self.parameter.get("packtype") == "binary":
not_found_pkg = str(to_search - searched_pkgs)
self.log_msg = f"binary packages {not_found_pkg} not found in {self.database}"
LOGGER.warning(self.log_msg)
is_init = False
to_search = next_search - searched_pkgs
def get_subpack_info(self, pkgname, database, pkgname_info):
"""
get subpacks info for source package
Args:
pkgname: package name
database: database
pkgname_info: package info which include subpacks
Returns:
subpacks: subpack list
"""
subpacks_bin_list = pkgname_info[pkgname].get('subpacks')
if not subpacks_bin_list:
_msg = "Error in getting subpack info."
LOGGER.error(_msg)
return []
subpacks = []
for subpack_name in subpacks_bin_list:
subpack_dict = {
"bin_name": subpack_name,
"provides": self.get_provides(subpack_name, database),
"requires": self.get_requires(subpack_name, database)
}
subpacks.append(subpack_dict)
return subpacks
def __get_subpacks(self, pkg_name_lst, is_init=False):
"""get source packages's subpacks
Args:
pkg_name_lst ([list]): [source packagenames list]
Returns:
[set]: [source package's subpacks]
"""
binary_pkgs = set()
if not pkg_name_lst:
return binary_pkgs
searched_pkg = set()
for pkg_dict in self.query_pkg.get_src_info(pkg_name_lst,
self.database, 1,
len(pkg_name_lst))["data"]:
for _, pkg_info in pkg_dict.items():
searched_pkg.add(pkg_info.get("src_name"))
binary_pkgs.update(set(pkg_info.get("subpacks", [])))
self._search_set.update(searched_pkg)
if is_init:
not_found_pkg = str(set(pkg_name_lst) - searched_pkg)
self.log_msg = f"source packages {not_found_pkg} not found in {self.database}"
LOGGER.warning(self.log_msg)
return binary_pkgs
def get_build_info(self, pkgname, database):
"""
get build dependency info
Args:
pkgname: package name
database: database
Returns:
build_depend_info: build dependency for source package
Attributes:
TypeError: object does not support this property or method
"""
try:
build_obj = BuildRequires([database])
build_depend = build_obj.get_build_req([pkgname], database)
if not build_depend:
return []
build_depend_info = build_depend[0].get("requires")
build_package_list = []
for requires_info in build_depend_info:
build_package_list.append(requires_info.get("component"))
return build_package_list
except (TypeError, AttributeError) as e:
LOGGER.error(e)
return []
def _compare_data(self, base_depend_dict, compare_depend_list):
"""
Compare the dependency difference between the base database and other databases, and write it to a csv file
:param base_depend_dict: base database dependency info
:param compare_depend_list: other databases dependency info
:return: None
"""
if not compare_depend_list:
LOGGER.warning(
'There is only one database input, no comparison operation')
return
csv_file = os.path.join(self.out_path, 'compare.csv')
with open(csv_file, 'w', encoding='utf-8') as file:
csv_writer = csv.writer(file)
csv_writer.writerow(self.databases)
for rpm, dependency in base_depend_dict.items():
_all_dependency_list = [dependency]
for other_database in compare_depend_list:
_all_dependency_list.append(other_database.get(rpm, []))
_all_dependency_set = set([
info for single_db_info in _all_dependency_list
for info in single_db_info
])
self._write_single_field(_all_dependency_list,
_all_dependency_set, csv_writer)
def _get_source_rpm_depend(self, dbs):
"""
Get one-level compilation dependency of all source packages in the database
:param dbs: Database to be queried
:return: None
"""
for database in dbs:
# First,query all source rpm name of specify database.
all_source_rpm = self.query_pkg.get_src_info(
src_list=None,
database=database,
page_num=DEFAULT_PAGE_NUM,
page_size=MAXIMUM_PAGE_SIZE,
command_line=True)
all_source_rpm_name = list()
for source_rpm in all_source_rpm.get('data'):
all_source_rpm_name.extend(source_rpm.keys())
LOGGER.info(
f'The number of source packages in the {database} database is {len(all_source_rpm_name)}'
)
# Second, query the one-level build dependencies of all packages based on the package name.
build_query_engine = BuildRequires([database])
all_source_rpm_depend = build_query_engine.get_build_req(
source_list=all_source_rpm_name)
# Third, format build dependencies
format_source_rpm_depend = self._format_depend(
all_source_rpm_depend, key_word=self.SOURCE_NAME)
self.result.append({database: format_source_rpm_depend})
def _get_bin_rpm_depend(self, dbs):
"""
Get one-level install dependency of all binary packages in the database
:param dbs: Database to be queried
:return:
"""
for database in dbs:
# First,query all binary rpm name of specify database.
all_bin_rpm = self.query_pkg.get_bin_info(
binary_list=None,
database=database,
page_num=DEFAULT_PAGE_NUM,
page_size=MAXIMUM_PAGE_SIZE,
command_line=True)
all_bin_rpm_name = list()
for bin_rpm in all_bin_rpm.get('data'):
all_bin_rpm_name.extend(bin_rpm.keys())
LOGGER.info(
f'The number of binary packages in the {database} database is {len(all_bin_rpm_name)}'
)
# Second, query the one-level install dependencies of all packages based on the package name.
install_query_engine = InstallRequires([database])
all_binary_rpm_depend = install_query_engine.get_install_req(
binary_list=all_bin_rpm_name)
# Third, format build dependencies
format_binary_rpm_depend = self._format_depend(
all_binary_rpm_depend, key_word=self.BINARY_NAME)
self.result.append({database: format_binary_rpm_depend})
def _import_error(self, error, record=True):
if record:
LOGGER.error(error)
self._fail.append(self.elastic_index)
fails = self._delete_index()
if fails:
LOGGER.warning("Delete the failed ES database:%s ." % fails)
def __init__(self, db_engine=None, host=None, port=None):
self.db_engine = db_engine or "elastic"
if self.db_engine not in self.__DATABASE_ENGINE_TYPE:
LOGGER.error("DataBase %s is not support" % self.db_engine)
raise DatabaseConfigException()
self._host = host or configuration.DATABASE_HOST
self._port = port or configuration.DATABASE_PORT
self.session = None
def src_package_info(self, src_name_list, database_list=None):
"""
get a source package info (provides, requires, etc)
Args:
src_name_list: source package list
database_list: database list
Returns:
src_package_info_res
Attributes:
AttributeError: Cannot find the attribute of the corresponding object
IndexError: list index out of range
TypeError: object does not support this property or method
Raises:
ElasticSearchQueryException: dataBase connect failed
DatabaseConfigException: dataBase config error
"""
try:
src_package_info_res = {}
query_package = QueryPackage()
for database in database_list:
single_db_src_info = query_package.get_src_info(
src_name_list, database, 1, 20).get("data")
if not single_db_src_info:
return {}
database_src_info_list = []
for pkg_info in single_db_src_info:
pkgname = list(pkg_info.keys())[0]
build_package_info = self.get_build_info(pkgname, database)
subpacks = self.get_subpack_info(pkgname, database,
pkg_info)
database_src_info_list.append({
"src_name":
pkgname,
"license":
pkg_info[pkgname].get("license"),
"version":
pkg_info[pkgname].get("version"),
"url":
pkg_info[pkgname].get("url"),
"summary":
pkg_info[pkgname].get("summary"),
"description":
pkg_info[pkgname].get("description"),
"build_dep":
build_package_info,
"subpacks":
subpacks
})
src_package_info_res[database] = database_src_info_list
return src_package_info_res
except (AttributeError, IndexError, TypeError) as e:
LOGGER.error(e)
return {}
def _redis():
try:
if REDIS_CONN.keys("pkgship_*"):
REDIS_CONN.delete(*REDIS_CONN.keys("pkgship_*"))
except redis.RedisError:
LOGGER.error(
"There is an exception in Redis service. Please check it later."
" After restart , please check the key value with pkgship_ prefix"
)
def __init__(self, host=None, port=None):
self._host = host
self._port = port
try:
self.client = Elasticsearch(
[{"host": self._host, "port": self._port}], timeout=60)
except LocationValueError:
LOGGER.error("The host of database in package.ini is empty")
raise DatabaseConfigException()
def update_setting(self):
"""
Update the ES configuration, currently the maximum number of modified queries
:return: None
"""
try:
self.client.indices.put_settings(index='_all', body={'index': {
'max_result_window': MAX_ES_QUERY_NUM}})
except ElasticsearchException:
LOGGER.error('Set max_result_window of all indies failed')
def inner(*args, **kwargs):
try:
return func(*args, **kwargs)
except (KeyError, TypeError, ValueError, AttributeError, IndexError,
TypeError) as error:
LOGGER.error(error)
raise ValueError("input json_data is error! please check")
except (IOError, OSError) as error:
LOGGER.error(error)
raise ValueError("input json_data is error! please check")
def all_bin_packages(self,
database,
page_num=1,
page_size=20,
package_list=None,
command_line=False):
"""
get all binary package info
Args:
database: database
page_num: paging query index
page_size: paging query size
package_list: package list name
command_line： command_line or UI
Returns:
all_bin_dict: all binary package information dict
for example::
{
"total": "",
"data": ""
}
Attributes:
AttributeError: Cannot find the attribute of the corresponding object
KeyError: key not exists
TypeError: object does not support this property or method
Raises:
ElasticSearchQueryException: dataBase connect failed
DatabaseConfigException: dataBase config error
"""
try:
# query all binary package info from database
query_package = QueryPackage()
all_bin_info = query_package.get_bin_info(package_list, database,
page_num, page_size,
command_line)
if not all_bin_info["data"]:
_msg = "An error occurred when getting bianry package info."
raise PackageInfoGettingError(_msg)
parsed_all_bin_info = []
total_num = all_bin_info.get("total")
for pkg_info in all_bin_info.get("data"):
single_pkg = []
for pkgname, info_values in pkg_info.items():
single_pkg = self.__parse_pkg_info(info_values, pkgname,
database)
single_pkg["source_name"] = info_values["src_name"]
parsed_all_bin_info.append(single_pkg)
all_bin_dict = {"total": total_num, "data": parsed_all_bin_info}
return all_bin_dict
except (AttributeError, KeyError, TypeError) as e:
LOGGER.error(e)
return {}
def create_engine(db_engine, **kwargs):
"""
Get database class by engines name
"""
try:
module = importlib.import_module("packageship.application.database.engines.%s" % db_engine)
except ModuleNotFoundError:
# Log here, the caller throws an exception
LOGGER.error("DataBase engine module not exist")
return None
cls = getattr(module, db_engine)
return cls(**kwargs)
def self_depend(self,
pkg_name,
pkgtype="binary",
self_build=False,
with_subpack=False):
"""
Description: get source(binary) rpm package(s) self depend relation
Args:
pkg_name: the list of package names needed to be searched
db_priority: database name list
packtype: the type of query package (source/binary)
with_subpack: whether to query subpackages
self_build: whether to query self build
Exception:
AttributeError: the input value is invalid
"""
if not isinstance(pkg_name, list):
raise AttributeError("the input is invalid")
install = InstallDepend(self.db_list, self)
build = BuildDepend(self.db_list, self)
self.subpack = with_subpack
for pkg in pkg_name:
if pkg:
if pkgtype == "source":
self.search_subpack_dict.get("non_db").add(pkg)
else:
self.search_install_dict.get("non_db").add(pkg)
if pkgtype == "source":
self.__query_subpack(pkg_list=pkg_name, is_init=True)
# end this loop while those three dictionry's value are None
is_init = True
while self._check_search(self.search_install_dict) \
or self._check_search(self.search_build_dict) \
or self._check_search(self.search_subpack_dict):
while self._check_search(self.search_install_dict):
install.install_depend([])
if is_init and pkgtype == "binary":
fmt = str(set(pkg_name) - set(self.binary_dict.keys()))
self.log_msg = f"binary name {fmt} not found in all database"
LOGGER.warning(self.log_msg)
is_init = False
while self._check_search(self.search_build_dict):
build.build_depend([], self_build=self_build)
while with_subpack and self._check_search(
self.search_subpack_dict):
self.__query_subpack()
def _repo_files(self):
xml = True if "db_file" in self._repo else False
try:
self._find_files(xml=xml)
except (FileNotFoundError, ValueError) as error:
LOGGER.error(error)
return False
if xml:
exists = all([self._repo["file_list"], self._repo["db_file"]])
else:
exists = self._sqlite()
return exists
def get_provides(self, pkgname, database):
"""
get provides info from bedepend query result
Args:
pkgname: package name
database: database
Returns:
provides_info_res: which include component, required_by_bin, required_by_src
for example:
[{'component': 'comp1',
'required_by_bin': ['A1', 'B1'],
'required_by_src': ['T', 'R']}]
"""
try:
# getting bedend query res
be_depend_obj = BeDependRequires()
be_depend_info = be_depend_obj.get_be_req([pkgname], database)
if not be_depend_info:
return []
provides_info_res = []
depend_info = be_depend_info[0].get("provides")
for component_info in depend_info:
# getting required_by_bin
install_require_info = component_info.get("install_require")
required_by_bin = []
for req_info in install_require_info:
if req_info.get("req_bin_name"):
required_by_bin.append(req_info.get("req_bin_name"))
# getting required_by_src
build_require_info = component_info.get("build_require")
required_by_src = []
for req_info in build_require_info:
if req_info.get("req_src_name"):
required_by_src.append(req_info.get("req_src_name"))
component_dict = {
"component": component_info.get("component"),
"required_by_bin": required_by_bin,
"required_by_src": required_by_src
}
provides_info_res.append(component_dict)
return provides_info_res
except (TypeError, IndexError) as e:
LOGGER.error(e)
return []
def _delete_file_path(self, folder_path):
"""
Delete folder
Args:
folder_path: Folder path
Returns:
None
"""
try:
if folder_path and os.path.exists(folder_path):
shutil.rmtree(folder_path)
except (IOError, OSError) as error:
LOGGER.error(error)
raise IOError("Failed to delete folder")
def all_depend_info(self, depend_type, dbs):
"""
Get all rpm info ,include source rpm(query build depend) and binary rpm (query install rpm)
:param depend_type:
:param dbs:
:return:
"""
if depend_type == BUILD_DEPEND_TYPE:
LOGGER.info("Start to query all source rpm dependency information")
self._get_source_rpm_depend(dbs)
elif depend_type == INSTALL_DEPEND_TYPE:
LOGGER.info("Start to query all binary rpm dependency information")
self._get_bin_rpm_depend(dbs)
return self.result
def connection(self):
"""
Returns: specify database connection
Raises: DatabaseConfigException, database class not found
"""
self.session = engine.create_engine(db_engine=self.db_engine,
host=self._host,
port=self._port)
if self.session is None:
LOGGER.error(
"Failed to create database engine %s, please check database configuration"
% self.db_engine)
raise DatabaseConfigException()
return self.session
def count(self, index, body):
"""
Obtain data volume of specify index
Args:
index: index of elasticsearch
body: query body of elasticsearch
Returns: data volume
Raises: ElasticSearchQueryException,including connection timeout, server unreachable, index does not exist, etc.
"""
try:
data_count = self.client.count(index=index, body=body)
return data_count
except ElasticsearchException as e:
LOGGER.error(str(e))
raise ElasticSearchQueryException()
def query(self, index, body):
"""
Elasticsearch query function
Args:
index: index of elasticsearch
body: query body of elasticsearch
Returns: elasticsearch data
Raises: ElasticSearchQueryException,including connection timeout, server unreachable, index does not exist, etc.
"""
try:
result = self.client.search(index=index, body=body)
return result
except ElasticsearchException as e:
LOGGER.error(str(e))
raise ElasticSearchQueryException()
def insert(self, index, body, doc_type="_doc"):
"""
Single insert ES data
Args:
index: Index to ES database
body: inserted data set
doc_type: document type
Returns:
Raises: ElasticSearchQueryException,including connection timeout, server unreachable, index does not exist, etc.
"""
try:
self.client.index(index=index, body=body, doc_type=doc_type)
except ElasticsearchException as e:
LOGGER.error(str(e))
raise ElasticSearchQueryException()
def del_temporary_file(path, folder=False):
"""
Description: Delete temporary files or folders
Args:
path: temporary files or folders
folder: file or folder, fsiles are deleted by default
"""
if not os.path.exists(path):
return
try:
if folder:
shutil.rmtree(path)
else:
os.remove(path)
except IOError as error:
LOGGER.error(error)
def import_depend(self, path=None):
"""
Description: Initializes import dependency data
Args:
path: repo source file
"""
# Initialize the judgment of the process
self._process()
if not path:
path = configuration.INIT_CONF_PATH
try:
self._config.load_config(path)
except (ValueError, FileNotFoundError) as error:
raise InitializeError(str(error)) from error
if not self._config.validate:
raise InitializeError(self._config.message)
self._clear_all_index()
# Clear the cached value for a particular key
self._redis()
for repo in self._config:
self._data = ESJson()
self._repo = repo
try:
if not self._repo_files():
raise RepoError("Repo source data error: %s" %
self.elastic_index)
xml_files = self._xml()
if xml_files["xml"] and xml_files["filelist"]:
self._xml_parse(**xml_files)
self._save()
self._session.update_setting()
except (RepoError, ElasticsearchException) as error:
LOGGER.error(error)
self._fail.append(self.elastic_index)
if isinstance(error, ElasticsearchException):
self._delete_index()
finally:
# delete temporary directory
del_temporary_file(configuration.TEMPORARY_DIRECTORY,
folder=True)
self._repo = None
def scan(self, index, body):
"""
Elasticsearch scan function, obtain all data
Args:
index: index of elasticsearch
body: query body of elasticsearch
Returns: elasticsearch all data
Raises: ElasticSearchQueryException,including connection timeout, server unreachable, index does not exist, etc.
"""
try:
result = helpers.scan(
client=self.client, index=index, query=body, scroll='3m', timeout='1m')
result_list = [res for res in result]
return result_list
except ElasticsearchException as e:
LOGGER.error(str(e))
raise ElasticSearchQueryException()
def _set_val(self, key):
"""
Description: Gets the cached hash value and assigns it to
the corresponding dependent instance
Args:
key: cached key
"""
self._depend.source_dict = json.loads(
constant.REDIS_CONN.hget(key, "source_dict"))
self._depend.binary_dict = json.loads(
constant.REDIS_CONN.hget(key, "binary_dict"))
self._depend.log_msg = constant.REDIS_CONN.hget(key, "log_msg")
if not self._depend.log_msg:
return
LOGGER.warning(self._depend.log_msg)