def get_packages(self, source, parser, downloader, list_or_file_path, property_validate=True): """ :param source: :param parser: :param downloader: :param list_or_file_path: :param property_validate: for `root` packages we need check property, bad if we find packages from `lock` file, we can skip validate part :return: """ _auth_type = source.args['auth_type'].lower( ) if 'auth_type' in source.args else 'simple' _art_auth_etc = {} if 'auth' in source.args: self.search_auth(list_or_file_path, source) if _auth_type == 'simple': _art_auth_etc['auth'] = HTTPBasicAuth( *tuple(source.args['auth'])) session.auth = _art_auth_etc['auth'] # elif _auth_type == 'cert': # _art_auth_etc['cert'] = os.path.realpath(os.path.expanduser(source.args['auth'])) if 'auth' not in _art_auth_etc: msg = 'You have to set auth parameter for sources with artifactory-aql adapter' # self._log.error(msg) raise CrosspmException(CROSSPM_ERRORCODE_ADAPTER_ERROR, msg) if 'verify' in source.args: _art_auth_etc['verify'] = source.args['verify'].lower in [ 'true', 'yes', '1' ] else: _art_auth_etc['verify'] = False _pkg_name_column = self._config.name_column _secret_variables = self._config.secret_variables _packages_found = OrderedDict() _pkg_name_old = "" _packed_exist = False _packed_cache_params = None self._log.info('parser: {}'.format(parser._name)) for _paths in parser.get_paths(list_or_file_path, source): _packages = [] _params_found = {} _params_found_raw = {} last_error = '' _pkg_name = _paths['params'][_pkg_name_column] if _pkg_name != _pkg_name_old: _pkg_name_old = _pkg_name self._log.info('{}: {}'.format( _pkg_name, { k: v for k, v in _paths['params'].items() if (k not in (_pkg_name_column, 'repo') and k not in _secret_variables) })) for _sub_paths in _paths['paths']: _tmp_params = dict(_paths['params']) self._log.info('repo: {}'.format(_sub_paths['repo'])) for _path in _sub_paths['paths']: _tmp_params['repo'] = _sub_paths['repo'] # ------ START ---- # HACK for prefer-local if self._config.prefer_local and not parser.has_rule( 'properties'): params = parser.get_params_with_extra( 'path', _paths['params']) for param in params: param['repo'] = _tmp_params['repo'] _path_packed = downloader.cache.path_packed( None, param) _packed_exist = os.path.isfile(_path_packed) if _packed_exist: self._log.info( "Skip searching, use package cache in path {}" .format(_path_packed)) _packed_cache_params = param break # break check local cache if _packed_exist: break # break connect to artifactory # ------ END ---- _path_fixed, _path_pattern, _file_name_pattern = parser.split_fixed_pattern_with_file_name( _path) try: _artifactory_server = _tmp_params['server'] _search_repo = _tmp_params['repo'] # Get AQL path pattern, with fixed part path, without artifactory url and repository name _aql_path_pattern = _path_fixed[len(_artifactory_server ) + 1 + len(_search_repo) + 1:] if _path_pattern: _aql_path_pattern = _aql_path_pattern + "/" + _path_pattern _aql_query_url = '{}/api/search/aql'.format( _artifactory_server) _aql_query_dict = { "repo": { "$eq": _search_repo, }, "path": { "$match": _aql_path_pattern, }, "name": { "$match": _file_name_pattern, }, } # Remove path if is empty string if not _aql_path_pattern: _aql_query_dict.pop('path') query = 'items.find({query_dict}).include("*", "property")'.format( query_dict=json.dumps(_aql_query_dict)) session.auth = _art_auth_etc['auth'] r = session.post(_aql_query_url, data=query, verify=_art_auth_etc['verify']) r.raise_for_status() _found_paths = r.json() for _found in _found_paths['results']: _repo_path = "{artifactory}/{repo}/{path}/{file_name}".format( artifactory=_artifactory_server, repo=_found['repo'], path=_found['path'], file_name=_found['name']) _repo_path = ArtifactoryPath( _repo_path, **_art_auth_etc) _mark = 'found' _matched, _params, _params_raw = parser.validate_path( str(_repo_path), _tmp_params) if _matched: _params_found[_repo_path] = { k: v for k, v in _params.items() } _params_found_raw[_repo_path] = { k: v for k, v in _params_raw.items() } _mark = 'match' # Check if it's `root` packages or from `lock` file # ALSO, if from `lock` and have * in name - validate with property property_validate_tmp = property_validate or '*' in _file_name_pattern # If have not rule in config, skip this part if parser.has_rule('properties' ) and property_validate_tmp: _found_properties = { x['key']: x.get('value', '') for x in _found['properties'] } _valid, _params = parser.validate( _found_properties, 'properties', _tmp_params, return_params=True) else: _valid, _params = True, {} if _valid: _mark = 'valid' _packages += [_repo_path] _params_found[_repo_path].update( {k: v for k, v in _params.items()}) _params_found[_repo_path][ 'filename'] = str(_repo_path.name) self._log.debug(' {}: {}'.format( _mark, str(_repo_path))) except RuntimeError as e: try: err = json.loads(e.args[0]) except: err = {} if isinstance(err, dict): # Check errors # :e.args[0]: { # "errors" : [ { # "status" : 404, # "message" : "Not Found" # } ] # } for error in err.get('errors', []): err_status = error.get('status', -1) err_msg = error.get('message', '') if err_status == 401: msg = 'Authentication error[{}]{}'.format( err_status, (': {}'.format(err_msg)) if err_msg else '') elif err_status == 404: msg = last_error else: msg = 'Error[{}]{}'.format( err_status, (': {}'.format(err_msg)) if err_msg else '') if last_error != msg: self._log.error(msg) last_error = msg _package = None # HACK for prefer-local if _packed_exist: # HACK - Normalize params for cached archive for key, value in _packed_cache_params.items(): if isinstance(value, list): value = ['' if x is None else x for x in value] _packed_cache_params[key] = value _package = Package(_pkg_name, None, _paths['params'], downloader, self, parser, _packed_cache_params, list_or_file_path['raw'], {}, in_cache=True) # END HACK if _packages: _tmp = copy.deepcopy(_params_found) _packages = parser.filter_one(_packages, _paths['params'], _tmp) if isinstance(_packages, dict): _packages = [_packages] if len(_packages) == 1: _stat_pkg = self.pkg_stat(_packages[0]['path']) _params_raw = _params_found_raw.get( _packages[0]['path'], {}) _params_tmp = _params_found.get(_packages[0]['path'], {}) _params_tmp.update({ k: v for k, v in _packages[0]['params'].items() if k not in _params_tmp }) _package = Package(_pkg_name, _packages[0]['path'], _paths['params'], downloader, self, parser, _params_tmp, _params_raw, _stat_pkg) _mark = 'chosen' self._log.info(' {}: {}'.format( _mark, str(_packages[0]['path']))) elif len(_packages) > 1: raise CrosspmException( CROSSPM_ERRORCODE_MULTIPLE_DEPS, 'Multiple instances found for package [{}] not found.'. format(_pkg_name)) else: # Package not found: may be error, but it could be in other source. pass else: # Package not found: may be error, but it could be in other source. pass if (_package is not None) or (not self._config.no_fails): _added, _package = downloader.add_package(_pkg_name, _package) else: _added = False if _package is not None: _pkg_name = _package.name if _added or (_package is not None): if (_package is not None) or (not self._config.no_fails): if (_package is not None) or (_packages_found.get( _pkg_name, None) is None): _packages_found[_pkg_name] = _package if _added and (_package is not None): if downloader.do_load: _package.download() _deps_file = _package.get_file( self._config.deps_lock_file_name) if _deps_file: _package.find_dependencies(_deps_file, property_validate=False) elif self._config.deps_file_name: _deps_file = _package.get_file( self._config.deps_file_name) if _deps_file and os.path.isfile(_deps_file): _package.find_dependencies(_deps_file, property_validate=False) # HACK for not found packages _package_names = [ x[self._config.name_column] for x in list_or_file_path['raw'] ] _packages_found_names = [x.name for x in _packages_found.values()] for package in _package_names: if package not in _packages_found_names: _packages_found[package] = None return _packages_found
def get_packages(self, source, parser, downloader, list_or_file_path): _auth_type = source.args['auth_type'].lower() if 'auth_type' in source.args else 'simple' _art_auth = {} if 'auth' in source.args: if _auth_type == 'simple': _art_auth['auth'] = tuple(source.args['auth']) elif _auth_type == 'cert': _art_auth['cert'] = os.path.realpath(source.args['auth']) if 'verify' in source.args: _art_auth['verify'] = source.args['verify'].lower in ['true', 'yes', '1'] _pkg_name_col = self._config.name_column _packages_found = {} _pkg_name_old = "" for _paths in parser.get_paths(list_or_file_path, source): _packages = [] _pkg_name = _paths['params'][_pkg_name_col] if _pkg_name != _pkg_name_old: _pkg_name_old = _pkg_name print_stdout( '{}: {}'.format(_pkg_name, {k: v for k, v in _paths['params'].items() if k != _pkg_name_col})) for _path in _paths['paths']: _path_fixed, _path_pattern = parser.split_fixed_pattern(_path) _repo_paths = ArtifactoryPath(_path_fixed, **_art_auth) for _repo_path in _repo_paths.glob(_path_pattern): _mark = 'found' if parser.validate_path(str(_repo_path), _paths['params']): _mark = 'match' if parser.validate(_repo_path.properties, 'properties', _paths['params']): _mark = 'valid' _packages += [_repo_path] print_stdout(' {}: {}'.format(_mark, str(_repo_path))) _package = None if _packages: _packages = parser.filter_one(_packages, _paths['params']) if type(_packages) is dict: _packages = [_packages] if len(_packages) == 1: # one package found: ok! # _stat = _packages[0]['path'].stat() # _stat = {k: getattr(_stat, k, None) for k in ('ctime', # 'mtime', # 'md5', # 'sha1', # 'size')} _package = Package(_pkg_name, _packages[0]['path'], _paths['params'], downloader, self, parser, _packages[0]['params']) # , _stat) _mark = 'chosen' print_stdout(' {}: {}'.format(_mark, str(_packages[0]['path']))) elif len(_packages) > 1: # TODO: multiple packages found: wtf?! raise CrosspmException( CROSSPM_ERRORCODE_MULTIPLE_DEPS, 'Multiple instances found for package [{}] not found.'.format(_pkg_name) ) else: # Package not found: may be error, but it could be in other source. pass else: # Package not found: may be error, but it could be in other source. pass # _pkg_name = self._config.get_column_name(0) # raise CrosspmException( # CROSSPM_ERRORCODE_PACKAGE_NOT_FOUND, # 'Package [{}] not found.'.format(_pkg_name) # ) if (_package is not None) or (not self._config.no_fails): _added, _package = downloader.add_package(_pkg_name, _package) else: _added = False if _package is not None: _pkg_name = _package.get_name_and_path(True) if _added or (_package is not None): if (_package is not None) or (not self._config.no_fails): if (_package is not None) or (_packages_found.get(_pkg_name, None) is None): _packages_found[_pkg_name] = _package if _added and (_package is not None): if downloader.do_load: _package.download(downloader.packed_path) _deps_file = _package.get_file(self._config.deps_lock_file_name, downloader.temp_path) if _deps_file: _package.find_dependencies(_deps_file) return _packages_found
def get_packages(self, source, parser, downloader, list_or_file_path): _pkg_name_col = self._config.name_column _packages_found = {} _pkg_name_old = "" for _paths in parser.get_paths(list_or_file_path, source): _packages = [] _params_found = {} _params_found_raw = {} last_error = '' _pkg_name = _paths['params'][_pkg_name_col] if _pkg_name != _pkg_name_old: _pkg_name_old = _pkg_name self._log.info('{}: {}'.format( _pkg_name, { k: v for k, v in _paths['params'].items() if k not in (_pkg_name_col, 'repo') })) for _sub_paths in _paths['paths']: self._log.info('repo: {}'.format(_sub_paths['repo'])) for _path in _sub_paths['paths']: _tmp_params = dict(_paths['params']) _tmp_params['repo'] = _sub_paths['repo'] _path_fixed, _path_pattern = parser.split_fixed_pattern( _path) _repo_paths = FilesPath(_path_fixed) try: for _repo_path in _repo_paths.glob(_path_pattern): _mark = 'found' _matched, _params, _params_raw = parser.validate_path( str(_repo_path), _tmp_params) if _matched: _params_found[_repo_path] = { k: v for k, v in _params.items() } _params_found_raw[_repo_path] = { k: v for k, v in _params_raw.items() } _mark = 'match' _valid, _params = parser.validate( _repo_path.properties, 'properties', _tmp_params, return_params=True) if _valid: _mark = 'valid' _packages += [_repo_path] _params_found[_repo_path].update( {k: v for k, v in _params.items()}) _params_found[_repo_path][ 'filename'] = str(_repo_path.name) self._log.debug(' {}: {}'.format( _mark, str(_repo_path))) except RuntimeError as e: try: err = json.loads(e.args[0]) except: err = {} if isinstance(err, dict): # TODO: Check errors # e.args[0] = '''{ # "errors" : [ { # "status" : 404, # "message" : "Not Found" # } ] # }''' for error in err.get('errors', []): err_status = error.get('status', -1) err_msg = error.get('message', '') if err_status == 401: msg = 'Authentication error[{}]{}'.format( err_status, (': {}'.format(err_msg)) if err_msg else '') elif err_status == 404: msg = last_error else: msg = 'Error[{}]{}'.format( err_status, (': {}'.format(err_msg)) if err_msg else '') if last_error != msg: self._log.error(msg) last_error = msg _package = None if _packages: _packages = parser.filter_one(_packages, _paths['params'], _params_found) if isinstance(_packages, dict): _packages = [_packages] if len(_packages) == 1: _stat_pkg = self.pkg_stat(_packages[0]['path']) _params_raw = _params_found_raw.get( _packages[0]['path'], {}) _params_tmp = _params_found.get(_packages[0]['path'], {}) _params_tmp.update({ k: v for k, v in _packages[0]['params'].items() if k not in _params_tmp }) _package = Package(_pkg_name, _packages[0]['path'], _paths['params'], downloader, self, parser, _params_tmp, _params_raw, _stat_pkg) _mark = 'chosen' self._log.info(' {}: {}'.format( _mark, str(_packages[0]['path']))) elif len(_packages) > 1: raise CrosspmException( CROSSPM_ERRORCODE_MULTIPLE_DEPS, 'Multiple instances found for package [{}] not found.'. format(_pkg_name)) else: # Package not found: may be error, but it could be in other source. pass else: # Package not found: may be error, but it could be in other source. pass if (_package is not None) or (not self._config.no_fails): _added, _package = downloader.add_package(_pkg_name, _package) else: _added = False if _package is not None: _pkg_name = _package.name if _added or (_package is not None): if (_package is not None) or (not self._config.no_fails): if (_package is not None) or (_packages_found.get( _pkg_name, None) is None): _packages_found[_pkg_name] = _package if _added and (_package is not None): if downloader.do_load: _package.download() _deps_file = _package.get_file( self._config.deps_lock_file_name) if _deps_file: _package.find_dependencies(_deps_file) elif self._config.deps_file_name: _deps_file = _package.get_file( self._config.deps_file_name) if _deps_file and os.path.isfile(_deps_file): _package.find_dependencies(_deps_file) return _packages_found
def get_packages(self, source, parser, downloader, list_or_file_path): _auth_type = source.args['auth_type'].lower( ) if 'auth_type' in source.args else 'simple' _art_auth = {} if 'auth' in source.args: if _auth_type == 'simple': _art_auth['auth'] = tuple(source.args['auth']) elif _auth_type == 'cert': _art_auth['cert'] = os.path.realpath(source.args['auth']) if 'verify' in source.args: _art_auth['verify'] = source.args['verify'].lower in [ 'true', 'yes', '1' ] _pkg_name_col = self._config.name_column _packages_found = {} _pkg_name_old = "" for _paths in parser.get_paths(list_or_file_path, source): _packages = [] _pkg_name = _paths['params'][_pkg_name_col] if _pkg_name != _pkg_name_old: _pkg_name_old = _pkg_name print_stdout('{}: {}'.format( _pkg_name, { k: v for k, v in _paths['params'].items() if k != _pkg_name_col })) for _path in _paths['paths']: _path_fixed, _path_pattern = parser.split_fixed_pattern(_path) _repo_paths = ArtifactoryPath(_path_fixed, **_art_auth) for _repo_path in _repo_paths.glob(_path_pattern): _mark = 'found' if parser.validate_path(str(_repo_path), _paths['params']): _mark = 'match' if parser.validate(_repo_path.properties, 'properties', _paths['params']): _mark = 'valid' _packages += [_repo_path] print_stdout(' {}: {}'.format(_mark, str(_repo_path))) _package = None if _packages: _packages = parser.filter_one(_packages, _paths['params']) if type(_packages) is dict: _packages = [_packages] if len(_packages) == 1: # one package found: ok! # _stat = _packages[0]['path'].stat() # _stat = {k: getattr(_stat, k, None) for k in ('ctime', # 'mtime', # 'md5', # 'sha1', # 'size')} _package = Package(_pkg_name, _packages[0]['path'], _paths['params'], downloader, self, parser, _packages[0]['params']) # , _stat) _mark = 'chosen' print_stdout(' {}: {}'.format(_mark, str(_packages[0]['path']))) elif len(_packages) > 1: # TODO: multiple packages found: wtf?! raise CrosspmException( CROSSPM_ERRORCODE_MULTIPLE_DEPS, 'Multiple instances found for package [{}] not found.'. format(_pkg_name)) else: # Package not found: may be error, but it could be in other source. pass else: # Package not found: may be error, but it could be in other source. pass # _pkg_name = self._config.get_column_name(0) # raise CrosspmException( # CROSSPM_ERRORCODE_PACKAGE_NOT_FOUND, # 'Package [{}] not found.'.format(_pkg_name) # ) if (_package is not None) or (not self._config.no_fails): _added, _package = downloader.add_package(_pkg_name, _package) else: _added = False if _package is not None: _pkg_name = _package.get_name_and_path(True) if _added or (_package is not None): if (_package is not None) or (not self._config.no_fails): if (_package is not None) or (_packages_found.get( _pkg_name, None) is None): _packages_found[_pkg_name] = _package if _added and (_package is not None): if downloader.do_load: _package.download(downloader.packed_path) _deps_file = _package.get_file( self._config.deps_lock_file_name, downloader.temp_path) if _deps_file: _package.find_dependencies(_deps_file) return _packages_found