def update_request_with_deps(request_id,
deps,
env_vars=None,
pkg_manager=None,
packages=None):
"""
Update the Cachito request with the resolved dependencies.
:param int request_id: the ID of the Cachito request
:param list deps: the list of dependency dictionaries to record
:param dict env_vars: mapping of environment variables to record
:param str pkg_manager: a package manager to add to the request if auto-detection was used
:param list packages: the list of packages that were resolved
:raise CachitoError: if the request to the Cachito API fails
"""
# Import this here to avoid a circular import
from cachito.workers.requests import requests_auth_session
config = get_worker_config()
request_url = f'{config.cachito_api_url.rstrip("/")}/requests/{request_id}'
log.info('Adding %d dependencies to request %d', len(deps), request_id)
for index in range(0, len(deps), config.cachito_deps_patch_batch_size):
batch_upper_limit = index + config.cachito_deps_patch_batch_size
payload = {'dependencies': deps[index:batch_upper_limit]}
if index == 0:
if env_vars:
log.info('Adding environment variables to the request %d: %s',
request_id, env_vars)
payload['environment_variables'] = env_vars
if pkg_manager:
log.info(
'Adding the package manager "%s" to the request %d',
pkg_manager,
request_id,
)
payload['pkg_managers'] = [pkg_manager]
if packages:
log.info('Adding the packages "%s" to the request %d',
packages, request_id)
payload['packages'] = packages
try:
log.info('Patching deps {} through {} out of {}'.format(
index + 1, min(batch_upper_limit, len(deps)), len(deps)))
rv = requests_auth_session.patch(
request_url, json=payload, timeout=config.cachito_api_timeout)
except requests.RequestException:
msg = f'The connection failed when setting the dependencies on request {request_id}'
log.exception(msg)
raise CachitoError(msg)
if not rv.ok:
log.error(
'The worker failed to set the dependencies on request %d. The status was %d. '
'The text was:\n%s',
request_id,
rv.status_code,
rv.text,
)
raise CachitoError(
f'Setting the dependencies on request {request_id} failed')
def run_download_cmd(cmd: Iterable[str], params: Dict[str, str]) -> str:
"""Run gomod command that downloads dependencies.
Such commands may fail due to network errors (go is bad at retrying), so the entire operation
will be retried a configurable number of times.
The backoff is constant. The download commands are typically used for multiple dependencies
at the same time, and the failure can be caused by any of them. Exponential backoff would make
more sense if we were downloading dependencies individually.
"""
n_tries = get_worker_config().cachito_gomod_download_max_tries
@backoff.on_exception(
backoff.constant,
CachitoCalledProcessError,
jitter=
None, # use the constant 1s backoff rather than a random value between 0 and 1
max_tries=n_tries,
logger=log,
)
def run_go(_cmd, _params) -> str:
log.debug(f"Running {_cmd}")
return run_gomod_cmd(_cmd, _params)
try:
return run_go(cmd, params)
except CachitoCalledProcessError:
err_msg = (
f"Processing gomod dependencies failed. Cachito tried the {' '.join(cmd)} command "
f"{n_tries} times. This may indicate a problem with your repository or Cachito itself."
)
raise CachitoError(err_msg)
def setup_task_logging(task_id, task, **kwargs):
"""
Set up the logging for the task via adding a file log handler.
If ``cachito_request_file_logs_dir`` is set, a temporary log handler is added before the
task is invoked.
If ``cahito_request_file_logs_dir`` is not set, the temporary log handler will not be added.
:param str task_id: the task ID
:param class task: the class of the task being executed
"""
worker_config = get_worker_config()
log_dir = worker_config.cachito_request_file_logs_dir
log_level = worker_config.cachito_request_file_logs_level
log_format = worker_config.cachito_request_file_logs_format
request_log_handler = None
if log_dir:
log_formatter = logging.Formatter(log_format)
request_id = _get_function_arg_value("request_id", task.__wrapped__,
kwargs["args"], kwargs["kwargs"])
if not request_id:
raise CachitoError("Unable to get 'request_id'")
log_file_path = os.path.join(log_dir, f"{request_id}.log")
request_log_handler = logging.FileHandler(log_file_path)
request_log_handler.setLevel(log_level)
request_log_handler.setFormatter(log_formatter)
os.chmod(log_file_path, worker_config.cachito_request_file_logs_perm)
logger = logging.getLogger()
logger.addHandler(request_log_handler)
def setup_task_logging_customization(task_id, task, **kwargs):
"""
Customize the logging for the task.
This adds a filter that sets ``request_id`` on the log record. If the request ID
cannot be determined, "unknown" is set instead. This also sets the log format to
the ``cachito_task_log_format`` config.
:param str task_id: the task ID
:param class task: the class of the task being executed
"""
conf = get_worker_config()
request_id = _get_function_arg_value("request_id", task, kwargs["args"],
kwargs["kwargs"])
log_filter = AddRequestIDFilter(str(request_id) or "unknown")
root_logger = logging.getLogger()
for handler in root_logger.handlers:
if not isinstance(handler, logging.StreamHandler):
continue
handler.addFilter(log_filter)
if isinstance(handler.formatter, ColorFormatter):
formatter = ColorFormatter(conf.cachito_task_log_format,
use_color=handler.formatter.use_color)
else:
formatter = logging.Formatter(conf.cachito_task_log_format)
handler.setFormatter(formatter)
def update_request_with_config_files(request_id, config_files):
"""
Update the Cachito request with the input configuration files.
:param list config_files: the list of configuration files to add to the request
:raise CachitoError: if the request to the Cachito API fails
"""
# Import this here to avoid a circular import
from cachito.workers.requests import requests_auth_session
log.info("Adding %d configuration files to the request %d", len(config_files), request_id)
config = get_worker_config()
request_url = _get_request_url(request_id) + "/configuration-files"
try:
rv = requests_auth_session.post(
request_url, json=config_files, timeout=config.cachito_api_timeout
)
except requests.RequestException:
msg = f"The connection failed when adding configuration files to the request {request_id}"
log.exception(msg)
raise CachitoError(msg)
if not rv.ok:
log.error(
"The worker failed to add configuration files to the request %d. The status was %d. "
"The text was:\n%s",
request_id,
rv.status_code,
rv.text,
)
raise CachitoError(f"Adding configuration files on request {request_id} failed")
def get_requests_session(auth=False):
"""
Create a requests session with authentication (when enabled).
:param bool auth: configure authentication on the session
:return: the configured requests session
:rtype: requests.Session
"""
config = get_worker_config()
session = requests.Session()
if auth:
if config.cachito_auth_type == "kerberos":
session.auth = requests_kerberos.HTTPKerberosAuth(
mutual_authentication=requests_kerberos.OPTIONAL)
elif config.cachito_auth_type == "cert":
session.cert = config.cachito_auth_cert
retry = Retry(total=5,
read=5,
connect=5,
backoff_factor=1.3,
status_forcelist=(500, 502, 503, 504))
adapter = requests.adapters.HTTPAdapter(max_retries=retry)
session.mount("http://", adapter)
session.mount("https://", adapter)
return session
def run_download_cmd(cmd: Iterable[str], params: Dict[str, str]) -> str:
"""Run gomod command that downloads dependencies.
Such commands may fail due to network errors (go is bad at retrying), so the entire operation
will be retried a configurable number of times.
Cachito will reuse the same cache directory between retries, so Go will not have to download
the same dependency twice. The backoff is exponential, Cachito will wait 1s -> 2s -> 4s -> ...
before retrying.
"""
n_tries = get_worker_config().cachito_gomod_download_max_tries
@backoff.on_exception(
backoff.expo,
CachitoCalledProcessError,
jitter=None, # use deterministic backoff, do not apply jitter
max_tries=n_tries,
logger=log,
)
def run_go(_cmd, _params) -> str:
log.debug(f"Running {_cmd}")
return run_gomod_cmd(_cmd, _params)
try:
return run_go(cmd, params)
except CachitoCalledProcessError:
err_msg = (
f"Processing gomod dependencies failed. Cachito tried the {' '.join(cmd)} command "
f"{n_tries} times. This may indicate a problem with your repository or Cachito itself."
)
raise CachitoError(err_msg)
def assemble_source_code_archive(app_archive_path, deps_path,
bundle_archive_path):
"""
Creates an archive with the source code for application and its dependencies.
:param str app_archive_path: the path to the archive of the application source code
:param str deps_path: the path to the directory containing the dependencies source code
:param str bundle_archive_path: the destination path of the assembled archive
"""
log.info('Assembling source code archive in "%s"', bundle_archive_path)
cachito_shared_dir = get_worker_config().cachito_shared_dir
absolute_app_archive_path = os.path.join(cachito_shared_dir,
app_archive_path)
absolute_deps_path = os.path.join(cachito_shared_dir, deps_path)
bundle_archive_path = os.path.join(cachito_shared_dir, bundle_archive_path)
# Generate a tarball containing the application and dependencies source code
with tarfile.open(bundle_archive_path, mode='w:gz') as bundle_archive:
with tarfile.open(absolute_app_archive_path,
mode='r:*') as app_archive:
for member in app_archive.getmembers():
bundle_archive.addfile(member,
app_archive.extractfile(member.name))
bundle_archive.add(absolute_deps_path, 'deps')
def update_request_with_deps(request_id, deps):
"""
Update the Cachito request with the resolved dependencies.
:param int request_id: the ID of the Cachito request
:param list deps: the list of dependency dictionaries to record
:raise CachitoError: if the request to the Cachito API fails
"""
# Import this here to avoid a circular import
from cachito.workers.requests import requests_auth_session
config = get_worker_config()
request_url = f'{config.cachito_api_url.rstrip("/")}/requests/{request_id}'
log.info('Adding %d dependencies to request %d', len(deps), request_id)
payload = {'dependencies': deps}
try:
rv = requests_auth_session.patch(request_url, json=payload, timeout=30)
except requests.RequestException:
msg = f'The connection failed when setting the dependencies on request {request_id}'
log.exception(msg)
raise CachitoError(msg)
if not rv.ok:
log.error(
'The worker failed to set the dependencies on request %d. The status was %d. '
'The text was:\n%s',
request_id, rv.status_code, rv.text,
)
raise CachitoError(f'Setting the dependencies on request {request_id} failed')
def _patch_request_or_fail(request_id: int, payload: dict,
connect_error_msg: str,
status_error_msg: str) -> None:
"""
Try to update the specified request using the Cachito PATCH API.
Both error messages can contain the {exc} placeholder which will be replaced by the actual
exception.
:param request_id: ID of the request to get
:param payload: the JSON data to send to the PATCH endpoint
:param connect_error_msg: error message to raise if the connection fails
:param status_error_msg: error message to raise if the response status is 4xx or 5xx
:raises CachitoError: if the connection fails or the API returns an error response
"""
config = get_worker_config()
request_url = f'{config.cachito_api_url.rstrip("/")}/requests/{request_id}'
try:
rv = requests_auth_session.patch(request_url,
json=payload,
timeout=config.cachito_api_timeout)
rv.raise_for_status()
except requests.HTTPError as e:
msg = status_error_msg.format(exc=e)
log.exception(msg)
raise CachitoError(msg)
except requests.RequestException as e:
msg = connect_error_msg.format(exc=e)
log.exception(msg)
raise CachitoError(msg)
def search_components(in_nexus_hoster=True, **query_params):
"""
Search for components using the Nexus REST API.
:param in_nexus_hoster: whether to search in the Nexus hoster instance, if one is
available. If set to false, will search in the self hosted instance
:param query_params: the query parameters to filter
:return: the list of components returned by the search
:rtype: list<dict>
:raise CachitoError: if the search fails
"""
config = get_worker_config()
if in_nexus_hoster:
username, password = get_nexus_hoster_credentials()
nexus_url = _get_nexus_hoster_url()
else:
username = config.cachito_nexus_username
password = config.cachito_nexus_password
nexus_url = config.cachito_nexus_url
auth = requests.auth.HTTPBasicAuth(username, password)
url = f"{nexus_url}/service/rest/v1/search"
# Create a copy so that the original query parameters are unaltered later on
params = copy.deepcopy(query_params)
log.debug(
"Searching Nexus for components using the following query parameters: %r", query_params
)
items = []
while True:
try:
rv = nexus_requests_session.get(
url, auth=auth, params=params, timeout=config.cachito_nexus_timeout
)
except requests.RequestException:
msg = "Could not connect to the Nexus instance to search for components"
log.exception(msg)
raise CachitoError(msg)
if not rv.ok:
log.error(
"Failed to search for components (%r) in Nexus with the status code %d and the "
"text: %s",
query_params,
rv.status_code,
rv.text,
)
raise CachitoError("Failed to search for components in Nexus")
rv_json = rv.json()
items.extend(rv_json["items"])
# Handle pagination
if rv_json["continuationToken"]:
log.debug("Getting the next page of Nexus component search results")
params["continuationToken"] = rv_json["continuationToken"]
else:
break
return items
def run_cmd(cmd, params, exc_msg=None):
"""
Run the given command with provided parameters.
:param iter cmd: iterable representing command to be executed
:param dict params: keyword parameters for command execution
:param str exc_msg: an optional exception message when the command fails
:returns: the command output
:rtype: str
:raises CachitoError: if the command fails
"""
params.setdefault("capture_output", True)
params.setdefault("universal_newlines", True)
params.setdefault("encoding", "utf-8")
conf = get_worker_config()
params.setdefault("timeout", conf.cachito_subprocess_timeout)
try:
response = subprocess.run(cmd, **params) # nosec
except subprocess.TimeoutExpired as e:
raise CachitoError(str(e))
if response.returncode != 0:
log.error('The command "%s" failed with: %s', " ".join(cmd),
response.stderr)
raise CachitoCalledProcessError(
exc_msg or "An unexpected error occurred", response.returncode)
return response.stdout
def set_request_state(request_id, state, state_reason):
"""
Set the state of the request using the Cachito API.
:param int request_id: the ID of the Cachito request
:param str state: the state to set the Cachito request to
:param str state_reason: the state reason to set the Cachito request to
:raise CachitoError: if the request to the Cachito API fails
"""
# Import this here to avoid a circular import
from cachito.workers.requests import requests_auth_session
config = get_worker_config()
request_url = f'{config.cachito_api_url.rstrip("/")}/requests/{request_id}'
log.info(
'Setting the state of request %d to "%s" with the reason "%s"',
request_id, state, state_reason
)
payload = {'state': state, 'state_reason': state_reason}
try:
rv = requests_auth_session.patch(
request_url, json=payload, timeout=config.cachito_api_timeout)
except requests.RequestException:
msg = f'The connection failed when setting the state to "{state}" on request {request_id}'
log.exception(msg)
raise CachitoError(msg)
if not rv.ok:
log.error(
'The worker failed to set request %d to the "%s" state. The status was %d. '
'The text was:\n%s',
request_id, state, rv.status_code, rv.text,
)
raise CachitoError(f'Setting the state to "{state}" on request {request_id} failed')
def execute_script(script_name, payload):
"""
Execute a script using the Nexus REST API.
:param str script_name: the name of the script to execute
:param dict payload: the JSON payload to send as arguments to the script
:raise NexusScriptError: if the script execution fails
"""
config = get_worker_config()
auth = requests.auth.HTTPBasicAuth(config.cachito_nexus_username, config.cachito_nexus_password)
script_url = f"{config.cachito_nexus_url.rstrip('/')}/service/rest/v1/script/{script_name}/run"
log.info("Executing the Nexus script %s", script_name)
try:
rv = nexus_requests_session.post(
script_url, auth=auth, json=payload, timeout=config.cachito_nexus_timeout
)
except requests.RequestException:
error_msg = f"Could not connect to the Nexus instance to execute the script {script_name}"
log.exception(error_msg)
raise NexusScriptError(error_msg)
if not rv.ok:
log.error(
"The Nexus script %s failed with the status code %d and the text: %s",
script_name,
rv.status_code,
rv.text,
)
raise NexusScriptError(f"The Nexus script {script_name} failed with: {rv.text}")
log.info("The Nexus script %s executed successfully", script_name)
def _get_request_or_fail(request_id: int, connect_error_msg: str, status_error_msg: str) -> dict:
"""
Try to download the JSON data for a request from the Cachito API.
Both error messages can contain the {exc} placeholder which will be replaced by the actual
exception.
:param request_id: ID of the request to get
:param connect_error_msg: error message to raise if the connection fails
:param status_error_msg: error message to raise if the response status is 4xx or 5xx
:raises CachitoError: if the connection fails or the API returns an error response
"""
# Import this here to avoid a circular import (tasks -> requests -> tasks)
from cachito.workers.requests import requests_session
config = get_worker_config()
request_url = f'{config.cachito_api_url.rstrip("/")}/requests/{request_id}'
try:
rv = requests_session.get(request_url, timeout=config.cachito_api_timeout)
rv.raise_for_status()
except requests.HTTPError as e:
msg = status_error_msg.format(exc=e)
log.exception(msg)
raise CachitoError(msg)
except requests.RequestException as e:
msg = connect_error_msg.format(exc=e)
log.exception(msg)
raise CachitoError(msg)
return rv.json()
def create_bundle_archive(app_archive_path, request_id):
"""
Create the bundle archive to be downloaded by the user.
:param str app_archive_path: the path to the source archive; this is used as the base of the
bundle archive
:param int request_id: the request the bundle is for
"""
set_request_state(request_id, 'in_progress', 'Assembling the bundle archive')
config = get_worker_config()
bundle_archive_path = os.path.join(config.cachito_bundles_dir, f'{request_id}.tar.gz')
deps_path = os.path.join(config.cachito_bundles_dir, 'temp', str(request_id), 'deps')
log.debug(
'Using %s as the source and %s as the deps for creating the bundle for request %d',
app_archive_path, deps_path, request_id,
)
if not os.path.isdir(deps_path):
log.debug('No deps are present at %s, creating an empty directory', deps_path)
os.makedirs(deps_path, exist_ok=True)
# Python can't append to a compressed tar file, so we must copy the contents of the app archive
# to create the bundle archive rather than starting with a copy of the app archive
log.info('Creating %s', bundle_archive_path)
with tarfile.open(bundle_archive_path, mode='w:gz') as bundle_archive:
# Copy over the existing app archive
with tarfile.open(app_archive_path, mode='r:*') as app_archive:
for member in app_archive.getmembers():
bundle_archive.addfile(member, app_archive.extractfile(member.name))
# Add the dependencies to the bundle
bundle_archive.add(deps_path, 'deps')
def update_request_env_vars(request_id: int,
env_vars: Dict[str, Dict[str, str]]) -> None:
"""Update environment variables of a request.
:param int request_id: the id of a request to update the environment variables.
:param dict env_vars: mapping of environment variables to record. The keys represent
the environment variable name, and its value should be another map with the "value" and
"kind" attributes, e.g. {"NAME": {"value": "VALUE", "kind": "KIND"}}.
:raise CachitoError: if the request to the Cachito API fails
"""
config = get_worker_config()
request_url = _get_request_url(request_id)
payload = {"environment_variables": env_vars}
try:
rv = requests_auth_session.patch(request_url,
json=payload,
timeout=config.cachito_api_timeout)
except requests.RequestException:
msg = (
f"The connection failed when updating environment variables on the request {request_id}"
)
log.exception(msg)
raise CachitoError(msg)
if not rv.ok:
log.error(
"The worker failed to update environment variables on the request %d. "
"The status was %d. The text was:\n%s",
request_id,
rv.status_code,
rv.text,
)
raise CachitoError(
f"Updating environment variables on request {request_id} failed")
def get_package_and_deps(package_json_path, package_lock_path):
"""
Get the main package and dependencies based on the lock file.
If the lockfile contains non-registry dependencies, the lock file will be modified to use ones
in Nexus. Non-registry dependencies will have the "version_in_nexus" key set.
:param str package_json_path: the path to the package.json file
:param str package_lock_path: the path to the lock file
:return: a dictionary that has the keys "deps" which is the list of dependencies,
"lock_file" which is the lock file if it was modified, "package" which is the
dictionary describing the main package, and "package.json" which is the package.json file if
it was modified.
:rtype: dict
"""
with open(package_lock_path, "r") as f:
package_lock = json.load(f)
package_lock_original = copy.deepcopy(package_lock)
package = {"name": package_lock["name"], "type": "npm", "version": package_lock["version"]}
file_deps_allowlist = set(
get_worker_config().cachito_npm_file_deps_allowlist.get(package["name"], [])
)
name_to_deps, top_level_replacements = _get_deps(
package_lock.get("dependencies", {}), file_deps_allowlist
)
# Convert the name_to_deps mapping to a list now that it's fully populated
deps = [dep_info for deps_info in name_to_deps.values() for dep_info in deps_info]
rv = {"deps": deps, "lock_file": None, "package": package, "package.json": None}
# If top level replacements are returned, the package.json may need to be updated to use
# the replaced dependencies in the lock file. If these updates don't occur, running
# `npm install` causes the lock file to be updated since it's assumed that it's out of date.
if top_level_replacements:
with open(package_json_path, "r") as f:
package_json = json.load(f)
package_json_original = copy.deepcopy(package_json)
for dep_name, dep_version in top_level_replacements:
for dep_type in ("dependencies", "devDependencies"):
if dep_name in package_json.get(dep_type, {}):
log.info(
"Replacing the version of %s in %s from %s to %s in package.json",
dep_name,
dep_type,
package_json[dep_type][dep_name],
dep_version,
)
package_json[dep_type][dep_name] = dep_version
if package_json != package_json_original:
rv["package.json"] = package_json
if package_lock != package_lock_original:
rv["lock_file"] = package_lock
return rv
def get_js_hosted_repo_name():
"""
Get the name of NPM hosted repository.
:return: the name of NPM hosted repository
:rtype: str
"""
config = get_worker_config()
return config.cachito_nexus_js_hosted_repo_name
def get_request_bundle_path(request_id):
"""
Get the path to the request's bundle.
:param int request_id: the request ID to get the bundle path for
:return: the path to the request's bundle
:rtype: str
"""
config = get_worker_config()
return os.path.join(config.cachito_bundles_dir, f'{request_id}.tar.gz')
def _get_request_url(request_id):
"""
Get the API URL for the Cachito request.
:param int request_id: the request ID to use when constructing the API URL
:return: the API URL of the Cachito request
:rtype: str
"""
config = get_worker_config()
return f'{config.cachito_api_url.rstrip("/")}/requests/{request_id}'
def __new__(cls, request_id):
"""Create a new Path object."""
root_dir = get_worker_config().cachito_bundles_dir
self = super().__new__(cls, request_id, root_dir)
log.debug("Ensure directory %s exists.", self)
log.debug("Ensure directory %s exists.", self.deps_dir)
self.deps_dir.mkdir(parents=True, exist_ok=True)
return self
def get_yarn_proxy_repo_name(request_id):
"""
Get the name of yarn proxy repository for the request.
:param int request_id: the ID of the request this repository is for
:return: the cachito-yarn-<REQUEST_ID> string, representing the temporary repository name
:rtype: str
"""
config = get_worker_config()
return f"{config.cachito_nexus_request_repo_prefix}yarn-{request_id}"
def get_request_bundle_dir(request_id):
"""
Get the path to the directory where the bundle is being created for the request.
:param int request_id: the request ID to get the directory for
:return: the path to the where the bundle is being created
:rtype: str
"""
config = get_worker_config()
return os.path.join(config.cachito_bundles_dir, 'temp', str(request_id))
def get_npm_proxy_repo_name(request_id):
"""
Get the name of npm proxy repository for the request.
:param int request_id: the ID of the request this repository is for
:return: the name of npm proxy repository for the request
:rtype: str
"""
config = get_worker_config()
return f"{config.cachito_nexus_request_repo_prefix}npm-{request_id}"
def get_npm_proxy_repo_url(request_id):
"""
Get the URL for the Nexus npm proxy repository for the request.
:param int request_id: the ID of the request this repository is for
:return: the URL for the Nexus npm proxy repository for the request
:rtype: str
"""
config = get_worker_config()
repo_name = get_npm_proxy_repo_name(request_id)
return f"{config.cachito_nexus_url.rstrip('/')}/repository/{repo_name}/"
def get_ca_cert():
"""
Get the CA certificate that signed the Nexus instance's SSL certificate.
:return: the string of the CA certificate or None
:rtype: str or None
"""
config = get_worker_config()
if config.cachito_nexus_ca_cert and os.path.exists(config.cachito_nexus_ca_cert):
with open(config.cachito_nexus_ca_cert, "r") as f:
return f.read()
def __new__(cls, repo_name, ref):
"""Create a new Path object."""
self = super().__new__(cls, get_worker_config().cachito_sources_dir)
repo_relative_dir = pathlib.Path(*repo_name.split("/"))
self.package_dir = self.joinpath(repo_relative_dir)
self.archive_path = self.joinpath(repo_relative_dir, f"{ref}.tar.gz")
log.debug("Ensure directory %s exists.", self.package_dir)
self.package_dir.mkdir(parents=True, exist_ok=True)
return self
def upload_component(params, payload, to_nexus_hoster, additional_data=None):
"""
Push a payload to the Nexus upload endpoint.
See https://help.sonatype.com/repomanager3/rest-and-integration-api/components-api for further
reference.
:param dict params: the request parameters to the upload endpoint (e.g. {"repository": NAME})
:param dict payload: Nexus API compliant file payload
:param bool to_nexus_hoster: Use the nexus hoster instance, if available
:param dict additional_data: non-file Nexus API compliant file payload. This is needed for
string params that would be passed in the "file" param. Note that python requests does not
support sending non-files in the file payload. See
https://issues.sonatype.org/browse/NEXUS-21946 for further reference.
:raise CachitoError: if the upload fails
"""
# Import this here to avoid a circular import
from cachito.workers.requests import requests_session
config = get_worker_config()
if to_nexus_hoster:
username, password = get_nexus_hoster_credentials()
nexus_url = _get_nexus_hoster_url()
else:
username = config.cachito_nexus_username
password = config.cachito_nexus_password
nexus_url = config.cachito_nexus_url
auth = requests.auth.HTTPBasicAuth(username, password)
endpoint = f"{nexus_url}/service/rest/v1/components"
try:
rv = requests_session.post(
endpoint,
auth=auth,
files=payload,
data=additional_data,
params=params,
timeout=config.cachito_nexus_timeout,
)
except requests.RequestException:
log.exception(
"Could not connect to the Nexus instance to upload the component")
raise CachitoError(
"Could not connect to the Nexus instance to upload a component")
if not rv.ok:
log.error(
"Failed to upload a component with the status code %d and the text: %s",
rv.status_code,
rv.text,
)
raise CachitoError("Failed to upload a component to Nexus")
def _get_nexus_hoster_url():
"""
Get the Nexus instance with the hosted repositories.
:return: the URL to the Nexus instance
:rtype: str
"""
config = get_worker_config()
if config.cachito_nexus_hoster_url:
return config.cachito_nexus_hoster_url.rstrip("/")
return config.cachito_nexus_url.rstrip("/")