def run(self, display_callback):
log.debug('INFO self.collection_info: %s', self.collection_info)
# ie, 'v1.2.3.tar.gz', not full path
archive_filename_basename = \
ARCHIVE_FILENAME_TEMPLATE.format(namespace=self.collection_info.namespace,
name=self.collection_info.name,
version=self.collection_info.version,
extension=ARCHIVE_FILENAME_EXTENSION)
archive_path = os.path.join(self.build_context.output_path,
archive_filename_basename)
log.debug('Building archive into archive_path: %s', archive_path)
# The name of the top level dir in the tar file, ie, there isnt one.
archive_top_dir = ""
log.debug('archive_top_dir: %s', archive_top_dir)
# 'x:gz' is 'create exclusive gzipped'
tar_file = tarfile.open(archive_path, mode='w:gz')
# Find collection files, build a file manifest, serialize to json and add to the tarfile
file_walker = collection_members.FileWalker(
collection_path=self.build_context.collection_path)
col_members = collection_members.CollectionMembers(walker=file_walker)
log.debug('col_members: %s', col_members)
col_file_names = col_members.run()
col_files = collection_artifact_file_manifest.gen_file_manifest_items(
col_file_names, self.build_context.collection_path)
file_manifest = CollectionArtifactFileManifest(files=col_files)
log.debug('file_manifest: %s', file_manifest)
for col_member_file in file_manifest.files:
top_dir = False
# arcname will be a relative path not an abspath at this point
rel_path = col_member_file.name or col_member_file.src_name
if rel_path == '.':
rel_path = ''
archive_member_path = rel_path
log.debug('adding %s to %s (from %s)', archive_member_path,
archive_path, col_member_file.name)
log.debug('name=%s, arcname=%s, top_dir=%s', col_member_file.name,
archive_member_path, top_dir)
# if top_dir:
# tar_file.add(col_member_file.name, arcname=archive_top_dir, recursive=False)
# else:
# tar_file.add(col_member_file.name, arcname=archive_member_path, recursive=False)
tar_file.add(col_member_file.src_name,
arcname=archive_member_path,
recursive=False)
# Generate FILES.json contents
# TODO/FIXME: find and use some streamable file format for the filelist (csv?)
file_manifest_buf = json.dumps(attr.asdict(
file_manifest, filter=filter_artifact_file_name),
indent=4)
log.debug('file_manifest_buf: %s', file_manifest_buf)
b_file_manifest_buf = to_bytes(file_manifest_buf)
b_file_manifest_buf_bytesio = six.BytesIO(b_file_manifest_buf)
archive_manifest_path = collection_artifact_manifest.COLLECTION_MANIFEST_FILENAME
log.debug('archive_manifest_path: %s', archive_manifest_path)
archive_file_manifest_path = collection_artifact_file_manifest.COLLECTION_FILE_MANIFEST_FILENAME
log.debug('archive_file_manifest_path: %s', archive_file_manifest_path)
file_manifest_tar_info = tar_file.gettarinfo(
os.path.join(self.build_context.collection_path,
COLLECTION_INFO_FILENAME))
file_manifest_tar_info.name = archive_file_manifest_path
file_manifest_tar_info.size = len(b_file_manifest_buf)
# Add FILES.json contents to tarball
tar_file.addfile(tarinfo=file_manifest_tar_info,
fileobj=b_file_manifest_buf_bytesio)
# addfile reads to end of bytesio, seek back to begin
b_file_manifest_buf_bytesio.seek(0)
file_manifest_file_chksum = chksums.sha256sum_from_fo(
b_file_manifest_buf_bytesio)
log.debug('file_manifest_file_chksum: %s', file_manifest_file_chksum)
# file_manifest_file_name_in_archive = os.path.relpath(archive_file_manifest_path, self.build_context.collection_path)
file_manifest_file_item = CollectionArtifactFile(
src_name=collection_artifact_file_manifest.
COLLECTION_FILE_MANIFEST_FILENAME,
# The path where the file will live inside the archive
name=collection_artifact_file_manifest.
COLLECTION_FILE_MANIFEST_FILENAME,
ftype='file',
chksum_type='sha256',
chksum_sha256=file_manifest_file_chksum)
# Generage MANIFEST.json contents
manifest = CollectionArtifactManifest(
collection_info=self.collection_info,
file_manifest_file=file_manifest_file_item)
log.debug('manifest: %s', manifest)
manifest_buf = json.dumps(
attr.asdict(manifest, filter=filter_artifact_file_name),
# sort_keys=True,
indent=4)
log.debug('manifest buf: %s', manifest_buf)
# add MANIFEST.yml to archive
b_manifest_buf = to_bytes(manifest_buf)
b_manifest_buf_bytesio = six.BytesIO(b_manifest_buf)
archive_manifest_path = os.path.join(
archive_top_dir,
collection_artifact_manifest.COLLECTION_MANIFEST_FILENAME)
log.debug('archive_manifest_path: %s', archive_manifest_path)
# copy the uid/gid/perms for galaxy.yml to use on the manifes. Need sep instances for manifest and file_manifest
# TODO: decide on what the generators owner/group/perms should be (root.root 644?)
manifest_tar_info = tar_file.gettarinfo(
os.path.join(self.build_context.collection_path,
COLLECTION_INFO_FILENAME))
manifest_tar_info.name = archive_manifest_path
manifest_tar_info.size = len(b_manifest_buf)
# TODO: set mtime equal to the 'build time' / build_info when we start creating that.
tar_file.addfile(tarinfo=manifest_tar_info,
fileobj=b_manifest_buf_bytesio)
log.debug('populated tarfile %s: %s', archive_path,
pprint.pformat(tar_file.getmembers))
tar_file.close()
# could in theory make creating the release artifact work much the same
# as serializing some object (I mean, that is what it is... but
messages = [
'Building collection: %s' % self.build_context.collection_path,
'Created artifact: %s' % archive_path
]
result = BuildResult(
status=BuildStatuses.success,
messages=messages,
# errors=[],
errors=col_members.walker.file_errors,
manifest=manifest,
file_manifest=file_manifest,
artifact_file_path=archive_path)
for message in result.messages:
log.info(message)
display_callback(message)
for error in result.errors:
log.error(error)
display_callback(error, level='warning')
return result
def run(self, display_callback):
file_walker = collection_members.FileWalker(collection_path=self.build_context.collection_path)
col_members = collection_members.CollectionMembers(walker=file_walker)
log.debug('col_members: %s', col_members)
log.debug('INFO self.collection_info: %s', self.collection_info)
col_file_names = col_members.run()
col_files = collection_artifact_manifest.gen_manifest_artifact_files(col_file_names,
self.build_context.collection_path)
manifest = CollectionArtifactManifest(collection_info=self.collection_info,
files=col_files)
log.debug('manifest: %s', manifest)
manifest_buf = json.dumps(attr.asdict(manifest,
filter=filter_artifact_file_name),
# sort_keys=True,
indent=4)
# manifest_buf = yaml.safe_dump(attr.asdict(manifest),
# default_flow_style=False)
log.debug('manifest buf: %s', manifest_buf)
# ie, 'v1.2.3.tar.gz', not full path
archive_filename_basename = \
ARCHIVE_FILENAME_TEMPLATE.format(namespace=self.collection_info.namespace,
name=self.collection_info.name,
version=self.collection_info.version,
extension=ARCHIVE_FILENAME_EXTENSION)
archive_path = os.path.join(self.build_context.output_path,
archive_filename_basename)
log.debug('Building archive into archive_path: %s', archive_path)
# The name of the top level dir in the tar file. It is
# in the format '{collection_name}-{version}'.
# NOTE: This doesnt follow convention of 'foo-bar-1.2.3.tar.gz -> foo-bar-1.2.3/*'
archive_top_dir = ARCHIVE_TOPDIR_TEMPLATE.format(collection_info=self.collection_info)
log.debug('archive_top_dir: %s', archive_top_dir)
# 'x:gz' is 'create exclusive gzipped'
tar_file = tarfile.open(archive_path, mode='w:gz')
# tar_file.add(archive_top_dir, arcname=archive_top_dir, recursive=False)
for col_member_file in manifest.files:
top_dir = False
# arcname will be a relative path not an abspath at this point
rel_path = col_member_file.name or col_member_file.src_name
if rel_path == '.':
rel_path = ''
archive_member_path = os.path.join(archive_top_dir, rel_path)
log.debug('adding %s to %s (from %s)', archive_member_path,
archive_path, col_member_file.name)
log.debug('name=%s, arcname=%s, top_dir=%s', col_member_file.name, archive_member_path, top_dir)
# if top_dir:
# tar_file.add(col_member_file.name, arcname=archive_top_dir, recursive=False)
# else:
# tar_file.add(col_member_file.name, arcname=archive_member_path, recursive=False)
tar_file.add(col_member_file.src_name, arcname=archive_member_path, recursive=False)
# add MANIFEST.yml to archive
b_manifest_buf = to_bytes(manifest_buf)
b_manifest_buf_bytesio = six.BytesIO(b_manifest_buf)
archive_manifest_path = os.path.join(archive_top_dir,
collection_artifact_manifest.COLLECTION_MANIFEST_FILENAME)
log.debug('archive_manifest_path: %s', archive_manifest_path)
# copy the uid/gid/perms for galaxy.yml to use on the manifest
# TODO: decide on what the generators owner/group/perms should be (root.root 644?)
manifest_tar_info = tar_file.gettarinfo(os.path.join(self.build_context.collection_path, COLLECTION_INFO_FILENAME))
manifest_tar_info.name = archive_manifest_path
manifest_tar_info.size = len(b_manifest_buf)
# TODO: set mtime equal to the 'build time' / build_info when we start creating that.
tar_file.addfile(tarinfo=manifest_tar_info,
fileobj=b_manifest_buf_bytesio)
log.debug('populated tarfile %s: %s', archive_path,
pprint.pformat(tar_file.getmembers))
tar_file.close()
# could in theory make creating the release artifact work much the same
# as serializing some object (I mean, that is what it is... but
messages = ['Building collection: %s' % self.build_context.collection_path,
'Created artifact: %s' % archive_path]
result = BuildResult(status=BuildStatuses.success,
messages=messages,
# errors=[],
errors=col_members.walker.file_errors,
manifest=manifest,
artifact_file_path=archive_path)
for message in result.messages:
log.info(message)
display_callback(message)
for error in result.errors:
log.error(error)
display_callback(error, level='warning')
return result
def http_request(self, req):
tmp_ca_cert_path, to_add_ca_cert_path, paths_checked = self.get_ca_certs(
)
https_proxy = os.environ.get('https_proxy')
context = None
try:
context = self._make_context(to_add_ca_cert_path)
except Exception:
# We'll make do with no context below
pass
# Detect if 'no_proxy' environment variable is set and if our URL is included
use_proxy = self.detect_no_proxy(req.get_full_url())
if not use_proxy:
# ignore proxy settings for this host request
if tmp_ca_cert_path:
try:
os.remove(tmp_ca_cert_path)
except OSError:
pass
if to_add_ca_cert_path:
try:
os.remove(to_add_ca_cert_path)
except OSError:
pass
return req
try:
if https_proxy:
proxy_parts = generic_urlparse(urlparse(https_proxy))
port = proxy_parts.get('port') or 443
s = socket.create_connection(
(proxy_parts.get('hostname'), port))
if proxy_parts.get('scheme') == 'http':
s.sendall(
to_bytes(self.CONNECT_COMMAND %
(self.hostname, self.port),
errors='surrogate_or_strict'))
if proxy_parts.get('username'):
credentials = "%s:%s" % (proxy_parts.get(
'username', ''), proxy_parts.get('password', ''))
s.sendall(
b'Proxy-Authorization: Basic %s\r\n' %
base64.b64encode(
to_bytes(
credentials,
errors='surrogate_or_strict')).strip())
s.sendall(b'\r\n')
connect_result = b""
while connect_result.find(b"\r\n\r\n") <= 0:
connect_result += s.recv(4096)
# 128 kilobytes of headers should be enough for everyone.
if len(connect_result) > 131072:
raise ProxyError(
'Proxy sent too verbose headers. Only 128KiB allowed.'
)
self.validate_proxy_response(connect_result)
if context:
ssl_s = context.wrap_socket(
s, server_hostname=self.hostname)
elif HAS_URLLIB3_SSL_WRAP_SOCKET:
ssl_s = ssl_wrap_socket(s,
ca_certs=tmp_ca_cert_path,
cert_reqs=ssl.CERT_REQUIRED,
ssl_version=PROTOCOL,
server_hostname=self.hostname)
else:
ssl_s = ssl.wrap_socket(s,
ca_certs=tmp_ca_cert_path,
cert_reqs=ssl.CERT_REQUIRED,
ssl_version=PROTOCOL)
match_hostname(ssl_s.getpeercert(), self.hostname)
else:
raise ProxyError(
'Unsupported proxy scheme: %s. Currently ansible only supports HTTP proxies.'
% proxy_parts.get('scheme'))
else:
s = socket.create_connection((self.hostname, self.port))
if context:
ssl_s = context.wrap_socket(s,
server_hostname=self.hostname)
elif HAS_URLLIB3_SSL_WRAP_SOCKET:
ssl_s = ssl_wrap_socket(s,
ca_certs=tmp_ca_cert_path,
cert_reqs=ssl.CERT_REQUIRED,
ssl_version=PROTOCOL,
server_hostname=self.hostname)
else:
ssl_s = ssl.wrap_socket(s,
ca_certs=tmp_ca_cert_path,
cert_reqs=ssl.CERT_REQUIRED,
ssl_version=PROTOCOL)
match_hostname(ssl_s.getpeercert(), self.hostname)
# close the ssl connection
# ssl_s.unwrap()
s.close()
except (ssl.SSLError, CertificateError) as e:
build_ssl_validation_error(self.hostname, self.port, paths_checked,
e)
except socket.error as e:
raise ConnectionError('Failed to connect to %s at port %s: %s' %
(self.hostname, self.port, to_native(e)))
try:
# cleanup the temp file created, don't worry
# if it fails for some reason
os.remove(tmp_ca_cert_path)
except:
pass
try:
# cleanup the temp file created, don't worry
# if it fails for some reason
if to_add_ca_cert_path:
os.remove(to_add_ca_cert_path)
except:
pass
return req
def open_url(url,
data=None,
headers=None,
method=None,
use_proxy=True,
force=False,
last_mod_time=None,
timeout=10,
validate_certs=True,
url_username=None,
url_password=None,
http_agent=None,
force_basic_auth=False,
follow_redirects='urllib2',
client_cert=None,
client_key=None,
cookies=None):
'''
Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3)
Does not require the module environment
'''
handlers = []
ssl_handler = maybe_add_ssl_handler(url, validate_certs)
if ssl_handler:
handlers.append(ssl_handler)
parsed = generic_urlparse(urlparse(url))
if parsed.scheme != 'ftp':
username = url_username
if headers is None:
headers = {}
if username:
password = url_password
netloc = parsed.netloc
elif '@' in parsed.netloc:
credentials, netloc = parsed.netloc.split('@', 1)
if ':' in credentials:
username, password = credentials.split(':', 1)
else:
username = credentials
password = ''
parsed_list = parsed.as_list()
parsed_list[1] = netloc
# reconstruct url without credentials
url = urlunparse(parsed_list)
if username and not force_basic_auth:
passman = urllib_request.HTTPPasswordMgrWithDefaultRealm()
# this creates a password manager
passman.add_password(None, netloc, username, password)
# because we have put None at the start it will always
# use this username/password combination for urls
# for which `theurl` is a super-url
authhandler = urllib_request.HTTPBasicAuthHandler(passman)
digest_authhandler = urllib_request.HTTPDigestAuthHandler(passman)
# create the AuthHandler
handlers.append(authhandler)
handlers.append(digest_authhandler)
elif username and force_basic_auth:
headers["Authorization"] = basic_auth_header(username, password)
else:
try:
rc = netrc.netrc(os.environ.get('NETRC'))
login = rc.authenticators(parsed.hostname)
except IOError:
login = None
if login:
username, _, password = login
if username and password:
headers["Authorization"] = basic_auth_header(
username, password)
if not use_proxy:
proxyhandler = urllib_request.ProxyHandler({})
handlers.append(proxyhandler)
if HAS_SSLCONTEXT and not validate_certs:
# In 2.7.9, the default context validates certificates
context = SSLContext(ssl.PROTOCOL_SSLv23)
context.options |= ssl.OP_NO_SSLv2
context.options |= ssl.OP_NO_SSLv3
context.verify_mode = ssl.CERT_NONE
context.check_hostname = False
handlers.append(
HTTPSClientAuthHandler(client_cert=client_cert,
client_key=client_key,
context=context))
elif client_cert:
handlers.append(
HTTPSClientAuthHandler(client_cert=client_cert,
client_key=client_key))
# pre-2.6 versions of python cannot use the custom https
# handler, since the socket class is lacking create_connection.
# Some python builds lack HTTPS support.
if hasattr(socket, 'create_connection') and CustomHTTPSHandler:
handlers.append(CustomHTTPSHandler)
handlers.append(RedirectHandlerFactory(follow_redirects, validate_certs))
# add some nicer cookie handling
if cookies is not None:
handlers.append(urllib_request.HTTPCookieProcessor(cookies))
opener = urllib_request.build_opener(*handlers)
urllib_request.install_opener(opener)
data = to_bytes(data, nonstring='passthru')
if method:
if method.upper() not in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT',
'DELETE', 'TRACE', 'CONNECT', 'PATCH'):
raise ConnectionError('invalid HTTP request method; %s' %
method.upper())
request = RequestWithMethod(url, method.upper(), data)
else:
request = urllib_request.Request(url, data)
# add the custom agent header, to help prevent issues
# with sites that block the default urllib agent string
if http_agent:
request.add_header('User-agent', http_agent)
# Cache control
# Either we directly force a cache refresh
if force:
request.add_header('cache-control', 'no-cache')
# or we do it if the original is more recent than our copy
elif last_mod_time:
tstamp = last_mod_time.strftime('%a, %d %b %Y %H:%M:%S +0000')
request.add_header('If-Modified-Since', tstamp)
# user defined headers now, which may override things we've set above
if headers:
if not isinstance(headers, dict):
raise ValueError("headers provided to fetch_url() must be a dict")
for header in headers:
request.add_header(header, headers[header])
urlopen_args = [request, None]
if sys.version_info >= (2, 6, 0):
# urlopen in python prior to 2.6.0 did not
# have a timeout parameter
urlopen_args.append(timeout)
r = urllib_request.urlopen(*urlopen_args)
return r
def basic_auth_header(username, password):
"""Takes a username and password and returns a byte string suitable for
using as value of an Authorization header to do basic auth.
"""
return b"Basic %s" % base64.b64encode(
to_bytes("%s:%s" % (username, password), errors='surrogate_or_strict'))