def main(self):
if not is_mac():
raise ProcessorError("This processor is Mac-only!")
# Clear and initiazize data structures
self.clear_vars()
# Ensure existence of necessary files, directories and paths
filename = self.get_filename()
if filename is None:
return
download_dir = self.get_download_dir()
self.env["pathname"] = os.path.join(download_dir, filename)
pathname_temporary = self.create_temp_file(download_dir)
# Prepare curl command
curl_cmd = self.prepare_download_curl_cmd(pathname_temporary)
# Execute curl command and parse headers
raw_headers = self.download_with_curl(curl_cmd)
header = self.parse_headers(raw_headers)
if self.download_changed(header):
self.env["download_changed"] = True
else:
# Discard the temp file
os.remove(pathname_temporary)
return
# New resource was downloaded. Move the temporary download file to the pathname
self.move_temp_file(pathname_temporary)
# Save last-modified and etag headers to files xattr
self.store_headers(header)
# Generate output messages and variables
self.output(f"Downloaded {self.env['pathname']}")
self.env["url_downloader_summary_result"] = {
"summary_text": "The following new items were downloaded:",
"data": {
"download_path": self.env["pathname"]
},
}
def test_is_mac_returns_false_on_not_mac(self, mock_platform):
"""On not-macOS, is_mac() should return False."""
mock_platform.return_value = "Windows-somethingsomething"
result = autopkglib.is_mac()
self.assertEqual(result, False)
def test_is_mac_returns_true_on_mac(self, mock_platform):
"""On macOS, is_mac() should return True."""
mock_platform.return_value = "Darwin-somethingsomething"
result = autopkglib.is_mac()
self.assertEqual(result, True)
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""See docstring for URLDownloader class"""
import os.path
import tempfile
from autopkglib import BUNDLE_ID, ProcessorError, is_mac
from autopkglib.URLGetter import URLGetter
if is_mac():
import xattr
__all__ = ["URLDownloader"]
class URLDownloader(URLGetter):
"""Downloads a URL to the specified download_dir using curl."""
description = __doc__
input_variables = {
"url": {
"required": True,
"description": "The URL to download."
},
"request_headers": {
def test_is_mac_returns_false_on_not_mac(self, mock_sys):
"""On not-macOS, is_mac() should return False."""
mock_sys.platform = "Win32-somethingsomething"
result = autopkglib.is_mac()
self.assertEqual(result, False)
def _default_use_python_native_extractor() -> bool:
if is_mac():
return False
return True
def main(self):
if not is_mac():
raise ProcessorError("This processor is Mac-only!")
# clear any pre-exising summary result
if "url_downloader_summary_result" in self.env:
del self.env["url_downloader_summary_result"]
self.env["last_modified"] = ""
self.env["etag"] = ""
existing_file_size = None
if "PKG" in self.env:
self.env["pathname"] = os.path.expanduser(self.env["PKG"])
self.env["download_changed"] = True
self.output("Given %s, no download needed." % self.env["pathname"])
return
if "filename" not in self.env:
# Generate filename.
filename = self.env["url"].rpartition("/")[2]
else:
filename = self.env["filename"]
download_dir = self.env.get("download_dir") or os.path.join(
self.env["RECIPE_CACHE_DIR"], "downloads"
)
pathname = os.path.join(download_dir, filename)
# Save pathname to environment
self.env["pathname"] = pathname
# create download_dir if needed
if not os.path.exists(download_dir):
try:
os.makedirs(download_dir)
except OSError as err:
raise ProcessorError(
"Can't create %s: %s" % (download_dir, err.strerror)
)
# Create a temp file
temporary_file = tempfile.NamedTemporaryFile(dir=download_dir, delete=False)
pathname_temporary = temporary_file.name
# Set permissions on the temp file as curl would set for a newly-downloaded
# file. NamedTemporaryFile uses mkstemp(), which sets a mode of 0600, and
# this can cause issues if this item is eventually copied to a Munki repo
# with the same permissions and the file is inaccessible by (for example)
# the webserver.
os.chmod(pathname_temporary, 0o644)
# construct curl command.
curl_cmd = [
self.env["CURL_PATH"],
"--silent",
"--show-error",
"--no-buffer",
"--fail",
"--dump-header",
"-",
"--speed-time",
"30",
"--location",
"--url",
self.env["url"],
"--output",
pathname_temporary,
]
if "request_headers" in self.env:
headers = self.env["request_headers"]
for header, value in headers.items():
curl_cmd.extend(["--header", "%s: %s" % (header, value)])
if "curl_opts" in self.env:
for item in self.env["curl_opts"]:
curl_cmd.extend([item])
# if file already exists and the size is 0, discard it and download
# again
if os.path.exists(pathname) and os.path.getsize(pathname) == 0:
os.remove(pathname)
# if file already exists, add some headers to the request
# so we don't retrieve the content if it hasn't changed
if os.path.exists(pathname):
existing_file_size = os.path.getsize(pathname)
etag = getxattr(pathname, XATTR_ETAG)
last_modified = getxattr(pathname, XATTR_LAST_MODIFIED)
if etag:
curl_cmd.extend(["--header", "If-None-Match: %s" % etag])
if last_modified:
curl_cmd.extend(["--header", "If-Modified-Since: %s" % last_modified])
# Open URL.
proc = subprocess.Popen(
curl_cmd,
shell=False,
bufsize=1,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
donewithheaders = False
maxheaders = 15
header = {}
header["http_result_code"] = "000"
header["http_result_description"] = ""
while True:
if not donewithheaders:
info = proc.stdout.readline().strip("\r\n")
if info.startswith("HTTP/"):
try:
header["http_result_code"] = info.split(None, 2)[1]
header["http_result_description"] = info.split(None, 2)[2]
except IndexError:
pass
elif ": " in info:
# got a header line
part = info.split(None, 1)
fieldname = part[0].rstrip(":").lower()
try:
header[fieldname] = part[1]
except IndexError:
header[fieldname] = ""
elif self.env["url"].startswith("ftp://"):
part = info.split(None, 1)
responsecode = part[0]
if responsecode == "213":
# This is the reply to curl's SIZE command on the file
# We can map it to the HTTP content-length header
try:
header["content-length"] = part[1]
except IndexError:
pass
elif responsecode.startswith("55"):
header["http_result_code"] = "404"
header["http_result_description"] = info
elif responsecode == "150" or responsecode == "125":
header["http_result_code"] = "200"
header["http_result_description"] = info
donewithheaders = True
elif info == "":
# we got an empty line; end of headers (or curl exited)
if header.get("http_result_code") in [
"301",
"302",
"303",
"307",
"308",
]:
# redirect, so more headers are coming.
# Throw away the headers we've received so far
header = {}
header["http_result_code"] = "000"
header["http_result_description"] = ""
else:
donewithheaders = True
else:
time.sleep(0.1)
if proc.poll() is not None:
# For small download files curl may exit before all headers
# have been parsed, don't immediately exit.
maxheaders -= 1
if donewithheaders or maxheaders <= 0:
break
retcode = proc.poll()
if retcode: # Non-zero exit code from curl => problem with download
curlerr = ""
try:
curlerr = proc.stderr.read().rstrip("\n")
curlerr = curlerr.split(None, 2)[2]
except IndexError:
pass
raise ProcessorError("Curl failure: %s (exit code %s)" % (curlerr, retcode))
# If Content-Length header is present and we had a cached
# file, see if it matches the size of the cached file.
# Useful for webservers that don't provide Last-Modified
# and ETag headers.
if (not header.get("etag") and not header.get("last-modified")) or self.env[
"CHECK_FILESIZE_ONLY"
]:
size_header = header.get("content-length")
if size_header and int(size_header) == existing_file_size:
self.env["download_changed"] = False
self.output(
"File size returned by webserver matches that "
"of the cached file: %s bytes" % size_header
)
self.output(
"WARNING: Matching a download by filesize is a "
"fallback mechanism that does not guarantee "
"that a build is unchanged."
)
self.output("Using existing %s" % pathname)
return
if header["http_result_code"] == "304":
# resource not modified
self.env["download_changed"] = False
self.output("Item at URL is unchanged.")
self.output("Using existing %s" % pathname)
# Discard the temp file
os.remove(pathname_temporary)
return
self.env["download_changed"] = True
# New resource was downloaded. Move the temporary download file
# to the pathname
if os.path.exists(pathname):
os.remove(pathname)
try:
os.rename(pathname_temporary, pathname)
except OSError:
raise ProcessorError("Can't move %s to %s" % (pathname_temporary, pathname))
# save last-modified header if it exists
if header.get("last-modified"):
self.env["last_modified"] = header.get("last-modified")
xattr.setxattr(pathname, XATTR_LAST_MODIFIED, header.get("last-modified"))
self.output(
"Storing new Last-Modified header: %s" % header.get("last-modified")
)
# save etag if it exists
self.env["etag"] = ""
if header.get("etag"):
self.env["etag"] = header.get("etag")
xattr.setxattr(pathname, XATTR_ETAG, header.get("etag"))
self.output("Storing new ETag header: %s" % header.get("etag"))
self.output("Downloaded %s" % pathname)
self.env["url_downloader_summary_result"] = {
"summary_text": "The following new items were downloaded:",
"data": {"download_path": pathname},
}
