def download_package(url):
filename = os.path.basename(url)
dir = cf_remote_packages_dir()
mkdir(dir)
location = os.path.join(dir, filename)
if os.path.exists(location):
print("Package already downloaded: '{}'".format(location))
return location
print("Downloading package: '{}'".format(location))
os.system("curl --silent -L '{}' -o '{}'".format(url, location))
return location
def download_package(url, path=None):
if not path:
filename = os.path.basename(url)
directory = cf_remote_packages_dir()
mkdir(directory)
path = os.path.join(directory, filename)
if os.path.exists(path):
print("Package already downloaded: '{}'".format(path))
return path
print("Downloading package: '{}'".format(path))
with open(path, "wb") as f:
f.write(urllib.request.urlopen(url).read())
return path
def download_package(url, path=None):
if not path:
filename = os.path.basename(url)
directory = cf_remote_packages_dir()
mkdir(directory)
path = os.path.join(directory, filename)
# Use "ab" to prevent truncation of the file in case it is already being
# downloaded by a different thread.
with open(path, "ab") as f:
# Get an exclusive lock. If the file size is != 0 then it's already
# downloaded, otherwise we download.
fcntl.flock(f.fileno(), fcntl.LOCK_EX)
st = os.stat(path)
if st.st_size != 0:
log.debug("Package '{}' already downloaded".format(path))
else:
print("Downloading package: '{}'".format(path))
f.write(urllib.request.urlopen(url).read())
f.flush()
fcntl.flock(f.fileno(), fcntl.LOCK_UN)
return path
def _download_urls(urls):
"""Download packages from URLs, replace URLs with filenames
Return a new list of packages where URLs are replaced with paths
to packages which have been downloaded. Other values, like None
and paths to local packages are preserved.
"""
urls_dir = cf_remote_packages_dir("url_specified")
downloaded_urls = []
downloaded_paths = []
paths = []
for package_url in urls:
# Skip anything that is not a package url:
if package_url is None or not is_package_url(package_url):
paths.append(package_url)
continue
if not os.path.isdir(urls_dir):
os.mkdir(urls_dir)
# separate name from url and construct path for downloaded file
url, name = package_url, get_package_name(package_url)
path = os.path.join(urls_dir, name)
# replace url with local path to package in list which will be returned
paths.append(path)
if path in downloaded_paths and url not in downloaded_urls:
user_error(
f"2 packages with the same name '{name}' from different URLs")
download_package(url, path)
downloaded_urls.append(url)
downloaded_paths.append(path)
return paths