def download(out_filename, url, work_dir, rm_on_error=True):
assert os.path.isabs(out_filename)
assert os.path.isabs(work_dir)
work_dir = work_dir.rstrip('/')
# Strip off whitespace to make it so scheme matching doesn't fail because
# of simple user whitespace.
url = url.strip()
# Handle file:// urls specially since requests doesn't know about them.
try:
if url.startswith('file://'):
src_filename = url[len('file://'):]
if not os.path.isabs(src_filename):
src_filename = work_dir + '/' + src_filename
shutil.copyfile(src_filename, out_filename)
else:
_download_remote_file(out_filename, url)
except Exception as fetch_exception:
if rm_on_error:
rm_passed = False
# try / except so if remove fails we don't get an exception during an exception.
# Sets rm_passed to true so if this fails we can include a special error message in the
# FetchError
try:
os.remove(out_filename)
rm_passed = True
except Exception:
pass
else:
rm_passed = True
raise FetchError(url, out_filename, fetch_exception,
rm_passed) from fetch_exception
def download(out_filename, url, work_dir, rm_on_error=True):
assert os.path.isabs(out_filename)
assert os.path.isabs(work_dir)
work_dir = work_dir.rstrip('/')
# Strip off whitespace to make it so scheme matching doesn't fail because
# of simple user whitespace.
url = url.strip()
# Handle file:// urls specially since requests doesn't know about them.
try:
if url.startswith('file://'):
src_filename = url[len('file://'):]
if not os.path.isabs(src_filename):
src_filename = work_dir + '/' + src_filename
shutil.copyfile(src_filename, out_filename)
else:
# Download the file.
with open(out_filename, "w+b") as f:
r = requests.get(url, stream=True)
if r.status_code == 301:
raise Exception("got a 301")
r.raise_for_status()
for chunk in r.iter_content(chunk_size=4096):
f.write(chunk)
except Exception as fetch_exception:
if rm_on_error:
rm_passed = False
# try / except so if remove fails we don't get an exception during an exception.
# Sets rm_passed to true so if this fails we can include a special error message in the
# FetchError
try:
os.remove(out_filename)
rm_passed = True
except Exception:
pass
else:
rm_passed = True
raise FetchError(url, out_filename, fetch_exception,
rm_passed) from fetch_exception
def download_inner(self, path, local_path):
local_path_tmp = '{}.tmp'.format(local_path)
url = self._get_absolute(path)
try:
with open(local_path_tmp, 'w+b') as f:
r = requests.get(url, stream=True)
r.raise_for_status()
for chunk in r.iter_content(chunk_size=4096):
f.write(chunk)
os.rename(local_path_tmp, local_path)
except Exception as fetch_exception:
rm_passed = False
# Delete the temp file, re-raise.
try:
os.remove(local_path_tmp)
rm_passed = True
except Exception:
pass
raise FetchError(url, local_path, fetch_exception,
rm_passed) from fetch_exception
def download_inner(self, path, local_path):
try:
subprocess.check_call(['cp', self.__full_path(path), local_path])
except Exception as fetch_exception:
raise FetchError(path, local_path, fetch_exception,
True) from fetch_exception