def ensure_service_directory(self, relative_name):
"""Create a directory in PROJECT_DIR/services with the given name.
The name should be unique to the ServiceRequirement creating the directory,
so usually the requirement's env var.
Args:
relative_name (str): name to distinguish this dir from other service directories
"""
path = _service_directory(self._local_state_file, relative_name)
makedirs_ok_if_exists(path)
return path
def do_test(dirname):
dir1 = os.path.join(dirname, "foo")
dir2 = os.path.join(dir1, "bar")
dir3 = os.path.join(dir2, "baz")
assert os.path.isdir(dirname)
assert not os.path.isdir(dir1)
assert not os.path.isdir(dir2)
assert not os.path.isdir(dir3)
makedirs_ok_if_exists(dir3)
assert os.path.isdir(dir1)
assert os.path.isdir(dir2)
assert os.path.isdir(dir3)
def _write_a_file(self, filename):
try:
makedirs.makedirs_ok_if_exists(os.path.dirname(filename))
with codecs.open(filename, 'w', encoding='utf-8') as f:
# we don't read the contents of the file for now, but
# recording the version in it in case in the future
# that is useful. We need to write something to the
# file to bump its mtime if it already exists...
f.write('{"anaconda_project_version": "%s"}\n' % version)
return True
except (IOError, OSError):
# ignore errors because this is just an optimization, if we
# fail we will survive
return False
def _save_file(yaml, filename):
contents = ryaml.dump(yaml, Dumper=ryaml.RoundTripDumper)
try:
# This is to ensure we don't corrupt the file, even if ruamel.yaml is broken
ryaml.load(contents, Loader=ryaml.RoundTripLoader)
except YAMLError as e: # pragma: no cover (should not happen)
print("ruamel.yaml bug; it failed to parse a file that it generated.", file=sys.stderr)
print(" the parse error was: " + str(e), file=sys.stderr)
print("Generated file was:", file=sys.stderr)
print(contents, file=sys.stderr)
raise RuntimeError("Bug in ruamel.yaml library; failed to parse a file that it generated: " + str(e))
if not os.path.isfile(filename):
# might have to make the directory
dirname = os.path.dirname(filename)
makedirs_ok_if_exists(dirname)
_atomic_replace(filename, contents)
def _extract_files_tar(tar_path, src_and_dest, frontend):
with tarfile.open(tar_path, mode='r') as tf:
for (src, dest) in src_and_dest:
frontend.info("Unpacking %s to %s" % (src, dest))
member = tf.getmember(src)
# we could also use tf._extract_member here, but the
# solution below with only the public API isn't that
# bad.
if member.isreg():
makedirs_ok_if_exists(os.path.dirname(dest))
tf.makefile(member, dest)
else:
assert member.isdir() # we filtered out other types
makedirs_ok_if_exists(dest)
try:
tf.chown(member, dest, False) # pragma: no cover (python 3.5 has another param)
except TypeError: # pragma: no cover
tf.chown(member, dest) # pragma: no cover (python 2.7, 3.4)
tf.chmod(member, dest)
tf.utime(member, dest)
def _extract_files_zip(zip_path, src_and_dest, frontend):
# the zipfile API has no way to extract to a filename of
# our choice, so we have to unpack to a temporary location,
# then copy those files over.
tmpdir = tempfile.mkdtemp()
try:
with zipfile.ZipFile(zip_path, mode='r') as zf:
zf.extractall(tmpdir)
for (src, dest) in src_and_dest:
frontend.info("Unpacking %s to %s" % (src, dest))
src_path = os.path.join(tmpdir, src)
if os.path.isdir(src_path):
makedirs_ok_if_exists(dest)
shutil.copystat(src_path, dest)
else:
makedirs_ok_if_exists(os.path.dirname(dest))
shutil.copy2(src_path, dest)
finally:
try:
shutil.rmtree(tmpdir)
except (IOError, OSError):
pass
import sys
try:
from backports.tempfile import TemporaryDirectory
except ImportError:
from tempfile import TemporaryDirectory
import zipfile
import tempfile
from anaconda_project.internal.makedirs import makedirs_ok_if_exists
from anaconda_project.local_state_file import LocalStateFile
from anaconda_project.yaml_file import _load_string
from anaconda_project.project_file import (possible_project_file_names,
DEFAULT_PROJECT_FILENAME)
local_tmp = os.path.abspath("./build/tmp")
makedirs_ok_if_exists(local_tmp)
def with_directory_contents(contents, func):
tempd = TemporaryDirectory(prefix="test-")
dirname = tempd.name
try:
for filename, file_content in contents.items():
path = os.path.join(dirname, filename)
if file_content is None:
# make a directory
makedirs_ok_if_exists(path)
else:
makedirs_ok_if_exists(os.path.dirname(path))
with codecs.open(path, 'w', 'utf-8') as f:
f.write(file_content)
def run(self, io_loop):
"""Run the download on the given io_loop."""
assert self._client is None
dirname = os.path.dirname(self._filename)
try:
makedirs.makedirs_ok_if_exists(dirname)
except Exception as e:
self._errors.append("Could not create directory '%s': %s" %
(dirname, e))
raise gen.Return(None)
if self._hash_algorithm is not None:
hasher = getattr(hashlib, self._hash_algorithm)()
self._client = httpclient.AsyncHTTPClient(
io_loop=io_loop,
max_clients=1,
# without this we buffer a huge amount
# of stuff and then call the streaming_callback
# once.
max_buffer_size=1024 * 1024,
# without this we 599 on large downloads
max_body_size=100 * 1024 * 1024 * 1024,
force_instance=True)
tmp_filename = self._filename + ".part"
try:
_file = open(tmp_filename, 'wb')
except EnvironmentError as e:
self._errors.append("Failed to open %s: %s" % (tmp_filename, e))
raise gen.Return(None)
def cleanup_tmp():
try:
_file.close()
# future: we could save it in order to try
# resuming a failed download midstream, but
# pointless until the download code above
# knows how to resume.
os.remove(tmp_filename)
except EnvironmentError:
pass
def writer(chunk):
if len(self._errors) > 0:
return
if self._hash_algorithm is not None:
hasher.update(chunk)
try:
_file.write(chunk)
except EnvironmentError as e:
# we can't actually throw this error or Tornado freaks out, so instead
# we ignore all future chunks once we have an error, which does mean
# we continue to download bytes that we don't use. yuck.
self._errors.append("Failed to write to %s: %s" %
(tmp_filename, e))
try:
timeout_in_seconds = 60 * 10 # pretty long because we could be dealing with huge files
request = httpclient.HTTPRequest(
url=self._url,
streaming_callback=writer,
request_timeout=timeout_in_seconds)
try:
response = yield self._client.fetch(request)
except Exception as e:
self._errors.append("Failed download to %s: %s" %
(self._filename, str(e)))
raise gen.Return(None)
# assert fetch() was supposed to throw the error, not leave it here unthrown
assert response.error is None
if len(self._errors) == 0:
try:
_file.close() # be sure tmp_filename is flushed
rename.rename_over_existing(tmp_filename, self._filename)
except EnvironmentError as e:
self._errors.append("Failed to rename %s to %s: %s" %
(tmp_filename, self._filename, str(e)))
if len(self._errors) == 0 and self._hash_algorithm is not None:
self._hash = hasher.hexdigest()
raise gen.Return(response)
finally:
cleanup_tmp()
