def get(location, local_copy=False):
"""Get a file or directory to a local temporary directory.
Args:
location: the source of the material to get. This source may
be one of:
* a local file or directory
* a URL (http or ftp)
* a python file-like object
Returns:
The location of the file or directory where the requested
content was saved. This will be contained in a temporary
directory on the local host. If the material to get was a
directory, the location will contain a trailing '/'
"""
tmpdir = get_tmp_dir()
# location is a file-like object
if hasattr(location, "read"):
tmpfile = os.path.join(tmpdir, "file")
tmpfileobj = file(tmpfile, 'w')
shutil.copyfileobj(location, tmpfileobj)
tmpfileobj.close()
return tmpfile
if isinstance(location, types.StringTypes):
# location is a URL
if location.startswith('http') or location.startswith('ftp'):
tmpfile = os.path.join(tmpdir, os.path.basename(location))
utils.urlretrieve(location, tmpfile)
return tmpfile
# location is a local path
elif os.path.exists(os.path.abspath(location)):
if not local_copy:
if os.path.isdir(location):
return location.rstrip('/') + '/'
else:
return location
tmpfile = os.path.join(tmpdir, os.path.basename(location))
if os.path.isdir(location):
tmpfile += '/'
shutil.copytree(location, tmpfile, symlinks=True)
return tmpfile
shutil.copyfile(location, tmpfile)
return tmpfile
# location is just a string, dump it to a file
else:
tmpfd, tmpfile = tempfile.mkstemp(dir=tmpdir)
tmpfileobj = os.fdopen(tmpfd, 'w')
tmpfileobj.write(location)
tmpfileobj.close()
return tmpfile
def get(location, local_copy = False):
"""Get a file or directory to a local temporary directory.
Args:
location: the source of the material to get. This source may
be one of:
* a local file or directory
* a URL (http or ftp)
* a python file-like object
Returns:
The location of the file or directory where the requested
content was saved. This will be contained in a temporary
directory on the local host. If the material to get was a
directory, the location will contain a trailing '/'
"""
tmpdir = get_tmp_dir()
# location is a file-like object
if hasattr(location, "read"):
tmpfile = os.path.join(tmpdir, "file")
tmpfileobj = file(tmpfile, 'w')
shutil.copyfileobj(location, tmpfileobj)
tmpfileobj.close()
return tmpfile
if isinstance(location, types.StringTypes):
# location is a URL
if location.startswith('http') or location.startswith('ftp'):
tmpfile = os.path.join(tmpdir, os.path.basename(location))
utils.urlretrieve(location, tmpfile)
return tmpfile
# location is a local path
elif os.path.exists(os.path.abspath(location)):
if not local_copy:
if os.path.isdir(location):
return location.rstrip('/') + '/'
else:
return location
tmpfile = os.path.join(tmpdir, os.path.basename(location))
if os.path.isdir(location):
tmpfile += '/'
shutil.copytree(location, tmpfile, symlinks=True)
return tmpfile
shutil.copyfile(location, tmpfile)
return tmpfile
# location is just a string, dump it to a file
else:
tmpfd, tmpfile = tempfile.mkstemp(dir=tmpdir)
tmpfileobj = os.fdopen(tmpfd, 'w')
tmpfileobj.write(location)
tmpfileobj.close()
return tmpfile
def test_urlopen_passed_arguments(self):
self.god.stub_function(utils, "urlopen")
self.god.stub_function(utils.shutil, "copyfileobj")
self.god.stub_function(utils, "open")
url = "url"
dest = "somefile"
data = object()
timeout = 10
src_file = self.god.create_mock_class(file, "file")
dest_file = self.god.create_mock_class(file, "file")
(utils.urlopen.expect_call(url, data=data, timeout=timeout).and_return(src_file))
utils.open.expect_call(dest, "wb").and_return(dest_file)
utils.shutil.copyfileobj.expect_call(src_file, dest_file)
dest_file.close.expect_call()
src_file.close.expect_call()
utils.urlretrieve(url, dest, data=data, timeout=timeout)
self.god.check_playback()
def test_urlopen_passed_arguments(self):
self.god.stub_function(utils, "urlopen")
self.god.stub_function(utils.shutil, "copyfileobj")
self.god.stub_function(utils, "open")
url = "url"
dest = "somefile"
data = object()
timeout = 10
src_file = self.god.create_mock_class(file, "file")
dest_file = self.god.create_mock_class(file, "file")
(utils.urlopen.expect_call(url, data=data,
timeout=timeout).and_return(src_file))
utils.open.expect_call(dest, "wb").and_return(dest_file)
utils.shutil.copyfileobj.expect_call(src_file, dest_file)
dest_file.close.expect_call()
src_file.close.expect_call()
utils.urlretrieve(url, dest, data=data, timeout=timeout)
self.god.check_playback()
def __init__(self, cfg, tmpdir='/tmp', raise_errors=False):
"""
Instantiate ConfigParser and provide the file like object that we'll
use to read configuration data from.
@param cfg: Where we'll get configuration data. It can be either:
* A URL containing the file
* A valid file path inside the filesystem
* A string containing configuration data
@param tmpdir: Where we'll dump the temporary conf files.
@param raise_errors: Whether config value absences will raise
ValueError exceptions.
"""
# Base Parser
self.parser = ConfigParser()
# Raise errors when lacking values
self.raise_errors = raise_errors
# File is already a file like object
if hasattr(cfg, 'read'):
self.cfg = cfg
self.parser.readfp(self.cfg)
elif isinstance(cfg, types.StringTypes):
# Config file is a URL. Download it to a temp dir
if cfg.startswith('http') or cfg.startswith('ftp'):
self.cfg = path.join(tmpdir, path.basename(cfg))
utils.urlretrieve(cfg, self.cfg)
self.parser.read(self.cfg)
# Config is a valid filesystem path to a file.
elif path.exists(path.abspath(cfg)):
if path.isfile(cfg):
self.cfg = path.abspath(cfg)
self.parser.read(self.cfg)
else:
e_msg = 'Invalid config file path: %s' % cfg
raise IOError(e_msg)
# Config file is just a string, convert it to a python file like
# object using StringIO
else:
self.cfg = StringIO(cfg)
self.parser.readfp(self.cfg)
def __init__(self, cfg, tmpdir='/tmp', raise_errors=False):
"""
Instantiate ConfigParser and provide the file like object that we'll
use to read configuration data from.
@param cfg: Where we'll get configuration data. It can be either:
* A URL containing the file
* A valid file path inside the filesystem
* A string containing configuration data
@param tmpdir: Where we'll dump the temporary conf files.
@param raise_errors: Whether config value absences will raise
ValueError exceptions.
"""
# Base Parser
self.parser = ConfigParser()
# Raise errors when lacking values
self.raise_errors = raise_errors
# File is already a file like object
if hasattr(cfg, 'read'):
self.cfg = cfg
self.parser.readfp(self.cfg)
elif isinstance(cfg, types.StringTypes):
# Config file is a URL. Download it to a temp dir
if cfg.startswith('http') or cfg.startswith('ftp'):
self.cfg = path.join(tmpdir, path.basename(cfg))
utils.urlretrieve(cfg, self.cfg)
self.parser.read(self.cfg)
# Config is a valid filesystem path to a file.
elif path.exists(path.abspath(cfg)):
if path.isfile(cfg):
self.cfg = path.abspath(cfg)
self.parser.read(self.cfg)
else:
e_msg = 'Invalid config file path: %s' % cfg
raise IOError(e_msg)
# Config file is just a string, convert it to a python file like
# object using StringIO
else:
self.cfg = StringIO(cfg)
self.parser.readfp(self.cfg)
