def test_urlretrieve_netrc(self): root_directory = tempfile.mkdtemp() examplename = os.path.join(root_directory, "foo") outname = os.path.join(root_directory, "fooout") with open(examplename, "w") as fhand: fhand.write('content') mockget = Mock() mockopen = Mock() mock_fhand = Mock() backopen = vcstools.common.urlopen backget = vcstools.common._netrc_open try: # vcstools.common.urlopen = mockopen # vcstools.common.urlopen.return_value = mock_fhand # mock_fhand.read.return_value = 'content' mockopen.open.return_value vcstools.common._netrc_open = Mock() vcstools.common._netrc_open.return_value = mockget (fname, headers) = urlretrieve_netrc('file://' + examplename) self.assertTrue(fname) self.assertFalse(os.path.exists(outname)) (fname, headers) = urlretrieve_netrc('file://' + examplename, outname) self.assertEqual(outname, fname) self.assertTrue(os.path.isfile(outname)) finally: vcstools.common.urlopen = backopen vcstools.common._netrc_open = backget shutil.rmtree(root_directory)
def checkout(self, url, version='', verbose=False, shallow=False, timeout=None): """ untars tar at url to self.path. If version was given, only the subdirectory 'version' of the tar will end up in self.path. Also creates a file next to the checkout named *.tar which is a yaml file listing origin url and version arguments. """ if not ensure_dir_notexists(self.get_path()): self.logger.error("Can't remove %s" % self.get_path()) return False tempdir = None result = False try: tempdir = tempfile.mkdtemp() if os.path.isfile(url): filename = url else: (filename, _) = urlretrieve_netrc(url) # print "filename", filename temp_tarfile = tarfile.open(filename, 'r:*') members = None # means all members in extractall if version == '' or version is None: subdir = tempdir self.logger.warn("No tar subdirectory chosen via the 'version' argument for url: %s" % url) else: # getmembers lists all files contained in tar with # relative path subdirs = [] members = [] for m in temp_tarfile.getmembers(): if m.name.startswith(version + '/'): members.append(m) if m.name.split('/')[0] not in subdirs: subdirs.append(m.name.split('/')[0]) if not members: raise VcsError("%s is not a subdirectory with contents in members %s" % (version, subdirs)) subdir = os.path.join(tempdir, version) temp_tarfile.extractall(path=tempdir, members=members) if not os.path.isdir(subdir): raise VcsError("%s is not a subdirectory\n" % subdir) try: # os.makedirs(os.path.dirname(self._path)) shutil.move(subdir, self._path) except Exception as ex: raise VcsError("%s failed to move %s to %s" % (ex, subdir, self._path)) metadata = yaml.dump({'url': url, 'version': version}) with open(self.metadata_path, 'w') as mdat: mdat.write(metadata) result = True except Exception as exc: self.logger.error("Tarball download unpack failed: %s" % str(exc)) finally: if tempdir is not None and os.path.exists(tempdir): rmtree(tempdir) return result
def checkout(self, url, version='', verbose=False, shallow=False, timeout=None): """ untars tar at url to self.path. If version was given, only the subdirectory 'version' of the tar will end up in self.path. Also creates a file next to the checkout named *.tar which is a yaml file listing origin url and version arguments. """ if not ensure_dir_notexists(self.get_path()): self.logger.error("Can't remove %s" % self.get_path()) return False tempdir = None result = False try: tempdir = tempfile.mkdtemp() if os.path.isfile(url): filename = url else: (filename, _) = urlretrieve_netrc(url) # print "filename", filename temp_tarfile = tarfile.open(filename, 'r:*') members = None # means all members in extractall if version == '' or version is None: self.logger.warn("No tar subdirectory chosen via the 'version' argument for url: %s" % url) else: # getmembers lists all files contained in tar with # relative path subdirs = [] members = [] for m in temp_tarfile.getmembers(): if m.name.startswith(version + '/'): members.append(m) if m.name.split('/')[0] not in subdirs: subdirs.append(m.name.split('/')[0]) if not members: raise VcsError("%s is not a subdirectory with contents in members %s" % (version, subdirs)) temp_tarfile.extractall(path=tempdir, members=members) subdir = os.path.join(tempdir, version) if not os.path.isdir(subdir): raise VcsError("%s is not a subdirectory\n" % subdir) try: # os.makedirs(os.path.dirname(self._path)) shutil.move(subdir, self._path) except Exception as ex: raise VcsError("%s failed to move %s to %s" % (ex, subdir, self._path)) metadata = yaml.dump({'url': url, 'version': version}) with open(self.metadata_path, 'w') as mdat: mdat.write(metadata) result = True except Exception as exc: self.logger.error("Tarball download unpack failed: %s" % str(exc)) finally: if tempdir is not None and os.path.exists(tempdir): shutil.rmtree(tempdir) return result