def exists (self, filename):
"""Test whether `filename` is present at the remote host.
:Parameters:
`filename`: string
Path to remote file relative to remote repository.
:Return:
- True unless remote file cannot be found.
"""
# save full url in case of error
_url = urljoin (self.pkg_url, filename)
try:
self._perform ({
pycurl.URL: _url,
pycurl.WRITEDATA: open ('/dev/null', 'wb'), # discard headers
pycurl.RANGE: '0-0', # fetch 1st byte only
pycurl.NOPROGRESS: 1,
pycurl.FAILONERROR: 1,
})
except depot.DepotFileNotFoundError:
return False
return True
def xml_file_filter (self, valid_depots, dep, found_depot_db_xml):
# depot has suffixed file. Either:
# the package name + self.DB_SUFFIX
# or just self.XML_FILE
db_filename = ''
if dep.exists (self.XML_FILE):
db_filename = self.XML_FILE
else:
for package in dep.packages:
if dep.exists (package + self.DB_SUFFIX):
db_filename = package + self.DB_SUFFIX
break
else:
if found_depot_db_xml:
valid_depots.append (dep)
return True
valid_depots.append (dep)
self.msg_callback ('begin', 'reading ' +
urljoin (dep.pkg_url, db_filename), 2, VERBOSE_DEPOT)
dep.fetch (db_filename) # copy to local depot
try:
self.read_db_xml (dep, dep, db_filename)
self.msg_callback ('end', 'done', 2, VERBOSE_DEPOT)
except XMLError, msg:
# Finish the callback output, and reraise the exception
self.msg_callback ('end', 'failed', 2, VERBOSE_DEPOT)
raise XMLError (str (msg))
def exists (self, filename):
"""Test whether `filename` is present at the remote host.
:Parameters:
`filename`: string
Path to remote file relative to remote repository.
:Return:
- True unless remote file cannot be found.
"""
# save full url in case of error
_url = urljoin (self.pkg_url, filename)
try:
self._perform ({
pycurl.URL: _url,
pycurl.NOBODY: 1,
pycurl.NOPROGRESS: 1,
pycurl.FAILONERROR: 1,
})
except depot.DepotFileNotFoundError:
return False
return True
def __init__ (self, var, local_depot, dep, sub_depot):
"""Return an HTTP `Depot` object."""
# This is better, but doesn't work for classic classes:
# super (http, self).__init__ (dep)
# So, we're forced to hardcode the superclass for now:
Depot.__init__ (self, dep)
if type (dep) == types.InstanceType:
self.var = dep.var
else:
self.var = var
# full URL path where packages are to be found
pkg_path = None
if type (dep) == types.InstanceType:
self.pkg_url = urljoin (dep.pkg_url, sub_depot)
elif var is not None:
# path to "root" of package directory
pkg_path = var['dist']
if pkg_path:
self.pkg_url = urljoin (self.pkg_url, pkg_path)
if not local_depot:
return
# make local depot mirror the directory structure of the
# remote depot. This is to make it possible to specify the
# path of the local depot as a depot to future sessions.
if type (dep) == types.InstanceType:
self.local_path = os.path.join (dep.local_path, sub_depot)
else:
if pkg_path:
self.local_path = os.path.join (local_depot, pkg_path)
else:
self.local_path = local_depot
if not os.path.exists (self.local_path):
os.makedirs (self.local_path)
def __init__ (self, var, dep, sub_depot):
# This is better, but doesn't work for classic classes:
# super (file, self).__init__ (dep)
# So, we're forced to hardcode the superclass for now:
Depot.__init__ (self, dep)
if type (dep) == types.InstanceType:
url = dep.url
self.var = dep.var
else:
url = dep
self.var = var
self.local_path = urlparse (url).path
# add to existing depot path
if type (dep) == types.InstanceType:
self.local_path = os.path.join (dep.local_path, sub_depot)
self.pkg_url = urljoin (dep.pkg_url, sub_depot)
else:
if var and var['dist'] and os.path.exists (os.path.join (self.local_path, var['dist'])):
self.local_path = os.path.join (self.local_path, var['dist'])
self.pkg_url = urljoin (self.pkg_url, var['dist'])
def fetch (self, filename, progressmeter = None, fetched_bytes = 0,
additional_bytes = 0):
"""Fetch a local copy of `filename` from the remote machine.
:Parameters:
`filename`: string
Path to remote file relative to remote repository, also
used for the name of the local copy.
`progressmeter`: TextProgressMeter
Re-use `progressmeter` for fetching related files.
`fetched_bytes`: integer
Number of bytes already fetched as previous files using
this `progressmeter`.
`additional_bytes`: integer
Additional bytes yet to be downloaded with further files
using this `progressmeter`.
:Return:
- Running total number of bytes downloaded so far in this file
group.
"""
# save full url in case of error
_url = urljoin (self.pkg_url, filename)
options = {
pycurl.URL: _url,
pycurl.WRITEDATA: open (os.path.join (self.local_path, filename), 'wb'),
pycurl.NOPROGRESS: 1,
}
if progressmeter:
# instantiate a new ProgressAdaptor for every file in the group
adaptor = ProgressAdaptor (progressmeter, fetched_bytes,
additional_bytes)
options[pycurl.PROGRESSFUNCTION] = adaptor.progress_cb
options[pycurl.NOPROGRESS] = 0
self._perform (options)
if progressmeter is not None:
fetched_bytes = adaptor.get_running_total ()
return fetched_bytes
def set (self, depot_names, options = []):
valid_depots = [] # depots we can use
search_depots = False
search_package_names = [] # packages to search for
# make copy as invalid_package_callback might modify in place
for package_name in self.package_names[:]:
if not package_name.isfile ():
# Unless we already have a local file, try to fetch the file from
# (remote) url into a temporary local depot for processing later on.
try:
self.temp_package_dir = misclib.mkstempdir ()
except misclib.MisclibTempDirCreationError, msg:
raise DepotTempdirCreationError (src (msg))
mysig.register (self.remove_temp_package_dir)
dep = init (None, self.temp_package_dir,
package_name.remote_depot ())
dep.indent_progressmeter = 2
dep.verify_checksum = False
self.depot_init (dep)
if dep.exists (package_name.remote_path ()):
# mysig.register() call above avoids a race if the program
# exits with a signal, and the atexit.register() call below
# ensures we also clean up when the program exits normally.
atexit.register (self.remove_temp_package_dir, self.msg_callback)
# To minimize chatter, only admit that we created a temporary
# directory if we actually need it...
self.msg_callback ('begin', 'creating temporary \
directory', 0, VERBOSE_LOCAL_DEPOT)
self.msg_callback ('end', self.temp_package_dir,
0, VERBOSE_LOCAL_DEPOT)
io.stdout (0, '', 'retrieving ' +
package_name.geturl () + ' ...\n')
elif package_name.getscheme () != 'file':
raise DepotFileNotFoundError ('%s: File not found.' %
package_name.geturl ())
else:
# ...otherwise, quietly remove it and pretend it was never
# here (in which case we won't be needing the atexit handler
# either):
self.remove_temp_package_dir ()
self.assign_depot_vars (dep)
# If we successfully fetched a local copy of the remote file,
# replace the remote package_name object with a file:// scheme
# package_name:
pkgfile = package_name.basename ()
if dep.exists (pkgfile):
dep.get (pkgfile, show_progress = True)
package_name = \
PackageName (os.path.join (self.temp_package_dir,
pkgfile))
if package_name.isfile ():
path = package_name.normpath ()
dep = init (None, self.local_depot, os.path.dirname (path))
self.assign_depot_vars (dep)
self.msg_callback ('begin', 'reading ' +
urljoin (dep.pkg_url, os.path.basename (path)),
0, VERBOSE_DEPOT)
try:
archive = self.archive_init (path)
if archive:
archive.path = path
# set depot path to the archive file
dep.local_path = path
dep.url = package_name.getscheme () + "://" + path
dep.archive = archive
if archive.exists (self.XML_FILE):
db_entry = self.read_db_xml (archive, dep, self.XML_FILE)
elif archive.exists (os.path.basename (path)):
db_entry = self.read_db_xml (archive, dep,
os.path.basename (path))
else:
if self.invalid_package_callback:
self.invalid_package_callback (package_name.name)
self.msg_callback ('end', 'failed', 0, VERBOSE_DEPOT)
self.msg_callback ('error', 'an input archive \
was specified but did not contain either a "%s" or "%s". Archives \
must contain one of these files. skipping.' % (self.XML_FILE,
os.path.basename (path)))
continue
else:
db_entry = self.read_db_xml (dep, dep, path)
if not db_entry:
self.msg_callback ('end', 'failed', 0, VERBOSE_DEPOT)
self.msg_callback ('error', 'unable to parse the \
XML file. please check "%s". it does not appear to follow \
%s(4). skipping.' % (path, self.XML_FILE))
continue
except DepotArchiveInitError, msg:
if self.invalid_package_callback:
self.invalid_package_callback (package_name.name)
self.msg_callback ('end', 'failed', 0, VERBOSE_DEPOT)
self.msg_callback ('error', str (msg))
continue
except XMLError, msg:
self.msg_callback ('end', 'failed', 0, VERBOSE_DEPOT)
self.msg_callback ('error', str (msg))
continue
def open (self, path, mode):
if self.local_path and os.path.exists (self.local_path + '/' + path):
return open (self.local_path + '/' + path, 'r')
else:
return (self.urlopen (urljoin (self.pkg_url, path)))
self.depots.append (dep)
self.msg_callback ('end', 'reading depot databases', 0,
VERBOSE_DEPOT)
#??? self.depots = self.verify_depots_callback (self.depots, options)
for dep in self.depots:
depot_db_xml = False # if depot-db.xml exists
db_xml = False # if xx-db.xml exists
if dep.archive: # skip archives
continue
# if parent depot contains more depots, add to list
self.msg_callback ('begin', urljoin (dep.pkg_url, 'depot-db.xml'),
2, VERBOSE_DEPOT)
try:
if dep.exists ('depot-db.xml'):
depot_db_xml = True
dep.fetch ('depot-db.xml') # make copy in local depot
self.msg_callback ('end', 'done', 2, VERBOSE_DEPOT)
self.validate_xml_callback ('depot-db.xml', dep)
sub_depots = depotdb.read_depotdb_xml ('depot-db.xml', dep)
# create new depots
def sbdb_path (self, filename): return urljoin (self.pkg_url, filename)
