def pre_install(package_name, download_url):
#pre install - using smart.fetcher module to download package
print '\n***Downloding WinLibre package...***'
try:
fetcher = Fetcher()
fetcher.reset()
info = {}#{'uncomp': True}
item = fetcher.enqueue(download_url, **info)
if (fetcher.run(what=False) is False):
print 'Using cached download'
except:
print 'Download error occoured'
#extracting the package to temp using 7Z CLI
print '\n***Now extracting the package contents...***'
temp_dir = os.getenv('temp')
download_name = temp_dir + '\\' + pkg_name(download_url)
#extract arg is temprorary and system dependent
#appending $PATH to have 7za.exe will remove absolute path neccessity
extract_arg = 'C:\wpc\wpc2\winlibre1.0\wpkg\\7za.exe'+' e ' + '-o'+temp_dir+' -y '+ download_name
try:
p = Popen([extract_arg], stdout=PIPE, shell = True)
op = p.communicate()[0]
print 'Output : ' , op
except Exception, e:
print 'Errors occoured during extract', e
op = -3 #Error code
def setUp(self):
self.local_path = self.makeDir()
self.fetcher = Fetcher()
self.fetcher.setLocalPathPrefix(self.local_path + "/")
# Smart changes SIGPIPE handling due to a problem which otherwise
# happens when running external scripts. Check out smart/__init__.py.
# We want the normal handling here because in some cases we may
# get SIGPIPE due to broken sockets on tests.
signal.signal(signal.SIGPIPE, signal.SIG_IGN)
def downloader(self, URL):
#using smart.fetcher() to download the package
try:
fetcher = Fetcher()
fetcher.reset()
info = {}#{'uncomp': True}
item = fetcher.enqueue(URL, **info)
if (fetcher.run(what=False) is False):
print 'Using cached download'
except:
print 'Download error occoured'
def setUp(self):
self.progress = Progress()
self.fetcher = Fetcher()
self.cache = Cache()
self.download_dir = self.makeDir()
self.fetcher.setLocalPathPrefix(self.download_dir + "/")
# Disable caching so that things blow up when not found.
self.fetcher.setCaching(NEVER)
sysconf.set("deb-arch", "i386")
def setUp(self):
self.progress = Progress()
self.fetcher = Fetcher()
self.cache = Cache()
self.download_dir = self.makeDir()
self.fetcher.setLocalPathPrefix(self.download_dir + "/")
# Disable caching so that things blow up when not found.
self.fetcher.setCaching(NEVER)
# Make sure to trigger old bugs in debug error reporting.
sysconf.set("log-level", DEBUG)
def setUp(self):
self.progress = Progress()
self.fetcher = Fetcher()
self.cache = Cache()
self.download_dir = self.makeDir()
self.fetcher.setLocalPathPrefix(self.download_dir + "/")
# Disable caching so that things blow up when not found.
self.fetcher.setCaching(NEVER)
sysconf.set("deb-arch", "i386")
self.trustdb = open("%s/aptdeb/trustdb.gpg" % TESTDATADIR).read()
def __init__(self, confpath=None, forcelocks=False):
self._confpath = None
self._channels = {} # alias -> Channel()
self._sysconfchannels = {} # alias -> data dict
self._dynamicchannels = {} # alias -> Channel()
self._pathlocks = PathLocks(forcelocks)
self._cache = Cache()
self.loadSysConf(confpath)
self._fetcher = Fetcher()
self._mediaset = self._fetcher.getMediaSet()
self._achanset = AvailableChannelSet(self._fetcher)
self._cachechanged = False
class FetcherTest(MockerTestCase):
def setUp(self):
self.local_path = self.makeDir()
self.fetcher = Fetcher()
self.fetcher.setLocalPathPrefix(self.local_path + "/")
# Smart changes SIGPIPE handling due to a problem which otherwise
# happens when running external scripts. Check out smart/__init__.py.
# We want the normal handling here because in some cases we may
# get SIGPIPE due to broken sockets on tests.
signal.signal(signal.SIGPIPE, signal.SIG_IGN)
def tearDown(self):
# See above.
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
def start_server(self, handler, hide_errors=False):
startup_lock = threading.Lock()
startup_lock.acquire()
def server():
class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_GET(self):
return handler(self)
def log_message(self, format, *args):
pass
while True:
try:
httpd = HTTPServer(("127.0.0.1", PORT), Handler)
break
except socket.error, error:
if "Address already in use" not in str(error):
raise
time.sleep(1)
startup_lock.release()
httpd.hide_errors = hide_errors
httpd.handle_request()
self.server_thread = threading.Thread(target=server)
self.server_thread.start()
# Wait until thread is ready.
startup_lock.acquire()
def setUp(self):
self.local_path = self.makeDir()
self.fetcher = Fetcher()
self.fetcher.setLocalPathPrefix(self.local_path + "/")
# Smart changes SIGPIPE handling due to a problem which otherwise
# happens when running external scripts. Check out smart/__init__.py.
# We want the normal handling here because in some cases we may
# get SIGPIPE due to broken sockets on tests.
signal.signal(signal.SIGPIPE, signal.SIG_IGN)
class FetcherTest(MockerTestCase):
def setUp(self):
self.local_path = self.makeDir()
self.fetcher = Fetcher()
self.fetcher.setLocalPathPrefix(self.local_path + "/")
# Smart changes SIGPIPE handling due to a problem which otherwise
# happens when running external scripts. Check out smart/__init__.py.
# We want the normal handling here because in some cases we may
# get SIGPIPE due to broken sockets on tests.
signal.signal(signal.SIGPIPE, signal.SIG_IGN)
def tearDown(self):
# See above.
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
def start_server(self, handler, hide_errors=False):
startup_lock = threading.Lock()
startup_lock.acquire()
def server():
class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_GET(self):
return handler(self)
def log_message(self, format, *args):
pass
while True:
try:
httpd = HTTPServer(("127.0.0.1", PORT), Handler)
break
except socket.error, error:
if "Address already in use" not in str(error):
raise
time.sleep(1)
startup_lock.release()
httpd.hide_errors = hide_errors
httpd.handle_request()
self.server_thread = threading.Thread(target=server)
self.server_thread.start()
# Wait until thread is ready.
startup_lock.acquire()
def setUp(self):
self.progress = Progress()
self.fetcher = Fetcher()
self.cache = Cache()
self.download_dir = self.makeDir()
self.fetcher.setLocalPathPrefix(self.download_dir + "/")
# Disable caching so that things blow up when not found.
self.fetcher.setCaching(NEVER)
sysconf.set("deb-arch", "i386")
self.trustdb = open("%s/aptdeb/trustdb.gpg" % TESTDATADIR).read()
def setUp(self):
self.progress = Progress()
self.fetcher = Fetcher()
self.cache = Cache()
self.download_dir = self.makeDir()
self.fetcher.setLocalPathPrefix(self.download_dir + "/")
# Disable caching so that things blow up when not found.
self.fetcher.setCaching(NEVER)
# Make sure to trigger old bugs in debug error reporting.
sysconf.set("log-level", DEBUG)
def __init__(self, confpath=None, forcelocks=False):
self._confpath = None
self._channels = {} # alias -> Channel()
self._sysconfchannels = {} # alias -> data dict
self._dynamicchannels = {} # alias -> Channel()
self._pathlocks = PathLocks(forcelocks)
self._cache = Cache()
self.loadSysConf(confpath)
self._fetcher = Fetcher()
self._mediaset = self._fetcher.getMediaSet()
self._achanset = AvailableChannelSet(self._fetcher)
self._cachechanged = False
def setUp(self):
self.channel = createChannel(
"alias", {
"type": "apt-deb",
"baseurl": "file://%s/deb" % TESTDATADIR,
"distribution": "./"
})
class TestInterface(Interface):
output = []
def message(self, level, msg):
self.output.append((level, msg))
def showOutput(self, data):
self.output.append(data)
self.iface = TestInterface(ctrl)
self.progress = Progress()
self.fetcher = Fetcher()
self.cache = Cache()
self.channel.fetch(self.fetcher, self.progress)
self.loader = self.channel.getLoaders()[0]
self.cache.addLoader(self.loader)
self.old_iface = iface.object
self.old_sysconf = pickle.dumps(sysconf.object)
iface.object = self.iface
self.cache.load()
self.pm = DebPackageManager()
# skip test if dpkg is unavailable
dpkg = sysconf.get("dpkg", "dpkg")
output = tempfile.TemporaryFile()
status = self.pm.dpkg([dpkg, "--version"], output)
if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0:
if not hasattr(self, 'skipTest'): # Python < 2.7
self.skipTest = self.fail # error
self.skipTest("%s not found" % dpkg)
def getCachedSize(self):
fetcher = Fetcher()
localdir = os.path.join(sysconf.get("data-dir"), "packages/")
fetcher.setLocalDir(localdir, mangle=False)
total = 0
for pkg in self.install:
for loader in pkg.loaders:
if not loader.getInstalled():
break
else:
continue
info = loader.getInfo(pkg)
for url in info.getURLs():
mirror = fetcher.getMirrorSystem().get(url)
item = FetchItem(fetcher, url, mirror)
path = fetcher.getLocalPath(item)
if os.path.exists(path):
total += os.path.getsize(path)
return total
class AptDebChannelTest(MockerTestCase):
def setUp(self):
self.progress = Progress()
self.fetcher = Fetcher()
self.cache = Cache()
self.download_dir = self.makeDir()
self.fetcher.setLocalPathPrefix(self.download_dir + "/")
# Disable caching so that things blow up when not found.
self.fetcher.setCaching(NEVER)
sysconf.set("deb-arch", "i386")
self.trustdb = open("%s/aptdeb/trustdb.gpg" % TESTDATADIR).read()
def tearDown(self):
sysconf.remove("deb-arch")
open("%s/aptdeb/trustdb.gpg" % TESTDATADIR, 'w').write(self.trustdb)
def check_channel(self, channel):
self.assertEquals(channel.fetch(self.fetcher, self.progress), True)
loaders = channel.getLoaders()
self.assertEquals(len(loaders), 1)
self.cache.addLoader(loaders[0])
saved = sys.stdout
sys.stdout = StringIO()
try:
self.cache.load()
finally:
sys.stdout = saved
packages = sorted(self.cache.getPackages())
self.assertEquals(len(packages), 2)
self.assertEquals(packages[0].name, "name1")
self.assertEquals(packages[1].name, "name2")
def test_fetch_with_component(self):
channel = createChannel(
"alias", {
"type": "apt-deb",
"baseurl": "file://%s/aptdeb" % TESTDATADIR,
"distribution": "./",
"components": "component"
})
self.check_channel(channel)
def test_fetch_without_component(self):
channel = createChannel(
"alias", {
"type": "apt-deb",
"baseurl": "file://%s/aptdeb" % TESTDATADIR,
"distribution": "component-less"
})
self.check_channel(channel)
def test_fetch_without_component_and_release_file(self):
channel = createChannel(
"alias", {
"type": "apt-deb",
"baseurl": "file://%s/deb" % TESTDATADIR,
"distribution": "./"
})
self.check_channel(channel)
def test_fetch_without_component_and_release_file_with_keyring(self):
channel = createChannel(
"alias", {
"type": "apt-deb",
"baseurl": "file://%s/deb" % TESTDATADIR,
"distribution": "./",
"keyring": "/dev/null"
})
try:
self.check_channel(channel)
except Error, error:
self.assertEquals(
str(error), "Download of Release failed for channel 'alias': "
"File not found for validation")
else:
class Control(object):
def __init__(self, confpath=None, forcelocks=False):
self._confpath = None
self._channels = {} # alias -> Channel()
self._sysconfchannels = {} # alias -> data dict
self._dynamicchannels = {} # alias -> Channel()
self._pathlocks = PathLocks(forcelocks)
self._cache = Cache()
self.loadSysConf(confpath)
self._fetcher = Fetcher()
self._mediaset = self._fetcher.getMediaSet()
self._achanset = AvailableChannelSet(self._fetcher)
self._cachechanged = False
def getChannels(self):
return self._channels.values()
def removeChannel(self, alias):
channel = self._channels[alias]
if isinstance(channel, PackageChannel):
channel.removeLoaders()
del self._channels[alias]
if alias in self._sysconfchannels:
del self._sysconfchannels[alias]
if alias in self._dynamicchannels:
del self._dynamicchannels[alias]
def getFileChannels(self):
return [x for x in self._channels.values()
if isinstance(x, FileChannel)]
def addFileChannel(self, filename):
if not self._sysconfchannels:
# Give a chance for backends to register
# themselves on FileChannel hooks.
self.rebuildSysConfChannels()
found = False
for channel in hooks.call("create-file-channel", filename):
if channel:
if channel.getAlias() in self._channels:
raise Error, _("There's another channel with alias '%s'") \
% channel.getAlias()
self._channels[channel.getAlias()] = channel
found = True
if not found:
raise Error, _("Unable to create channel for file: %s") % filename
def removeFileChannel(self, filename):
filename = os.path.abspath(filename)
for channel in self._channels.values():
if (isinstance(channel, FileChannel) and
channel.getFileName() == filename):
channel.removeLoaders()
break
else:
raise Error, _("Channel not found for '%s'") % filename
def askForRemovableChannels(self, channels):
removable = [(str(x), x) for x in channels if x.isRemovable()]
if not removable:
return True
removable.sort()
removable = [x for name, x in removable]
self._mediaset.umountAll()
if not iface.insertRemovableChannels(removable):
return False
self._mediaset.mountAll()
return True
def getCache(self):
return self._cache
def getFetcher(self):
return self._fetcher
def getMediaSet(self):
return self._mediaset
def restoreMediaState(self):
self._mediaset.restoreState()
__stateversion__ = 2
def loadSysConf(self, confpath=None):
datadir = sysconf.get("data-dir")
if confpath:
confpath = os.path.expanduser(confpath)
if not os.path.isfile(confpath):
raise Error, _("Configuration file not found: %s") % confpath
sysconf.load(confpath)
else:
confpath = os.path.join(datadir, CONFFILE)
if os.path.isfile(confpath):
sysconf.load(confpath)
self._confpath = confpath
if os.path.isdir(datadir):
writable = os.access(datadir, os.W_OK)
else:
try:
os.makedirs(datadir)
writable = True
except OSError:
raise Error, _("Can't create datadir at %s") % datadir
if writable and not self._pathlocks.lock(datadir, exclusive=True):
writable = False
sysconf.setReadOnly(not writable)
def saveSysConf(self, confpath=None):
msys = self._fetcher.getMirrorSystem()
if msys.getHistoryChanged() and not sysconf.getReadOnly():
sysconf.set("mirrors-history", msys.getHistory())
if confpath:
confpath = os.path.expanduser(confpath)
else:
if sysconf.getReadOnly():
return
if self._cachechanged:
cachepath = os.path.join(sysconf.get("data-dir"), "cache")
if sysconf.get("disk-cache", True):
iface.showStatus(_("Saving cache..."))
cachefile = open(cachepath+".new", "w")
state = (self.__stateversion__,
self._cache,
self._channels,
self._sysconfchannels)
cPickle.dump(state, cachefile, 2)
if os.name == 'nt':
os.close(cachefile.fileno())
else: # 'posix'
cachefile.close()
os.rename(cachepath+".new", cachepath)
iface.hideStatus()
elif os.path.isfile(cachepath):
os.unlink(cachepath)
if not sysconf.getModified():
return
sysconf.resetModified()
confpath = self._confpath
sysconf.save(confpath)
def reloadMirrors(self):
mirrors = sysconf.get("mirrors", {})
for channel in self._channels.values():
if isinstance(channel, MirrorsChannel):
cmirrors = channel.getMirrors()
if cmirrors:
for origin in cmirrors:
set = dict.fromkeys(cmirrors[origin])
set.update(dict.fromkeys(mirrors.get(origin, [])))
mirrors[origin] = set.keys()
msys = self._fetcher.getMirrorSystem()
msys.setMirrors(mirrors)
if not msys.getHistory():
msys.setHistory(sysconf.get("mirrors-history", []))
def rebuildSysConfChannels(self):
channels = sysconf.get("channels", ())
forcechannels = sysconf.get("force-channels", "")
if forcechannels:
forcechannels = forcechannels.split(",")
def isEnabled(alias, data):
if forcechannels:
return alias in forcechannels
return not data.get("disabled")
if channels and not self._channels:
cachepath = os.path.join(sysconf.get("data-dir"), "cache")
if os.path.isfile(cachepath) and sysconf.get("disk-cache", True):
iface.showStatus(_("Loading cache..."))
cachefile = open(cachepath)
try:
state = cPickle.load(cachefile)
if state[0] != self.__stateversion__:
raise StateVersionError
except:
if sysconf.get("log-level") == DEBUG:
import traceback
traceback.print_exc()
if os.access(os.path.dirname(cachepath), os.W_OK):
if os.name == 'nt':
os.close(cachefile.fileno())
else: # 'posix'
cachefile.close()
os.unlink(cachepath)
else:
(__stateversion__,
self._cache,
self._channels,
self._sysconfchannels) = state
for alias in self._channels.keys():
if (alias not in channels or
not isEnabled(alias, channels[alias])):
self.removeChannel(alias)
cachefile.close()
iface.hideStatus()
for alias in channels:
data = channels[alias]
if not isEnabled(alias, data):
continue
if alias in self._sysconfchannels.keys():
if self._sysconfchannels[alias] == data:
continue
else:
channel = self._channels[alias]
if isinstance(channel, PackageChannel):
channel.removeLoaders()
del self._channels[alias]
del self._sysconfchannels[alias]
channel = createChannel(alias, data)
self._sysconfchannels[alias] = data
self._channels[alias] = channel
for alias in self._sysconfchannels.keys():
if alias not in channels or channels[alias].get("disabled"):
self.removeChannel(alias)
def rebuildDynamicChannels(self):
for alias in self._dynamicchannels.keys():
self.removeChannel(alias)
newchannels = {}
for channels in hooks.call("rebuild-dynamic-channels"):
if channels:
for channel in channels:
alias = channel.getAlias()
if alias in self._channels:
raise Error, _("There's another channel with "
"alias '%s'") % alias
newchannels[alias] = channel
self._channels.update(newchannels)
self._dynamicchannels.update(newchannels)
def reloadChannels(self, channels=None, caching=ALWAYS):
if channels is None:
manual = False
self.rebuildSysConfChannels()
self.rebuildDynamicChannels()
channels = self._channels.values()
hooks.call("reload-channels", channels)
else:
manual = True
# Get channels directory and check the necessary locks.
channelsdir = os.path.join(sysconf.get("data-dir"), "channels/")
userchannelsdir = os.path.join(sysconf.get("user-data-dir"),
"channels/")
if not os.path.isdir(channelsdir):
try:
os.makedirs(channelsdir)
except OSError:
raise Error, _("Unable to create channel directory.")
if caching is ALWAYS:
if sysconf.getReadOnly() and os.access(channelsdir, os.W_OK):
iface.warning(_("The Smart library is already in use by "
"another process."))
iface.warning(_("Configuration is in readonly mode!"))
if not self._pathlocks.lock(channelsdir):
raise Error, _("Channel information is locked for writing.")
elif sysconf.getReadOnly():
raise Error, _("Can't update channels in readonly mode.")
elif not self._pathlocks.lock(channelsdir, exclusive=True):
raise Error, _("Can't update channels with active readers.")
self._fetcher.setLocalDir(channelsdir, mangle=True)
# Prepare progress. If we're reading from the cache, we don't want
# too much information being shown. Otherwise, ask for a full-blown
# progress for the interface, and build information of currently
# available packages to compare later.
if caching is ALWAYS:
progress = Progress()
else:
progress = iface.getProgress(self._fetcher, True)
oldpkgs = {}
for pkg in self._cache.getPackages():
oldpkgs[(pkg.name, pkg.version)] = True
progress.start()
steps = 0
for channel in channels:
steps += channel.getFetchSteps()
progress.set(0, steps)
# Rebuild mirror information.
self.reloadMirrors()
self._fetcher.setForceMountedCopy(True)
self._cache.reset()
# Do the real work.
result = True
for channel in channels:
digest = channel.getDigest()
if not manual and channel.hasManualUpdate():
self._fetcher.setCaching(ALWAYS)
else:
self._fetcher.setCaching(caching)
if channel.getFetchSteps() > 0:
progress.setTopic(_("Fetching information for '%s'...") %
(channel.getName() or channel.getAlias()))
progress.show()
self._fetcher.setForceCopy(channel.isRemovable())
self._fetcher.setLocalPathPrefix(channel.getAlias()+"%%")
try:
if not channel.fetch(self._fetcher, progress):
iface.debug(_("Failed fetching channel '%s'") % channel)
result = False
except Error, e:
iface.error(unicode(e))
iface.debug(_("Failed fetching channel '%s'") % channel)
result = False
if (channel.getDigest() != digest and
isinstance(channel, PackageChannel)):
channel.addLoaders(self._cache)
if channel.getAlias() in self._sysconfchannels:
self._cachechanged = True
if result and caching is not ALWAYS:
sysconf.set("last-update", time.time())
self._fetcher.setForceMountedCopy(False)
self._fetcher.setForceCopy(False)
self._fetcher.setLocalPathPrefix(None)
# Finish progress.
progress.setStopped()
progress.show()
progress.stop()
# Build cache with the new information.
self._cache.load()
# Compare new packages with what we had available, and mark
# new packages.
if caching is not ALWAYS:
pkgconf.clearFlag("new")
for pkg in self._cache.getPackages():
if (pkg.name, pkg.version) not in oldpkgs:
pkgconf.setFlag("new", pkg.name, "=", pkg.version)
# Remove unused files from channels directory.
for dir in (channelsdir, userchannelsdir):
if os.access(dir, os.W_OK):
aliases = self._channels.copy()
aliases.update(dict.fromkeys(sysconf.get("channels", ())))
for entry in os.listdir(dir):
sep = entry.find("%%")
if sep == -1 or entry[:sep] not in aliases:
os.unlink(os.path.join(dir, entry))
# Change back to a shared lock.
self._pathlocks.lock(channelsdir)
return result
class YumRpmChannelTest(MockerTestCase):
def setUp(self):
self.progress = Progress()
self.fetcher = Fetcher()
self.cache = Cache()
self.download_dir = self.makeDir()
self.fetcher.setLocalPathPrefix(self.download_dir + "/")
# Disable caching so that things blow up when not found.
self.fetcher.setCaching(NEVER)
# Make sure to trigger old bugs in debug error reporting.
sysconf.set("log-level", DEBUG)
def tearDown(self):
sysconf.remove("log-level")
def check_channel(self, channel):
self.assertEquals(channel.fetch(self.fetcher, self.progress), True)
loaders = channel.getLoaders()
self.assertEquals(len(loaders), 1)
self.cache.addLoader(loaders[0])
saved = sys.stdout
sys.stdout = StringIO()
try:
self.cache.load()
finally:
sys.stdout = saved
packages = sorted(self.cache.getPackages())
self.assertEquals(len(packages), 2)
self.assertEquals(packages[0].name, "name1")
self.assertEquals(packages[1].name, "name2")
def test_fetch(self):
channel = createChannel("alias",
{"type": "rpm-md",
"baseurl": "file://%s/yumrpm" % TESTDATADIR})
self.check_channel(channel)
def test_fetch_with_broken_mirrorlist(self):
def fail_open(filename, mode='r', bufsize=-1):
raise IOError("emulating a broken mirrorlist...")
old_open = __builtin__.open
__builtin__.open = fail_open
channel = createChannel("alias",
{"type": "rpm-md",
"baseurl": "file://%s/yumrpm" % TESTDATADIR,
"mirrorlist": "file://%s/yumrpm/mirrorlist-broken.txt" % TESTDATADIR})
try:
try:
self.check_channel(channel)
except AttributeError, error:
# AttributeError: 'exceptions.IOError' object has no attribute 'split'
self.fail(error)
except IOError:
pass
finally:
class YumRpmChannelTest(MockerTestCase):
def setUp(self):
self.progress = Progress()
self.fetcher = Fetcher()
self.cache = Cache()
self.download_dir = self.makeDir()
self.fetcher.setLocalPathPrefix(self.download_dir + "/")
# Disable caching so that things blow up when not found.
self.fetcher.setCaching(NEVER)
# Make sure to trigger old bugs in debug error reporting.
sysconf.set("log-level", DEBUG)
def tearDown(self):
sysconf.remove("log-level")
def check_channel(self, channel):
self.assertEquals(channel.fetch(self.fetcher, self.progress), True)
loaders = channel.getLoaders()
self.assertEquals(len(loaders), 1)
self.cache.addLoader(loaders[0])
saved = sys.stdout
sys.stdout = StringIO()
try:
self.cache.load()
finally:
sys.stdout = saved
packages = sorted(self.cache.getPackages())
self.assertEquals(len(packages), 2)
self.assertEquals(packages[0].name, "name1")
self.assertEquals(packages[1].name, "name2")
def test_fetch(self):
channel = createChannel("alias", {
"type": "rpm-md",
"baseurl": "file://%s/yumrpm" % TESTDATADIR
})
self.check_channel(channel)
def test_fetch_with_broken_mirrorlist(self):
def fail_open(filename, mode='r', bufsize=-1):
raise IOError("emulating a broken mirrorlist...")
old_open = __builtin__.open
__builtin__.open = fail_open
channel = createChannel(
"alias", {
"type":
"rpm-md",
"baseurl":
"file://%s/yumrpm" % TESTDATADIR,
"mirrorlist":
"file://%s/yumrpm/mirrorlist-broken.txt" % TESTDATADIR
})
try:
try:
self.check_channel(channel)
except AttributeError, error:
# AttributeError: 'exceptions.IOError' object has no attribute 'split'
self.fail(error)
except IOError:
pass
finally:
class Control(object):
def __init__(self, confpath=None, forcelocks=False):
self._confpath = None
self._channels = {} # alias -> Channel()
self._sysconfchannels = {} # alias -> data dict
self._dynamicchannels = {} # alias -> Channel()
self._pathlocks = PathLocks(forcelocks)
self._cache = Cache()
self.loadSysConf(confpath)
self._fetcher = Fetcher()
self._mediaset = self._fetcher.getMediaSet()
self._achanset = AvailableChannelSet(self._fetcher)
self._cachechanged = False
def getChannels(self):
return self._channels.values()
def removeChannel(self, alias):
channel = self._channels[alias]
if isinstance(channel, PackageChannel):
channel.removeLoaders()
del self._channels[alias]
if alias in self._sysconfchannels:
del self._sysconfchannels[alias]
if alias in self._dynamicchannels:
del self._dynamicchannels[alias]
def getFileChannels(self):
return [
x for x in self._channels.values() if isinstance(x, FileChannel)
]
def checkPackageFile(self, arg):
if os.path.exists(arg):
if filter(None, hooks.call("check-package-file", arg)):
return True
if (os.path.isfile(arg) and os.path.getsize(arg) > 0
and tarfile.is_tarfile(arg)):
return True
return False
def addFileChannel(self, filename):
if not self._sysconfchannels:
# Give a chance for backends to register
# themselves on FileChannel hooks.
self.rebuildSysConfChannels()
found = False
for channel in hooks.call("create-file-channel", filename):
if channel:
if channel.getAlias() in self._channels:
raise Error, _("There's another channel with alias '%s'") \
% channel.getAlias()
self._channels[channel.getAlias()] = channel
found = True
if not found and tarfile.is_tarfile(filename):
tempdir = tempfile.mkdtemp()
iface.debug("Extracting from %s to %s" % (filename, tempdir))
tarball = tarfile.open(filename, 'r')
names = tarball.getnames()
for name in names:
tarball.extract(name, path=tempdir)
tarball.close()
files = os.listdir(tempdir)
for file in files:
path = os.path.join(tempdir, file)
if os.path.isdir(path) and file == "RPMS":
for rpm in os.listdir(path):
files.append(os.path.join(path, rpm))
if os.path.isfile(path) and self.checkPackageFile(path):
self.addFileChannel(path)
found = True
if not found:
raise Error, _("Unable to create channel for file: %s") % filename
def removeFileChannel(self, filename):
filename = os.path.abspath(filename)
for channel in self._channels.values():
if (isinstance(channel, FileChannel)
and channel.getFileName() == filename):
channel.removeLoaders()
break
else:
raise Error, _("Channel not found for '%s'") % filename
def askForRemovableChannels(self, channels):
removable = [(str(x), x) for x in channels if x.isRemovable()]
if not removable:
return True
removable.sort()
removable = [x for name, x in removable]
self._mediaset.umountAll()
if not iface.insertRemovableChannels(removable):
return False
self._mediaset.mountAll()
return True
def getCache(self):
return self._cache
def getFetcher(self):
return self._fetcher
def getMediaSet(self):
return self._mediaset
def restoreMediaState(self):
self._mediaset.restoreState()
__stateversion__ = 2
def loadSysConf(self, confpath=None):
datadir = sysconf.get("data-dir")
if confpath:
confpath = os.path.expanduser(confpath)
if not os.path.isfile(confpath):
raise Error, _("Configuration file not found: %s") % confpath
sysconf.load(confpath)
else:
confpath = os.path.join(datadir, CONFFILE)
if os.path.isfile(confpath):
sysconf.load(confpath)
self._confpath = confpath
if os.path.isdir(datadir):
writable = os.access(datadir, os.W_OK)
else:
try:
os.makedirs(datadir)
writable = True
except OSError:
raise Error, _("Can't create datadir at %s") % datadir
if writable and not self._pathlocks.lock(datadir, exclusive=True):
writable = False
sysconf.setReadOnly(not writable)
def saveSysConf(self, confpath=None):
msys = self._fetcher.getMirrorSystem()
if msys.getHistoryChanged() and not sysconf.getReadOnly():
sysconf.set("mirrors-history", msys.getHistory())
if confpath:
confpath = os.path.expanduser(confpath)
else:
if sysconf.getReadOnly():
return
if self._cachechanged:
cachepath = os.path.join(sysconf.get("data-dir"), "cache")
if sysconf.get("disk-cache", True):
iface.showStatus(_("Saving cache..."))
cachefile = open(cachepath + ".new", "w")
state = (self.__stateversion__, self._cache,
self._channels, self._sysconfchannels)
cPickle.dump(state, cachefile, 2)
cachefile.close()
os.rename(cachepath + ".new", cachepath)
iface.hideStatus()
elif os.path.isfile(cachepath):
os.unlink(cachepath)
if not sysconf.getModified():
return
sysconf.resetModified()
confpath = self._confpath
sysconf.save(confpath)
def reloadMirrors(self):
mirrors = sysconf.get("mirrors", {})
for channel in self._channels.values():
if isinstance(channel, MirrorsChannel):
cmirrors = channel.getMirrors()
if cmirrors:
for origin in cmirrors:
set = dict.fromkeys(cmirrors[origin])
set.update(dict.fromkeys(mirrors.get(origin, [])))
mirrors[origin] = set.keys()
msys = self._fetcher.getMirrorSystem()
msys.setMirrors(mirrors)
if not msys.getHistory():
msys.setHistory(sysconf.get("mirrors-history", []))
def rebuildSysConfChannels(self):
channels = sysconf.get("channels", ())
forcechannels = sysconf.get("force-channels", "")
if forcechannels:
forcechannels = forcechannels.split(",")
def isEnabled(alias, data):
if forcechannels:
return alias in forcechannels
return not data.get("disabled")
if channels and not self._channels:
cachepath = os.path.join(sysconf.get("data-dir"), "cache")
if os.path.isfile(cachepath) and sysconf.get("disk-cache", True):
iface.showStatus(_("Loading cache..."))
cachefile = open(cachepath)
try:
state = cPickle.load(cachefile)
if state[0] != self.__stateversion__:
raise StateVersionError
except:
if sysconf.get("log-level") == DEBUG:
import traceback
traceback.print_exc()
if os.access(os.path.dirname(cachepath), os.W_OK):
os.unlink(cachepath)
else:
(__stateversion__, self._cache, self._channels,
self._sysconfchannels) = state
for alias in self._channels.keys():
if (alias not in channels
or not isEnabled(alias, channels[alias])):
self.removeChannel(alias)
cachefile.close()
iface.hideStatus()
for alias in channels:
data = channels[alias]
if not isEnabled(alias, data):
continue
if alias in self._sysconfchannels.keys():
if self._sysconfchannels[alias] == data:
continue
else:
channel = self._channels[alias]
if isinstance(channel, PackageChannel):
channel.removeLoaders()
del self._channels[alias]
del self._sysconfchannels[alias]
channel = createChannel(alias, data)
self._sysconfchannels[alias] = data
self._channels[alias] = channel
for alias in self._sysconfchannels.keys():
if alias not in channels or channels[alias].get("disabled"):
self.removeChannel(alias)
def rebuildDynamicChannels(self):
for alias in self._dynamicchannels.keys():
self.removeChannel(alias)
newchannels = {}
for channels in hooks.call("rebuild-dynamic-channels"):
if channels:
for channel in channels:
alias = channel.getAlias()
if alias in self._channels:
raise Error, _("There's another channel with "
"alias '%s'") % alias
newchannels[alias] = channel
self._channels.update(newchannels)
self._dynamicchannels.update(newchannels)
def reloadChannels(self, channels=None, caching=ALWAYS):
if channels is None:
manual = False
self.rebuildSysConfChannels()
self.rebuildDynamicChannels()
channels = self._channels.values()
hooks.call("reload-channels", channels)
else:
manual = True
# Get channels directory and check the necessary locks.
channelsdir = os.path.join(sysconf.get("data-dir"), "channels/")
userchannelsdir = os.path.join(sysconf.get("user-data-dir"),
"channels/")
if not os.path.isdir(channelsdir):
try:
os.makedirs(channelsdir)
except OSError:
raise Error, _("Unable to create channel directory.")
if caching is ALWAYS:
if sysconf.getReadOnly() and os.access(channelsdir, os.W_OK):
iface.warning(
_("The Smart library is already in use by "
"another process."))
iface.warning(_("Configuration is in readonly mode!"))
if not self._pathlocks.lock(channelsdir):
raise Error, _("Channel information is locked for writing.")
elif sysconf.getReadOnly():
raise Error, _("Can't update channels in readonly mode.")
elif not self._pathlocks.lock(channelsdir, exclusive=True):
raise Error, _("Can't update channels with active readers.")
self._fetcher.setLocalDir(channelsdir, mangle=True)
# Prepare progress. If we're reading from the cache, we don't want
# too much information being shown. Otherwise, ask for a full-blown
# progress for the interface, and build information of currently
# available packages to compare later.
if caching is ALWAYS:
progress = Progress()
else:
progress = iface.getProgress(self._fetcher, True)
oldpkgs = {}
for pkg in self._cache.getPackages():
oldpkgs[(pkg.name, pkg.version)] = True
progress.start()
steps = 0
for channel in channels:
steps += channel.getFetchSteps()
progress.set(0, steps)
# Rebuild mirror information.
self.reloadMirrors()
self._fetcher.setForceMountedCopy(True)
self._cache.reset()
# Do the real work.
result = True
for channel in channels:
digest = channel.getDigest()
if not manual and channel.hasManualUpdate():
self._fetcher.setCaching(ALWAYS)
else:
self._fetcher.setCaching(caching)
if channel.getFetchSteps() > 0:
progress.setTopic(
_("Fetching information for '%s'...") %
(channel.getName() or channel.getAlias()))
progress.show()
self._fetcher.setForceCopy(channel.isRemovable())
self._fetcher.setLocalPathPrefix(channel.getAlias() + "%%")
try:
if not channel.fetch(self._fetcher, progress):
iface.debug(_("Failed fetching channel '%s'") % channel)
result = False
except Error, e:
iface.error(unicode(e))
iface.debug(_("Failed fetching channel '%s'") % channel)
result = False
if (channel.getDigest() != digest
and isinstance(channel, PackageChannel)):
channel.addLoaders(self._cache)
if channel.getAlias() in self._sysconfchannels:
self._cachechanged = True
if result and caching is not ALWAYS:
sysconf.set("last-update", time.time())
self._fetcher.setForceMountedCopy(False)
self._fetcher.setForceCopy(False)
self._fetcher.setLocalPathPrefix(None)
# Finish progress.
progress.setStopped()
progress.show()
progress.stop()
# Build cache with the new information.
self._cache.load()
# Compare new packages with what we had available, and mark
# new packages.
if caching is not ALWAYS:
pkgconf.clearFlag("new")
for pkg in self._cache.getPackages():
if (pkg.name, pkg.version) not in oldpkgs:
pkgconf.setFlag("new", pkg.name, "=", pkg.version)
# Remove unused files from channels directory.
for dir in (channelsdir, userchannelsdir):
if os.access(dir, os.W_OK):
aliases = self._channels.copy()
aliases.update(dict.fromkeys(sysconf.get("channels", ())))
for entry in os.listdir(dir):
sep = entry.find("%%")
if sep == -1 or entry[:sep] not in aliases:
os.unlink(os.path.join(dir, entry))
# Change back to a shared lock.
self._pathlocks.lock(channelsdir)
return result
class AptDebChannelTest(MockerTestCase):
def setUp(self):
self.progress = Progress()
self.fetcher = Fetcher()
self.cache = Cache()
self.download_dir = self.makeDir()
self.fetcher.setLocalPathPrefix(self.download_dir + "/")
# Disable caching so that things blow up when not found.
self.fetcher.setCaching(NEVER)
sysconf.set("deb-arch", "i386")
self.trustdb = open("%s/aptdeb/trustdb.gpg" % TESTDATADIR).read()
def tearDown(self):
sysconf.remove("deb-arch")
open("%s/aptdeb/trustdb.gpg" % TESTDATADIR, 'w').write(self.trustdb)
def check_channel(self, channel):
self.assertEquals(channel.fetch(self.fetcher, self.progress), True)
loaders = channel.getLoaders()
self.assertEquals(len(loaders), 1)
self.cache.addLoader(loaders[0])
saved = sys.stdout
sys.stdout = StringIO()
try:
self.cache.load()
finally:
sys.stdout = saved
packages = sorted(self.cache.getPackages())
self.assertEquals(len(packages), 2)
self.assertEquals(packages[0].name, "name1")
self.assertEquals(packages[1].name, "name2")
def test_fetch_with_component(self):
channel = createChannel("alias",
{"type": "apt-deb",
"baseurl": "file://%s/aptdeb" % TESTDATADIR,
"distribution": "./",
"components": "component"})
self.check_channel(channel)
def test_fetch_without_component(self):
channel = createChannel("alias",
{"type": "apt-deb",
"baseurl": "file://%s/aptdeb" % TESTDATADIR,
"distribution": "component-less"})
self.check_channel(channel)
def test_fetch_without_component_and_release_file(self):
channel = createChannel("alias",
{"type": "apt-deb",
"baseurl": "file://%s/deb" % TESTDATADIR,
"distribution": "./"})
self.check_channel(channel)
def test_fetch_without_component_and_release_file_with_keyring(self):
channel = createChannel("alias",
{"type": "apt-deb",
"baseurl": "file://%s/deb" % TESTDATADIR,
"distribution": "./",
"keyring": "/dev/null"})
try:
self.check_channel(channel)
except Error, error:
self.assertEquals(str(error),
"Download of Release failed for channel 'alias': "
"File not found for validation")
else:
def fetch_urpmi_cfg(filename):
fetcher = Fetcher()
if os.path.isfile(URPMI_MIRRORS_CACHE):
mirrorsfile = open(URPMI_MIRRORS_CACHE)
mirrorscache = eval(mirrorsfile.read().replace('=>',
':').replace(';', ''))
mirrorsfile.close()
else:
mirrorscache = None
config = []
in_block = False
skip = False
urpmi_cfg_f = open(filename)
for line in urpmi_cfg_f.readlines():
line = line.strip()
if len(line) <= 0:
continue
if line[-1] == "}":
in_block = False
skip = False
if skip:
continue
if line[-1] == "{":
in_block = True
s = line.split(" ")
if len(s) == 1:
skip = True
continue
config.append(dict())
config[-1]["name"] = " ".join(s[0:-2]).replace("\\", "")
if s[-2]:
config[-1]["url"] = s[-2]
continue
if not in_block:
continue
if line.find(": ") > 0:
(key, value) = line.strip().split(": ")
config[-1][key.strip()] = value
if key == "mirrorlist":
if mirrorscache:
if mirrorscache.has_key(value):
value = mirrorscache[value]
# Make sure to find a mirror using a protocol that we
# actually handle...
scheme = value["chosen"].split(":")[0]
if fetcher.getHandler(scheme, None):
config[-1]["url"] = value["chosen"]
else:
config[-1]["url"] = None
for mirror in value["list"]:
scheme = mirror["url"].split(":")[0]
if fetcher.getHandler(scheme, None):
if not config[-1]["url"]:
config[-1]["url"] = mirror["url"]
else:
sysconf.add(("mirrors", config[-1]["url"]),
mirror["url"],
unique=True)
else:
config[-1][line.strip()] = True
urpmi_cfg_f.close()
return config
def fetch_urpmi_cfg(filename):
fetcher = Fetcher()
if os.path.isfile(URPMI_MIRRORS_CACHE):
mirrorsfile = open(URPMI_MIRRORS_CACHE)
mirrorscache = eval(mirrorsfile.read().replace('=>', ':').replace(';',''))
mirrorsfile.close()
else:
mirrorscache = None
config=[]
in_block = False
skip = False
urpmi_cfg_f=open(filename)
for line in urpmi_cfg_f.readlines():
line = line.strip()
if len(line) <= 0:
continue
if line[-1] == "}":
in_block = False
skip = False
if skip:
continue
if line[-1] == "{":
in_block = True
s = line.split(" ")
if len(s) == 1:
skip = True
continue
config.append(dict())
config[-1]["name"] = " ".join(s[0:-2]).replace("\\", "")
if s[-2]:
config[-1]["url"] = s[-2]
continue
if not in_block:
continue
if line.find(": ") > 0:
(key,value) = line.strip().split(": ")
config[-1][key.strip()] = value
if key == "mirrorlist":
if mirrorscache:
if mirrorscache.has_key(value):
value = mirrorscache[value]
# Make sure to find a mirror using a protocol that we
# actually handle...
scheme = value["chosen"].split(":")[0]
if fetcher.getHandler(scheme, None):
config[-1]["url"] = value["chosen"]
else:
config[-1]["url"] = None
for mirror in value["list"]:
scheme = mirror["url"].split(":")[0]
if fetcher.getHandler(scheme, None):
if not config[-1]["url"]:
config[-1]["url"] = mirror["url"]
else:
sysconf.add(("mirrors", config[-1]["url"]), mirror["url"], unique=True)
else:
config[-1][line.strip()] = True
urpmi_cfg_f.close()
return config
