def generatePackages(self, urls):
"""Generates :class:`Package` instances and names from urls. Useful for many different links and no\
given package name.
:param urls: list of urls
:return: list of `Package`
"""
return [Package(name, purls) for name, purls in parseNames([(url,url) for url in urls]).iteritems()]
def generatePackages(self, links):
""" Parses links, generates packages names from urls
:param links: list of urls
:return: package names mapped to urls
"""
result = parseNames((x, x) for x in links)
return result
def generatePackages(self, links):
""" Parses links, generates packages names from urls
:param links: list of urls
:return: package names mapped to urls
"""
result = parseNames((x, x) for x in links)
return result
def generatePackages(self, urls):
"""Generates :class:`Package` instances and names from urls. Useful for many different links and no\
given package name.
:param urls: list of urls
:return: list of `Package`
"""
return [Package(name, purls) for name, purls in parseNames([(url,url) for url in urls]).iteritems()]
def updateResult(self, plugin, result, force=False):
# parse package name and generate result
# accumulate results
self.cache.extend(result)
if len(self.cache) >= 20 or force:
# used for package generating
tmp = [(name, (url, OnlineStatus(name, plugin, "unknown", status, int(size))))
for name, size, status, url in self.cache]
data = parseNames(tmp)
result = {}
for k, v in data.iteritems():
for url, status in v:
status.packagename = k
result[url] = status
self.m.setInfoResults(self.rid, result)
self.cache = []
def checkOnlineStatus(self, urls):
""" initiates online status check, will also decrypt files.
:param urls:
:return: initial set of data as :class:`OnlineCheck` instance containing the result id
"""
data, crypter = self.core.pluginManager.parseUrls(urls)
# initial result does not contain the crypter links
tmp = [(url, (url, LinkStatus(url, pluginname, "unknown", 3, 0))) for url, pluginname in data]
data = parseNames(tmp)
result = {}
for k, v in data.iteritems():
for url, status in v:
status.packagename = k
result[url] = status
data.update(crypter) # hoster and crypter will be processed
rid = self.core.threadManager.createResultThread(data, False)
return OnlineCheck(rid, result)
def checkOnlineStatus(self, urls):
""" initiates online status check, will also decrypt files.
:param urls:
:return: initial set of data as :class:`OnlineCheck` instance containing the result id
"""
data, crypter = self.core.pluginManager.parseUrls(urls)
# initial result does not contain the crypter links
tmp = [(url, (url, LinkStatus(url, pluginname, "unknown", 3, 0)))
for url, pluginname in data]
data = parseNames(tmp)
result = {}
for k, v in data.iteritems():
for url, status in v:
status.packagename = k
result[url] = status
data.update(crypter) # hoster and crypter will be processed
rid = self.core.threadManager.createResultThread(data, False)
return OnlineCheck(rid, result)
def updateResult(self, plugin, result, force=False):
#parse package name and generate result
#accumulate results
self.cache.extend(result)
if len(self.cache) >= 20 or force:
#used for package generating
tmp = [(name, (url,
LinkStatus(name, plugin, "unknown", status,
int(size))))
for name, size, status, url in self.cache]
data = parseNames(tmp)
result = {}
for k, v in data.iteritems():
for url, status in v:
status.packagename = k
result[url] = status
self.m.setInfoResults(self.rid, result)
self.cache = []
def run(self):
"""run method"""
plugins = {}
container = []
for url, plugin in self.data:
if plugin in plugins:
plugins[plugin].append(url)
else:
plugins[plugin] = [url]
# filter out container plugins
for name in self.m.core.pluginManager.containerPlugins:
if name in plugins:
container.extend([(name, url) for url in plugins[name]])
del plugins[name]
# directly write to database
if self.pid > -1:
for pluginname, urls in plugins.iteritems():
plugin = self.m.core.pluginManager.getPlugin(pluginname, True)
if hasattr(plugin, "getInfo"):
self.fetchForPlugin(pluginname, plugin, urls, self.updateDB)
self.m.core.files.save()
elif self.add:
for pluginname, urls in plugins.iteritems():
plugin = self.m.core.pluginManager.getPlugin(pluginname, True)
if hasattr(plugin, "getInfo"):
self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True)
else:
# generate default result
result = [(url, 0, 3, url) for url in urls]
self.updateCache(pluginname, result)
packs = parseNames([(name, url) for name, x, y, url in self.cache])
self.m.log.debug("Fetched and generated %d packages" % len(packs))
for k, v in packs:
self.m.core.api.addPackage(k, v)
# empty cache
del self.cache[:]
else: # post the results
for name, url in container:
# attach container content
try:
data = self.decryptContainer(name, url)
except Exception:
print_exc()
self.m.log.error("Could not decrypt container.")
data = []
for url, plugin in data:
if plugin in plugins:
plugins[plugin].append(url)
else:
plugins[plugin] = [url]
self.m.infoResults[self.rid] = {}
for pluginname, urls in plugins.iteritems():
plugin = self.m.core.pluginManager.getPlugin(pluginname, True)
if hasattr(plugin, "getInfo"):
self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True)
# force to process cache
if self.cache:
self.updateResult(pluginname, [], True)
else:
# generate default result
result = [(url, 0, 3, url) for url in urls]
self.updateResult(pluginname, result, True)
self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = {}
self.m.timestamp = time() + 5 * 60
def run(self):
"""run method"""
plugins = {}
container = []
for url, plugin in self.data:
if plugin in plugins:
plugins[plugin].append(url)
else:
plugins[plugin] = [url]
# filter out container plugins
for name in self.m.core.pluginManager.containerPlugins:
if name in plugins:
container.extend([(name, url) for url in plugins[name]])
del plugins[name]
#directly write to database
if self.pid > -1:
for pluginname, urls in plugins.iteritems():
plugin = self.m.core.pluginManager.getPlugin(pluginname, True)
if hasattr(plugin, "getInfo"):
self.fetchForPlugin(pluginname, plugin, urls,
self.updateDB)
self.m.core.files.save()
elif self.add:
for pluginname, urls in plugins.iteritems():
plugin = self.m.core.pluginManager.getPlugin(pluginname, True)
if hasattr(plugin, "getInfo"):
self.fetchForPlugin(pluginname, plugin, urls,
self.updateCache, True)
else:
#generate default result
result = [(url, 0, 3, url) for url in urls]
self.updateCache(pluginname, result)
packs = parseNames([(name, url) for name, x, y, url in self.cache])
self.m.log.debug("Fetched and generated %d packages" % len(packs))
for k, v in packs:
self.m.core.api.addPackage(k, v)
#empty cache
del self.cache[:]
else: #post the results
for name, url in container:
#attach container content
try:
data = self.decryptContainer(name, url)
except:
print_exc()
self.m.log.error("Could not decrypt container.")
data = []
for url, plugin in data:
if plugin in plugins:
plugins[plugin].append(url)
else:
plugins[plugin] = [url]
self.m.infoResults[self.rid] = {}
for pluginname, urls in plugins.iteritems():
plugin = self.m.core.pluginManager.getPlugin(pluginname, True)
if hasattr(plugin, "getInfo"):
self.fetchForPlugin(pluginname, plugin, urls,
self.updateResult, True)
#force to process cache
if self.cache:
self.updateResult(pluginname, [], True)
else:
#generate default result
result = [(url, 0, 3, url) for url in urls]
self.updateResult(pluginname, result, True)
self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = {}
self.m.timestamp = time() + 5 * 60
