def __init__(self, *args, **kwargs): global content_db content_db = self self.filled = {} # Site addresses that already filled from content.json self.need_filling = False # file_optional table just created, fill data from content.json files self.time_peer_numbers_updated = 0 self.my_optional_files = {} # Last 50 site_address/inner_path called by fileWrite (auto-pinning these files) self.optional_files = collections.defaultdict(dict) self.optional_files_loading = False helper.timer(60 * 5, self.checkOptionalLimit) super(ContentDbPlugin, self).__init__(*args, **kwargs)
def openArchive(archive_path, path_within): if archive_path not in archive_cache: if archive_path.endswith("tar.gz"): import tarfile archive_cache[archive_path] = tarfile.open(archive_path, "r:gz") elif archive_path.endswith("tar.bz2"): import tarfile archive_cache[archive_path] = tarfile.open(archive_path, "r:bz2") else: import zipfile archive_cache[archive_path] = zipfile.ZipFile(archive_path) helper.timer(5, lambda: closeArchive(archive_path)) # Close after 5 sec archive = archive_cache[archive_path] if archive_path.endswith(".zip"): return archive.open(path_within) else: return archive.extractfile(path_within.encode("utf8"))
def openArchive(archive_path, path_within): if archive_path not in archive_cache: if archive_path.endswith("tar.gz"): import tarfile archive_cache[archive_path] = tarfile.open(archive_path, "r:gz") elif archive_path.endswith("tar.bz2"): import tarfile archive_cache[archive_path] = tarfile.open(archive_path, "r:bz2") else: import zipfile archive_cache[archive_path] = zipfile.ZipFile(archive_path) helper.timer(5, lambda: closeArchive(archive_path)) # Close after 5 sec archive = archive_cache[archive_path] if archive_path.endswith(".zip"): return archive.open(path_within) else: return archive.extractfile(path_within.encode("utf8"))
"UPDATE file_optional SET uploaded = uploaded + %s WHERE ?" % uploaded, { "site_id": site_id, "inner_path": inner_path }) num += 1 cur.execute("END") request_log.clear() if "access_log" not in locals().keys(): # To keep between module reloads access_log = collections.defaultdict( dict) # {site_id: {inner_path1: 1, inner_path2: 1...}} request_log = collections.defaultdict(lambda: collections.defaultdict( int)) # {site_id: {inner_path1: 1, inner_path2: 1...}} helper.timer(61, processAccessLog) helper.timer(60, processRequestLog) @PluginManager.registerTo("WorkerManager") class WorkerManagerPlugin(object): def doneTask(self, task): if task["optional_hash_id"]: content_db = self.site.content_manager.contents.db content_db.executeDelayed( "UPDATE file_optional SET time_downloaded = :now, is_downloaded = 1, peer = peer + 1 WHERE site_id = :site_id AND inner_path = :inner_path", { "now": int(time.time()), "site_id": content_db.site_ids[self.site.address], "inner_path": task["inner_path"] })
import time import itertools import gevent from Config import config from util import helper from Plugin import PluginManager from ChartDb import ChartDb from ChartCollector import ChartCollector if "db" not in locals().keys(): # Share on reloads db = ChartDb() gevent.spawn_later(10 * 60 * 60, db.archive) helper.timer(60 * 60 * 6, db.archive) collector = ChartCollector(db) @PluginManager.registerTo("SiteManager") class SiteManagerPlugin(object): def load(self, *args, **kwargs): back = super(SiteManagerPlugin, self).load(*args, **kwargs) collector.setInitialLastValues(self.sites.values()) return back def delete(self, address, *args, **kwargs): db.deleteSite(address) return super(SiteManagerPlugin, self).delete(address, *args, **kwargs) @PluginManager.registerTo("UiWebsocket")
import time import itertools import gevent from Config import config from util import helper from Plugin import PluginManager from ChartDb import ChartDb from ChartCollector import ChartCollector if "db" not in locals().keys(): # Share on reloads db = ChartDb() gevent.spawn_later(10 * 60 * 60, db.archive) helper.timer(60 * 60 * 6, db.archive) collector = ChartCollector(db) @PluginManager.registerTo("SiteManager") class SiteManagerPlugin(object): def load(self, *args, **kwargs): back = super(SiteManagerPlugin, self).load(*args, **kwargs) collector.setInitialLastValues(self.sites.values()) return back def delete(self, address, *args, **kwargs): db.deleteSite(address) return super(SiteManagerPlugin, self).delete(address, *args, **kwargs) @PluginManager.registerTo("UiWebsocket") class UiWebsocketPlugin(object): def actionChartDbQuery(self, to, query, params=None):
cur.execute("BEGIN") for site_id in request_log: for inner_path, uploaded in request_log[site_id].iteritems(): content_db.execute( "UPDATE file_optional SET uploaded = uploaded + %s WHERE ?" % uploaded, {"site_id": site_id, "inner_path": inner_path} ) num += 1 cur.execute("END") request_log.clear() if "access_log" not in locals().keys(): # To keep between module reloads access_log = collections.defaultdict(dict) # {site_id: {inner_path1: 1, inner_path2: 1...}} request_log = collections.defaultdict(lambda: collections.defaultdict(int)) # {site_id: {inner_path1: 1, inner_path2: 1...}} helper.timer(61, processAccessLog) helper.timer(60, processRequestLog) @PluginManager.registerTo("ContentManager") class ContentManagerPlugin(object): def __init__(self, *args, **kwargs): self.cache_is_pinned = {} super(ContentManagerPlugin, self).__init__(*args, **kwargs) def optionalDownloaded(self, inner_path, hash_id, size=None, own=False): if "|" in inner_path: # Big file piece file_inner_path, file_range = inner_path.split("|") else: file_inner_path = inner_path