def __init__(self, *args, **kwargs):
global content_db
content_db = self
self.filled = {} # Site addresses that already filled from content.json
self.need_filling = False # file_optional table just created, fill data from content.json files
self.time_peer_numbers_updated = 0
self.my_optional_files = {} # Last 50 site_address/inner_path called by fileWrite (auto-pinning these files)
self.optional_files = collections.defaultdict(dict)
self.optional_files_loading = False
helper.timer(60 * 5, self.checkOptionalLimit)
super(ContentDbPlugin, self).__init__(*args, **kwargs)
def openArchive(archive_path, path_within):
if archive_path not in archive_cache:
if archive_path.endswith("tar.gz"):
import tarfile
archive_cache[archive_path] = tarfile.open(archive_path, "r:gz")
elif archive_path.endswith("tar.bz2"):
import tarfile
archive_cache[archive_path] = tarfile.open(archive_path, "r:bz2")
else:
import zipfile
archive_cache[archive_path] = zipfile.ZipFile(archive_path)
helper.timer(5, lambda: closeArchive(archive_path)) # Close after 5 sec
archive = archive_cache[archive_path]
if archive_path.endswith(".zip"):
return archive.open(path_within)
else:
return archive.extractfile(path_within.encode("utf8"))
def openArchive(archive_path, path_within):
if archive_path not in archive_cache:
if archive_path.endswith("tar.gz"):
import tarfile
archive_cache[archive_path] = tarfile.open(archive_path, "r:gz")
elif archive_path.endswith("tar.bz2"):
import tarfile
archive_cache[archive_path] = tarfile.open(archive_path, "r:bz2")
else:
import zipfile
archive_cache[archive_path] = zipfile.ZipFile(archive_path)
helper.timer(5, lambda: closeArchive(archive_path)) # Close after 5 sec
archive = archive_cache[archive_path]
if archive_path.endswith(".zip"):
return archive.open(path_within)
else:
return archive.extractfile(path_within.encode("utf8"))
"UPDATE file_optional SET uploaded = uploaded + %s WHERE ?"
% uploaded, {
"site_id": site_id,
"inner_path": inner_path
})
num += 1
cur.execute("END")
request_log.clear()
if "access_log" not in locals().keys(): # To keep between module reloads
access_log = collections.defaultdict(
dict) # {site_id: {inner_path1: 1, inner_path2: 1...}}
request_log = collections.defaultdict(lambda: collections.defaultdict(
int)) # {site_id: {inner_path1: 1, inner_path2: 1...}}
helper.timer(61, processAccessLog)
helper.timer(60, processRequestLog)
@PluginManager.registerTo("WorkerManager")
class WorkerManagerPlugin(object):
def doneTask(self, task):
if task["optional_hash_id"]:
content_db = self.site.content_manager.contents.db
content_db.executeDelayed(
"UPDATE file_optional SET time_downloaded = :now, is_downloaded = 1, peer = peer + 1 WHERE site_id = :site_id AND inner_path = :inner_path",
{
"now": int(time.time()),
"site_id": content_db.site_ids[self.site.address],
"inner_path": task["inner_path"]
})
import time
import itertools
import gevent
from Config import config
from util import helper
from Plugin import PluginManager
from ChartDb import ChartDb
from ChartCollector import ChartCollector
if "db" not in locals().keys(): # Share on reloads
db = ChartDb()
gevent.spawn_later(10 * 60 * 60, db.archive)
helper.timer(60 * 60 * 6, db.archive)
collector = ChartCollector(db)
@PluginManager.registerTo("SiteManager")
class SiteManagerPlugin(object):
def load(self, *args, **kwargs):
back = super(SiteManagerPlugin, self).load(*args, **kwargs)
collector.setInitialLastValues(self.sites.values())
return back
def delete(self, address, *args, **kwargs):
db.deleteSite(address)
return super(SiteManagerPlugin, self).delete(address, *args, **kwargs)
@PluginManager.registerTo("UiWebsocket")
import time
import itertools
import gevent
from Config import config
from util import helper
from Plugin import PluginManager
from ChartDb import ChartDb
from ChartCollector import ChartCollector
if "db" not in locals().keys(): # Share on reloads
db = ChartDb()
gevent.spawn_later(10 * 60 * 60, db.archive)
helper.timer(60 * 60 * 6, db.archive)
collector = ChartCollector(db)
@PluginManager.registerTo("SiteManager")
class SiteManagerPlugin(object):
def load(self, *args, **kwargs):
back = super(SiteManagerPlugin, self).load(*args, **kwargs)
collector.setInitialLastValues(self.sites.values())
return back
def delete(self, address, *args, **kwargs):
db.deleteSite(address)
return super(SiteManagerPlugin, self).delete(address, *args, **kwargs)
@PluginManager.registerTo("UiWebsocket")
class UiWebsocketPlugin(object):
def actionChartDbQuery(self, to, query, params=None):
cur.execute("BEGIN")
for site_id in request_log:
for inner_path, uploaded in request_log[site_id].iteritems():
content_db.execute(
"UPDATE file_optional SET uploaded = uploaded + %s WHERE ?" % uploaded,
{"site_id": site_id, "inner_path": inner_path}
)
num += 1
cur.execute("END")
request_log.clear()
if "access_log" not in locals().keys(): # To keep between module reloads
access_log = collections.defaultdict(dict) # {site_id: {inner_path1: 1, inner_path2: 1...}}
request_log = collections.defaultdict(lambda: collections.defaultdict(int)) # {site_id: {inner_path1: 1, inner_path2: 1...}}
helper.timer(61, processAccessLog)
helper.timer(60, processRequestLog)
@PluginManager.registerTo("ContentManager")
class ContentManagerPlugin(object):
def __init__(self, *args, **kwargs):
self.cache_is_pinned = {}
super(ContentManagerPlugin, self).__init__(*args, **kwargs)
def optionalDownloaded(self, inner_path, hash_id, size=None, own=False):
if "|" in inner_path: # Big file piece
file_inner_path, file_range = inner_path.split("|")
else:
file_inner_path = inner_path
