def __init__(self, manager, fid, name, size, filestatus, media, added, fileorder, url, pluginname, hash, status, error, package, owner): self.m = manager self.fid = int(fid) self._name = safe_filename(name) self._size = size self.filestatus = filestatus self.media = media self.added = added self.fileorder = fileorder self.url = url self.pluginname = pluginname self.hash = hash self.status = status self.error = error self.owner = owner self.packageid = package # database information ends here self.lock = ReadWriteLock() self.plugin = None self.waitUntil = 0 # time() + time to wait # status attributes self.abort = False self.reconnected = False self.statusname = None
def __init__(self, bits=16, load=None): self.useBits(bits) self.mapping = {} self.unused_keys = [] # keys are added to this as they are deleted self.last_key = 0 self.rwlock = ReadWriteLock() if load: self.load(load)
def __init__(self, core): """Constructor""" self.core = core self.evm = core.eventManager # translations self.statusMsg = [_("none"), _("offline"), _("online"), _("queued"), _("paused"), _("finished"), _("skipped"), _("failed"), _("starting"), _("waiting"), _("downloading"), _("temp. offline"), _("aborted"), _("not possible"), _("missing"), _("file mismatch"), _("occupied"), _("decrypting"), _("processing"), _("custom"), _("unknown")] self.files = {} # holds instances for files self.packages = {} # same for packages self.jobCache = {} # locking the caches, db is already locked implicit self.lock = ReadWriteLock() #self.lock._Verbose__verbose = True self.downloadstats = {} # cached dl stats self.queuestats = {} # cached queue stats self.db = self.core.db
def __init__(self, core): self.core = core self.log = core.log #: won't start download when true self.paused = True #: each thread is in exactly one category self.free = [] #: a thread that in working must have a pyfile as active attribute self.working = [] #: holds the decrypter threads self.decrypter = [] #: indicates when reconnect has occurred self.reconnecting = Event() self.reconnecting.clear() self.lock = ReadWriteLock()
class PyFile(object): """ Represents a file object at runtime """ __slots__ = ("m", "fid", "_name", "_size", "filestatus", "media", "added", "fileorder", "url", "pluginname", "hash", "status", "error", "packageid", "owner", "lock", "plugin", "waitUntil", "abort", "statusname", "reconnected", "pluginclass") @staticmethod def fromInfoData(m, info): f = PyFile(m, info.fid, info.name, info.size, info.status, info.media, info.added, info.fileorder, "", "", "", DownloadStatus.NA, "", info.package, info.owner) if info.download: f.url = info.download.url f.pluginname = info.download.plugin f.hash = info.download.hash f.status = info.download.status f.error = info.download.error return f def __init__(self, manager, fid, name, size, filestatus, media, added, fileorder, url, pluginname, hash, status, error, package, owner): self.m = manager self.fid = int(fid) self._name = safe_filename(name) self._size = size self.filestatus = filestatus self.media = media self.added = added self.fileorder = fileorder self.url = url self.pluginname = pluginname self.hash = hash self.status = status self.error = error self.owner = owner self.packageid = package # database information ends here self.lock = ReadWriteLock() self.plugin = None self.waitUntil = 0 # time() + time to wait # status attributes self.abort = False self.reconnected = False self.statusname = None @property def id(self): self.m.core.log.debug("Deprecated attr .id, use .fid instead") return self.fid def setSize(self, value): self._size = int(value) # will convert all sizes to ints size = property(lambda self: self._size, setSize) def getName(self): try: if self.plugin.req.name: return self.plugin.req.name else: return self._name except: return self._name def setName(self, name): """ Only set unicode or utf8 strings as name """ if type(name) == str: name = name.decode("utf8") name = safe_filename(name) # media type is updated if needed if self._name != name: self.media = guess_type(name) self._name = name name = property(getName, setName) def __repr__(self): return "<PyFile %s: %s@%s>" % (self.id, self.name, self.pluginname) @lock def initPlugin(self): """ inits plugin instance """ if not self.plugin: self.pluginclass = self.m.core.pluginManager.getPluginClass("hoster", self.pluginname) self.plugin = self.pluginclass(self) @read_lock def hasPlugin(self): """Thread safe way to determine this file has initialized plugin attribute""" return self.plugin is not None def package(self): """ return package instance""" return self.m.getPackage(self.packageid) def setStatus(self, status): self.status = statusMap[status] # needs to sync so status is written to database self.sync() def setCustomStatus(self, msg, status="processing"): self.statusname = msg self.setStatus(status) def getStatusName(self): if self.status not in (15, 16) or not self.statusname: return self.m.statusMsg[self.status] else: return self.statusname def hasStatus(self, status): return statusMap[status] == self.status def sync(self): """sync PyFile instance with database""" self.m.updateFile(self) @lock def release(self): """sync and remove from cache""" if self.plugin is not None: self.plugin.clean() self.plugin = None self.m.releaseFile(self.fid) def toInfoData(self): return FileInfo(self.fid, self.getName(), self.packageid, self.owner, self.getSize(), self.filestatus, self.media, self.added, self.fileorder, DownloadInfo( self.url, self.pluginname, self.hash, self.status, self.getStatusName(), self.error ) ) def getPath(self): pass def move(self, pid): pass def abortDownload(self): """abort pyfile if possible""" # TODO: abort timeout, currently dead locks while self.fid in self.m.core.dlm.processingIds(): self.lock.acquire(shared=True) self.abort = True if self.plugin and self.plugin.req: self.plugin.req.abort() if self.plugin.dl: self.plugin.dl.abort() self.lock.release() sleep(0.5) self.abort = False self.setStatus("aborted") self.release() def finishIfDone(self): """set status to finish and release file if every thread is finished with it""" # TODO: this is wrong now, it should check if addons are using it if self.id in self.m.core.dlm.processingIds(): return False self.setStatus("finished") self.release() self.m.checkAllLinksFinished() return True def checkIfProcessed(self): self.m.checkAllLinksProcessed(self.id) def getSpeed(self): """ calculates speed """ try: return self.plugin.dl.speed except: return 0 def getETA(self): """ gets established time of arrival / or waiting time""" try: if self.status == DownloadStatus.Waiting: return self.waitUntil - time() return self.getBytesLeft() / self.getSpeed() except: return 0 def getBytesArrived(self): """ gets bytes arrived """ try: return self.plugin.dl.arrived except: return 0 def getBytesLeft(self): """ gets bytes left """ try: return self.plugin.dl.size - self.plugin.dl.arrived except: return 0 def getSize(self): """ get size of download """ try: if self.plugin.dl.size: return self.plugin.dl.size else: return self.size except: return self.size def getProgressInfo(self): return ProgressInfo(self.pluginname, self.name, self.getStatusName(), self.getETA(), self.getBytesArrived(), self.getSize(), self.owner, ProgressType.Download, DownloadProgress(self.fid, self.packageid, self.getSpeed(), self.status))
class BitKeyMap: """Map a string to a limited number of bits""" def __init__(self, bits=16, load=None): self.useBits(bits) self.mapping = {} self.unused_keys = [] # keys are added to this as they are deleted self.last_key = 0 self.rwlock = ReadWriteLock() if load: self.load(load) def useBits(self, bits): self.bits = bits self.max_key = 2 ** bits def size(self): return len(self.mapping) def lastKey(self): return self.last_key def get(self, key_string): self.rwlock.acquire_read() if key_string not in self.mapping: self.rwlock.release_read() self.rwlock.acquire_write() try: if self.last_key < self.max_key-1: self.last_key += 1 self.mapping[key_string] = self.last_key else: if len(self.unused_keys) == 0: raise Exception("Mapping is full; all bits have been exhausted") self.mapping[key_string] = self.unused_keys.pop() return self.mapping[key_string] finally: self.rwlock.release_write() #print "BITKEYMAP_NEW_KEY\t%d\t%s\t%d" % (threading.current_thread().ident, key_string, self.mapping[key_string]) k = self.mapping[key_string] self.rwlock.release_read() return k def contains(self, key_string): self.rwlock.acquire_read() b = key_string in self.mapping self.rwlock.release_read() return b def free(self, key_string): self.rwlock.acquire_write() if key_string in self.mapping: self.unused_keys.append(self.mapping[key_string]) del self.mapping[key_string] self.rwlock.release_write() def dump(self, file_or_str): d = dict(max_key=self.max_key, mapping=self.mapping, unused_keys=self.unused_keys, last_key=self.last_key) if type(file_or_str) is str: with open(file_or_str, 'w') as f: pickle.dump(d, f) else: pickle.dump(d, file_or_str) def load(self, file_or_str): if type(file_or_str) is str: with open(file_or_str, 'r') as f: d = pickle.load(f) else: d = pickle.load(file_or_str) self.max_key, self.mapping, self.unused_keys, self.last_key = d['max_key'], d['mapping'], d['unused_keys'], d['last_key']
import web import json import numpy as np from pymongo import MongoClient from rbm import RBM from ReadWriteLock import ReadWriteLock urls = ("/query", "query", "/update", "update", "/train", "train") app = web.application(urls, globals()) rbm = RBM() rw = ReadWriteLock() class query: def __init__(self): self.userPref = [] def POST(self): data = json.loads(web.data()) client = MongoClient() db = client.data usersDB = db.users # Get user preference list items = usersDB.find_one({"_id": data['userID']}) if items is not None: for item in items["items"]: self.userPref.append( [float(x) for x in item["itemValue"].split(",")])
def __init__(self, maxLen): self.l = ReadWriteLock() self.nextT = int((time() - 1312742671) * 4) self.maxLen = maxLen self.m = {} self.s = []
class PhotoCache: def __init__(self, maxLen): self.l = ReadWriteLock() self.nextT = int((time() - 1312742671) * 4) self.maxLen = maxLen self.m = {} self.s = [] def update(self, photoId, data): self.l.acquireWrite() if photoId in self.m: oldT, oldData = self.m[photoId] data['tagging_id'] = oldData['tagging_id'] newInfo = (oldT, data) else: data['tagging_id'] = self.nextT newInfo = (self.nextT, data) self.nextT = self.nextT + 1 self.s.append(photoId) if len(self.s) > self.maxLen: self.s.pop(0) self.m[photoId] = newInfo self.l.release() return newInfo def dumpLatest(self, n): self.l.acquireRead() print >> sys.stderr, map(lambda photoId: self.m[photoId][1]['tagging_id'], self.s[-n:]) self.l.release() def getLatest(self, n): self.l.acquireRead() output = [] for photoId in self.s[-n:]: output.append(self.m[photoId][1]) self.l.release() return output
class PyFile(object): """ Represents a file object at runtime """ __slots__ = ("m", "fid", "_name", "_size", "filestatus", "media", "added", "fileorder", "url", "pluginname", "hash", "status", "error", "packageid", "owner", "lock", "plugin", "waitUntil", "abort", "statusname", "reconnected", "pluginclass") @staticmethod def fromInfoData(m, info): f = PyFile(m, info.fid, info.name, info.size, info.status, info.media, info.added, info.fileorder, "", "", "", DownloadStatus.NA, "", info.package, info.owner) if info.download: f.url = info.download.url f.pluginname = info.download.plugin f.hash = info.download.hash f.status = info.download.status f.error = info.download.error return f def __init__(self, manager, fid, name, size, filestatus, media, added, fileorder, url, pluginname, hash, status, error, package, owner): self.m = manager self.fid = int(fid) self._name = safe_filename(name) self._size = size self.filestatus = filestatus self.media = media self.added = added self.fileorder = fileorder self.url = url self.pluginname = pluginname self.hash = hash self.status = status self.error = error self.owner = owner self.packageid = package # database information ends here self.lock = ReadWriteLock() self.plugin = None self.waitUntil = 0 # time() + time to wait # status attributes self.abort = False self.reconnected = False self.statusname = None @property def id(self): self.m.core.log.debug("Deprecated attr .id, use .fid instead") return self.fid def setSize(self, value): self._size = int(value) # will convert all sizes to ints size = property(lambda self: self._size, setSize) def getName(self): try: if self.plugin.req.name: return self.plugin.req.name else: return self._name except: return self._name def setName(self, name): """ Only set unicode or utf8 strings as name """ if type(name) == str: name = name.decode("utf8") name = safe_filename(name) # media type is updated if needed if self._name != name: self.media = guess_type(name) self._name = name name = property(getName, setName) def __repr__(self): return "<PyFile %s: %s@%s>" % (self.id, self.name, self.pluginname) @lock def initPlugin(self): """ inits plugin instance """ if not self.plugin: self.pluginclass = self.m.core.pluginManager.getPluginClass( "hoster", self.pluginname) self.plugin = self.pluginclass(self) @read_lock def hasPlugin(self): """Thread safe way to determine this file has initialized plugin attribute""" return self.plugin is not None def package(self): """ return package instance""" return self.m.getPackage(self.packageid) def setStatus(self, status): self.status = statusMap[status] # needs to sync so status is written to database self.sync() def setCustomStatus(self, msg, status="processing"): self.statusname = msg self.setStatus(status) def getStatusName(self): if self.status not in (15, 16) or not self.statusname: return self.m.statusMsg[self.status] else: return self.statusname def hasStatus(self, status): return statusMap[status] == self.status def sync(self): """sync PyFile instance with database""" self.m.updateFile(self) @lock def release(self): """sync and remove from cache""" if self.plugin is not None: self.plugin.clean() self.plugin = None self.m.releaseFile(self.fid) def toInfoData(self): return FileInfo( self.fid, self.getName(), self.packageid, self.owner, self.getSize(), self.filestatus, self.media, self.added, self.fileorder, DownloadInfo(self.url, self.pluginname, self.hash, self.status, self.getStatusName(), self.error)) def getPath(self): pass def move(self, pid): pass def abortDownload(self): """abort pyfile if possible""" while self.fid in self.m.core.dlm.processingIds(): self.lock.acquire(shared=True) self.abort = True if self.plugin and self.plugin.req: self.plugin.req.abort() if self.plugin.dl: self.plugin.dl.abort() self.lock.release() sleep(0.5) self.abort = False self.setStatus("aborted") self.release() def finishIfDone(self): """set status to finish and release file if every thread is finished with it""" # TODO: this is wrong now, it should check if addons are using it if self.id in self.m.core.dlm.processingIds(): return False self.setStatus("finished") self.release() self.m.checkAllLinksFinished() return True def checkIfProcessed(self): self.m.checkAllLinksProcessed(self.id) @try_catch(0) def getSpeed(self): """ calculates speed """ return self.plugin.dl.speed @try_catch(0) def getETA(self): """ gets estimated time of arrival / or waiting time""" if self.status == DownloadStatus.Waiting: return self.waitUntil - time() return self.getBytesLeft() / self.getSpeed() @try_catch(0) def getBytesArrived(self): """ gets bytes arrived """ return self.plugin.dl.arrived @try_catch(0) def getBytesLeft(self): """ gets bytes left """ return self.plugin.dl.size - self.plugin.dl.arrived def getSize(self): """ get size of download """ try: if self.plugin.dl.size: return self.plugin.dl.size else: return self.size except: return self.size @try_catch(0) def getFlags(self): return self.plugin.dl.flags def getProgressInfo(self): return ProgressInfo( self.pluginname, self.name, self.getStatusName(), self.getETA(), self.getBytesArrived(), self.getSize(), self.owner, ProgressType.Download, DownloadProgress(self.fid, self.packageid, self.getSpeed(), self.getFlags(), self.status))