Example #1
0
    def __init__(self, manager, fid, name, size, filestatus, media, added,
                 fileorder, url, pluginname, hash, status, error, package,
                 owner):

        self.m = manager

        self.fid = int(fid)
        self._name = safe_filename(name)
        self._size = size
        self.filestatus = filestatus
        self.media = media
        self.added = added
        self.fileorder = fileorder
        self.url = url
        self.pluginname = pluginname
        self.hash = hash
        self.status = status
        self.error = error
        self.owner = owner
        self.packageid = package
        # database information ends here

        self.lock = ReadWriteLock()

        self.plugin = None

        self.waitUntil = 0  # time() + time to wait

        # status attributes
        self.abort = False
        self.reconnected = False
        self.statusname = None
Example #2
0
    def __init__(self, bits=16, load=None):
        self.useBits(bits)
        self.mapping = {}
        self.unused_keys = [] # keys are added to this as they are deleted
        self.last_key = 0
        self.rwlock = ReadWriteLock()

        if load: self.load(load)
Example #3
0
    def __init__(self, manager, fid, name, size, filestatus, media, added, fileorder,
                 url, pluginname, hash, status, error, package, owner):

        self.m = manager

        self.fid = int(fid)
        self._name = safe_filename(name)
        self._size = size
        self.filestatus = filestatus
        self.media = media
        self.added = added
        self.fileorder = fileorder
        self.url = url
        self.pluginname = pluginname
        self.hash = hash
        self.status = status
        self.error = error
        self.owner = owner
        self.packageid = package
        # database information ends here

        self.lock = ReadWriteLock()

        self.plugin = None

        self.waitUntil = 0 # time() + time to wait

        # status attributes
        self.abort = False
        self.reconnected = False
        self.statusname = None
Example #4
0
    def __init__(self, core):
        """Constructor"""
        self.core = core
        self.evm = core.eventManager

        # translations
        self.statusMsg = [_("none"), _("offline"), _("online"), _("queued"), _("paused"),
                          _("finished"), _("skipped"), _("failed"), _("starting"), _("waiting"),
                          _("downloading"), _("temp. offline"), _("aborted"), _("not possible"), _("missing"),
                          _("file mismatch"), _("occupied"), _("decrypting"), _("processing"), _("custom"),
                          _("unknown")]

        self.files = {} # holds instances for files
        self.packages = {}  # same for packages

        self.jobCache = {}

        # locking the caches, db is already locked implicit
        self.lock = ReadWriteLock()
        #self.lock._Verbose__verbose = True

        self.downloadstats = {} # cached dl stats
        self.queuestats = {} # cached queue stats

        self.db = self.core.db
Example #5
0
    def __init__(self, core):
        self.core = core
        self.log = core.log

        #: won't start download when true
        self.paused = True

        #: each thread is in exactly one category
        self.free = []
        #: a thread that in working must have a pyfile as active attribute
        self.working = []
        #: holds the decrypter threads
        self.decrypter = []

        #: indicates when reconnect has occurred
        self.reconnecting = Event()
        self.reconnecting.clear()

        self.lock = ReadWriteLock()
Example #6
0
class PyFile(object):
    """
    Represents a file object at runtime
    """
    __slots__ = ("m", "fid", "_name", "_size", "filestatus", "media", "added", "fileorder",
                 "url", "pluginname", "hash", "status", "error", "packageid", "owner",
                 "lock", "plugin", "waitUntil", "abort", "statusname",
                 "reconnected", "pluginclass")

    @staticmethod
    def fromInfoData(m, info):
        f = PyFile(m, info.fid, info.name, info.size, info.status, info.media, info.added, info.fileorder,
                   "", "", "", DownloadStatus.NA, "", info.package, info.owner)
        if info.download:
            f.url = info.download.url
            f.pluginname = info.download.plugin
            f.hash = info.download.hash
            f.status = info.download.status
            f.error = info.download.error

        return f

    def __init__(self, manager, fid, name, size, filestatus, media, added, fileorder,
                 url, pluginname, hash, status, error, package, owner):

        self.m = manager

        self.fid = int(fid)
        self._name = safe_filename(name)
        self._size = size
        self.filestatus = filestatus
        self.media = media
        self.added = added
        self.fileorder = fileorder
        self.url = url
        self.pluginname = pluginname
        self.hash = hash
        self.status = status
        self.error = error
        self.owner = owner
        self.packageid = package
        # database information ends here

        self.lock = ReadWriteLock()

        self.plugin = None

        self.waitUntil = 0 # time() + time to wait

        # status attributes
        self.abort = False
        self.reconnected = False
        self.statusname = None


    @property
    def id(self):
        self.m.core.log.debug("Deprecated attr .id, use .fid instead")
        return self.fid

    def setSize(self, value):
        self._size = int(value)

    # will convert all sizes to ints
    size = property(lambda self: self._size, setSize)

    def getName(self):
        try:
            if self.plugin.req.name:
                return self.plugin.req.name
            else:
                return self._name
        except:
            return self._name

    def setName(self, name):
        """ Only set unicode or utf8 strings as name """
        if type(name) == str:
            name = name.decode("utf8")

        name = safe_filename(name)

        # media type is updated if needed
        if self._name != name:
            self.media = guess_type(name)
            self._name = name

    name = property(getName, setName)

    def __repr__(self):
        return "<PyFile %s: %s@%s>" % (self.id, self.name, self.pluginname)

    @lock
    def initPlugin(self):
        """ inits plugin instance """
        if not self.plugin:
            self.pluginclass = self.m.core.pluginManager.getPluginClass("hoster", self.pluginname)
            self.plugin = self.pluginclass(self)

    @read_lock
    def hasPlugin(self):
        """Thread safe way to determine this file has initialized plugin attribute"""
        return self.plugin is not None

    def package(self):
        """ return package instance"""
        return self.m.getPackage(self.packageid)

    def setStatus(self, status):
        self.status = statusMap[status]
        # needs to sync so status is written to database
        self.sync()

    def setCustomStatus(self, msg, status="processing"):
        self.statusname = msg
        self.setStatus(status)

    def getStatusName(self):
        if self.status not in (15, 16) or not self.statusname:
            return self.m.statusMsg[self.status]
        else:
            return self.statusname

    def hasStatus(self, status):
        return statusMap[status] == self.status

    def sync(self):
        """sync PyFile instance with database"""
        self.m.updateFile(self)

    @lock
    def release(self):
        """sync and remove from cache"""
        if self.plugin is not None:
            self.plugin.clean()
            self.plugin = None

        self.m.releaseFile(self.fid)


    def toInfoData(self):
        return FileInfo(self.fid, self.getName(), self.packageid, self.owner, self.getSize(), self.filestatus,
                        self.media, self.added, self.fileorder, DownloadInfo(
                self.url, self.pluginname, self.hash, self.status, self.getStatusName(), self.error
            )
        )

    def getPath(self):
        pass

    def move(self, pid):
        pass

    def abortDownload(self):
        """abort pyfile if possible"""
        # TODO: abort timeout, currently dead locks
        while self.fid in self.m.core.dlm.processingIds():

            self.lock.acquire(shared=True)
            self.abort = True
            if self.plugin and self.plugin.req:
                self.plugin.req.abort()
                if self.plugin.dl:
                    self.plugin.dl.abort()
            self.lock.release()

            sleep(0.5)

        self.abort = False
        self.setStatus("aborted")
        self.release()

    def finishIfDone(self):
        """set status to finish and release file if every thread is finished with it"""

        # TODO: this is wrong now, it should check if addons are using it
        if self.id in self.m.core.dlm.processingIds():
            return False

        self.setStatus("finished")
        self.release()
        self.m.checkAllLinksFinished()
        return True

    def checkIfProcessed(self):
        self.m.checkAllLinksProcessed(self.id)

    def getSpeed(self):
        """ calculates speed """
        try:
            return self.plugin.dl.speed
        except:
            return 0

    def getETA(self):
        """ gets established time of arrival / or waiting time"""
        try:
            if self.status == DownloadStatus.Waiting:
                return self.waitUntil - time()

            return self.getBytesLeft() / self.getSpeed()
        except:
            return 0

    def getBytesArrived(self):
        """ gets bytes arrived """
        try:
            return self.plugin.dl.arrived
        except:
            return 0

    def getBytesLeft(self):
        """ gets bytes left """
        try:
            return self.plugin.dl.size - self.plugin.dl.arrived
        except:
            return 0

    def getSize(self):
        """ get size of download """
        try:
            if self.plugin.dl.size:
                return self.plugin.dl.size
            else:
                return self.size
        except:
            return self.size

    def getProgressInfo(self):
        return ProgressInfo(self.pluginname, self.name, self.getStatusName(), self.getETA(),
                            self.getBytesArrived(), self.getSize(), self.owner, ProgressType.Download,
                            DownloadProgress(self.fid, self.packageid, self.getSpeed(), self.status))
Example #7
0
class BitKeyMap:
    """Map a string to a limited number of bits"""

    def __init__(self, bits=16, load=None):
        self.useBits(bits)
        self.mapping = {}
        self.unused_keys = [] # keys are added to this as they are deleted
        self.last_key = 0
        self.rwlock = ReadWriteLock()

        if load: self.load(load)

    def useBits(self, bits):
        self.bits = bits
        self.max_key = 2 ** bits

    def size(self):
        return len(self.mapping)

    def lastKey(self):
        return self.last_key

    def get(self, key_string):
        self.rwlock.acquire_read()
        if key_string not in self.mapping:
            self.rwlock.release_read()
            self.rwlock.acquire_write()
            try:
                if self.last_key < self.max_key-1:
                    self.last_key += 1
                    self.mapping[key_string] = self.last_key
                else:
                    if len(self.unused_keys) == 0:
                        raise Exception("Mapping is full; all bits have been exhausted")
                    self.mapping[key_string] = self.unused_keys.pop()
                return self.mapping[key_string]
            finally:
                self.rwlock.release_write()
            #print "BITKEYMAP_NEW_KEY\t%d\t%s\t%d" % (threading.current_thread().ident, key_string, self.mapping[key_string])
        k = self.mapping[key_string]
        self.rwlock.release_read()
        return k

    def contains(self, key_string):
        self.rwlock.acquire_read()
        b = key_string in self.mapping
        self.rwlock.release_read()
        return b

    def free(self, key_string):
        self.rwlock.acquire_write()
        if key_string in self.mapping:
            self.unused_keys.append(self.mapping[key_string])
            del self.mapping[key_string]
        self.rwlock.release_write()

    def dump(self, file_or_str):
        d = dict(max_key=self.max_key, mapping=self.mapping, unused_keys=self.unused_keys, last_key=self.last_key)
        if type(file_or_str) is str:
            with open(file_or_str, 'w') as f:
                pickle.dump(d, f)
        else: pickle.dump(d, file_or_str)

    def load(self, file_or_str):
        if type(file_or_str) is str:
            with open(file_or_str, 'r') as f:
                d = pickle.load(f)
        else: d = pickle.load(file_or_str)
        self.max_key, self.mapping, self.unused_keys, self.last_key = d['max_key'], d['mapping'], d['unused_keys'], d['last_key']
Example #8
0
import web
import json
import numpy as np
from pymongo import MongoClient
from rbm import RBM
from ReadWriteLock import ReadWriteLock

urls = ("/query", "query", "/update", "update", "/train", "train")
app = web.application(urls, globals())

rbm = RBM()
rw = ReadWriteLock()


class query:
    def __init__(self):
        self.userPref = []

    def POST(self):
        data = json.loads(web.data())
        client = MongoClient()
        db = client.data
        usersDB = db.users

        # Get user preference list
        items = usersDB.find_one({"_id": data['userID']})
        if items is not None:
            for item in items["items"]:
                self.userPref.append(
                    [float(x) for x in item["itemValue"].split(",")])
Example #9
0
 def __init__(self, maxLen):
     self.l = ReadWriteLock()
     self.nextT = int((time() - 1312742671) * 4)
     self.maxLen = maxLen
     self.m = {}
     self.s = []
Example #10
0
class PhotoCache:
    def __init__(self, maxLen):
        self.l = ReadWriteLock()
        self.nextT = int((time() - 1312742671) * 4)
        self.maxLen = maxLen
        self.m = {}
        self.s = []
    
    def update(self, photoId, data):
        self.l.acquireWrite()
        if photoId in self.m:
            oldT, oldData = self.m[photoId]
            data['tagging_id'] = oldData['tagging_id']
            newInfo = (oldT, data)
        else:
            data['tagging_id'] = self.nextT
            newInfo = (self.nextT, data)
            self.nextT = self.nextT + 1
            self.s.append(photoId)
            if len(self.s) > self.maxLen:
                self.s.pop(0)
        self.m[photoId] = newInfo
        self.l.release()
        return newInfo
    
    def dumpLatest(self, n):
        self.l.acquireRead()
        print >> sys.stderr, map(lambda photoId: self.m[photoId][1]['tagging_id'], self.s[-n:])
        self.l.release()
    
    def getLatest(self, n):
        self.l.acquireRead()
        output = []
        for photoId in self.s[-n:]:
            output.append(self.m[photoId][1])
        self.l.release()
        return output
Example #11
0
class PyFile(object):
    """
    Represents a file object at runtime
    """
    __slots__ = ("m", "fid", "_name", "_size", "filestatus", "media", "added",
                 "fileorder", "url", "pluginname", "hash", "status", "error",
                 "packageid", "owner", "lock", "plugin", "waitUntil", "abort",
                 "statusname", "reconnected", "pluginclass")

    @staticmethod
    def fromInfoData(m, info):
        f = PyFile(m, info.fid, info.name, info.size, info.status, info.media,
                   info.added, info.fileorder, "", "", "", DownloadStatus.NA,
                   "", info.package, info.owner)
        if info.download:
            f.url = info.download.url
            f.pluginname = info.download.plugin
            f.hash = info.download.hash
            f.status = info.download.status
            f.error = info.download.error

        return f

    def __init__(self, manager, fid, name, size, filestatus, media, added,
                 fileorder, url, pluginname, hash, status, error, package,
                 owner):

        self.m = manager

        self.fid = int(fid)
        self._name = safe_filename(name)
        self._size = size
        self.filestatus = filestatus
        self.media = media
        self.added = added
        self.fileorder = fileorder
        self.url = url
        self.pluginname = pluginname
        self.hash = hash
        self.status = status
        self.error = error
        self.owner = owner
        self.packageid = package
        # database information ends here

        self.lock = ReadWriteLock()

        self.plugin = None

        self.waitUntil = 0  # time() + time to wait

        # status attributes
        self.abort = False
        self.reconnected = False
        self.statusname = None

    @property
    def id(self):
        self.m.core.log.debug("Deprecated attr .id, use .fid instead")
        return self.fid

    def setSize(self, value):
        self._size = int(value)

    # will convert all sizes to ints
    size = property(lambda self: self._size, setSize)

    def getName(self):
        try:
            if self.plugin.req.name:
                return self.plugin.req.name
            else:
                return self._name
        except:
            return self._name

    def setName(self, name):
        """ Only set unicode or utf8 strings as name """
        if type(name) == str:
            name = name.decode("utf8")

        name = safe_filename(name)

        # media type is updated if needed
        if self._name != name:
            self.media = guess_type(name)
            self._name = name

    name = property(getName, setName)

    def __repr__(self):
        return "<PyFile %s: %s@%s>" % (self.id, self.name, self.pluginname)

    @lock
    def initPlugin(self):
        """ inits plugin instance """
        if not self.plugin:
            self.pluginclass = self.m.core.pluginManager.getPluginClass(
                "hoster", self.pluginname)
            self.plugin = self.pluginclass(self)

    @read_lock
    def hasPlugin(self):
        """Thread safe way to determine this file has initialized plugin attribute"""
        return self.plugin is not None

    def package(self):
        """ return package instance"""
        return self.m.getPackage(self.packageid)

    def setStatus(self, status):
        self.status = statusMap[status]
        # needs to sync so status is written to database
        self.sync()

    def setCustomStatus(self, msg, status="processing"):
        self.statusname = msg
        self.setStatus(status)

    def getStatusName(self):
        if self.status not in (15, 16) or not self.statusname:
            return self.m.statusMsg[self.status]
        else:
            return self.statusname

    def hasStatus(self, status):
        return statusMap[status] == self.status

    def sync(self):
        """sync PyFile instance with database"""
        self.m.updateFile(self)

    @lock
    def release(self):
        """sync and remove from cache"""
        if self.plugin is not None:
            self.plugin.clean()
            self.plugin = None

        self.m.releaseFile(self.fid)

    def toInfoData(self):
        return FileInfo(
            self.fid, self.getName(), self.packageid, self.owner,
            self.getSize(), self.filestatus, self.media, self.added,
            self.fileorder,
            DownloadInfo(self.url, self.pluginname, self.hash, self.status,
                         self.getStatusName(), self.error))

    def getPath(self):
        pass

    def move(self, pid):
        pass

    def abortDownload(self):
        """abort pyfile if possible"""
        while self.fid in self.m.core.dlm.processingIds():

            self.lock.acquire(shared=True)
            self.abort = True
            if self.plugin and self.plugin.req:
                self.plugin.req.abort()
                if self.plugin.dl:
                    self.plugin.dl.abort()
            self.lock.release()

            sleep(0.5)

        self.abort = False
        self.setStatus("aborted")
        self.release()

    def finishIfDone(self):
        """set status to finish and release file if every thread is finished with it"""

        # TODO: this is wrong now, it should check if addons are using it
        if self.id in self.m.core.dlm.processingIds():
            return False

        self.setStatus("finished")
        self.release()
        self.m.checkAllLinksFinished()
        return True

    def checkIfProcessed(self):
        self.m.checkAllLinksProcessed(self.id)

    @try_catch(0)
    def getSpeed(self):
        """ calculates speed """
        return self.plugin.dl.speed

    @try_catch(0)
    def getETA(self):
        """ gets estimated time of arrival / or waiting time"""
        if self.status == DownloadStatus.Waiting:
            return self.waitUntil - time()

        return self.getBytesLeft() / self.getSpeed()

    @try_catch(0)
    def getBytesArrived(self):
        """ gets bytes arrived """
        return self.plugin.dl.arrived

    @try_catch(0)
    def getBytesLeft(self):
        """ gets bytes left """
        return self.plugin.dl.size - self.plugin.dl.arrived

    def getSize(self):
        """ get size of download """
        try:
            if self.plugin.dl.size:
                return self.plugin.dl.size
            else:
                return self.size
        except:
            return self.size

    @try_catch(0)
    def getFlags(self):
        return self.plugin.dl.flags

    def getProgressInfo(self):
        return ProgressInfo(
            self.pluginname, self.name, self.getStatusName(), self.getETA(),
            self.getBytesArrived(), self.getSize(), self.owner,
            ProgressType.Download,
            DownloadProgress(self.fid, self.packageid, self.getSpeed(),
                             self.getFlags(), self.status))