Beispiel #1
0
 def __init__(self, id, title=''):
     """Initialize Repository object"""
     StorageManager.__init__(self, id)
     self.OAI = OAIHandler('OAI')
     self.title = title
     self._create_catalog()
     self._create_cache() #results cache needs better invalidation code - consider fake=True if you're getting conflicts on publish
    def start_task(self):
        print("Webparser is starting")
        # network request
        request = NetworkRequest()
        page = request.get_with_retry(WebParser.default_retry_count,
                                      WebParser.url_gmb)

        # # parsing page for images
        parser = DocumentParser(page)
        listofImages = parser.get_images_from_reddit()

        # creating folder for storing image
        currentWorkingDir = os.getcwd()
        full_directory_path = currentWorkingDir + "\wallpaper\\"
        storageManager = StorageManager(full_directory_path)
        storageManager.createRootStorageDir()

        # # downloading image
        selected_image = random.choice(listofImages)
        while True:
            selected_image = random.choice(listofImages)
            print("Checking if " + selected_image + " is valid Image")
            if NetworkRequest().check_if_valid_image(selected_image):
                request.startImageDownload(selected_image, full_directory_path)
                break
 def __init__(self, id, title=''):
     """Initialize Repository object"""
     StorageManager.__init__(self, id)
     self.OAI = OAIHandler('OAI')
     self.title = title
     self._create_catalog()
     self._create_cache(
     )  #results cache needs better invalidation code - consider fake=True if you're getting conflicts on publish
Beispiel #4
0
    def startAll(self):
        """Setup and start all threads."""
        try:
            # Create StorageManager and Interpreter for BufferListener
            storMan = StorageManager()
            interpr = Interpreter(storMan)

            # Create BufferListener
            buffLis = self.createBufferListener(interpr)

            if buffLis.conn:
                self.hsThreads.append(buffLis)

            # Check scheduler
            # Get the nagios configuration section from config file
            nagiosConf = self.config.itemsdict('NagiosPush')
            machine = re.search('([a-z0-9]+).zip',
                                self.config.get('Station', 'Certificate'))
            nagiosConf['machine_name'] = machine.group(1)
            checkSched = self.createCheckScheduler(interpr, nagiosConf)
            eventRate = checkSched.getEventRate()
            storMan.addObserver(eventRate)
            self.hsThreads.append(checkSched)

            # Uploader central
            up = self.createUploader(0, "Upload-datastore", nagiosConf)
            self.hsThreads.append(up)
            storMan.addObserver(up)
            up.setNumServer(self.numServers)

            # Try local server
            #try:
            #    up2 = self.createUploader(1, "Upload-local", nagiosConf)
            #    self.hsThreads.append(up2)
            #    storMan.addObserver(up2)
            #    self.numServers += 1
            #    up.setNumServer(self.numServers)
            #    up2.setNumServer(self.numServers)
            #except Exception, msg:
            #    logger.debug("Error while parsing local server: %s." % msg)
            #    logger.debug("Will not upload to a local server.")#

            # Set number of servers for our own StorageManager
            storMan.setNumServer(self.numServers)
            storMan.clearOldUploadedEvents()

            # Start all threads, running their run() function.
            for thread in self.hsThreads:
                thread.start()

        except Exception, msg:
            logger.critical("Error HsMonitor: %s" % msg)
            sys.exit(1)
    def __init__(self, config, interpreter):
        super(CheckScheduler, self).__init__(name='CheckScheduler')
        self.stop_event = threading.Event()

        self.status = None
        self.sched = Scheduler(self.status)
        self.dicConfig = config

        self.nagiosPush = NagiosPush(config)
        self.storageManager = StorageManager()
        self.interpreter = interpreter

        self.eventRate = EventRate()
    def __init__(self, config, interpreter):
        # invoke constructor of parent class (threading)
        threading.Thread.__init__(self)
        self.stop_event = threading.Event()

        self.status = None
        self.sched = Scheduler(self.status)
        self.dicConfig = config

        # create a nagios push object
        self.nagiosPush = NagiosPush(config)
        self.storageManager = StorageManager()
        self.interpreter = interpreter

        ### Event rate:
        self.eventRate = EventRate()
    def setUp(self):
        # create a few dummy events
        self.e1 = Event(0)
        self.e2 = Event(4)
        self.e3 = Event(5)
        self.e4 = Event(6)

        self.elist = [self.e1, self.e2, self.e3, self.e4]
        # add bogus information to an event
        self.e1.data = "hello world"
        self.e2.data = "snd message"
        self.e3.data = "non-funny message"
        self.e4.data = "whatever"

        # setup storagemanager
        self.sm = StorageManager()
        self.sm.setNumServer(2)
        self.sm.openConnection()
def init():

    global db

    # global session_management
    global user_management
    global storage_handler
    global cart_management
    global order_management
    global product_management
    global sales_management
    global product_management

    storage_handler = StorageHandler()
    db = StorageManager()

    user_management = UserManagement(storage_handler)
    cart_management = CartManagement(storage_handler)
    product_management = ProductManagement(storage_handler)
    sales_management = SalesManagement(storage_handler)
    order_management = OrderManagement(storage_handler, sales_management)
Beispiel #9
0
    def __init__(self, serverID, stationID, password, URL, config,
                 retryAfter=MINWAIT, maxWait=MAXWAIT,
                 minBatchSize=BATCHSIZE, maxBatchSize=BATCHSIZE):
        self.storageManager = StorageManager()
        self.serverID = serverID
        self.stationID = stationID
        self.password = password
        self.URL = URL
        self.nagiosPush = NagiosPush(config)
        self.minBatchSize = minBatchSize
        self.maxBatchSize = maxBatchSize
        self.retryAfter = MINWAIT
        self.maxWait = MAXWAIT

        # lock to protect numEvents
        self.numEventsLock = Semaphore(1)

        # Semaphore to block if the number of events drops below minBatchSize
        self.noEventsSem = Semaphore(0)

        super(Uploader, self).__init__(name='Uploader')
        self.stop_event = Event()
        self.isRunning = False
Beispiel #10
0
def get_test_storage_manager(root_db):
    """
    Generate testing storage manager instance
    :param root_db: indicate root database
    :return: the testing storage manager with local filesystem and a mongoDB
    """
    from storages.file_storage import file_storage
    from storages.mongo_db import mongo_conn, mongo_writer, mongo_reader

    fs_hot = file_storage({'name': '/mnt/hotsemantic/',
                           'pathname': '/mnt/hotsemantic/'})
    fs_warm = file_storage({'name': '/mnt/warmsemantic/',
                            'pathname': '/mnt/warmsemantic/'})
    fs_cold = file_storage({'name': '/mnt/coldsemantic/',
                            'pathname': '/mnt/coldsemantic/'})
    mongo_db2 = mongo_conn({
        '__DB_ADDR__': 'localhost:27027',
    })
    storage_manager = StorageManager()

    storage_manager.add_storage(root_db,
                                name='mongo_db',
                                write_handler=mongo_writer,
                                read_handler=mongo_reader
                                )
    storage_manager.add_storage(mongo_db2,
                                name='mongo_db2',
                                write_handler=mongo_writer,
                                read_handler=mongo_reader)

    storage_manager.add_storage(fs_hot,
                                name='ceph_hot',
                                storage_type='fs',
                                write_handler=fs_hot.write,
                                read_handler=fs_hot.read)
    storage_manager.add_storage(fs_warm,
                                name='ceph_warm',
                                storage_type='fs',
                                write_handler=fs_warm.write,
                                read_handler=fs_warm.read)
    storage_manager.add_storage(fs_cold,
                                name='ceph_cold',
                                storage_type='fs',
                                write_handler=fs_cold.write,
                                read_handler=fs_cold.read)

    storage_manager.set_default_storage('mongo_db2')
    print 'Pool Databases:', storage_manager.get_databases()
    print 'Pool File systems:', storage_manager.get_filesystems()

    return storage_manager
Beispiel #11
0
class Uploader(Observer, Thread):
    def __init__(self, serverID, stationID, password, URL, config,
                 retryAfter=MINWAIT, maxWait=MAXWAIT,
                 minBatchSize=BATCHSIZE, maxBatchSize=BATCHSIZE):
        self.storageManager = StorageManager()
        self.serverID = serverID
        self.stationID = stationID
        self.password = password
        self.URL = URL
        self.nagiosPush = NagiosPush(config)
        self.minBatchSize = minBatchSize
        self.maxBatchSize = maxBatchSize
        self.retryAfter = MINWAIT
        self.maxWait = MAXWAIT

        # lock to protect numEvents
        self.numEventsLock = Semaphore(1)

        # Semaphore to block if the number of events drops below minBatchSize
        self.noEventsSem = Semaphore(0)

        super(Uploader, self).__init__(name='Uploader')
        self.stop_event = Event()
        self.isRunning = False

    def setNumServer(self, numServer):
        """Sets the number of servers to upload to.

        Need to be set before changing the UploadedTo-status
        of events in the StorageManager.

        """
        self.storageManager.setNumServer(numServer)

    def stop(self):
        self.stop_event.set()
        # release semaphore
        self.noEventsSem.release()

    crashes = []

    def init_restart(self):
        """Support for restarting crashed threads."""

        if len(self.crashes) > 3 and time() - self.crashes[-3] < 60.:
            raise ThreadCrashError("Thread has crashed three times in "
                                   "less than a minute")
        else:
            super(Uploader, self).__init__()
            self.crashes.append(time())

    def notify(self, count=1):
        """Notify the uploader that count events were received."""

        if (self.isRunning):
            shouldRelease = 0
            self.numEventsLock.acquire()

            oldNumEvents = self.numEvents
            self.numEvents += count
            logger.debug('%i: %i events pending.' %
                         (self.serverID, self.numEvents))

            # calculate if uploader-thread should be unblocked
            if (self.numEvents >= self.minBatchSize and
                    oldNumEvents < self.minBatchSize):
                shouldRelease = 1

            self.numEventsLock.release()

            if (shouldRelease):
                self.noEventsSem.release()

    def __getNumEventsToUpload(self):
        """Gives the number of events that the Uploader can upload now.

        The result will be between min and max batch size. If insufficient
        events are available this function will block on noEventSem.

        """

        shouldBlock = False
        self.numEventsLock.acquire()
        res = min(self.numEvents, self.maxBatchSize)
        if (res < self.minBatchSize):
            shouldBlock = True
        self.numEventsLock.release()

        if shouldBlock:
            logger.debug('%i: Blocked: Too few events' % self.serverID)
            self.noEventsSem.acquire()
            logger.debug('%i: Unblocked' % self.serverID)
            return self.minBatchSize
        else:
            return res

    def __upload(self, elist):
        """Upload a list of events to the database server."""

        data = dumps(elist)
        checksum = md5(data).hexdigest()

        params = urlencode({'station_id': self.stationID,
                            'password': self.password,
                            'data': data,
                            'checksum': checksum})

        # Open the connection and send our data. Exceptions are caught
        # explicitly to make sure we understand the implications of errors.
        try:
            f = urlopen(self.URL, params)
        except (URLError, HTTPError), msg:
            # For example: connection refused or internal server error
            returncode = str(msg)
        except Exception, msg:
            returncode = ('Uncatched exception occured in function '
                          '__upload: %s' % str(msg))
class TestStorageManager(unittest.TestCase):

    def setUp(self):
        # create a few dummy events
        self.e1 = Event(0)
        self.e2 = Event(4)
        self.e3 = Event(5)
        self.e4 = Event(6)

        self.elist = [self.e1, self.e2, self.e3, self.e4]
        # add bogus information to an event
        self.e1.data = "hello world"
        self.e2.data = "snd message"
        self.e3.data = "non-funny message"
        self.e4.data = "whatever"

        # setup storagemanager
        self.sm = StorageManager()
        self.sm.setNumServer(2)
        self.sm.openConnection()

    def tearDown(self):
        global observer_calls
        # remove all events to get back in virgin state
        self.sm.clear()
        observer_calls = 0

    def testAddOnebyOne(self):
        num = self.sm.getNumEvents()
        self.sm.addEvent(self.e1)
        num1 = self.sm.getNumEvents()
        self.sm.addEvent(self.e2)
        num2 = self.sm.getNumEvents()
        self.sm.addEvent(self.e3)
        num3 = self.sm.getNumEvents()
        self.sm.addEvent(self.e4)
        num4 = self.sm.getNumEvents()

    # test the numbers
        self.assertEqual(num1, num + 1)
        self.assertEqual(num2, num + 2)
        self.assertEqual(num3, num + 3)
        self.assertEqual(num4, num + 4)

    def testAddMultiple(self):
        num = self.sm.getNumEvents()
        self.sm.addEvents(self.elist)
        num4 = self.sm.getNumEvents()
        self.assertEqual(num + 4, num4)

    def testGetOneByOne(self):
        # this test also sets the status so that you get a new event each time
        self.sm.addEvent(self.e1)
        self.sm.addEvent(self.e2)
        self.sm.addEvent(self.e3)
        self.sm.addEvent(self.e4)

        self.assertEqual(4, self.sm.getNumEventsServer(0))

        (e1, id1) = self.sm.getEvent(0)
        self.assertEqual(self.e1.data, e1)
        self.sm.setUploaded(0, [id1])
        self.assertEqual(3, self.sm.getNumEventsServer(0))

        (e2, id2) = self.sm.getEvent(0)
        self.assertEqual(self.e2.data, e2)
        self.sm.setUploaded(0, [id2])
        self.assertEqual(2, self.sm.getNumEventsServer(0))

        (e3, id3) = self.sm.getEvent(0)
        self.assertEqual(self.e3.data, e3)
        self.sm.setUploaded(0, [id3])
        self.assertEqual(1, self.sm.getNumEventsServer(0))

        (e4, id4) = self.sm.getEvent(0)
        self.assertEqual(self.e4.data, e4)
        self.sm.setUploaded(0, [id4])
        self.assertEqual(0, self.sm.getNumEventsServer(0))

    def testUploadStatus(self):
        # add events
        self.sm.addEvents(self.elist)
        # get the ids
        (elist, eids) = self.sm.getEvents(0, 4)
        self.assertEqual(4, self.sm.getNumEvents())
        self.assertEqual(4, self.sm.getNumEventsServer(0))
        self.assertEqual(4, self.sm.getNumEventsServer(1))
        # set server 1 uploaded
        self.sm.setUploaded(0, eids)
        self.assertEqual(4, self.sm.getNumEvents())
        self.assertEqual(0, self.sm.getNumEventsServer(0))
        self.assertEqual(4, self.sm.getNumEventsServer(1))
        # set server 2 uploaded
        self.sm.setUploaded(1, eids)
        self.assertEqual(0, self.sm.getNumEvents())
        self.assertEqual(0, self.sm.getNumEventsServer(0))
        self.assertEqual(0, self.sm.getNumEventsServer(1))

    def testUploadStatusWrongID(self):
        self.sm.addEvents(self.elist)
        self.sm.setUploaded(1, [-1, -4, 999999999999])
        self.assertEqual(4, self.sm.getNumEvents())
        self.assertEqual(4, self.sm.getNumEventsServer(0))
        self.assertEqual(4, self.sm.getNumEventsServer(1))

    def testUploadStatusWrongServerID(self):
        self.sm.addEvents(self.elist)
        (elist, eids) = self.sm.getEvents(0, 4)
        self.sm.setUploaded(11, eids)
        self.assertEqual(4, self.sm.getNumEvents())
        self.assertEqual(4, self.sm.getNumEventsServer(0))
        self.assertEqual(4, self.sm.getNumEventsServer(1))

        # now see if we can still remove it
        self.sm.setUploaded(0, eids)
        self.sm.setUploaded(1, eids)
        self.assertEqual(0, self.sm.getNumEvents())

    def testGetMultiple(self):
        self.sm.addEvents(self.elist)
        (elist, eidlist) = self.sm.getEvents(0, 4)
        i = 0
        for e in elist:
            self.assertEqual(e, self.elist[i].data)
            i = i + 1

    def testOtherBatchSize(self):
        self.sm.addEvents(self.elist)

        (elist, eidlist) = self.sm.getEvents(0, 1)
        self.assertEqual(1, len(elist))
        (elist, eidlist) = self.sm.getEvents(0, 2)
        self.assertEqual(2, len(elist))
        (elist, eidlist) = self.sm.getEvents(0, 3)
        self.assertEqual(3, len(elist))
        (elist, eidlist) = self.sm.getEvents(0, 4)
        self.assertEqual(4, len(elist))
        (elist, eidlist) = self.sm.getEvents(0, 5)
        self.assertEqual(4, len(elist))
        (elist, eidlist) = self.sm.getEvents(0, 0)
        self.assertEqual(0, len(elist))

        # remove 2 and test again
        (elist, eidlist) = self.sm.getEvents(0, 5)
        self.sm.setUploaded(0, [eidlist[0], eidlist[1]])

        (elist, eidlist) = self.sm.getEvents(0, 1)
        self.assertEqual(1, len(elist))
        (elist, eidlist) = self.sm.getEvents(0, 2)
        self.assertEqual(2, len(elist))
        (elist, eidlist) = self.sm.getEvents(0, 3)
        self.assertEqual(2, len(elist))

    def testObserver(self):
        global observer_calls
        obs = Obs()
        self.sm.addObserver(obs)
        self.sm.addEvent(self.e1)
        self.assertEqual(1, observer_calls)
        self.sm.addEvents(self.elist)
        self.assertEqual(5, observer_calls)

    def testTwoObservers(self):
        global observer_calls
        self.sm.addObserver(Obs())
        self.sm.addObserver(Obs())
        self.sm.addEvent(self.e1)
        self.assertEqual(2, observer_calls)
        self.sm.addEvents(self.elist)
        self.assertEqual(10, observer_calls)
class CheckScheduler(threading.Thread):
    def __init__(self, config, interpreter):
        # invoke constructor of parent class (threading)
        threading.Thread.__init__(self)
        self.stop_event = threading.Event()

        self.status = None
        self.sched = Scheduler(self.status)
        self.dicConfig = config

        # create a nagios push object
        self.nagiosPush = NagiosPush(config)
        self.storageManager = StorageManager()
        self.interpreter = interpreter

        ### Event rate:
        self.eventRate = EventRate()

    def getEventRate(self):
        return self.eventRate

    def stop(self):
        self.stop_event.set()

    crashes = []

    def init_restart(self):
        """Support for restarting crashed threads"""

        if len(self.crashes) > 3 and time.time() - self.crashes[-3] < 60.0:
            raise ThreadCrashError("Thread has crashed three times in " "less than a minute")
        else:
            super(CheckScheduler, self).__init__()
            self.crashes.append(time.time())

    # This is what the thread actually runs; the required name is run().
    # The threading.Thread.start() calls threading.Thread.run(),
    # which is always overridden.
    def run(self):
        log("CheckScheduler: Thread started!", severity=2)
        self.storageManager.openConnection()

        ### Trigger rate:
        triggerRate = TriggerRate(self.interpreter)
        TR_interval = int(self.dicConfig["triggerrate_interval"])
        self.sched.addJob(triggerRate.check, interval=TR_interval, args=self.dicConfig)
        ### Storage size:
        storageSize = StorageSize(self.storageManager)
        SS_interval = int(self.dicConfig["storagesize_interval"])
        self.sched.addJob(storageSize.check, interval=SS_interval, args=self.dicConfig)
        ### Event rate:
        ER_interval = int(self.dicConfig["eventrate_interval"])
        self.sched.addJob(self.eventRate.check, interval=ER_interval, args=self.dicConfig)
        ### Storage growth:
        storageGrowth = StorageGrowth(self.storageManager)
        SG_interval = int(self.dicConfig["storagegrowth_interval"])
        self.sched.addJob(storageGrowth.check, interval=SG_interval, args=self.dicConfig)

        while not self.stop_event.isSet():
            # run all checks
            self.sched.schedule(self.nagiosPush, self.dicConfig)

            try:
                time.sleep(1)
            except KeyboardInterrupt:
                break
            except:
                pass
        log("CheckScheduler: Thread stopped!", severity=2)
Beispiel #14
0
from StorageManager import StorageManager

AVAILABLE_STORAGES = StorageManager()
default_app_config = 'opencsp.apps.OpenCSPAppConfig'
def reset():
    StorageManager.reset()
def init():
    global db
    db = StorageManager()
 def getImageName(self, image_url, full_save_path):
     print("Starting download for " + image_url)
     numberOfPresentWallpapers = StorageManager(
         full_save_path).count_files()
     filename = "wallpaper_" + str(numberOfPresentWallpapers)
     return filename