def test_can_get_worker_by_uuid(self): worker = WorkerFactory.create() WorkerFactory.create() loaded_worker = Worker.by_uuid(worker.uuid, self.db) expect(loaded_worker.id).to_equal(worker.id) invalid_worker = Worker.by_uuid(uuid4(), self.db) expect(invalid_worker).to_be_null()
def _ping_api(self): self._remove_zombie_workers() worker = Worker.by_uuid(self.uuid, self.db) self.db.begin(subtransactions=True) try: if worker: worker.last_ping = datetime.now() worker.current_url = self.working_url else: worker = Worker(uuid=self.uuid, current_url=self.working_url) self.db.add(worker) self.db.flush() self.db.commit() except OperationalError: self.db.rollback() self.publish(dumps({ 'type': 'worker-status', 'workerId': str(worker.uuid) })) return True
def _start_job(self, url): self.update_otto_limiter() self.working_url = url self.db.begin(subtransactions=True) worker = Worker.by_uuid(self.uuid, self.db) worker.current_url = url worker.last_ping = datetime.now() self.db.flush() self.db.commit() return True
def test_worker_alive_can_add_new_worker(self): worker_uuid = uuid4() response = yield self.http_client.fetch( self.get_url('/worker/%s/alive' % str(worker_uuid)), method='POST', body='' ) worker = Worker.by_uuid(worker_uuid, self.db) expect(worker).not_to_be_null() expect(response.code).to_equal(200) expect(response.body).to_be_like(str(worker.uuid))
def test_worker_complete_work(self): worker = WorkerFactory.create(current_url="http://www.globo.com/") self.db.flush() response = yield self.http_client.fetch( self.get_url('/worker/%s/complete' % str(worker.uuid)), method='POST', body='' ) worker = Worker.by_uuid(worker.uuid, self.db) expect(worker).not_to_be_null() expect(response.code).to_equal(200) expect(response.body).to_be_like('OK') expect(worker.current_url).to_be_null()
def test_worker_alive_can_ping_existing_worker(self): date = datetime.now() worker = WorkerFactory.create(last_ping=date) self.db.flush() response = yield self.http_client.fetch( self.get_url('/worker/%s/alive' % str(worker.uuid)), method='POST', body='' ) worker = Worker.by_uuid(worker.uuid, self.db) expect(worker).not_to_be_null() expect(response.code).to_equal(200) expect(response.body).to_be_like(str(worker.uuid)) expect(worker.last_ping).to_be_greater_than(date)
def _complete_job(self, lock, error=None): self.working_url = None worker = Worker.by_uuid(self.uuid, self.db) if worker: for i in range(3): self.db.begin(subtransactions=True) try: self.cache.release_next_job(lock) worker.current_url = None worker.last_ping = datetime.utcnow() self.db.flush() self.db.commit() break except Exception: err = sys.exc_info()[1] if 'Deadlock found' in str(err): self.error('Deadlock happened! Trying again (try number %d)! (Details: %s)' % (i, str(err))) else: self.db.rollback() raise return True