def test_default_realms_config(self): portal = self.providing_stub([IPloneSiteRoot, IAttributeAnnotatable]) portal.portal_url.getPortalObject.return_value = portal config = IConfig(portal) self.assertTrue(config) self.assertTrue(config.is_update_realms_possible()) self.assertEquals(len(config.getRealms()), 0) config.appendRealm(Realm(1, 'http://site', 'foo', 'pw')) self.assertEquals(len(config.getRealms()), 1)
def test_default_realms_config(self): portal = self.providing_stub([IPloneSiteRoot, IAttributeAnnotatable]) self.expect(portal.portal_url.getPortalObject()).result(portal) self.replay() config = IConfig(portal) self.assertTrue(config) self.assertTrue(config.is_update_realms_possible()) self.assertEquals(len(config.getRealms()), 0) config.appendRealm(Realm(1, 'http://site', 'foo', 'pw')) self.assertEquals(len(config.getRealms()), 1)
def test_overridden_realms_config(self): self.layer.load_zcml_string( '\n'.join(( '<configure xmlns:publisher="http://namespaces.' + \ 'zope.org/ftw.publisher">', ' <publisher:override-realm', ' url="http://localhost:9090/site"', ' username="******"', ' password="******" />', '</configure>' ))) portal = self.providing_stub([IPloneSiteRoot, IAttributeAnnotatable]) self.expect(portal.portal_url.getPortalObject()).result(portal) self.replay() config = IConfig(portal) self.assertTrue(config) self.assertFalse(config.is_update_realms_possible()) self.assertEquals(len(config.getRealms()), 1) with self.assertRaises(AttributeError): config.appendRealm(Realm(1, 'http://site', 'foo', 'pw'))
def test_overridden_realms_config(self): self.layer.load_zcml_string( '\n'.join(( '<configure xmlns:publisher="http://namespaces.' + \ 'zope.org/ftw.publisher">', ' <publisher:override-realm', ' url="http://localhost:9090/site"', ' username="******"', ' password="******" />', '</configure>' ))) portal = self.providing_stub([IPloneSiteRoot, IAttributeAnnotatable]) portal.portal_url.getPortalObject.return_value = portal config = IConfig(portal) self.assertTrue(config) self.assertFalse(config.is_update_realms_possible()) self.assertEquals(len(config.getRealms()), 1) with self.assertRaises(AttributeError): config.appendRealm(Realm(1, 'http://site', 'foo', 'pw'))
def handleAdd(self, action): """ This handler handles a click on the "Add Realm"-Button. If no errors occured, it adds a new Realm to the Config. @param action: ActionInfo object provided by z3c.form @return: None (form is shown) or Response-redirect """ data, errors = self.extractData() config = IConfig(self.context) if len(errors)==0: assert config.is_update_realms_possible() # url + username has to be unique for realm in config.getRealms(): if realm.url==data['url'] and realm.username==data['username']: self.statusMessage( 'This URL / Username combination already exists!', 'error') return kwargs = { 'active': data['active'] and 1 or 0, 'url': data['url'], 'username': data['username'], 'password': data['password'], } realm = Realm(**kwargs) config.appendRealm(realm) self.statusMessage('Added realm successfully') return self.request.RESPONSE.redirect('./@@publisher-config')
def handleSave(self, action): """ """ data, errors = self.extractData() config = IConfig(self.context) assert config.is_update_realms_possible() if len(errors)==0: # get realm currentRealm = self.getRealmById(data['id']) if not currentRealm: raise Exception('Could not find realm') # no other realm should have same url+username for realm in config.getRealms(): if realm!=currentRealm: if realm.username==data['username'] and\ realm.url==data['url']: self.statusMessage( 'This URL / Username combination already exists!', ) return # update realm currentRealm.active = data['active'] and 1 or 0 currentRealm.url = data['url'] currentRealm.username = data['username'] if data['password'] and len(data['password'])>0: currentRealm.password = data['password'] self.statusMessage('Updated realm successfully') return self.request.RESPONSE.redirect('./@@publisher-config')
def handleAdd(self, action): """ This handler handles a click on the "Add Realm"-Button. If no errors occured, it adds a new Realm to the Config. @param action: ActionInfo object provided by z3c.form @return: None (form is shown) or Response-redirect """ data, errors = self.extractData() config = IConfig(self.context) if len(errors)==0: assert config.is_update_realms_possible() # url + username has to be unique for realm in config.getRealms(): if realm.url==data['url'] and realm.username==data['username']: self.statusMessage( 'This URL / Username combination already exists!', 'error') return kwargs = { 'active': data['active'] and 1 or 0, 'url': data['url'], 'username': data['username'], 'password': data['password'], } realm = Realm(**kwargs) config.appendRealm(realm) self.statusMessage('Added realm successfully') return self.request.RESPONSE.redirect('./@@publisher-config')
def handleSave(self, action): """ """ data, errors = self.extractData() config = IConfig(self.context) assert config.is_update_realms_possible() if len(errors)==0: # get realm currentRealm = self.getRealmById(data['id']) if not currentRealm: raise Exception('Could not find realm') # no other realm should have same url+username for realm in config.getRealms(): if realm!=currentRealm: if realm.username==data['username'] and\ realm.url==data['url']: self.statusMessage( 'This URL / Username combination already exists!', ) return # update realm currentRealm.active = data['active'] and 1 or 0 currentRealm.url = data['url'] currentRealm.username = data['username'] if data['password'] and len(data['password'])>0: currentRealm.password = data['password'] self.statusMessage('Updated realm successfully') return self.request.RESPONSE.redirect('./@@publisher-config')
def get_realm_options(self): portal = self.context.portal_url.getPortalObject() config = IConfig(portal) controller = IStatisticsCacheController(portal) current_realm = controller.get_current_realm() for realm in config.getRealms(): if realm.active: label = '%s : %s' % (realm.url, realm.username) yield {'id': self.make_realm_id(realm), 'label': label, 'selected': current_realm==realm, }
def download(self, REQUEST=None, RESPONSE=None): """Download the saved data """ url_tool = getToolByName(self, 'portal_url') config = IConfig(url_tool.getPortalObject()) pub_state = getMultiAdapter((self, REQUEST), IPublisherContextState) realms = config.getRealms() download_format = getattr(self, 'DownloadFormat', 'csv') if len(realms) == 0 or not pub_state.is_parent_published(): if download_format == 'tsv': return self.download_tsv(REQUEST, RESPONSE) else: assert download_format == 'csv', 'Unknown download format' return self.download_csv(REQUEST, RESPONSE) elif len(realms) == 1: data = {'uid': self.UID(), 'download_format': download_format} return_data_realm = sendRequestToRealm(data, realms[0], 'formgen_get_saved_data') return_data_this = self.getSavedFormInputForEdit() return_data = '{}{}'.format(return_data_realm, return_data_this) filename = self.id if filename.find('.') < 0: filename = '%s.%s' % (filename, download_format) header_value = contentDispositionHeader('attachment', self.getCharset(), filename=filename) RESPONSE.setHeader("Content-Disposition", header_value) sep_type = download_format == 'csv' and 'comma' or 'tab' RESPONSE.setHeader("Content-Type", 'text/%s-separated-values;' 'charset=%s' % (sep_type, self.getCharset())) return return_data else: messages = IStatusMessage(self.request) messages.add(_(u"couldn't determine correct realm to fetch from."), type=u"error") return RESPONSE.redirect(self.context.absolute_url())
class PublisherConfigletView(BrowserView): def __init__(self, *args, **kwargs): super(PublisherConfigletView, self).__init__(*args, **kwargs) self.config = IConfig(self.context) self.queue = IQueue(self.context) def makeRealmId(self, realm): return md5.md5('%s-%s' % (realm.url, realm.username)).hexdigest() def getRealmById(self, id): for realm in self.config.getRealms(): if self.makeRealmId(realm)==id: return realm return None def statusMessage(self, message, type='info'): IStatusMessage(self.request).addStatusMessage( message, type=type)
class PublisherConfigletView(BrowserView): def __init__(self, *args, **kwargs): super(PublisherConfigletView, self).__init__(*args, **kwargs) self.config = IConfig(self.context) self.queue = IQueue(self.context) def makeRealmId(self, realm): return md5.md5('%s-%s' % (realm.url, realm.username)).hexdigest() def getRealmById(self, id): for realm in self.config.getRealms(): if self.makeRealmId(realm)==id: return realm return None def statusMessage(self, message, type='info'): IStatusMessage(self.request).addStatusMessage( message, type=type)
def download(self, REQUEST=None, RESPONSE=None): """Download the saved data """ url_tool = getToolByName(self, 'portal_url') config = IConfig(url_tool.getPortalObject()) pub_state = getMultiAdapter((self, REQUEST), IPublisherContextState) realms = config.getRealms() download_format = getattr(self, 'DownloadFormat', 'csv') if len(realms) == 0 or not pub_state.is_parent_published(): if download_format == 'tsv': return self.download_tsv(REQUEST, RESPONSE) else: assert download_format == 'csv', 'Unknown download format' return self.download_csv(REQUEST, RESPONSE) elif len(realms) == 1: data = {'uid': self.UID(), 'download_format': download_format} return_data = sendRequestToRealm(data, realms[0], 'formgen_get_saved_data') filename = self.id if filename.find('.') < 0: filename = '%s.%s' % (filename, download_format) header_value = contentDispositionHeader('attachment', self.getCharset(), filename=filename) RESPONSE.setHeader("Content-Disposition", header_value) sep_type = download_format == 'csv' and 'comma' or 'tab' RESPONSE.setHeader("Content-Type", 'text/%s-separated-values;' 'charset=%s' % (sep_type, self.getCharset())) return return_data else: messages = IStatusMessage(self.request) messages.add(_(u"couldn't determine correct realm to fetch from."), type=u"error") return RESPONSE.redirect(self.context.absolute_url())
def get_realm_by_id(self, id): config = IConfig(self.context.portal_url.getPortalObject()) for realm in config.getRealms(): if self.make_realm_id(realm) == id: return realm return None
class ExecuteQueue(BrowserView): """Executes the Queue and sends all Jobs to the target realms. """ def execute_single_job(self, job): """ Executes a single job without calling the view """ self.logger = getLogger() self.error_logger = getErrorLogger() portal = self.context.portal_url.getPortalObject() self.config = IConfig(portal) self.queue = IQueue(portal) # remove job from queue if job in self.queue.getJobs(): self.queue.removeJob(job) elif job in self.queue.get_executed_jobs(): self.queue.remove_executed_job(job) # execute it self.executeJob(job) # move json file job.move_jsonfile_to(self.config.get_executed_folder()) # add to executed list return self.queue.append_executed_job(job) def __call__(self): """ Handles logging purposes and calls execute() method. """ # get config and queue self.config = IConfig(self.context) portal = self.context.portal_url.getPortalObject() self.queue = IQueue(portal) event.notify(BeforeQueueExecutionEvent(portal, self.queue)) # prepare logger self.logger = getLogger() self.error_logger = getErrorLogger() # is it allowed to publish? if not self.config.publishing_enabled(): self.logger.warning('PUBLISHING IS DISABLED') return 'PUBLISHING IS DISABLED' if self.config.locking_enabled(): self.logger.info('LOCKING IS ENABLED') else: self.logger.info('LOCKING IS DISABLED') # lock - check for locking flag if self.config.locking_enabled() and not self.get_lock_object().acquire(0): self.logger.warning('Already publishing') return 'Already publishing' # register our own logging handler for returning logs afterwards logStream = StringIO() logHandler = logging.StreamHandler(logStream) self.logger.addHandler(logHandler) # be sure to remove the handler! try: # execute queue self.execute() except: self.logger.removeHandler(logHandler) if self.config.locking_enabled(): self.get_lock_object().release() # re-raise exception raise # get logs self.logger.removeHandler(logHandler) logStream.seek(0) log = logStream.read() del logStream del logHandler # unlock if self.config.locking_enabled(): self.get_lock_object().release() event.notify(QueueExecutedEvent(portal, log)) return log def get_lock_object(self): if getattr(self.__class__, '_lock', None) == None: self.__class__._lock = RLock() return self.__class__._lock def getActiveRealms(self): """ @return: a list of active Realms @rtype: list """ if '_activeRealms' not in dir(self): self._activeRealms = [r for r in self.config.getRealms() if r.active] return self._activeRealms def execute(self): """ Executes the jobs from the queue. @return: None """ jobs = self.queue.countJobs() self.queue.move_to_worker_queue() self.logger.info('Executing Queue: %i of %i objects to %i realms' % ( jobs, self.queue.countJobs(), len(self.getActiveRealms()), )) while len(self.queue.get_worker_queue()): job = self.queue.popJob() if not job.json_file_exists(): continue try: # execute job self.executeJob(job) except (ConflictError, Retry): raise except URLError: raise except ReceiverTimeoutError: raise except: # print the exception to the publisher error log exc = ''.join(traceback.format_exception(*sys.exc_info())) self.error_logger.error(exc) job.executed_exception = exc job.move_jsonfile_to(self.config.get_executed_folder()) self.queue.append_executed_job(job) transaction.commit() def executeJob(self, job): """ Executes a Job: sends the job to all available realms. @param job: Job object to execute @type job: Job """ objTitle = job.objectTitle if isinstance(objTitle, unicode): objTitle = objTitle.encode('utf8') # is the object blacklisted? if IPathBlacklist(self.context).is_blacklisted(job.objectPath): self.logger.error('blacklisted: "%s" on "%s" (at %s | UID %s)' % ( job.action, objTitle, job.objectPath, job.objectUID, )) self.error_logger.error( 'blacklisted: "%s" on "%s" (at %s | UID %s)' % ( job.action, objTitle, job.objectPath, job.objectUID, )) return False # get data from chache file state = None json = job.getData() self.logger.info('-' * 100) self.logger.info('executing "%s" on "%s" (at %s | UID %s)' % ( job.action, objTitle, job.objectPath, job.objectUID, )) self.logger.info('... request data length: %i' % len(json)) state_entries = {'date': datetime.now()} for realm in self.getActiveRealms(): self.logger.info('... to realm %s' % ( realm.url, )) # send data to each realm state = sendJsonToRealm(json, realm, 'publisher.receive') if isinstance(state, states.ErrorState): self.logger.error('... got result: %s' % state.toString()) self.error_logger.error( 'executing "%s" on "%s" (at %s | UID %s)' % ( job.action, objTitle, job.objectPath, job.objectUID, )) self.error_logger.error('... got result: %s' % state.toString()) else: self.logger.info('... got result: %s' % state.toString()) state_entries[realm] = state job.executed_with_states(state_entries) # fire AfterPushEvent reference_catalog = getToolByName(self.context, 'reference_catalog') obj = reference_catalog.lookupObject(job.objectUID) if state is not None: event.notify(AfterPushEvent(obj, state, job))
class ExecuteQueue(BrowserView): """Executes the Queue and sends all Jobs to the target realms. """ def execute_single_job(self, job): """ Executes a single job without calling the view """ self.logger = getLogger() self.error_logger = getErrorLogger() portal = self.context.portal_url.getPortalObject() self.config = IConfig(portal) self.queue = IQueue(portal) # remove job from queue if job in self.queue.getJobs(): self.queue.removeJob(job) elif job in self.queue.get_executed_jobs(): self.queue.remove_executed_job(job) # execute it self.executeJob(job) # move json file job.move_jsonfile_to(self.config.get_executed_folder()) # add to executed list return self.queue.append_executed_job(job) def __call__(self): """ Handles logging purposes and calls execute() method. """ # get config and queue self.config = IConfig(self.context) portal = self.context.portal_url.getPortalObject() self.queue = IQueue(portal) event.notify(BeforeQueueExecutionEvent(portal, self.queue)) # prepare logger self.logger = getLogger() self.error_logger = getErrorLogger() # is it allowed to publish? if not self.config.publishing_enabled(): self.logger.warning('PUBLISHING IS DISABLED') return 'PUBLISHING IS DISABLED' if self.config.locking_enabled(): self.logger.info('LOCKING IS ENABLED') else: self.logger.info('LOCKING IS DISABLED') # lock - check for locking flag if self.config.locking_enabled( ) and not self.get_lock_object().acquire(0): self.logger.warning('Already publishing') return 'Already publishing' # register our own logging handler for returning logs afterwards logStream = StringIO() logHandler = logging.StreamHandler(logStream) self.logger.addHandler(logHandler) # be sure to remove the handler! try: # execute queue self.execute() except Exception: self.logger.removeHandler(logHandler) if self.config.locking_enabled(): self.get_lock_object().release() # re-raise exception raise # get logs self.logger.removeHandler(logHandler) logStream.seek(0) log = logStream.read() del logStream del logHandler # unlock if self.config.locking_enabled(): self.get_lock_object().release() event.notify(QueueExecutedEvent(portal, log)) return log def get_lock_object(self): if getattr(self.__class__, '_lock', None) is None: self.__class__._lock = RLock() return self.__class__._lock def getActiveRealms(self): """ @return: a list of active Realms @rtype: list """ if '_activeRealms' not in dir(self): self._activeRealms = [ r for r in self.config.getRealms() if r.active ] return self._activeRealms def execute(self): """ Executes the jobs from the queue. @return: None """ jobs = self.queue.countJobs() self.queue.move_to_worker_queue() self.logger.info('Executing Queue: %i of %i objects to %i realms' % ( jobs, self.queue.countJobs(), len(self.getActiveRealms()), )) while len(self.queue.get_worker_queue()): job = self.queue.popJob() if not job.json_file_exists(): continue try: # execute job self.executeJob(job) except (ConflictError, Retry): raise except URLError: raise except ReceiverTimeoutError: raise except Exception: # print the exception to the publisher error log exc = ''.join(traceback.format_exception(*sys.exc_info())) self.error_logger.error(exc) job.executed_exception = exc job.move_jsonfile_to(self.config.get_executed_folder()) self.queue.append_executed_job(job) transaction.commit() def executeJob(self, job): """ Executes a Job: sends the job to all available realms. @param job: Job object to execute @type job: Job """ objTitle = job.objectTitle if isinstance(objTitle, unicode): objTitle = objTitle.encode('utf8') # is the object blacklisted? if IPathBlacklist(self.context).is_blacklisted(job.objectPath): self.logger.error('blacklisted: "%s" on "%s" (at %s | UID %s)' % ( job.action, objTitle, job.objectPath, job.objectUID, )) self.error_logger.error( 'blacklisted: "%s" on "%s" (at %s | UID %s)' % ( job.action, objTitle, job.objectPath, job.objectUID, )) return False # get data from chache file state = None json = job.getData() self.logger.info('-' * 100) self.logger.info('executing "%s" on "%s" (at %s | UID %s)' % ( job.action, objTitle, job.objectPath, job.objectUID, )) self.logger.info('... request data length: %i' % len(json)) state_entries = {'date': datetime.now()} for realm in self.getActiveRealms(): self.logger.info('... to realm %s' % (realm.url, )) # send data to each realm state = sendJsonToRealm(json, realm, 'publisher.receive') if isinstance(state, states.ErrorState): self.logger.error('... got result: %s' % state.toString()) self.error_logger.error( 'executing "%s" on "%s" (at %s | UID %s)' % ( job.action, objTitle, job.objectPath, job.objectUID, )) self.error_logger.error('... got result: %s' % state.toString()) else: self.logger.info('... got result: %s' % state.toString()) state_entries[realm] = state job.executed_with_states(state_entries) # fire AfterPushEvent obj = uuidToObject(job.objectUID) if state is not None: event.notify(AfterPushEvent(obj, state, job))