def test_default_realms_config(self):
portal = self.providing_stub([IPloneSiteRoot, IAttributeAnnotatable])
portal.portal_url.getPortalObject.return_value = portal
config = IConfig(portal)
self.assertTrue(config)
self.assertTrue(config.is_update_realms_possible())
self.assertEquals(len(config.getRealms()), 0)
config.appendRealm(Realm(1, 'http://site', 'foo', 'pw'))
self.assertEquals(len(config.getRealms()), 1)
def test_default_realms_config(self):
portal = self.providing_stub([IPloneSiteRoot, IAttributeAnnotatable])
self.expect(portal.portal_url.getPortalObject()).result(portal)
self.replay()
config = IConfig(portal)
self.assertTrue(config)
self.assertTrue(config.is_update_realms_possible())
self.assertEquals(len(config.getRealms()), 0)
config.appendRealm(Realm(1, 'http://site', 'foo', 'pw'))
self.assertEquals(len(config.getRealms()), 1)
def test_overridden_realms_config(self):
self.layer.load_zcml_string(
'\n'.join((
'<configure xmlns:publisher="http://namespaces.' + \
'zope.org/ftw.publisher">',
' <publisher:override-realm',
' url="http://localhost:9090/site"',
' username="******"',
' password="******" />',
'</configure>'
)))
portal = self.providing_stub([IPloneSiteRoot, IAttributeAnnotatable])
self.expect(portal.portal_url.getPortalObject()).result(portal)
self.replay()
config = IConfig(portal)
self.assertTrue(config)
self.assertFalse(config.is_update_realms_possible())
self.assertEquals(len(config.getRealms()), 1)
with self.assertRaises(AttributeError):
config.appendRealm(Realm(1, 'http://site', 'foo', 'pw'))
def test_overridden_realms_config(self):
self.layer.load_zcml_string(
'\n'.join((
'<configure xmlns:publisher="http://namespaces.' + \
'zope.org/ftw.publisher">',
' <publisher:override-realm',
' url="http://localhost:9090/site"',
' username="******"',
' password="******" />',
'</configure>'
)))
portal = self.providing_stub([IPloneSiteRoot, IAttributeAnnotatable])
portal.portal_url.getPortalObject.return_value = portal
config = IConfig(portal)
self.assertTrue(config)
self.assertFalse(config.is_update_realms_possible())
self.assertEquals(len(config.getRealms()), 1)
with self.assertRaises(AttributeError):
config.appendRealm(Realm(1, 'http://site', 'foo', 'pw'))
def handleAdd(self, action):
"""
This handler handles a click on the "Add Realm"-Button.
If no errors occured, it adds a new Realm to the Config.
@param action: ActionInfo object provided by z3c.form
@return: None (form is shown) or Response-redirect
"""
data, errors = self.extractData()
config = IConfig(self.context)
if len(errors)==0:
assert config.is_update_realms_possible()
# url + username has to be unique
for realm in config.getRealms():
if realm.url==data['url'] and realm.username==data['username']:
self.statusMessage(
'This URL / Username combination already exists!',
'error')
return
kwargs = {
'active': data['active'] and 1 or 0,
'url': data['url'],
'username': data['username'],
'password': data['password'],
}
realm = Realm(**kwargs)
config.appendRealm(realm)
self.statusMessage('Added realm successfully')
return self.request.RESPONSE.redirect('./@@publisher-config')
def handleSave(self, action):
"""
"""
data, errors = self.extractData()
config = IConfig(self.context)
assert config.is_update_realms_possible()
if len(errors)==0:
# get realm
currentRealm = self.getRealmById(data['id'])
if not currentRealm:
raise Exception('Could not find realm')
# no other realm should have same url+username
for realm in config.getRealms():
if realm!=currentRealm:
if realm.username==data['username'] and\
realm.url==data['url']:
self.statusMessage(
'This URL / Username combination already exists!',
)
return
# update realm
currentRealm.active = data['active'] and 1 or 0
currentRealm.url = data['url']
currentRealm.username = data['username']
if data['password'] and len(data['password'])>0:
currentRealm.password = data['password']
self.statusMessage('Updated realm successfully')
return self.request.RESPONSE.redirect('./@@publisher-config')
def handleAdd(self, action):
"""
This handler handles a click on the "Add Realm"-Button.
If no errors occured, it adds a new Realm to the Config.
@param action: ActionInfo object provided by z3c.form
@return: None (form is shown) or Response-redirect
"""
data, errors = self.extractData()
config = IConfig(self.context)
if len(errors)==0:
assert config.is_update_realms_possible()
# url + username has to be unique
for realm in config.getRealms():
if realm.url==data['url'] and realm.username==data['username']:
self.statusMessage(
'This URL / Username combination already exists!',
'error')
return
kwargs = {
'active': data['active'] and 1 or 0,
'url': data['url'],
'username': data['username'],
'password': data['password'],
}
realm = Realm(**kwargs)
config.appendRealm(realm)
self.statusMessage('Added realm successfully')
return self.request.RESPONSE.redirect('./@@publisher-config')
def handleSave(self, action):
"""
"""
data, errors = self.extractData()
config = IConfig(self.context)
assert config.is_update_realms_possible()
if len(errors)==0:
# get realm
currentRealm = self.getRealmById(data['id'])
if not currentRealm:
raise Exception('Could not find realm')
# no other realm should have same url+username
for realm in config.getRealms():
if realm!=currentRealm:
if realm.username==data['username'] and\
realm.url==data['url']:
self.statusMessage(
'This URL / Username combination already exists!',
)
return
# update realm
currentRealm.active = data['active'] and 1 or 0
currentRealm.url = data['url']
currentRealm.username = data['username']
if data['password'] and len(data['password'])>0:
currentRealm.password = data['password']
self.statusMessage('Updated realm successfully')
return self.request.RESPONSE.redirect('./@@publisher-config')
def get_realm_options(self):
portal = self.context.portal_url.getPortalObject()
config = IConfig(portal)
controller = IStatisticsCacheController(portal)
current_realm = controller.get_current_realm()
for realm in config.getRealms():
if realm.active:
label = '%s : %s' % (realm.url, realm.username)
yield {'id': self.make_realm_id(realm),
'label': label,
'selected': current_realm==realm,
}
def download(self, REQUEST=None, RESPONSE=None):
"""Download the saved data
"""
url_tool = getToolByName(self, 'portal_url')
config = IConfig(url_tool.getPortalObject())
pub_state = getMultiAdapter((self, REQUEST), IPublisherContextState)
realms = config.getRealms()
download_format = getattr(self, 'DownloadFormat', 'csv')
if len(realms) == 0 or not pub_state.is_parent_published():
if download_format == 'tsv':
return self.download_tsv(REQUEST, RESPONSE)
else:
assert download_format == 'csv', 'Unknown download format'
return self.download_csv(REQUEST, RESPONSE)
elif len(realms) == 1:
data = {'uid': self.UID(), 'download_format': download_format}
return_data_realm = sendRequestToRealm(data,
realms[0],
'formgen_get_saved_data')
return_data_this = self.getSavedFormInputForEdit()
return_data = '{}{}'.format(return_data_realm, return_data_this)
filename = self.id
if filename.find('.') < 0:
filename = '%s.%s' % (filename, download_format)
header_value = contentDispositionHeader('attachment',
self.getCharset(),
filename=filename)
RESPONSE.setHeader("Content-Disposition", header_value)
sep_type = download_format == 'csv' and 'comma' or 'tab'
RESPONSE.setHeader("Content-Type",
'text/%s-separated-values;'
'charset=%s' % (sep_type, self.getCharset()))
return return_data
else:
messages = IStatusMessage(self.request)
messages.add(_(u"couldn't determine correct realm to fetch from."),
type=u"error")
return RESPONSE.redirect(self.context.absolute_url())
class PublisherConfigletView(BrowserView):
def __init__(self, *args, **kwargs):
super(PublisherConfigletView, self).__init__(*args, **kwargs)
self.config = IConfig(self.context)
self.queue = IQueue(self.context)
def makeRealmId(self, realm):
return md5.md5('%s-%s' % (realm.url, realm.username)).hexdigest()
def getRealmById(self, id):
for realm in self.config.getRealms():
if self.makeRealmId(realm)==id:
return realm
return None
def statusMessage(self, message, type='info'):
IStatusMessage(self.request).addStatusMessage(
message,
type=type)
class PublisherConfigletView(BrowserView):
def __init__(self, *args, **kwargs):
super(PublisherConfigletView, self).__init__(*args, **kwargs)
self.config = IConfig(self.context)
self.queue = IQueue(self.context)
def makeRealmId(self, realm):
return md5.md5('%s-%s' % (realm.url, realm.username)).hexdigest()
def getRealmById(self, id):
for realm in self.config.getRealms():
if self.makeRealmId(realm)==id:
return realm
return None
def statusMessage(self, message, type='info'):
IStatusMessage(self.request).addStatusMessage(
message,
type=type)
def download(self, REQUEST=None, RESPONSE=None):
"""Download the saved data
"""
url_tool = getToolByName(self, 'portal_url')
config = IConfig(url_tool.getPortalObject())
pub_state = getMultiAdapter((self, REQUEST), IPublisherContextState)
realms = config.getRealms()
download_format = getattr(self, 'DownloadFormat', 'csv')
if len(realms) == 0 or not pub_state.is_parent_published():
if download_format == 'tsv':
return self.download_tsv(REQUEST, RESPONSE)
else:
assert download_format == 'csv', 'Unknown download format'
return self.download_csv(REQUEST, RESPONSE)
elif len(realms) == 1:
data = {'uid': self.UID(), 'download_format': download_format}
return_data = sendRequestToRealm(data,
realms[0],
'formgen_get_saved_data')
filename = self.id
if filename.find('.') < 0:
filename = '%s.%s' % (filename, download_format)
header_value = contentDispositionHeader('attachment',
self.getCharset(),
filename=filename)
RESPONSE.setHeader("Content-Disposition", header_value)
sep_type = download_format == 'csv' and 'comma' or 'tab'
RESPONSE.setHeader("Content-Type",
'text/%s-separated-values;'
'charset=%s' % (sep_type, self.getCharset()))
return return_data
else:
messages = IStatusMessage(self.request)
messages.add(_(u"couldn't determine correct realm to fetch from."),
type=u"error")
return RESPONSE.redirect(self.context.absolute_url())
def get_realm_by_id(self, id):
config = IConfig(self.context.portal_url.getPortalObject())
for realm in config.getRealms():
if self.make_realm_id(realm) == id:
return realm
return None
class ExecuteQueue(BrowserView):
"""Executes the Queue and sends all Jobs to the target
realms.
"""
def execute_single_job(self, job):
""" Executes a single job without calling the view
"""
self.logger = getLogger()
self.error_logger = getErrorLogger()
portal = self.context.portal_url.getPortalObject()
self.config = IConfig(portal)
self.queue = IQueue(portal)
# remove job from queue
if job in self.queue.getJobs():
self.queue.removeJob(job)
elif job in self.queue.get_executed_jobs():
self.queue.remove_executed_job(job)
# execute it
self.executeJob(job)
# move json file
job.move_jsonfile_to(self.config.get_executed_folder())
# add to executed list
return self.queue.append_executed_job(job)
def __call__(self):
"""
Handles logging purposes and calls execute() method.
"""
# get config and queue
self.config = IConfig(self.context)
portal = self.context.portal_url.getPortalObject()
self.queue = IQueue(portal)
event.notify(BeforeQueueExecutionEvent(portal, self.queue))
# prepare logger
self.logger = getLogger()
self.error_logger = getErrorLogger()
# is it allowed to publish?
if not self.config.publishing_enabled():
self.logger.warning('PUBLISHING IS DISABLED')
return 'PUBLISHING IS DISABLED'
if self.config.locking_enabled():
self.logger.info('LOCKING IS ENABLED')
else:
self.logger.info('LOCKING IS DISABLED')
# lock - check for locking flag
if self.config.locking_enabled() and not self.get_lock_object().acquire(0):
self.logger.warning('Already publishing')
return 'Already publishing'
# register our own logging handler for returning logs afterwards
logStream = StringIO()
logHandler = logging.StreamHandler(logStream)
self.logger.addHandler(logHandler)
# be sure to remove the handler!
try:
# execute queue
self.execute()
except:
self.logger.removeHandler(logHandler)
if self.config.locking_enabled(): self.get_lock_object().release()
# re-raise exception
raise
# get logs
self.logger.removeHandler(logHandler)
logStream.seek(0)
log = logStream.read()
del logStream
del logHandler
# unlock
if self.config.locking_enabled(): self.get_lock_object().release()
event.notify(QueueExecutedEvent(portal, log))
return log
def get_lock_object(self):
if getattr(self.__class__, '_lock', None) == None:
self.__class__._lock = RLock()
return self.__class__._lock
def getActiveRealms(self):
"""
@return: a list of active Realms
@rtype: list
"""
if '_activeRealms' not in dir(self):
self._activeRealms = [r for r in self.config.getRealms()
if r.active]
return self._activeRealms
def execute(self):
"""
Executes the jobs from the queue.
@return: None
"""
jobs = self.queue.countJobs()
self.queue.move_to_worker_queue()
self.logger.info('Executing Queue: %i of %i objects to %i realms' % (
jobs,
self.queue.countJobs(),
len(self.getActiveRealms()),
))
while len(self.queue.get_worker_queue()):
job = self.queue.popJob()
if not job.json_file_exists():
continue
try:
# execute job
self.executeJob(job)
except (ConflictError, Retry):
raise
except URLError:
raise
except ReceiverTimeoutError:
raise
except:
# print the exception to the publisher error log
exc = ''.join(traceback.format_exception(*sys.exc_info()))
self.error_logger.error(exc)
job.executed_exception = exc
job.move_jsonfile_to(self.config.get_executed_folder())
self.queue.append_executed_job(job)
transaction.commit()
def executeJob(self, job):
"""
Executes a Job: sends the job to all available realms.
@param job: Job object to execute
@type job: Job
"""
objTitle = job.objectTitle
if isinstance(objTitle, unicode):
objTitle = objTitle.encode('utf8')
# is the object blacklisted?
if IPathBlacklist(self.context).is_blacklisted(job.objectPath):
self.logger.error('blacklisted: "%s" on "%s" (at %s | UID %s)' % (
job.action,
objTitle,
job.objectPath,
job.objectUID,
))
self.error_logger.error(
'blacklisted: "%s" on "%s" (at %s | UID %s)' % (
job.action,
objTitle,
job.objectPath,
job.objectUID,
))
return False
# get data from chache file
state = None
json = job.getData()
self.logger.info('-' * 100)
self.logger.info('executing "%s" on "%s" (at %s | UID %s)' % (
job.action,
objTitle,
job.objectPath,
job.objectUID,
))
self.logger.info('... request data length: %i' % len(json))
state_entries = {'date': datetime.now()}
for realm in self.getActiveRealms():
self.logger.info('... to realm %s' % (
realm.url,
))
# send data to each realm
state = sendJsonToRealm(json, realm, 'publisher.receive')
if isinstance(state, states.ErrorState):
self.logger.error('... got result: %s' % state.toString())
self.error_logger.error(
'executing "%s" on "%s" (at %s | UID %s)' % (
job.action,
objTitle,
job.objectPath,
job.objectUID,
))
self.error_logger.error('... got result: %s' %
state.toString())
else:
self.logger.info('... got result: %s' % state.toString())
state_entries[realm] = state
job.executed_with_states(state_entries)
# fire AfterPushEvent
reference_catalog = getToolByName(self.context, 'reference_catalog')
obj = reference_catalog.lookupObject(job.objectUID)
if state is not None:
event.notify(AfterPushEvent(obj, state, job))
class ExecuteQueue(BrowserView):
"""Executes the Queue and sends all Jobs to the target
realms.
"""
def execute_single_job(self, job):
""" Executes a single job without calling the view
"""
self.logger = getLogger()
self.error_logger = getErrorLogger()
portal = self.context.portal_url.getPortalObject()
self.config = IConfig(portal)
self.queue = IQueue(portal)
# remove job from queue
if job in self.queue.getJobs():
self.queue.removeJob(job)
elif job in self.queue.get_executed_jobs():
self.queue.remove_executed_job(job)
# execute it
self.executeJob(job)
# move json file
job.move_jsonfile_to(self.config.get_executed_folder())
# add to executed list
return self.queue.append_executed_job(job)
def __call__(self):
"""
Handles logging purposes and calls execute() method.
"""
# get config and queue
self.config = IConfig(self.context)
portal = self.context.portal_url.getPortalObject()
self.queue = IQueue(portal)
event.notify(BeforeQueueExecutionEvent(portal, self.queue))
# prepare logger
self.logger = getLogger()
self.error_logger = getErrorLogger()
# is it allowed to publish?
if not self.config.publishing_enabled():
self.logger.warning('PUBLISHING IS DISABLED')
return 'PUBLISHING IS DISABLED'
if self.config.locking_enabled():
self.logger.info('LOCKING IS ENABLED')
else:
self.logger.info('LOCKING IS DISABLED')
# lock - check for locking flag
if self.config.locking_enabled(
) and not self.get_lock_object().acquire(0):
self.logger.warning('Already publishing')
return 'Already publishing'
# register our own logging handler for returning logs afterwards
logStream = StringIO()
logHandler = logging.StreamHandler(logStream)
self.logger.addHandler(logHandler)
# be sure to remove the handler!
try:
# execute queue
self.execute()
except Exception:
self.logger.removeHandler(logHandler)
if self.config.locking_enabled():
self.get_lock_object().release()
# re-raise exception
raise
# get logs
self.logger.removeHandler(logHandler)
logStream.seek(0)
log = logStream.read()
del logStream
del logHandler
# unlock
if self.config.locking_enabled():
self.get_lock_object().release()
event.notify(QueueExecutedEvent(portal, log))
return log
def get_lock_object(self):
if getattr(self.__class__, '_lock', None) is None:
self.__class__._lock = RLock()
return self.__class__._lock
def getActiveRealms(self):
"""
@return: a list of active Realms
@rtype: list
"""
if '_activeRealms' not in dir(self):
self._activeRealms = [
r for r in self.config.getRealms() if r.active
]
return self._activeRealms
def execute(self):
"""
Executes the jobs from the queue.
@return: None
"""
jobs = self.queue.countJobs()
self.queue.move_to_worker_queue()
self.logger.info('Executing Queue: %i of %i objects to %i realms' % (
jobs,
self.queue.countJobs(),
len(self.getActiveRealms()),
))
while len(self.queue.get_worker_queue()):
job = self.queue.popJob()
if not job.json_file_exists():
continue
try:
# execute job
self.executeJob(job)
except (ConflictError, Retry):
raise
except URLError:
raise
except ReceiverTimeoutError:
raise
except Exception:
# print the exception to the publisher error log
exc = ''.join(traceback.format_exception(*sys.exc_info()))
self.error_logger.error(exc)
job.executed_exception = exc
job.move_jsonfile_to(self.config.get_executed_folder())
self.queue.append_executed_job(job)
transaction.commit()
def executeJob(self, job):
"""
Executes a Job: sends the job to all available realms.
@param job: Job object to execute
@type job: Job
"""
objTitle = job.objectTitle
if isinstance(objTitle, unicode):
objTitle = objTitle.encode('utf8')
# is the object blacklisted?
if IPathBlacklist(self.context).is_blacklisted(job.objectPath):
self.logger.error('blacklisted: "%s" on "%s" (at %s | UID %s)' % (
job.action,
objTitle,
job.objectPath,
job.objectUID,
))
self.error_logger.error(
'blacklisted: "%s" on "%s" (at %s | UID %s)' % (
job.action,
objTitle,
job.objectPath,
job.objectUID,
))
return False
# get data from chache file
state = None
json = job.getData()
self.logger.info('-' * 100)
self.logger.info('executing "%s" on "%s" (at %s | UID %s)' % (
job.action,
objTitle,
job.objectPath,
job.objectUID,
))
self.logger.info('... request data length: %i' % len(json))
state_entries = {'date': datetime.now()}
for realm in self.getActiveRealms():
self.logger.info('... to realm %s' % (realm.url, ))
# send data to each realm
state = sendJsonToRealm(json, realm, 'publisher.receive')
if isinstance(state, states.ErrorState):
self.logger.error('... got result: %s' % state.toString())
self.error_logger.error(
'executing "%s" on "%s" (at %s | UID %s)' % (
job.action,
objTitle,
job.objectPath,
job.objectUID,
))
self.error_logger.error('... got result: %s' %
state.toString())
else:
self.logger.info('... got result: %s' % state.toString())
state_entries[realm] = state
job.executed_with_states(state_entries)
# fire AfterPushEvent
obj = uuidToObject(job.objectUID)
if state is not None:
event.notify(AfterPushEvent(obj, state, job))
