def email_worker():
connection = get_mq_connection()
if not connection:
return
channel = connection.channel()
channel.basic_qos(prefetch_count=1)
channel.basic_consume(email_notifications_callback,
queue="bungeni_email_queue")
channel.start_consuming()
def email_worker():
connection = get_mq_connection()
if not connection:
return
channel = connection.channel()
channel.basic_qos(prefetch_count=1)
channel.basic_consume(email_notifications_callback,
queue="bungeni_email_queue")
channel.start_consuming()
def serialization_worker():
connection = get_mq_connection()
if not connection:
return
channel = connection.channel()
channel.basic_qos(prefetch_count=1)
channel.basic_consume(serialization_notifications_callback,
queue=SERIALIZE_QUEUE)
channel.start_consuming()
def serialization_worker():
connection = get_mq_connection()
if not connection:
return
channel = connection.channel()
channel.basic_qos(prefetch_count=1)
channel.basic_consume(serialization_notifications_callback,
queue=SERIALIZE_QUEUE)
channel.start_consuming()
def serialization_notifications():
"""Set up bungeni serialization worker as a daemon.
"""
mq_utility = zope.component.getUtility(IMessageQueueConfig)
connection = get_mq_connection()
if not connection:
#delay
delay = TIMER_DELAYS["serialize_setup"]
if delay > MAX_DELAY:
log.error(
"Could not set up amqp workers - No rabbitmq "
"connection found after %d seconds.", delay)
else:
log.info(
"Attempting to setup serialization AMQP consumers "
"in %d seconds", delay)
timer = Timer(TIMER_DELAYS["serialize_setup"],
serialization_notifications)
timer.daemon = True
timer.start()
TIMER_DELAYS["serialize_setup"] *= 2
return
channel = connection.channel()
#create exchange
channel.exchange_declare(exchange=SERIALIZE_EXCHANGE,
type="fanout",
durable=True)
channel.queue_declare(queue=SERIALIZE_QUEUE, durable=True)
channel.queue_bind(queue=SERIALIZE_QUEUE,
exchange=SERIALIZE_EXCHANGE,
routing_key=SERIALIZE_ROUTING_KEY)
#xml outputs channel and queue
channel.exchange_declare(exchange=SERIALIZE_OUTPUT_EXCHANGE,
type="direct",
passive=False)
channel.queue_declare(queue=SERIALIZE_OUTPUT_QUEUE,
durable=True,
exclusive=False,
auto_delete=False)
channel.queue_bind(queue=SERIALIZE_OUTPUT_QUEUE,
exchange=SERIALIZE_OUTPUT_EXCHANGE,
routing_key=SERIALIZE_OUTPUT_ROUTING_KEY)
for i in range(mq_utility.get_number_of_workers()):
init_thread()
def load_email():
for type_key, ti in capi.iter_type_info():
workflow = ti.workflow
if workflow and workflow.has_feature("email"):
if not workflow.has_feature("notification"):
raise EmailError("Email notifications feature for %r cannot be "
"enabled without first enabling the notification "
"feature" % (type_key)) # !+FEATURE_DEPENDENCIES
mq_utility = component.getUtility(IMessageQueueConfig)
connection = get_mq_connection()
if not connection:
return
channel = connection.channel()
channel.queue_declare(queue="bungeni_email_queue", durable=True)
channel.queue_bind(queue="bungeni_email_queue",
exchange=str(mq_utility.get_message_exchange()),
routing_key="")
for i in range(mq_utility.get_number_of_workers()):
task_thread = Thread(target=email_worker)
task_thread.daemon = True
task_thread.start()
def serialization_notifications():
"""Set up bungeni serialization worker as a daemon.
"""
mq_utility = zope.component.getUtility(IMessageQueueConfig)
connection = get_mq_connection()
if not connection:
#delay
delay = TIMER_DELAYS["serialize_setup"]
if delay > MAX_DELAY:
log.error("Could not set up amqp workers - No rabbitmq "
"connection found after %d seconds.", delay)
else:
log.info("Attempting to setup serialization AMQP consumers "
"in %d seconds", delay
)
timer = Timer(TIMER_DELAYS["serialize_setup"],
serialization_notifications
)
timer.daemon = True
timer.start()
TIMER_DELAYS["serialize_setup"] *= 2
return
channel = connection.channel()
#create exchange
channel.exchange_declare(exchange=SERIALIZE_EXCHANGE,
type="fanout", durable=True)
channel.queue_declare(queue=SERIALIZE_QUEUE, durable=True)
channel.queue_bind(queue=SERIALIZE_QUEUE,
exchange=SERIALIZE_EXCHANGE,
routing_key=SERIALIZE_ROUTING_KEY)
#xml outputs channel and queue
channel.exchange_declare(exchange=SERIALIZE_OUTPUT_EXCHANGE,
type="direct", passive=False)
channel.queue_declare(queue=SERIALIZE_OUTPUT_QUEUE,
durable=True, exclusive=False, auto_delete=False)
channel.queue_bind(queue=SERIALIZE_OUTPUT_QUEUE,
exchange=SERIALIZE_OUTPUT_EXCHANGE,
routing_key=SERIALIZE_OUTPUT_ROUTING_KEY)
for i in range(mq_utility.get_number_of_workers()):
init_thread()
def load_email():
for type_key, ti in capi.iter_type_info():
workflow = ti.workflow
if workflow and workflow.has_feature("email"):
if not workflow.has_feature("notification"):
raise EmailError(
"Email notifications feature for %r cannot be "
"enabled without first enabling the notification "
"feature" % (type_key)) # !+FEATURE_DEPENDENCIES
mq_utility = component.getUtility(IMessageQueueConfig)
connection = get_mq_connection()
if not connection:
return
channel = connection.channel()
channel.queue_declare(queue="bungeni_email_queue", durable=True)
channel.queue_bind(queue="bungeni_email_queue",
exchange=str(mq_utility.get_message_exchange()),
routing_key="")
for i in range(mq_utility.get_number_of_workers()):
task_thread = Thread(target=email_worker)
task_thread.daemon = True
task_thread.start()
def queue_object_serialization(obj):
"""Send a message to the serialization queue for non-draft documents
"""
connection = get_mq_connection()
if not connection:
log.warn("Could not get rabbitmq connection. Will not send "
"AMQP message for this change.")
log.warn("Publishing XML directly - this will slow things down")
try:
publish_to_xml(obj)
except Exception, e:
notify_serialization_failure(SERIALIZE_FAILURE_TEMPLATE,
obj=obj,
message="",
error=e)
notify_serialization_failure(
None,
body="Failed to find running RabbitMQ",
subject="Notice - RabbitMQ")
notify_serialization_failure(None,
body="Failed to find running RabbitMQ",
subject="Notice - RabbitMQ")
return
def queue_object_serialization(obj):
"""Send a message to the serialization queue for non-draft documents
"""
connection = get_mq_connection()
if not connection:
log.warn("Could not get rabbitmq connection. Will not send "
"AMQP message for this change."
)
log.warn("Publishing XML directly - this will slow things down")
try:
publish_to_xml(obj)
except Exception, e:
notify_serialization_failure(SERIALIZE_FAILURE_TEMPLATE,
obj=obj, message="", error=e
)
notify_serialization_failure(None,
body="Failed to find running RabbitMQ",
subject="Notice - RabbitMQ"
)
notify_serialization_failure(None,
body="Failed to find running RabbitMQ",
subject="Notice - RabbitMQ"
)
return
def publish_to_xml(context):
"""Generates XML for object and saves it to the file. If object contains
attachments - XML is saved in zip archive with all attached files.
"""
#create a fake interaction to ensure items requiring a participation
#are serialized
#!+SERIALIZATION(mb, Jan-2013) review this approach
try:
zope.security.management.getInteraction()
except zope.security.interfaces.NoInteraction:
principal = zope.security.testing.Principal('user', 'manager', ())
zope.security.management.newInteraction(create_participation(principal))
include = []
# list of files to zip
files = []
# data dict to be published
data = {}
context = zope.security.proxy.removeSecurityProxy(context)
if interfaces.IFeatureVersion.providedBy(context):
include.append("versions")
if interfaces.IFeatureAudit.providedBy(context):
include.append("event")
exclude = ["data", "event", "attachments", "changes"]
# include binary fields and include them in the zip of files for this object
for column in class_mapper(context.__class__).columns:
if column.type.__class__ == Binary:
exclude.append(column.key)
content = getattr(context, column.key, None)
if content:
bfile = tmp(delete=False)
bfile.write(content)
files.append(bfile.name)
data[column.key] = dict(
saved_file=os.path.basename(bfile.name)
)
bfile.close()
data.update(
obj2dict(context, 1,
parent=None,
include=include,
exclude=exclude
)
)
obj_type = IWorkflow(context).name
tags = IStateController(context).get_state().tags
if tags:
data["tags"] = tags
permissions = get_object_state_rpm(context).permissions
data["permissions"] = get_permissions_dict(permissions)
data["changes"] = []
for change in getattr(context, "changes", []):
change_dict = obj2dict(change, 0, parent=context)
change_permissions = get_head_object_state_rpm(change).permissions
change_dict["permissions"] = get_permissions_dict(change_permissions)
data["changes"].append(change_dict)
# setup path to save serialized data
path = os.path.join(setupStorageDirectory(), obj_type)
if not os.path.exists(path):
os.makedirs(path)
# xml file path
file_path = os.path.join(path, stringKey(context))
if interfaces.IFeatureAttachment.providedBy(context):
attachments = getattr(context, "attachments", None)
if attachments:
data["attachments"] = []
for attachment in attachments:
# serializing attachment
attachment_dict = obj2dict(attachment, 1,
parent=context,
exclude=["data", "event", "versions"])
permissions = get_object_state_rpm(attachment).permissions
attachment_dict["permissions"] = \
get_permissions_dict(permissions)
# saving attachment to tmp
attached_file = tmp(delete=False)
attached_file.write(attachment.data)
attached_file.flush()
attached_file.close()
files.append(attached_file.name)
attachment_dict["saved_file"] = os.path.basename(
attached_file.name
)
data["attachments"].append(attachment_dict)
# zipping xml, attached files plus any binary fields
# also remove the temporary files
if files:
#generate temporary xml file
temp_xml = tmp(delete=False)
temp_xml.write(serialize(data, name=obj_type))
temp_xml.close()
#write attachments/binary fields to zip
zip_file = ZipFile("%s.zip" % (file_path), "w")
for f in files:
zip_file.write(f, os.path.basename(f))
os.remove(f)
#write the xml
zip_file.write(temp_xml.name, "%s.xml" % os.path.basename(file_path))
zip_file.close()
#placed remove after zip_file.close !+ZIP_FILE_CRC_FAILURE
os.remove(temp_xml.name)
else:
# save serialized xml to file
with open("%s.xml" % (file_path), "w") as xml_file:
xml_file.write(serialize(data, name=obj_type))
xml_file.close()
#publish to rabbitmq outputs queue
connection = get_mq_connection()
if not connection:
return
channel = connection.channel()
publish_file_path = "%s.%s" %(file_path, ("zip" if files else "xml"))
channel.basic_publish(
exchange=SERIALIZE_OUTPUT_EXCHANGE,
routing_key=SERIALIZE_OUTPUT_ROUTING_KEY,
body=simplejson.dumps({"type": "file", "location": publish_file_path }),
properties=pika.BasicProperties(content_type="text/plain",
delivery_mode=2
)
)
#clean up - remove any files if zip was created
if files:
prev_xml_file = "%s.%s" %(file_path, "xml")
if os.path.exists(prev_xml_file):
os.remove(prev_xml_file)
def publish_to_xml(context):
"""Generates XML for object and saves it to the file. If object contains
attachments - XML is saved in zip archive with all attached files.
"""
#create a fake interaction to ensure items requiring a participation
#are serialized
#!+SERIALIZATION(mb, Jan-2013) review this approach
try:
zope.security.management.getInteraction()
except zope.security.interfaces.NoInteraction:
principal = zope.security.testing.Principal('user', 'manager', ())
zope.security.management.newInteraction(
create_participation(principal))
include = []
# list of files to zip
files = []
# data dict to be published
data = {}
context = zope.security.proxy.removeSecurityProxy(context)
if interfaces.IFeatureVersion.providedBy(context):
include.append("versions")
if interfaces.IFeatureAudit.providedBy(context):
include.append("event")
exclude = ["data", "event", "attachments", "changes"]
# include binary fields and include them in the zip of files for this object
for column in class_mapper(context.__class__).columns:
if column.type.__class__ == Binary:
exclude.append(column.key)
content = getattr(context, column.key, None)
if content:
bfile = tmp(delete=False)
bfile.write(content)
files.append(bfile.name)
data[column.key] = dict(
saved_file=os.path.basename(bfile.name))
bfile.close()
data.update(
obj2dict(context, 1, parent=None, include=include, exclude=exclude))
obj_type = IWorkflow(context).name
tags = IStateController(context).get_state().tags
if tags:
data["tags"] = tags
permissions = get_object_state_rpm(context).permissions
data["permissions"] = get_permissions_dict(permissions)
data["changes"] = []
for change in getattr(context, "changes", []):
change_dict = obj2dict(change, 0, parent=context)
change_permissions = get_head_object_state_rpm(change).permissions
change_dict["permissions"] = get_permissions_dict(change_permissions)
data["changes"].append(change_dict)
# setup path to save serialized data
path = os.path.join(setupStorageDirectory(), obj_type)
if not os.path.exists(path):
os.makedirs(path)
# xml file path
file_path = os.path.join(path, stringKey(context))
if interfaces.IFeatureAttachment.providedBy(context):
attachments = getattr(context, "attachments", None)
if attachments:
data["attachments"] = []
for attachment in attachments:
# serializing attachment
attachment_dict = obj2dict(
attachment,
1,
parent=context,
exclude=["data", "event", "versions"])
permissions = get_object_state_rpm(attachment).permissions
attachment_dict["permissions"] = \
get_permissions_dict(permissions)
# saving attachment to tmp
attached_file = tmp(delete=False)
attached_file.write(attachment.data)
attached_file.flush()
attached_file.close()
files.append(attached_file.name)
attachment_dict["saved_file"] = os.path.basename(
attached_file.name)
data["attachments"].append(attachment_dict)
# zipping xml, attached files plus any binary fields
# also remove the temporary files
if files:
#generate temporary xml file
temp_xml = tmp(delete=False)
temp_xml.write(serialize(data, name=obj_type))
temp_xml.close()
#write attachments/binary fields to zip
zip_file = ZipFile("%s.zip" % (file_path), "w")
for f in files:
zip_file.write(f, os.path.basename(f))
os.remove(f)
#write the xml
zip_file.write(temp_xml.name, "%s.xml" % os.path.basename(file_path))
zip_file.close()
#placed remove after zip_file.close !+ZIP_FILE_CRC_FAILURE
os.remove(temp_xml.name)
else:
# save serialized xml to file
with open("%s.xml" % (file_path), "w") as xml_file:
xml_file.write(serialize(data, name=obj_type))
xml_file.close()
#publish to rabbitmq outputs queue
connection = get_mq_connection()
if not connection:
return
channel = connection.channel()
publish_file_path = "%s.%s" % (file_path, ("zip" if files else "xml"))
channel.basic_publish(exchange=SERIALIZE_OUTPUT_EXCHANGE,
routing_key=SERIALIZE_OUTPUT_ROUTING_KEY,
body=simplejson.dumps({
"type": "file",
"location": publish_file_path
}),
properties=pika.BasicProperties(
content_type="text/plain", delivery_mode=2))
#clean up - remove any files if zip was created
if files:
prev_xml_file = "%s.%s" % (file_path, "xml")
if os.path.exists(prev_xml_file):
os.remove(prev_xml_file)
