def publish_to_xml(context, type="", include=None):
""" Generates XML for object and saves it to the file. If object contains
attachments - XML is saved in zip archive with all attached files.
"""
if include is None:
include = []
context = removeSecurityProxy(context)
if IVersionable.implementedBy(context.__class__):
include.append("versions")
if IAuditable.implementedBy(context.__class__):
include.append("event")
data = obj2dict(context,1,parent=None,include=include,
exclude=["file_data", "image", "logo_data","event_item"])
if type=="":
type = getattr(context,"type", None)
data["permissions"]= []
permissions = get_object_state_rpm(context).permissions
for x in permissions:
data["permissions"].append({"role":x[2],
"permission":x[1],
"setting":x[0] and "Allow" or "Deny"})
assert type, "%s has no 'type' field. Use 'type' function parameter." % context.__class__
files = []
path = os.path.join(setupStorageDirectory(), type)
if not os.path.exists(path):
os.makedirs(path)
file_path = os.path.join(path,stringKey(context))
files.append(file_path+".xml")
with open(file_path+".xml","w") as file:
file.write(serialize(data, name=type))
if IAttachmentable.implementedBy(context.__class__):
attached_files = getattr(context, "attached_files", None)
if attached_files:
for attachment in attached_files:
attachment_path = os.path.join(path, attachment.file_name)
files.append(attachment_path)
with open(os.path.join(path, attachment.file_name), "wb") as file:
file.write(attachment.file_data)
zip = ZipFile(file_path+".zip", "w")
for file in files:
zip.write(file, os.path.split(file)[-1])
os.remove(file)
zip.close()
def obj2dict(obj, depth, parent=None, include=[], exclude=[], lang=None, root_key=None):
""" Returns dictionary representation of a domain object.
"""
if lang is None:
lang = getattr(obj, "language", capi.default_language)
result = {}
obj = zope.security.proxy.removeSecurityProxy(obj)
descriptor = None
if IAlchemistContent.providedBy(obj):
try:
descriptor = utils.get_descriptor(obj)
except KeyError:
log.error("Could not get descriptor for IAlchemistContent %r", obj)
if parent is not None and IWorkflowed.providedBy(obj):
permissions = get_object_state_rpm(obj).permissions
result["permissions"] = get_permissions_dict(permissions)
result["tags"] = IStateController(obj).get_state().tags
# Get additional attributes
for name in include:
value = getattr(obj, name, None)
if value is None:
continue
if not name.endswith("s"):
name += "s"
if isinstance(value, collections.Iterable):
res = []
# !+ allowance for non-container-api-conformant alchemist containers
if IAlchemistContainer.providedBy(value):
value = value.values()
for item in value:
i = obj2dict(item, 0, lang=lang, root_key=root_key)
res.append(i)
result[name] = res
else:
result[name] = value
# Get mapped attributes
mapper = class_mapper(obj.__class__)
for property in mapper.iterate_properties:
if property.key in exclude:
continue
value = getattr(obj, property.key)
if value == parent:
continue
if value is None:
continue
if isinstance(property, RelationshipProperty) and depth > 0:
if isinstance(value, collections.Iterable):
result[property.key] = []
for item in value:
result[property.key].append(obj2dict(item, 1,
parent=obj,
include=[],
exclude=exclude + INNER_EXCLUDES,
lang=lang,
root_key=root_key
))
else:
result[property.key] = obj2dict(value, depth-1,
parent=obj,
include=[],
exclude=exclude + INNER_EXCLUDES,
lang=lang,
root_key=root_key
)
else:
if isinstance(property, RelationshipProperty):
continue
elif isinstance(property, ColumnProperty):
columns = property.columns
if len(columns) == 1:
if is_column_binary(columns[0]):
if (parent and
interfaces.ISerializable.providedBy(obj)):
#skip serialization of binary fields
#that have already been serialized elsewhere
continue
#save files
result[columns[0].key] = dict(
saved_file=PersistFiles.store_file(
obj, columns[0], root_key
)
)
continue
if descriptor:
columns = property.columns
is_foreign = False
if len(columns) == 1:
if len(columns[0].foreign_keys):
is_foreign = True
if (not is_foreign) and (property.key in descriptor.keys()):
field = descriptor.get(property.key)
if (field and field.property and
(schema.interfaces.IChoice.providedBy(field.property)
or IVocabularyTextField.providedBy(field.property))
):
factory = (field.property.vocabulary or
field.property.source
)
if factory is None:
vocab_name = getattr(field.property,
"vocabularyName", None)
factory = zope.component.getUtility(
schema.interfaces.IVocabularyFactory,
vocab_name
)
# !+VOCABULARIES(mb, aug-2012)some vocabularies
# expect an interaction to generate values
# todo - update these vocabularies to work
# with no request e.g. in notification threads
display_name = None
try:
vocabulary = factory(obj)
#handle vdex hierarchical terms
if ITreeVocabulary.providedBy(factory):
values = value.splitlines()
term_values = []
for val in values:
term_values.append(dict(
name=property.key,
value=val,
displayAs=factory.vdex.getTermCaption(
factory.getTermById(val),
lang
)
))
result[property.key] = term_values
continue
term = vocabulary.getTerm(value)
if lang:
if hasattr(factory, "vdex"):
display_name = (
factory.vdex.getTermCaption(
factory.getTermById(value),
lang
))
else:
display_name = translate(
(term.title or term.value),
target_language=lang,
domain="bungeni"
)
else:
display_name = term.title or term.value
except zope.security.interfaces.NoInteraction:
log.error("This vocabulary %s expects an"
"interaction to generate terms.",
factory
)
# try to use dc adapter lookup
try:
_prop = mapper.get_property_by_column(
property.columns[0])
_prop_value = getattr(obj, _prop.key)
dc = IDCDescriptiveProperties(
_prop_value, None)
if dc:
display_name = (
IDCDescriptiveProperties(
_prop_value).title
)
except KeyError:
log.warn("No display text found for %s"
" on object %s. Unmapped in orm.",
property.key, obj
)
except Exception, e:
log.error("Could not instantiate"
" vocabulary %s. Exception: %s",
factory, e
)
finally:
# fallback we cannot look up vocabularies/dc
if display_name is None:
if not isinstance(value, unicode):
display_name = unicode(value)
if display_name is not None:
result[property.key] = dict(
name=property.key,
value=value,
displayAs=display_name
)
continue
def publish_to_xml(context):
"""Generates XML for object and saves it to the file. If object contains
attachments - XML is saved in zip archive with all attached files.
"""
context = zope.security.proxy.removeSecurityProxy(context)
obj_type = IWorkflow(context).name
#locking
lock_name = "%s-%s" %(obj_type, stringKey(context))
with LockStore.get_lock(lock_name):
#root key (used to cache files to zip)
root_key = make_key()
#create a fake interaction to ensure items requiring a participation
#are serialized
#!+SERIALIZATION(mb, Jan-2013) review this approach
try:
zope.security.management.getInteraction()
except zope.security.interfaces.NoInteraction:
principal = zope.security.testing.Principal('user', 'manager', ())
zope.security.management.newInteraction(create_participation(principal))
include = []
# data dict to be published
data = {}
if interfaces.IFeatureVersion.providedBy(context):
include.append("versions")
if interfaces.IFeatureAudit.providedBy(context):
include.append("event")
exclude = ["data", "event", "attachments"]
data.update(
obj2dict(context, 1,
parent=None,
include=include,
exclude=exclude,
root_key=root_key
)
)
tags = IStateController(context).get_state().tags
if tags:
data["tags"] = tags
permissions = get_object_state_rpm(context).permissions
data["permissions"] = get_permissions_dict(permissions)
# setup path to save serialized data
path = os.path.join(setupStorageDirectory(), obj_type)
if not os.path.exists(path):
os.makedirs(path)
# xml file path
file_path = os.path.join(path, stringKey(context))
#files to zip
files = []
if interfaces.IFeatureAttachment.providedBy(context):
attachments = getattr(context, "attachments", None)
if attachments:
data["attachments"] = []
for attachment in attachments:
# serializing attachment
attachment_dict = obj2dict(attachment, 1,
parent=context,
exclude=["data", "event", "versions"])
# saving attachment to tmp
attached_file = tmp(delete=False)
attached_file.write(attachment.data)
attached_file.flush()
attached_file.close()
files.append(attached_file.name)
attachment_dict["saved_file"] = os.path.basename(
attached_file.name
)
data["attachments"].append(attachment_dict)
#add explicit origin chamber for this object (used to partition data in
#if more than one parliament exists)
data["origin_parliament"] = get_origin_parliament(context)
#add any additional files to file list
files = files + PersistFiles.get_files(root_key)
# zipping xml, attached files plus any binary fields
# also remove the temporary files
if files:
#generate temporary xml file
temp_xml = tmp(delete=False)
temp_xml.write(serialize(data, name=obj_type))
temp_xml.close()
#write attachments/binary fields to zip
with ZipFile("%s.zip" % (file_path), "w") as zip_file:
for f in files:
zip_file.write(f, os.path.basename(f))
# write the xml
zip_file.write(temp_xml.name, "%s.xml" % os.path.basename(file_path))
files.append(temp_xml.name)
else:
# save serialized xml to file
with open("%s.xml" % (file_path), "w") as xml_file:
xml_file.write(serialize(data, name=obj_type))
xml_file.close()
# publish to rabbitmq outputs queue
connection = bungeni.core.notifications.get_mq_connection()
if not connection:
return
channel = connection.channel()
publish_file_path = "%s.%s" %(file_path, ("zip" if files else "xml"))
channel.basic_publish(
exchange=SERIALIZE_OUTPUT_EXCHANGE,
routing_key=SERIALIZE_OUTPUT_ROUTING_KEY,
body=simplejson.dumps({"type": "file", "location": publish_file_path }),
properties=pika.BasicProperties(content_type="text/plain",
delivery_mode=2
)
)
#clean up - remove any files if zip was/was not created
if files:
files.append("%s.%s" %(file_path, "xml"))
else:
files.append("%s.%s" %(file_path, "zip"))
remove_files(files)
#clear the cache
PersistFiles.clear_files(root_key)
def publish_to_xml(context):
"""Generates XML for object and saves it to the file. If object contains
attachments - XML is saved in zip archive with all attached files.
"""
#create a fake interaction to ensure items requiring a participation
#are serialized
#!+SERIALIZATION(mb, Jan-2013) review this approach
try:
zope.security.management.getInteraction()
except zope.security.interfaces.NoInteraction:
principal = zope.security.testing.Principal('user', 'manager', ())
zope.security.management.newInteraction(create_participation(principal))
include = []
# list of files to zip
files = []
# data dict to be published
data = {}
context = zope.security.proxy.removeSecurityProxy(context)
if interfaces.IFeatureVersion.providedBy(context):
include.append("versions")
if interfaces.IFeatureAudit.providedBy(context):
include.append("event")
exclude = ["data", "event", "attachments", "changes"]
# include binary fields and include them in the zip of files for this object
for column in class_mapper(context.__class__).columns:
if column.type.__class__ == Binary:
exclude.append(column.key)
content = getattr(context, column.key, None)
if content:
bfile = tmp(delete=False)
bfile.write(content)
files.append(bfile.name)
data[column.key] = dict(
saved_file=os.path.basename(bfile.name)
)
bfile.close()
data.update(
obj2dict(context, 1,
parent=None,
include=include,
exclude=exclude
)
)
obj_type = IWorkflow(context).name
tags = IStateController(context).get_state().tags
if tags:
data["tags"] = tags
permissions = get_object_state_rpm(context).permissions
data["permissions"] = get_permissions_dict(permissions)
data["changes"] = []
for change in getattr(context, "changes", []):
change_dict = obj2dict(change, 0, parent=context)
change_permissions = get_head_object_state_rpm(change).permissions
change_dict["permissions"] = get_permissions_dict(change_permissions)
data["changes"].append(change_dict)
# setup path to save serialized data
path = os.path.join(setupStorageDirectory(), obj_type)
if not os.path.exists(path):
os.makedirs(path)
# xml file path
file_path = os.path.join(path, stringKey(context))
if interfaces.IFeatureAttachment.providedBy(context):
attachments = getattr(context, "attachments", None)
if attachments:
data["attachments"] = []
for attachment in attachments:
# serializing attachment
attachment_dict = obj2dict(attachment, 1,
parent=context,
exclude=["data", "event", "versions"])
permissions = get_object_state_rpm(attachment).permissions
attachment_dict["permissions"] = \
get_permissions_dict(permissions)
# saving attachment to tmp
attached_file = tmp(delete=False)
attached_file.write(attachment.data)
attached_file.flush()
attached_file.close()
files.append(attached_file.name)
attachment_dict["saved_file"] = os.path.basename(
attached_file.name
)
data["attachments"].append(attachment_dict)
# zipping xml, attached files plus any binary fields
# also remove the temporary files
if files:
#generate temporary xml file
temp_xml = tmp(delete=False)
temp_xml.write(serialize(data, name=obj_type))
temp_xml.close()
#write attachments/binary fields to zip
zip_file = ZipFile("%s.zip" % (file_path), "w")
for f in files:
zip_file.write(f, os.path.basename(f))
os.remove(f)
#write the xml
zip_file.write(temp_xml.name, "%s.xml" % os.path.basename(file_path))
zip_file.close()
#placed remove after zip_file.close !+ZIP_FILE_CRC_FAILURE
os.remove(temp_xml.name)
else:
# save serialized xml to file
with open("%s.xml" % (file_path), "w") as xml_file:
xml_file.write(serialize(data, name=obj_type))
xml_file.close()
#publish to rabbitmq outputs queue
connection = get_mq_connection()
if not connection:
return
channel = connection.channel()
publish_file_path = "%s.%s" %(file_path, ("zip" if files else "xml"))
channel.basic_publish(
exchange=SERIALIZE_OUTPUT_EXCHANGE,
routing_key=SERIALIZE_OUTPUT_ROUTING_KEY,
body=simplejson.dumps({"type": "file", "location": publish_file_path }),
properties=pika.BasicProperties(content_type="text/plain",
delivery_mode=2
)
)
#clean up - remove any files if zip was created
if files:
prev_xml_file = "%s.%s" %(file_path, "xml")
if os.path.exists(prev_xml_file):
os.remove(prev_xml_file)
def publish_to_xml(context):
"""Generates XML for object and saves it to the file. If object contains
attachments - XML is saved in zip archive with all attached files.
"""
context = zope.security.proxy.removeSecurityProxy(context)
obj_type = IWorkflow(context).name
#locking
random_name_sfx = generate_random_filename()
context_file_name = "%s-%s" % (stringKey(context), random_name_sfx)
#lock_name = "%s-%s" %(obj_type, context_file_name)
#!+LOCKING(AH, 25-01-2014) disabling file locking
#! locking was reqiured when the serializer used ta constant file name
#! for an object. Now serialized file names are unique, and non repeated
#with LockStore.get_lock(lock_name):
#
#root key (used to cache files to zip)
root_key = make_key()
# create a fake interaction to ensure items requiring a participation
# are serialized
#!+SERIALIZATION(mb, Jan-2013) review this approach
try:
zope.security.management.getInteraction()
except zope.security.interfaces.NoInteraction:
principal = zope.security.testing.Principal("user", "manager", ())
zope.security.management.newInteraction(create_participation(principal))
include = []
# data dict to be published
data = {}
if IFeatureVersion.providedBy(context):
include.append("versions")
if IFeatureEvent.providedBy(context):
include.append("event")
exclude = ["data", "event", "attachments"]
updated_dict = obj2dict(context, 1,
parent=None,
include=include,
exclude=exclude,
root_key=root_key
)
data.update(
updated_dict
)
tags = IStateController(context).get_state().tags
if tags:
data["tags"] = tags
permissions = get_object_state_rpm(context).permissions
data["permissions"] = get_permissions_dict(permissions)
# setup path to save serialized data
path = os.path.join(setupStorageDirectory(), obj_type)
log.info("Setting up path to write to : %s", path)
if not os.path.exists(path):
#
# !+THREADSAFE(AH, 2014-09-24) making makedirs threadsafe,
# sometimes between checking for existence and execution
# of makedirs() the folder has already been created by
# another thread
try:
os.makedirs(path)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(path):
log.info("Error Folder : %s already exists, ignoring exception ", path)
else:
raise
# xml file path
file_path = os.path.join(path, context_file_name)
# files to zip
files = []
if IFeatureAttachment.providedBy(context):
attachments = getattr(context, "attachments", None)
if attachments:
data["attachments"] = []
for attachment in attachments:
# serializing attachment
attachment_dict = obj2dict(attachment, 1,
parent=context,
exclude=["data", "event", "versions"])
# saving attachment to tmp
attached_file = tmp(delete=False)
attached_file.write(attachment.data)
attached_file.flush()
attached_file.close()
files.append(attached_file.name)
attachment_dict["saved_file"] = os.path.basename(
attached_file.name
)
data["attachments"].append(attachment_dict)
# add explicit origin chamber for this object (used to partition data
# in if more than one chamber exists)
if obj_type == "Legislature":
data["origin_chamber"] = None
else:
data["origin_chamber"] = get_origin_chamber(context)
# add any additional files to file list
files = files + PersistFiles.get_files(root_key)
# zipping xml, attached files plus any binary fields
# also remove the temporary files
if files:
# generate temporary xml file
temp_xml = tmp(delete=False)
temp_xml.write(serialize(data, name=obj_type))
temp_xml.close()
# write attachments/binary fields to zip
with ZipFile("%s.zip" % (file_path), "w") as zip_file:
for f in files:
zip_file.write(f, os.path.basename(f))
# write the xml
zip_file.write(temp_xml.name, "%s.xml" % os.path.basename(file_path))
files.append(temp_xml.name)
else:
# save serialized xml to file
with open("%s.xml" % (file_path), "w") as xml_file:
xml_file.write(serialize(data, name=obj_type))
xml_file.close()
# publish to rabbitmq outputs queue
connection = bungeni.core.notifications.get_mq_connection()
if not connection:
return
channel = connection.channel()
#channel.confirm_delivery()
publish_file_path = "%s.%s" %(file_path, ("zip" if files else "xml"))
#channel_delivery =
channel.basic_publish(
exchange=SERIALIZE_OUTPUT_EXCHANGE,
routing_key=SERIALIZE_OUTPUT_ROUTING_KEY,
body=simplejson.dumps({"type": "file", "location": publish_file_path }),
properties=pika.BasicProperties(content_type="text/plain",
delivery_mode=2
)
)
#if channel_delivery:
# log.info("Message published to exchange %s with key %s for %s" %
# (SERIALIZE_OUTPUT_EXCHANGE, SERIALIZE_OUTPUT_ROUTING_KEY, publish_file_path))
#else:
# log.error("Message publication failed for %r", publish_file_path)
#clean up - remove any files if zip was/was not created
if files:
files.append("%s.%s" %(file_path, "xml"))
else:
files.append("%s.%s" %(file_path, "zip"))
remove_files(files)
# clear the cache
PersistFiles.clear_files(root_key)
def obj2dict(obj, depth, parent=None, include=[], exclude=[], lang=None, root_key=None):
""" Returns dictionary representation of a domain object.
"""
if lang is None:
lang = getattr(obj, "language", capi.default_language)
result = {}
obj = zope.security.proxy.removeSecurityProxy(obj)
descriptor = None
if IAlchemistContent.providedBy(obj):
try:
descriptor = utils.get_descriptor(obj)
except KeyError:
log.error("Could not get descriptor for IAlchemistContent %r", obj)
if parent is not None and IWorkflowed.providedBy(obj):
permissions = get_object_state_rpm(obj).permissions
result["permissions"] = get_permissions_dict(permissions)
result["tags"] = IStateController(obj).get_state().tags
# Get additional attributes
for name in include:
value = getattr(obj, name, None)
if value is None:
continue
if not name.endswith("s"):
name += "s"
if isinstance(value, collections.Iterable):
res = []
# !+ allowance for non-container-api-conformant alchemist containers
if IAlchemistContainer.providedBy(value):
value = value.values()
for item in value:
i = obj2dict(item, 0, lang=lang, root_key=root_key)
res.append(i)
result[name] = res
else:
result[name] = value
# Get mapped attributes
seen_keys = []
mapper = class_mapper(obj.__class__)
for mproperty in mapper.iterate_properties:
if mproperty.key.startswith("_vp"):
#skip vertical props
continue
if mproperty.key in exclude:
continue
seen_keys.append(mproperty.key)
value = getattr(obj, mproperty.key)
if value == parent:
continue
if value is None:
continue
if isinstance(mproperty, RelationshipProperty) and depth > 0:
if isinstance(value, collections.Iterable):
result[mproperty.key] = []
for item in value:
# !+DEPTH(ah, 2014-09-19) depth was set to 1 here, this causes
# a very deep branching for upper level groups like legislature and chamber
# and legislature times out occasionally. Doesnt seem neccessary to go depth=1
# for child objects, because they get serialized independently anyway, changing
# depth to depth-1 so all dependent objects are iterated 1 level lower than the
# parent.
# UPDATE(ah, 2014-11-03) Item Schedule is an exceptional case of an object
# whose context is within a parent container but is not visible outside of the sitting
# it is not a type defined in types.xml and does not have its own
# wokflow so we need to handle that in a unique way
# we don't decrement the depth and instead process it as is
active_depth = depth
if item.__class__.__name__ == "ItemSchedule":
active_depth = depth
else:
active_depth = depth-1
result[mproperty.key].append(
obj2dict(
item,
active_depth,
parent=obj,
include=["owner", "item_schedule", "item_schedule_discussion"],
exclude=exclude + INNER_EXCLUDES,
lang=lang,
root_key=root_key
)
)
else:
result[mproperty.key] = obj2dict(value, depth-1,
parent=obj,
include=["owner"],
exclude=exclude + INNER_EXCLUDES,
lang=lang,
root_key=root_key
)
else:
if isinstance(mproperty, RelationshipProperty):
continue
elif isinstance(mproperty, ColumnProperty):
columns = mproperty.columns
if len(columns) == 1:
if is_column_binary(columns[0]):
fname = PersistFiles.store_file(obj, parent,
columns[0].key, root_key)
if fname:
result[columns[0].key] = dict(saved_file=fname)
continue
if descriptor:
columns = mproperty.columns
is_foreign = False
if len(columns) == 1:
if len(columns[0].foreign_keys):
is_foreign = True
if (not is_foreign) and (mproperty.key in descriptor.keys()):
field = descriptor.get(mproperty.key)
if (field and field.property and
(schema.interfaces.IChoice.providedBy(field.property) or
IVocabularyTextField.providedBy(field.property))
):
factory = field.property.vocabulary or field.property.source
if factory is None:
vocab_name = getattr(field.property, "vocabularyName", None)
factory = get_vocabulary(vocab_name)
# !+VOCABULARIES(mb, aug-2012)some vocabularies
# expect an interaction to generate values
# todo - update these vocabularies to work
# with no request e.g. in notification threads
display_name = None
try:
vocabulary = factory(obj)
# handle vdex hierarchical terms
if ITreeVocabulary.providedBy(factory):
values = value.splitlines()
term_values = []
for val in values:
term_values.append(dict(
name=mproperty.key,
value=val,
displayAs=factory.getTermCaption(
factory.getTermById(val), lang=lang)))
result[mproperty.key] = term_values
continue
term = vocabulary.getTerm(value)
if lang:
if hasattr(factory, "getTermCaption"):
display_name = factory.getTermCaption(
factory.getTermById(value), lang=lang)
else:
display_name = translate(
term.title or term.value,
target_language=lang,
domain="bungeni")
else:
display_name = term.title or term.value
except zope.security.interfaces.NoInteraction:
log.error("This vocabulary %s expects an interaction "
"to generate terms.", factory)
# try to use dc adapter lookup
try:
_prop = mapper.get_property_by_column(
mproperty.columns[0])
_prop_value = getattr(obj, _prop.key)
dc = IDCDescriptiveProperties(_prop_value, None)
if dc:
display_name = IDCDescriptiveProperties(
_prop_value).title
except KeyError:
log.warn("No display text found for %s on "
"object %s. Unmapped in orm.",
property.key, obj)
except Exception, e:
log.error("Could not instantiate vocabulary %s. "
"Exception: %s", factory, e)
finally:
# fallback we cannot look up vocabularies/dc
if display_name is None:
if not isinstance(value, unicode):
display_name = unicode(value)
if display_name is not None:
result[mproperty.key] = dict(
name=mproperty.key,
value=value,
displayAs=display_name)
continue
def publish_to_xml(context):
"""Generates XML for object and saves it to the file. If object contains
attachments - XML is saved in zip archive with all attached files.
"""
include = []
context = removeSecurityProxy(context)
if IVersionable.implementedBy(context.__class__):
include.append("versions")
if IAuditable.implementedBy(context.__class__):
include.append("event")
data = obj2dict(context,
1,
parent=None,
include=include,
exclude=[
"file_data", "image", "logo_data", "event",
"attached_files", "changes"
])
type = IWorkflow(context).name
tags = IStateController(context).get_state().tags
if tags:
data["tags"] = tags
permissions = get_object_state_rpm(context).permissions
data["permissions"] = get_permissions_dict(permissions)
data["changes"] = []
for change in getattr(context, "changes", []):
change_dict = obj2dict(change, 0, parent=context)
change_permissions = get_head_object_state_rpm(change).permissions
change_dict["permissions"] = get_permissions_dict(change_permissions)
data["changes"].append(change_dict)
# list of files to zip
files = []
# setup path to save serialized data
path = os.path.join(setupStorageDirectory(), type)
if not os.path.exists(path):
os.makedirs(path)
# xml file path
file_path = os.path.join(path, stringKey(context))
has_attachments = False
if IAttachmentable.implementedBy(context.__class__):
attached_files = getattr(context, "attached_files", None)
if attached_files:
has_attachments = True
# add xml file to list of files to zip
files.append("%s.xml" % (file_path))
data["attached_files"] = []
for attachment in attached_files:
# serializing attachment
attachment_dict = obj2dict(
attachment,
1,
parent=context,
exclude=["file_data", "event", "versions", "changes"])
permissions = get_object_state_rpm(attachment).permissions
attachment_dict["permissions"] = \
get_permissions_dict(permissions)
# saving attachment to tmp
with tmp(delete=False) as f:
f.write(attachment.file_data)
files.append(f.name)
attachment_dict["saved_file"] = \
os.path.split(f.name)[-1]
data["attached_files"].append(attachment_dict)
# saving xml file
with open("%s.xml" % (file_path), "w") as file:
file.write(serialize(data, name=type))
# zipping xml and attached files
# unzipped files are removed
if has_attachments:
zip = ZipFile("%s.zip" % (file_path), "w")
for f in files:
zip.write(f, os.path.split(f)[-1])
os.remove(f)
zip.close()
def obj2dict(obj,
depth,
parent=None,
include=[],
exclude=[],
lang=None,
root_key=None):
""" Returns dictionary representation of a domain object.
"""
if lang is None:
lang = getattr(obj, "language", capi.default_language)
result = {}
obj = zope.security.proxy.removeSecurityProxy(obj)
descriptor = None
if IAlchemistContent.providedBy(obj):
try:
descriptor = utils.get_descriptor(obj)
except KeyError:
log.error("Could not get descriptor for IAlchemistContent %r", obj)
if parent is not None and IWorkflowed.providedBy(obj):
permissions = get_object_state_rpm(obj).permissions
result["permissions"] = get_permissions_dict(permissions)
result["tags"] = IStateController(obj).get_state().tags
# Get additional attributes
for name in include:
value = getattr(obj, name, None)
if value is None:
continue
if not name.endswith("s"):
name += "s"
if isinstance(value, collections.Iterable):
res = []
# !+ allowance for non-container-api-conformant alchemist containers
if IAlchemistContainer.providedBy(value):
value = value.values()
for item in value:
i = obj2dict(item, 0, lang=lang, root_key=root_key)
res.append(i)
result[name] = res
else:
result[name] = value
# Get mapped attributes
mapper = class_mapper(obj.__class__)
for property in mapper.iterate_properties:
if property.key in exclude:
continue
value = getattr(obj, property.key)
if value == parent:
continue
if value is None:
continue
if isinstance(property, RelationshipProperty) and depth > 0:
if isinstance(value, collections.Iterable):
result[property.key] = []
for item in value:
result[property.key].append(
obj2dict(item,
1,
parent=obj,
include=[],
exclude=exclude + INNER_EXCLUDES,
lang=lang,
root_key=root_key))
else:
result[property.key] = obj2dict(value,
depth - 1,
parent=obj,
include=[],
exclude=exclude +
INNER_EXCLUDES,
lang=lang,
root_key=root_key)
else:
if isinstance(property, RelationshipProperty):
continue
elif isinstance(property, ColumnProperty):
columns = property.columns
if len(columns) == 1:
if is_column_binary(columns[0]):
if (parent
and interfaces.ISerializable.providedBy(obj)):
#skip serialization of binary fields
#that have already been serialized elsewhere
continue
#save files
result[columns[0].key] = dict(
saved_file=PersistFiles.store_file(
obj, columns[0], root_key))
continue
if descriptor:
columns = property.columns
is_foreign = False
if len(columns) == 1:
if len(columns[0].foreign_keys):
is_foreign = True
if (not is_foreign) and (property.key in descriptor.keys()):
field = descriptor.get(property.key)
if (field and field.property and
(schema.interfaces.IChoice.providedBy(field.property)
or IVocabularyTextField.providedBy(field.property))):
factory = (field.property.vocabulary
or field.property.source)
if factory is None:
vocab_name = getattr(field.property,
"vocabularyName", None)
factory = zope.component.getUtility(
schema.interfaces.IVocabularyFactory,
vocab_name)
# !+VOCABULARIES(mb, aug-2012)some vocabularies
# expect an interaction to generate values
# todo - update these vocabularies to work
# with no request e.g. in notification threads
display_name = None
try:
vocabulary = factory(obj)
#handle vdex hierarchical terms
if ITreeVocabulary.providedBy(factory):
values = value.splitlines()
term_values = []
for val in values:
term_values.append(
dict(name=property.key,
value=val,
displayAs=factory.vdex.
getTermCaption(
factory.getTermById(val),
lang)))
result[property.key] = term_values
continue
term = vocabulary.getTerm(value)
if lang:
if hasattr(factory, "vdex"):
display_name = (
factory.vdex.getTermCaption(
factory.getTermById(value), lang))
else:
display_name = translate(
(term.title or term.value),
target_language=lang,
domain="bungeni")
else:
display_name = term.title or term.value
except zope.security.interfaces.NoInteraction:
log.error(
"This vocabulary %s expects an"
"interaction to generate terms.", factory)
# try to use dc adapter lookup
try:
_prop = mapper.get_property_by_column(
property.columns[0])
_prop_value = getattr(obj, _prop.key)
dc = IDCDescriptiveProperties(
_prop_value, None)
if dc:
display_name = (IDCDescriptiveProperties(
_prop_value).title)
except KeyError:
log.warn(
"No display text found for %s"
" on object %s. Unmapped in orm.",
property.key, obj)
except Exception, e:
log.error(
"Could not instantiate"
" vocabulary %s. Exception: %s", factory, e)
finally:
# fallback we cannot look up vocabularies/dc
if display_name is None:
if not isinstance(value, unicode):
display_name = unicode(value)
if display_name is not None:
result[property.key] = dict(name=property.key,
value=value,
displayAs=display_name)
continue
def publish_to_xml(context):
"""Generates XML for object and saves it to the file. If object contains
attachments - XML is saved in zip archive with all attached files.
"""
context = zope.security.proxy.removeSecurityProxy(context)
obj_type = IWorkflow(context).name
#locking
lock_name = "%s-%s" % (obj_type, stringKey(context))
with LockStore.get_lock(lock_name):
#root key (used to cache files to zip)
root_key = make_key()
#create a fake interaction to ensure items requiring a participation
#are serialized
#!+SERIALIZATION(mb, Jan-2013) review this approach
try:
zope.security.management.getInteraction()
except zope.security.interfaces.NoInteraction:
principal = zope.security.testing.Principal('user', 'manager', ())
zope.security.management.newInteraction(
create_participation(principal))
include = []
# data dict to be published
data = {}
if interfaces.IFeatureVersion.providedBy(context):
include.append("versions")
if interfaces.IFeatureAudit.providedBy(context):
include.append("event")
exclude = ["data", "event", "attachments"]
data.update(
obj2dict(context,
1,
parent=None,
include=include,
exclude=exclude,
root_key=root_key))
tags = IStateController(context).get_state().tags
if tags:
data["tags"] = tags
permissions = get_object_state_rpm(context).permissions
data["permissions"] = get_permissions_dict(permissions)
# setup path to save serialized data
path = os.path.join(setupStorageDirectory(), obj_type)
if not os.path.exists(path):
os.makedirs(path)
# xml file path
file_path = os.path.join(path, stringKey(context))
#files to zip
files = []
if interfaces.IFeatureAttachment.providedBy(context):
attachments = getattr(context, "attachments", None)
if attachments:
data["attachments"] = []
for attachment in attachments:
# serializing attachment
attachment_dict = obj2dict(
attachment,
1,
parent=context,
exclude=["data", "event", "versions"])
# saving attachment to tmp
attached_file = tmp(delete=False)
attached_file.write(attachment.data)
attached_file.flush()
attached_file.close()
files.append(attached_file.name)
attachment_dict["saved_file"] = os.path.basename(
attached_file.name)
data["attachments"].append(attachment_dict)
#add explicit origin chamber for this object (used to partition data in
#if more than one parliament exists)
data["origin_parliament"] = get_origin_parliament(context)
#add any additional files to file list
files = files + PersistFiles.get_files(root_key)
# zipping xml, attached files plus any binary fields
# also remove the temporary files
if files:
#generate temporary xml file
temp_xml = tmp(delete=False)
temp_xml.write(serialize(data, name=obj_type))
temp_xml.close()
#write attachments/binary fields to zip
with ZipFile("%s.zip" % (file_path), "w") as zip_file:
for f in files:
zip_file.write(f, os.path.basename(f))
# write the xml
zip_file.write(temp_xml.name,
"%s.xml" % os.path.basename(file_path))
files.append(temp_xml.name)
else:
# save serialized xml to file
with open("%s.xml" % (file_path), "w") as xml_file:
xml_file.write(serialize(data, name=obj_type))
xml_file.close()
# publish to rabbitmq outputs queue
connection = bungeni.core.notifications.get_mq_connection()
if not connection:
return
channel = connection.channel()
publish_file_path = "%s.%s" % (file_path, ("zip" if files else "xml"))
channel.basic_publish(exchange=SERIALIZE_OUTPUT_EXCHANGE,
routing_key=SERIALIZE_OUTPUT_ROUTING_KEY,
body=simplejson.dumps({
"type":
"file",
"location":
publish_file_path
}),
properties=pika.BasicProperties(
content_type="text/plain", delivery_mode=2))
#clean up - remove any files if zip was/was not created
if files:
files.append("%s.%s" % (file_path, "xml"))
else:
files.append("%s.%s" % (file_path, "zip"))
remove_files(files)
#clear the cache
PersistFiles.clear_files(root_key)
def publish_to_xml(context):
"""Generates XML for object and saves it to the file. If object contains
attachments - XML is saved in zip archive with all attached files.
"""
#create a fake interaction to ensure items requiring a participation
#are serialized
#!+SERIALIZATION(mb, Jan-2013) review this approach
try:
zope.security.management.getInteraction()
except zope.security.interfaces.NoInteraction:
principal = zope.security.testing.Principal('user', 'manager', ())
zope.security.management.newInteraction(
create_participation(principal))
include = []
# list of files to zip
files = []
# data dict to be published
data = {}
context = zope.security.proxy.removeSecurityProxy(context)
if interfaces.IFeatureVersion.providedBy(context):
include.append("versions")
if interfaces.IFeatureAudit.providedBy(context):
include.append("event")
exclude = ["data", "event", "attachments", "changes"]
# include binary fields and include them in the zip of files for this object
for column in class_mapper(context.__class__).columns:
if column.type.__class__ == Binary:
exclude.append(column.key)
content = getattr(context, column.key, None)
if content:
bfile = tmp(delete=False)
bfile.write(content)
files.append(bfile.name)
data[column.key] = dict(
saved_file=os.path.basename(bfile.name))
bfile.close()
data.update(
obj2dict(context, 1, parent=None, include=include, exclude=exclude))
obj_type = IWorkflow(context).name
tags = IStateController(context).get_state().tags
if tags:
data["tags"] = tags
permissions = get_object_state_rpm(context).permissions
data["permissions"] = get_permissions_dict(permissions)
data["changes"] = []
for change in getattr(context, "changes", []):
change_dict = obj2dict(change, 0, parent=context)
change_permissions = get_head_object_state_rpm(change).permissions
change_dict["permissions"] = get_permissions_dict(change_permissions)
data["changes"].append(change_dict)
# setup path to save serialized data
path = os.path.join(setupStorageDirectory(), obj_type)
if not os.path.exists(path):
os.makedirs(path)
# xml file path
file_path = os.path.join(path, stringKey(context))
if interfaces.IFeatureAttachment.providedBy(context):
attachments = getattr(context, "attachments", None)
if attachments:
data["attachments"] = []
for attachment in attachments:
# serializing attachment
attachment_dict = obj2dict(
attachment,
1,
parent=context,
exclude=["data", "event", "versions"])
permissions = get_object_state_rpm(attachment).permissions
attachment_dict["permissions"] = \
get_permissions_dict(permissions)
# saving attachment to tmp
attached_file = tmp(delete=False)
attached_file.write(attachment.data)
attached_file.flush()
attached_file.close()
files.append(attached_file.name)
attachment_dict["saved_file"] = os.path.basename(
attached_file.name)
data["attachments"].append(attachment_dict)
# zipping xml, attached files plus any binary fields
# also remove the temporary files
if files:
#generate temporary xml file
temp_xml = tmp(delete=False)
temp_xml.write(serialize(data, name=obj_type))
temp_xml.close()
#write attachments/binary fields to zip
zip_file = ZipFile("%s.zip" % (file_path), "w")
for f in files:
zip_file.write(f, os.path.basename(f))
os.remove(f)
#write the xml
zip_file.write(temp_xml.name, "%s.xml" % os.path.basename(file_path))
zip_file.close()
#placed remove after zip_file.close !+ZIP_FILE_CRC_FAILURE
os.remove(temp_xml.name)
else:
# save serialized xml to file
with open("%s.xml" % (file_path), "w") as xml_file:
xml_file.write(serialize(data, name=obj_type))
xml_file.close()
#publish to rabbitmq outputs queue
connection = get_mq_connection()
if not connection:
return
channel = connection.channel()
publish_file_path = "%s.%s" % (file_path, ("zip" if files else "xml"))
channel.basic_publish(exchange=SERIALIZE_OUTPUT_EXCHANGE,
routing_key=SERIALIZE_OUTPUT_ROUTING_KEY,
body=simplejson.dumps({
"type": "file",
"location": publish_file_path
}),
properties=pika.BasicProperties(
content_type="text/plain", delivery_mode=2))
#clean up - remove any files if zip was created
if files:
prev_xml_file = "%s.%s" % (file_path, "xml")
if os.path.exists(prev_xml_file):
os.remove(prev_xml_file)
def publish_to_xml(context):
"""Generates XML for object and saves it to the file. If object contains
attachments - XML is saved in zip archive with all attached files.
"""
include = []
context = removeSecurityProxy(context)
if interfaces.IFeatureVersion.providedBy(context):
include.append("versions")
if interfaces.IFeatureAudit.providedBy(context):
include.append("event")
data = obj2dict(context, 1,
parent=None,
include=include,
exclude=["data", "image", "logo_data", "event", "attachments",
"changes"]
)
# !+please do not use python builtin names as variable names
type = IWorkflow(context).name
# !+IWorkflow(context).get_state(context.status).tags
tags = IStateController(context).get_state().tags
if tags:
data["tags"] = tags
permissions = get_object_state_rpm(context).permissions
data["permissions"] = get_permissions_dict(permissions)
data["changes"] = []
for change in getattr(context, "changes", []):
change_dict = obj2dict(change, 0, parent=context)
change_permissions = get_head_object_state_rpm(change).permissions
change_dict["permissions"] = get_permissions_dict(change_permissions)
data["changes"].append(change_dict)
# list of files to zip
files = []
# setup path to save serialized data
path = os.path.join(setupStorageDirectory(), type)
if not os.path.exists(path):
os.makedirs(path)
# xml file path
file_path = os.path.join(path, stringKey(context))
has_attachments = False
if interfaces.IFeatureAttachment.providedBy(context):
attachments = getattr(context, "attachments", None)
if attachments:
has_attachments = True
# add xml file to list of files to zip
files.append("%s.xml" % (file_path))
data["attachments"] = []
for attachment in attachments:
# serializing attachment
attachment_dict = obj2dict(attachment, 1,
parent=context,
exclude=["data", "event", "versions"])
permissions = get_object_state_rpm(attachment).permissions
attachment_dict["permissions"] = \
get_permissions_dict(permissions)
# saving attachment to tmp
with tmp(delete=False) as f:
f.write(attachment.data)
files.append(f.name)
attachment_dict["saved_file"] = \
os.path.split(f.name)[-1]
data["attachments"].append(attachment_dict)
# saving xml file
with open("%s.xml" % (file_path), "w") as file:
file.write(serialize(data, name=type))
# zipping xml and attached files
# unzipped files are removed
if has_attachments:
zip = ZipFile("%s.zip" % (file_path), "w")
for f in files:
zip.write(f, os.path.split(f)[-1])
os.remove(f)
zip.close()
