def __init__(self, context):
preference_tool = getToolByName(context, 'portal_preferences')
uri = getattr(preference_tool,
"getPreferredDocumentConversionServerUrl", str)()
if uri in ('', None):
address = preference_tool.getPreferredOoodocServerAddress()
port = preference_tool.getPreferredOoodocServerPortNumber()
if address in ('', None) or port in ('', None):
raise ConversionError(
'OOoDocument: cannot proceed with conversion:'
' conversion server url is not defined in preferences')
LOG('OOoDocument', WARNING, 'PreferredOoodocServer{Address,PortNumber}' + \
' are DEPRECATED please use PreferredDocumentServerUrl instead', error=True)
scheme = "http"
uri = 'http://%s:%d' % (address, port)
else:
if uri.startswith("http://"):
scheme = "http"
elif uri.startswith("https://"):
scheme = "https"
else:
raise ConversionError(
'OOoDocument: cannot proceed with conversion:'
' preferred conversion server url is invalid')
timeout = preference_tool.getPreferredOoodocServerTimeout() \
or OOO_SERVER_PROXY_TIMEOUT
transport = TimeoutTransport(timeout=timeout, scheme=scheme)
ServerProxy.__init__(self, uri, allow_none=True, transport=transport)
def __init__(self, context):
self._serverproxy_list = []
preference_tool = getToolByName(context, 'portal_preferences')
self._ooo_server_retry = preference_tool.getPreferredDocumentConversionServerRetry() or OOO_SERVER_RETRY
uri_list = preference_tool.getPreferredDocumentConversionServerUrlList()
if not uri_list:
address = preference_tool.getPreferredOoodocServerAddress()
port = preference_tool.getPreferredOoodocServerPortNumber()
if not (address and port):
raise ConversionError('OOoDocument: cannot proceed with conversion:'
' conversion server url is not defined in preferences')
LOG('OOoDocument', WARNING, 'PreferredOoodocServer{Address,PortNumber}' + \
' are DEPRECATED please use PreferredDocumentServerUrl instead', error=True)
uri_list = ['%s://%s:%s' % ('http', address, port)]
timeout = preference_tool.getPreferredOoodocServerTimeout() \
or OOO_SERVER_PROXY_TIMEOUT
for uri in uri_list:
if uri.startswith("http://"):
scheme = "http"
elif uri.startswith("https://"):
scheme = "https"
else:
raise ConversionError('OOoDocument: cannot proceed with conversion:'
' preferred conversion server url is invalid')
transport = TimeoutTransport(timeout=timeout, scheme=scheme)
self._serverproxy_list.append((uri, ServerProxy(uri, allow_none=True, transport=transport)))
def cached_getTargetFormatItemList(content_type):
server_proxy = OOoServerProxy(self)
try:
allowed_target_item_list = server_proxy.getAllowedTargetItemList(
content_type)
try:
response_code, response_dict, response_message = \
allowed_target_item_list
except ValueError:
# Compatibility with older oood where getAllowedTargetItemList only
# returned response_dict
response_code, response_dict, response_message = \
200, dict(response_data=allowed_target_item_list), ''
if response_code == 200:
allowed = response_dict['response_data']
else:
# This is very temporary code - XXX needs to be changed
# so that the system can retry
raise ConversionError(
"OOoDocument: can not get list of allowed acceptable"
" formats for conversion (Code %s: %s)" %
(response_code, response_message))
except Fault, f:
allowed = server_proxy.getAllowedTargets(content_type)
warn(
'Your oood version is too old, using old method '
'getAllowedTargets instead of getAllowedTargetList',
DeprecationWarning)
def updateBaseMetadata(self, **kw):
"""
Updates metadata information in the converted OOo document
based on the values provided by the user. This is implemented
through the invocation of the conversion server.
"""
if not self.hasBaseData():
# XXX please pass a meaningful description of error as argument
raise NotConvertedError()
server_proxy = OOoServerProxy(self)
response_code, response_dict, response_message = \
server_proxy.run_setmetadata(self.getId(),
enc(str(self.getBaseData())),
kw)
if response_code == 200:
# successful meta data extraction
self._setBaseData(dec(response_dict['data']))
self.updateFileMetadata(
) # record in workflow history # XXX must put appropriate comments.
else:
# Explicitly raise the exception!
raise ConversionError(
"OOoDocument: error getting document metadata (Code %s: %s)" %
(response_code, response_message))
def _convertToHTML(self):
"""Convert the PDF text content to HTML with pdftohtml
"""
if not self.hasData():
return ''
tmp = tempfile.NamedTemporaryFile()
tmp.write(self.getData())
tmp.seek(0)
command_result = None
try:
command = ['pdftohtml', '-enc', 'UTF-8', '-stdout',
'-noframes', '-i', tmp.name]
try:
command_result = Popen(command, stdout=PIPE).communicate()[0]
except OSError, e:
if e.errno == errno.ENOENT:
raise ConversionError('pdftohtml was not found')
raise
finally:
tmp.close()
# Quick hack to remove bg color - XXX
h = command_result.replace('<BODY bgcolor="#A0A0A0"', '<BODY ')
# Make links relative
h = h.replace('href="%s.html' % tmp.name.split(os.sep)[-1],
'href="asEntireHTML')
return h
def getDataURI(url):
try:
data = urllib2.urlopen(url)
except Exception, e:
raise ConversionError(
"Error to transform url (%s) into data uri. ERROR = %s" %
(url, Exception(e)))
def _convertToBaseFormat(self):
"""
Converts the original document into ODF
by invoking the conversion server. Store the result
on the object. Update metadata information.
"""
server_proxy = OOoServerProxy(self)
response_code, response_dict, response_message = server_proxy.run_convert(
self.getFilename() or self.getId(),
enc(str(self.getData())),
None,
None,
self.getContentType())
if response_code == 200:
# sucessfully converted document
self._setBaseData(dec(response_dict['data']))
metadata = response_dict['meta']
self._base_metadata = metadata
if metadata.get('MIMEType', None) is not None:
self._setBaseContentType(metadata['MIMEType'])
else:
# Explicitly raise the exception!
raise ConversionError(
"OOoDocument: Error converting document to base format. (Code %s: %s)"
% (response_code, response_message))
def getContentInformation(self):
"""Returns the information about the PDF document with pdfinfo.
"""
if not self.hasData():
return dict()
try:
return self._content_information.copy()
except AttributeError:
pass
tmp = tempfile.NamedTemporaryFile()
tmp.write(self.getData())
tmp.seek(0)
command_result = None
try:
# First, we use pdfinfo to get standard metadata
command = ['pdfinfo', '-meta', '-box', tmp.name]
try:
command_result = Popen(command, stdout=PIPE).communicate()[0]
except OSError, e:
if e.errno == errno.ENOENT:
raise ConversionError('pdfinfo was not found')
raise
result = {}
for line in command_result.splitlines():
item_list = line.split(':')
key = item_list[0].strip()
value = ':'.join(item_list[1:]).strip()
result[key] = value
# Then we use PyPDF2 to get extra metadata
try:
from PyPDF2 import PdfFileReader
from PyPDF2.utils import PdfReadError
except ImportError:
# if PyPDF2 not found, pass
pass
else:
try:
pdf_file = PdfFileReader(tmp)
for info_key, info_value in (pdf_file.getDocumentInfo() or {}).iteritems():
info_key = info_key.lstrip("/")
if isinstance(info_value, unicode):
info_value = info_value.encode("utf-8")
# Ignore values that cannot be pickled ( such as AAPL:Keywords )
try:
pickle.dumps(info_value)
except pickle.PicklingError:
LOG("PDFDocument.getContentInformation", INFO,
"Ignoring non picklable document info on %s: %s (%r)" % (
self.getRelativeUrl(), info_key, info_value))
else:
result.setdefault(info_key, info_value)
except PdfReadError:
LOG("PDFDocument.getContentInformation", PROBLEM,
"PyPDF2 is Unable to read PDF, probably corrupted PDF here : %s" % \
(self.getRelativeUrl(),))
def _getFormatFromMimetype(self, mimetype):
"""
XXX: This should not be done here but Conversion Server API to get
supported Format/Extension is deprecated (topic under discussion)
"""
import mimetypes
extension = mimetypes.guess_extension(mimetype)
if extension is None:
raise ConversionError(
"Could not guess extension from mimetype '%s'" % mimetype)
return extension.split('.', 1)[1]
def _resize(self, quality, width, height, format, resolution, frame):
"""Resize and resample photo."""
parameter_list = [
'convert', '-colorspace', 'sRGB', '-depth', '8', '-quality',
str(quality), '-geometry',
'%sx%s' % (width, height)
]
if format not in VALID_TRANSPARENT_IMAGE_FORMAT_LIST:
# ImageMagick way to remove transparent that works with multiple
# images. http://www.imagemagick.org/Usage/masking/#remove
parameter_list += '-bordercolor', 'white', '-border', '0'
if resolution:
parameter_list += '-density', '%sx%s' % (resolution, resolution)
if frame is not None:
parameter_list.append('-[%s]' % frame)
else:
parameter_list.append('-')
if format:
# Is there a way to make 'convert' fail if the format is unknown,
# instead of treating this whole parameter as an output file path?
# As a workaround, we run 'convert' in a non-writeable directory.
if '/' in format or os.access('/', os.W_OK):
raise ConversionError
parameter_list.append('%s:-' % format)
else:
parameter_list.append('-')
data = str(self.getData())
if self.getContentType() == "image/svg+xml":
data = transformUrlToDataURI(data)
env = os.environ.copy()
env.update({'LC_NUMERIC': 'C'})
process = subprocess.Popen(parameter_list,
env=env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd='/',
close_fds=True)
try:
# XXX: The only portable way is to pass what stdin.write can accept,
# which is a string for PIPE.
image, err = process.communicate(data)
finally:
del process
if image:
return StringIO(image)
raise ConversionError('Image conversion failed (%s).' % err)
def _convertToText(self):
"""
Convert the PDF text content to text with pdftotext
"""
if not self.hasData():
return ''
mime_type = 'text/plain'
portal_transforms = self.getPortalObject().portal_transforms
filename = self.getFilename()
result = portal_transforms.convertToData(
mime_type,
str(self.getData()),
context=self,
filename=filename,
mimetype=self.getContentType())
if result:
return result
else:
# Try to use OCR
# As high dpi images are required, it may take some times to convert the
# pdf.
# It may be required to use activities to fill the cache and at the end,
# to calculate the final result
text = ''
content_information = self.getContentInformation()
page_count = int(content_information.get('Pages', 0))
for page_number in range(page_count):
src_mimetype, png_data = self._convert('png',
quality=100,
resolution=300,
frame=page_number,
display='identical')
if not src_mimetype.endswith('png'):
continue
content = str(png_data)
if content is not None:
filename = self.getStandardFilename(format='png')
result = portal_transforms.convertToData(
mime_type,
content,
context=self,
filename=filename,
mimetype=src_mimetype)
if result is None:
raise ConversionError(
'PDFDocument conversion error. '
'portal_transforms failed to convert to %s: %r' %
(mime_type, self))
text += result
return text
def convertTo(self, format):
server_proxy = OOoServerProxy(self.context)
response_code, response_dict, message = \
server_proxy.getAllowedTargetItemList(self.mimetype)
allowed_extension_list = response_dict['response_data']
if format in dict(allowed_extension_list):
response_code, response_dict, message = server_proxy.run_generate(
'', enc(self.data), None, format, self.mimetype)
data = dec(response_dict['data'])
if self.mimetype == 'text/html':
data = self.includeImageList(data)
return data
else:
raise ConversionError('Format not allowed %s' % format)
def __init__(self, context):
preference_tool = getToolByName(context, 'portal_preferences')
address = preference_tool.getPreferredOoodocServerAddress()
port = preference_tool.getPreferredOoodocServerPortNumber()
if address in ('', None) or port in ('', None):
raise ConversionError(
'OOoDocument: cannot proceed with conversion:'
' conversion server host and port is not defined in preferences'
)
uri = 'http://%s:%d' % (address, port)
timeout = preference_tool.getPreferredOoodocServerTimeout() \
or OOO_SERVER_PROXY_TIMEOUT
transport = TimeoutTransport(timeout=timeout, scheme='http')
ServerProxy.__init__(self, uri, allow_none=True, transport=transport)
def __call__(self, *args, **kw):
"""
Catch Protocol Errors (transport layer) and specifically
identify them as OOo server network/communication error
xml-rpc application level errors still go through: if a wrong method
is called, or with wrong parameters, xmlrpclib.Fault will be raised.
"""
try:
return self.__callable(*args, **kw)
except ProtocolError, e:
message = "%s %s" % (e.errcode, e.errmsg)
if e.errcode == -1:
message = "Connection refused"
raise ConversionError(
"Protocol error while contacting OOo conversion"
" server: %s" % (message))
def _resize(self, quality, width, height, format, resolution, frame):
"""Resize and resample photo."""
parameter_list = [
'convert', '-colorspace', 'sRGB', '-depth', '8', '-quality',
str(quality), '-geometry',
'%sx%s' % (width, height)
]
if format not in VALID_TRANSPARENT_IMAGE_FORMAT_LIST:
# ImageMagick way to remove transparent that works with multiple
# images. http://www.imagemagick.org/Usage/masking/#remove
parameter_list += '-bordercolor', 'white', '-border', '0'
if resolution:
parameter_list += '-density', '%sx%s' % (resolution, resolution)
if frame is not None:
parameter_list.append('-[%s]' % frame)
else:
parameter_list.append('-')
if format:
parameter_list.append('%s:-' % format)
else:
parameter_list.append('-')
data = str(self.getData())
if self.getContentType() == "image/svg+xml":
data = transformUrlToDataURI(data)
process = subprocess.Popen(parameter_list,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
try:
# XXX: The only portable way is to pass what stdin.write can accept,
# which is a string for PIPE.
image, err = process.communicate(data)
finally:
del process
if image:
return StringIO(image)
raise ConversionError('Image conversion failed (%s).' % err)
def _convertToDJVU(self):
"""Convert the PDF text content to DJVU with pdf2djvu
"""
if not self.hasData():
return ''
tmp = tempfile.NamedTemporaryFile()
tmp.write(self.getData())
tmp.seek(0)
command_result = None
try:
command = ['pdf2djvu', tmp.name]
try:
command_result = Popen(command, stdout=PIPE).communicate()[0]
except OSError, e:
if e.errno == errno.ENOENT:
raise ConversionError('pdf2djvu was not found')
raise
finally:
tmp.close()
return command_result
def convert(self, orig, data, context=None, **kwargs):
server_proxy = DocumentConversionServerProxy(context)
source_mimetype = self._getAllowedSourceMimetypeFromConversionServer(
server_proxy)
if source_mimetype is None:
raise ConversionError(
"Format(s) not allowed on Conversion Server %r" % self.inputs)
source_format = self._getFormatFromMimetype(source_mimetype)
destination_format = self._getFormatFromMimetype(self.output)
data.setData(
dec(
server_proxy.convertFile(
enc(orig),
source_format,
destination_format,
# Default values are ConversionServer default ones
kwargs.get('zip', False),
kwargs.get('refresh', False),
kwargs.get('conversion_kw', {}))))
return data
def _convert(self, format, substitution_method_parameter_dict=None,
safe_substitute=True, charset=None, text_content=None, substitute=True, **kw):
"""
Convert text using portal_transforms or oood
"""
# XXX 'or DEFAULT_CONTENT_TYPE' is compaptibility code used for old
# web_page that have neither content_type nor text_format. Migration
# should be done to make all web page having content_type property
src_mimetype = self.getContentType() or DEFAULT_CONTENT_TYPE
if not format and src_mimetype == 'text/html':
format = 'html' # Force safe_html
if not format:
# can return document without conversion
return src_mimetype, self.getTextContent()
portal = self.getPortalObject()
mime_type = portal.mimetypes_registry.lookupExtension('name.%s' % format)
original_mime_type = mime_type = str(mime_type)
if text_content is None:
# check if document has set text_content and convert if necessary
text_content = self.getTextContent()
if text_content:
kw['format'] = format
convert_kw = {}
# PortalTransforms does not accept empty values for 'encoding' parameter
if charset:
kw['charset'] = convert_kw['encoding'] = charset
if not self.hasConversion(**kw):
portal_transforms = portal.portal_transforms
filename = self.getFilename()
if mime_type == 'text/html':
mime_type = 'text/x-html-safe'
if src_mimetype != "image/svg+xml":
result = portal_transforms.convertToData(mime_type, text_content,
object=self, context=self,
filename=filename,
mimetype=src_mimetype,
**convert_kw)
if result is None:
raise ConversionError('TextDocument conversion error. '
'portal_transforms failed to convert '
'from %r to %s: %r' %
(src_mimetype, mime_type, self))
else:
result = text_content
if format in VALID_IMAGE_FORMAT_LIST:
# Include extra parameter for image conversions
temp_image = self.portal_contributions.newContent(
portal_type='Image',
file=cStringIO.StringIO(),
filename=self.getId(),
temp_object=1)
temp_image._setData(result)
mime, result = temp_image.convert(**kw)
self.setConversion(result, original_mime_type, **kw)
else:
mime_type, result = self.getConversion(**kw)
if substitute and format in VALID_TEXT_FORMAT_LIST:
# only textual content can be sustituted
if substitution_method_parameter_dict is None:
substitution_method_parameter_dict = {}
result = self._substituteTextContent(result, safe_substitute=safe_substitute,
**substitution_method_parameter_dict)
return original_mime_type, result
else:
# text_content is not set, return empty string instead of None
return original_mime_type, ''
def _convert(self, format, frame=0, **kw):
"""Convert the document to the given format.
If a conversion is already stored for this format, it is returned
directly, otherwise the conversion is stored for the next time.
frame: Only used for image conversion
XXX Cascading conversions must be delegated to conversion server,
not by OOoDocument._convert (ie: convert to pdf, then convert to image, then resize)
*OR* as an optimisation we can read cached intermediate conversions
instead of compute them each times.
1- odt->pdf->png
2- odt->cached(pdf)->jpg
"""
#XXX if document is empty, stop to try to convert.
#XXX but I don't know what is a appropriate mime-type.(Yusei)
if not self.hasData():
return 'text/plain', ''
# if no conversion asked (format empty)
# return raw data
if not format:
return self.getContentType(), self.getData()
# Check if we have already a base conversion
if not self.hasBaseData():
# XXX please pass a meaningful description of error as argument
raise NotConvertedError()
# Make sure we can support html and pdf by default
is_html = 0
requires_pdf_first = 0
original_format = format
allowed_format_list = self.getTargetFormatList()
if format == 'base-data':
return self.getBaseContentType(), str(self.getBaseData())
if format == 'pdf':
format_list = [x for x in allowed_format_list if x.endswith('pdf')]
format = format_list[0]
elif format in VALID_IMAGE_FORMAT_LIST:
format_list = [
x for x in allowed_format_list if x.endswith(format)
]
if len(format_list):
format = format_list[0]
else:
# We must fist make a PDF which will be used to produce an image out of it
requires_pdf_first = 1
format_list = [
x for x in allowed_format_list if x.endswith('pdf')
]
format = format_list[0]
elif format == 'html':
format_list = [
x for x in allowed_format_list
if x.startswith('html') or x.endswith('html')
]
format = format_list[0]
is_html = 1
elif format in ('txt', 'text', 'text-content'):
# if possible, we try to get utf8 text. ('enc.txt' will encode to utf8)
if 'enc.txt' in allowed_format_list:
format = 'enc.txt'
elif format not in allowed_format_list:
#Text conversion is not supported by oood, do it in other way
if not self.hasConversion(format=original_format):
#Do real conversion for text
mime, data = self._getConversionFromProxyServer(
format='text-content')
self.setConversion(data, mime, format=original_format)
return mime, data
return self.getConversion(format=original_format)
# Raise an error if the format is not supported
if not self.isTargetFormatAllowed(format):
raise ConversionError(
"OOoDocument: target format %s is not supported" % format)
has_format = self.hasConversion(format=original_format, **kw)
if not has_format:
# Do real conversion
mime, data = self._getConversionFromProxyServer(format)
if is_html:
# Extra processing required since
# we receive a zip file
cs = cStringIO.StringIO()
cs.write(str(data))
z = zipfile.ZipFile(
cs) # A disk file would be more RAM efficient
for f in z.infolist():
fn = f.filename
if fn.endswith('html'):
if self.getPortalType() == 'Presentation'\
and not (fn.find('impr') >= 0):
continue
data = z.read(fn)
break
mime = 'text/html'
self._populateConversionCacheWithHTML(
zip_file=z) # Maybe some parts should be asynchronous for
# better usability
z.close()
cs.close()
if original_format not in VALID_IMAGE_FORMAT_LIST \
and not requires_pdf_first:
self.setConversion(data, mime, format=original_format, **kw)
else:
# create temporary image and use it to resize accordingly
temp_image = self.portal_contributions.newContent(
portal_type='Image',
file=cStringIO.StringIO(),
filename=self.getId(),
temp_object=1)
temp_image._setData(data)
# we care for first page only but as well for image quality
mime, data = temp_image.convert(original_format,
frame=frame,
**kw)
# store conversion
self.setConversion(data, mime, format=original_format, **kw)
return self.getConversion(format=original_format, **kw)
# avoid using same server again
global_server_proxy_uri_failure_time[uri] = int(DateTime())
# All servers are failed
if count == self._ooo_server_retry or len(retry_server_list) == 0:
break
count += 1
serverproxy_list = retry_server_list
# Check error type
# Return only one error result for compability
if len(result_error_set_list):
return result_error_set_list[0]
if len(protocol_error_list):
raise ConversionError("Protocol error while contacting OOo conversion: "
"%s" % (','.join(protocol_error_list)))
if len(socket_error_list):
raise SocketError("%s" % (','.join(socket_error_list)))
if len(fault_error_list):
raise fault_error_list[0]
def __getattr__(self, attr):
return partial(self._proxy_function, attr)
class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixin, File,
TextConvertableMixin, Document):
"""
A file document able to convert OOo compatible files to
any OOo supported format, to capture metadata and to
update metadata in OOo documents.