def test_round_trip_rss():
now = datetime.utcnow().replace(microsecond=0)
feed = rss_gen.RSS2(
title='Feed Title',
link='http://example.com/link/',
description='feed description',
lastBuildDate=now,
items=[
rss_gen.RSSItem(
title='Item Title',
link='http://example.com/1/',
description='item description',
pubDate=now - timedelta(seconds=5),
),
rss_gen.RSSItem(
title='Second Item Title',
link='http://example.com/2/',
description='another item description',
pubDate=now - timedelta(seconds=10),
)
]
)
pseudofile = BytesIO()
feed.write_xml(pseudofile, encoding='utf-8')
pseudofile.flush()
pseudofile.seek(0)
parsed = parse_rss(pseudofile)
assert_feeds_equal(feed, parsed)
class Collector(object):
"""
Collector for map and reduce output values
"""
def __init__(self, scheme=None, outputClient=None):
"""
Parameters
---------------------------------------------
scheme - The scheme for the datums to output - can be a json string
- or an instance of Schema
outputClient - The output client used to send messages to the parent
"""
if not (isinstance(scheme, schema.Schema)):
scheme = schema.parse(scheme)
if (outputClient is None):
raise ValueError("output client can't be none.")
self.scheme = scheme
self.buff = StringIO()
self.encoder = avio.BinaryEncoder(self.buff)
self.datum_writer = avio.DatumWriter(writers_schema=self.scheme)
self.outputClient = outputClient
def collect(self, record, partition=None):
"""Collect a map or reduce output value
Parameters
------------------------------------------------------
record - The record to write
partition - Indicates the partition for a pre-partitioned map output
- currently not supported
"""
self.buff.truncate(0)
self.datum_writer.write(record, self.encoder)
self.buff.flush()
self.buff.seek(0)
# delete all the data in the buffer
if (partition is None):
# TODO: Is there a more efficient way to read the data in self.buff?
# we could use self.buff.read() but that returns the byte array as a string
# will that work? We can also use self.buff.readinto to read it into
# a bytearray but the byte array must be pre-allocated
# self.outputClient.output(self.buff.buffer.read())
#its not a StringIO
self.outputClient.request("output", {"datum": self.buff.read()})
else:
self.outputClient.request("outputPartitioned", {
"datum": self.buff.read(),
"partition": partition
})
class UniversalBytesIO(object):
def __init__(self, container=None, charset=None):
self.charset = charset or settings.DEFAULT_CHARSET
self._container = BytesIO() if container is None else container
# These methods partially implement the file-like object interface.
# See https://docs.python.org/3/library/io.html#io.IOBase
def close(self):
self._container.close()
def write(self, content):
self._container.write(self.make_bytes(content))
def flush(self):
self._container.flush()
def tell(self):
return self._container.tell()
def readable(self):
return False
def seekable(self):
return False
def writable(self):
return True
def writelines(self, lines):
for line in lines:
self.write(line)
def make_bytes(self, value):
"""Turn a value into a bytestring encoded in the output charset."""
if isinstance(value, bytes):
return bytes(value)
if isinstance(value, six.text_type):
return bytes(value.encode(self.charset))
# Handle non-string types
return force_bytes(value, self.charset)
def get_string_value(self):
return self._container.getvalue().decode(self.charset)
def getvalue(self):
return self._container.getvalue()
if sys.version_info[0:2] < (3, 5):
def seek(self, *args, **kwargs):
pass
class UniversalBytesIO(object):
def __init__(self, container=None, charset=None):
self.charset = charset or settings.DEFAULT_CHARSET
self._container = BytesIO() if container is None else container
# These methods partially implement the file-like object interface.
# See https://docs.python.org/3/library/io.html#io.IOBase
def close(self):
self._container.close()
def write(self, content):
self._container.write(self.make_bytes(content))
def flush(self):
self._container.flush()
def tell(self):
return self._container.tell()
def readable(self):
return False
def seekable(self):
return False
def writable(self):
return True
def writelines(self, lines):
for line in lines:
self.write(line)
def make_bytes(self, value):
"""Turn a value into a bytestring encoded in the output charset."""
if isinstance(value, bytes):
return bytes(value)
if isinstance(value, six.text_type):
return bytes(value.encode(self.charset))
# Handle non-string types
return force_bytes(value, self.charset)
def get_string_value(self):
return self._container.getvalue().decode(self.charset)
def getvalue(self):
return self._container.getvalue()
if sys.version_info[0:2] < (3, 5):
def seek(self, *args, **kwargs):
pass
def process(self, request, ids):
if isinstance(ids, six.string_types) and ids == "all":
return JsonResponse(
{"error": ugettext("Selecting all is not supported.")})
shipment_ids = set(
Shipment.objects.filter(order_id__in=ids).values_list("id",
flat=True))
if len(shipment_ids) == 1:
try:
response = get_delivery_pdf(request, ids[0])
response[
'Content-Disposition'] = 'attachment; filename=shipment_%s_delivery.pdf' % ids[
0]
return response
except Exception as e:
msg = e.message if hasattr(e, "message") else e
return JsonResponse({"error": force_text(msg)})
buff = BytesIO()
archive = zipfile.ZipFile(buff, 'w', zipfile.ZIP_DEFLATED)
added = 0
errors = []
for id in shipment_ids:
try:
pdf_file = get_delivery_pdf(request, id)
filename = "shipment_%d_delivery.pdf" % id
archive.writestr(filename, pdf_file.content)
added += 1
except Exception as e:
msg = e.message if hasattr(e, "message") else e
errors.append(force_text(msg))
continue
if added:
archive.close()
buff.flush()
ret_zip = buff.getvalue()
buff.close()
response = HttpResponse(content_type='application/zip')
response[
'Content-Disposition'] = 'attachment; filename=order_delivery_pdf.zip'
response.write(ret_zip)
return response
return JsonResponse({"errors": errors})
def process(self, request, ids):
if isinstance(ids, six.string_types) and ids == "all":
return JsonResponse(
{"error": ugettext("Error! Selecting all is not supported.")},
status=400)
if len(ids) == 1:
try:
response = get_confirmation_pdf(request, ids[0])
response[
'Content-Disposition'] = 'attachment; filename=order_%s_confirmation.pdf' % ids[
0]
return response
except Exception as e:
msg = e.message if hasattr(e, "message") else e
return JsonResponse({"error": force_text(msg)}, status=400)
buff = BytesIO()
archive = zipfile.ZipFile(buff, 'w', zipfile.ZIP_DEFLATED)
added = 0
errors = []
for id in ids:
try:
pdf_file = get_confirmation_pdf(request, id)
filename = "order_%d_confirmation.pdf" % id
archive.writestr(filename, pdf_file.content)
added += 1
except Exception as e:
msg = e.message if hasattr(e, "message") else e
errors.append(force_text(msg))
continue
if added:
archive.close()
buff.flush()
ret_zip = buff.getvalue()
buff.close()
response = HttpResponse(content_type='application/zip')
response[
'Content-Disposition'] = 'attachment; filename=order_confirmation_pdf.zip'
response.write(ret_zip)
return response
return JsonResponse({"errors": errors}, status=400)
def process(self, request, ids):
if isinstance(ids, six.string_types) and ids == "all":
return JsonResponse({"error": ugettext("Selecting all is not supported.")})
shipment_ids = set(Shipment.objects.filter(order_id__in=ids).values_list("id", flat=True))
if len(shipment_ids) == 1:
try:
shipment_id = shipment_ids.pop()
response = get_delivery_pdf(request, shipment_id)
response['Content-Disposition'] = 'attachment; filename=shipment_%s_delivery.pdf' % shipment_id
return response
except Exception as e:
msg = e.message if hasattr(e, "message") else e
return JsonResponse({"error": force_text(msg)})
buff = BytesIO()
archive = zipfile.ZipFile(buff, 'w', zipfile.ZIP_DEFLATED)
added = 0
errors = []
for id in shipment_ids:
try:
pdf_file = get_delivery_pdf(request, id)
filename = "shipment_%d_delivery.pdf" % id
archive.writestr(filename, pdf_file.content)
added += 1
except Exception as e:
msg = e.message if hasattr(e, "message") else e
errors.append(force_text(msg))
continue
if added:
archive.close()
buff.flush()
ret_zip = buff.getvalue()
buff.close()
response = HttpResponse(content_type='application/zip')
response['Content-Disposition'] = 'attachment; filename=order_delivery_pdf.zip'
response.write(ret_zip)
return response
return JsonResponse({"errors": errors})
def process(self, request, ids):
if len(ids) == 1:
try:
response = get_delivery_pdf(request, ids[0])
response[
'Content-Disposition'] = 'attachment; filename=order_%s_delivery.pdf' % ids[
0]
return response
except Exception as e:
msg = e.message if hasattr(e, "message") else e
return JsonResponse({"error": force_text(msg)})
buff = BytesIO()
archive = zipfile.ZipFile(buff, 'w', zipfile.ZIP_DEFLATED)
added = 0
errors = []
for id in ids:
try:
pdf_file = get_delivery_pdf(request, id)
filename = "order_%d_delivery.pdf" % id
archive.writestr(filename, pdf_file.content)
added += 1
except Exception as e:
msg = e.message if hasattr(e, "message") else e
errors.append(force_text(msg))
continue
if added:
archive.close()
buff.flush()
ret_zip = buff.getvalue()
buff.close()
response = HttpResponse(content_type='application/zip')
response[
'Content-Disposition'] = 'attachment; filename=order_delivery_pdf.zip'
response.write(ret_zip)
return response
return JsonResponse({"errors": errors})
def process(self, request, ids):
if isinstance(ids, six.string_types) and ids == "all":
return JsonResponse({"error": ugettext("Selecting all is not supported.")}, status=400)
if len(ids) == 1:
try:
response = get_confirmation_pdf(request, ids[0])
response['Content-Disposition'] = 'attachment; filename=order_%s_confirmation.pdf' % ids[0]
return response
except Exception as e:
msg = e.message if hasattr(e, "message") else e
return JsonResponse({"error": force_text(msg)}, status=400)
buff = BytesIO()
archive = zipfile.ZipFile(buff, 'w', zipfile.ZIP_DEFLATED)
added = 0
errors = []
for id in ids:
try:
pdf_file = get_confirmation_pdf(request, id)
filename = "order_%d_confirmation.pdf" % id
archive.writestr(filename, pdf_file.content)
added += 1
except Exception as e:
msg = e.message if hasattr(e, "message") else e
errors.append(force_text(msg))
continue
if added:
archive.close()
buff.flush()
ret_zip = buff.getvalue()
buff.close()
response = HttpResponse(content_type='application/zip')
response['Content-Disposition'] = 'attachment; filename=order_confirmation_pdf.zip'
response.write(ret_zip)
return response
return JsonResponse({"errors": errors}, status=400)
class TeeInput(object):
CHUNK_SIZE = conn.CHUNK_SIZE
def __init__(self, stream):
self.buf = StringIO()
self.eof = False
if isinstance(stream, six.string_types):
stream = StringIO(stream)
self.tmp = StringIO()
else:
self.tmp = tempfile.TemporaryFile()
self.stream = stream
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, traceback):
return
def seek(self, offset, whence=0):
""" naive implementation of seek """
current_size = self._tmp_size()
diff = 0
if whence == 0:
diff = offset - current_size
elif whence == 2:
diff = (self.tmp.tell() + offset) - current_size
elif whence == 3 and not self.eof:
# we read until the end
while True:
self.tmp.seek(0, 2)
if not self._tee(self.CHUNK_SIZE):
break
if not self.eof and diff > 0:
self._ensure_length(StringIO(), diff)
self.tmp.seek(offset, whence)
def flush(self):
self.tmp.flush()
def read(self, length=-1):
""" read """
if self.eof:
return self.tmp.read(length)
if length < 0:
buf = StringIO()
buf.write(self.tmp.read())
while True:
chunk = self._tee(self.CHUNK_SIZE)
if not chunk:
break
buf.write(chunk)
return buf.getvalue()
else:
dest = StringIO()
diff = self._tmp_size() - self.tmp.tell()
if not diff:
dest.write(self._tee(length))
return self._ensure_length(dest, length)
else:
l = min(diff, length)
dest.write(self.tmp.read(l))
return self._ensure_length(dest, length)
def readline(self, size=-1):
if self.eof:
return self.tmp.readline()
orig_size = self._tmp_size()
if self.tmp.tell() == orig_size:
if not self._tee(self.CHUNK_SIZE):
return ''
self.tmp.seek(orig_size)
# now we can get line
line = self.tmp.readline()
if line.find("\n") >=0:
return line
buf = StringIO()
buf.write(line)
while True:
orig_size = self.tmp.tell()
data = self._tee(self.CHUNK_SIZE)
if not data:
break
self.tmp.seek(orig_size)
buf.write(self.tmp.readline())
if data.find("\n") >= 0:
break
return buf.getvalue()
def readlines(self, sizehint=0):
total = 0
lines = []
line = self.readline()
while line:
lines.append(line)
total += len(line)
if 0 < sizehint <= total:
break
line = self.readline()
return lines
def close(self):
if not self.eof:
# we didn't read until the end
self._close_unreader()
return self.tmp.close()
def next(self):
r = self.readline()
if not r:
raise StopIteration
return r
__next__ = next
def __iter__(self):
return self
def _tee(self, length):
""" fetch partial body"""
buf2 = self.buf
buf2.seek(0, 2)
chunk = self.stream.read(length)
if chunk:
self.tmp.write(chunk)
self.tmp.flush()
self.tmp.seek(0, 2)
return chunk
self._finalize()
return ""
def _finalize(self):
""" here we wil fetch final trailers
if any."""
self.eof = True
def _tmp_size(self):
if hasattr(self.tmp, 'fileno'):
return int(os.fstat(self.tmp.fileno())[6])
else:
return len(self.tmp.getvalue())
def _ensure_length(self, dest, length):
if len(dest.getvalue()) < length:
data = self._tee(length - len(dest.getvalue()))
dest.write(data)
return dest.getvalue()
class File(ClarityElement):
"""
This is a file in Clarity. It is also a Python file (more or less).
You can read, write, and do everything else you can normally do with a Python file.
NOTE: nothing will be committed to Clarity until you call close, or commit.
"""
UNIVERSAL_TAG = "{http://genologics.com/ri/file}file"
def __init__(self, lims, uri=None, xml_root=None, name=None, limsid=None):
super(File, self).__init__(lims, uri, xml_root, name, limsid)
self._data = None
self._dirty = False
self.content_type = 'text/plain'
self.writeable = True
self.only_write_locally = False
self.mode = "r"
@classmethod
def new_empty(cls, attachment_point_element, name=None):
"""
Create a new empty :class:`File`.
:param attachment_point_element: An element to attach the file to.
:type attachment_point_element: ClarityElement
:param name: A name for the file.
:type name: str
:rtype: File
"""
root = ETree.Element(cls.UNIVERSAL_TAG)
f = File(uri=None, xml_root=root, lims=attachment_point_element.lims)
if name is not None:
f.name = name
f.attached_to = attachment_point_element.uri
return f
@classmethod
def new_from_local(cls, attachment_point_element, local_file_path, mode="r+b"):
"""
Create a new :class:`File` from a local file.
:param attachment_point_element: An element to attach the file to.
:type attachment_point_element: ClarityElement
:param local_file_path: Path to the local file.
:type local_file_path: str
:param mode: Mode to open the file with.
:type mode: str
:rtype: File
"""
root = ETree.Element(cls.UNIVERSAL_TAG)
f = File(uri=None, xml_root=root, lims=attachment_point_element.lims)
f.name = local_file_path
f.attached_to = attachment_point_element.uri
f._data = open(local_file_path, mode)
f._dirty = True
return f
name = subnode_property('original-location')
attached_to = subnode_property('attached-to')
content_location = subnode_property('content-location')
is_published = subnode_property('is-published', typename=types.BOOLEAN)
@property
def is_binary_mode(self):
"""
:type: bool
"""
return "b" in self.mode
def pipe_to(self, target_file_object):
"""
:raises FileNotFoundException: if the file does not exist in Clarity.
"""
response = self.lims.raw_request('GET', self.uri + '/download')
self.content_type = response.headers.get("Content-Type")
if self.is_binary_mode:
file_contents = response.content
else:
file_contents = response.content if isinstance(response.content, string_types) else response.text
target_file_object.write(file_contents)
def replace_and_commit_from_local(self, local_file_path, content_type='text/plain', mode="r+b"):
self.mode = mode
other_file = open(local_file_path, self.mode)
self.replace_and_commit(other_file, local_file_path, content_type)
other_file.close()
def replace_and_commit(self, stream, name, content_type='text/plain'):
if not self.writeable:
raise Exception("file not writeable")
self.name = name
self.data.write(stream.read())
self.content_type = content_type
self._dirty = True
self.commit()
@property
def data(self):
"""
:return: The file data IO stream.
:rtype: io.IOBase
"""
if self._data is None:
if self.only_write_locally:
pathstrippedname = os.path.basename(self.name)
if os.path.exists(self.name):
file_name = self.name
else:
file_name = pathstrippedname
self._data = open(file_name, self.mode)
else:
self._data = BytesIO() if self.is_binary_mode else StringIO()
if self.uri is not None:
try:
log.debug("Getting file contents from lims...")
# convenient!
self.pipe_to(self._data)
self._data.seek(0)
except FileNotFoundException:
log.debug("File not found at %s" % self.uri)
# this is ok, we just leave the buffer empty.
# uri = None means we will need a new uri, later, allocated through glsstorage.
self.uri = None
return self._data
# Implementation for standard io.IOBase methods to support being used as a file:
def read(self, n=-1):
return self.data.read(n)
def readline(self, length=None):
return self.data.readline(length)
def readlines(self, sizehint=0):
return self.data.readlines(sizehint)
def write(self, s):
if not self.writeable:
raise Exception("file not writeable")
self._dirty = True
return self.data.write(s)
def writelines(self, iterable):
if not self.writeable:
raise Exception("file not writeable")
self._dirty = True
return self.data.writelines(iterable)
def flush(self):
# don't do anything at all
return
def getvalue(self):
return self.data.getvalue()
def truncate(self, size=None):
if not self.writeable:
raise Exception("file not writeable")
self._dirty = True
if size is None and self._data is None:
self._data = BytesIO() if self.is_binary_mode else StringIO()
else:
self._data.truncate(size)
def tell(self):
return self.data.tell()
def isatty(self):
return False
def close(self):
"""
Commit the file and close the data stream.
"""
self.commit()
return self.data.close()
def __iter__(self):
return self.data.__iter__()
def seek(self, pos, mode=0):
return self.data.seek(pos, mode)
def readable(self):
return self.data.readable()
def writable(self):
return self.data.writable()
def seekable(self):
return self.data.seekable()
# end file-like functions
def seek_to_end(self):
return self.data.seek(0, 2)
def commit(self):
if not self.writeable or self._data is None:
return
if self.only_write_locally:
self._data.flush()
return
if self.name is None:
raise Exception("Value for .name required.")
if self.uri is not None:
# If we are overwriting an existing file, first delete to
# allow name to be changed.
self.lims.raw_request('DELETE', self.uri)
self.uri = None
# first we get an allocation from glsstorage
self.post_and_parse(self.lims.root_uri + '/glsstorage')
# then we post ourselves to files, which gives us a uri.
self.post_and_parse(self.lims.root_uri + '/files')
if self._dirty:
old_pos = self.data.tell()
self.data.seek(0)
self.lims.raw_request('POST', self.uri + '/upload',
files={'file': (self.name, self.data, self.content_type)}
)
self._dirty = False
self.data.seek(old_pos)
class TeeInput(object):
CHUNK_SIZE = conn.CHUNK_SIZE
def __init__(self, stream):
self.buf = StringIO()
self.eof = False
if isinstance(stream, six.string_types):
stream = StringIO(stream)
self.tmp = StringIO()
else:
self.tmp = tempfile.TemporaryFile()
self.stream = stream
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, traceback):
return
def seek(self, offset, whence=0):
""" naive implementation of seek """
current_size = self._tmp_size()
diff = 0
if whence == 0:
diff = offset - current_size
elif whence == 2:
diff = (self.tmp.tell() + offset) - current_size
elif whence == 3 and not self.eof:
# we read until the end
while True:
self.tmp.seek(0, 2)
if not self._tee(self.CHUNK_SIZE):
break
if not self.eof and diff > 0:
self._ensure_length(StringIO(), diff)
self.tmp.seek(offset, whence)
def flush(self):
self.tmp.flush()
def read(self, length=-1):
""" read """
if self.eof:
return self.tmp.read(length)
if length < 0:
buf = StringIO()
buf.write(self.tmp.read())
while True:
chunk = self._tee(self.CHUNK_SIZE)
if not chunk:
break
buf.write(chunk)
return buf.getvalue()
else:
dest = StringIO()
diff = self._tmp_size() - self.tmp.tell()
if not diff:
dest.write(self._tee(length))
return self._ensure_length(dest, length)
else:
l = min(diff, length)
dest.write(self.tmp.read(l))
return self._ensure_length(dest, length)
def readline(self, size=-1):
if self.eof:
return self.tmp.readline()
orig_size = self._tmp_size()
if self.tmp.tell() == orig_size:
if not self._tee(self.CHUNK_SIZE):
return ''
self.tmp.seek(orig_size)
# now we can get line
line = self.tmp.readline()
if line.find("\n") >= 0:
return line
buf = StringIO()
buf.write(line)
while True:
orig_size = self.tmp.tell()
data = self._tee(self.CHUNK_SIZE)
if not data:
break
self.tmp.seek(orig_size)
buf.write(self.tmp.readline())
if data.find("\n") >= 0:
break
return buf.getvalue()
def readlines(self, sizehint=0):
total = 0
lines = []
line = self.readline()
while line:
lines.append(line)
total += len(line)
if 0 < sizehint <= total:
break
line = self.readline()
return lines
def close(self):
if not self.eof:
# we didn't read until the end
self._close_unreader()
return self.tmp.close()
def next(self):
r = self.readline()
if not r:
raise StopIteration
return r
__next__ = next
def __iter__(self):
return self
def _tee(self, length):
""" fetch partial body"""
buf2 = self.buf
buf2.seek(0, 2)
chunk = self.stream.read(length)
if chunk:
self.tmp.write(chunk)
self.tmp.flush()
self.tmp.seek(0, 2)
return chunk
self._finalize()
return ""
def _finalize(self):
""" here we wil fetch final trailers
if any."""
self.eof = True
def _tmp_size(self):
if hasattr(self.tmp, 'fileno'):
return int(os.fstat(self.tmp.fileno())[6])
else:
return len(self.tmp.getvalue())
def _ensure_length(self, dest, length):
if len(dest.getvalue()) < length:
data = self._tee(length - len(dest.getvalue()))
dest.write(data)
return dest.getvalue()
