def parse_attachment(self, message_part):
content_disposition = message_part.get("Content-Disposition", None)
if content_disposition:
dispo_type, dispo_dict = self.parse_dispositions(content_disposition)
if dispo_type == "attachment" or (dispo_type == 'inline' and
'filename' in dispo_dict):
file_data = message_part.get_payload(decode=True)
if file_data is None:
file_data = ""
attachment = StringIO(file_data)
attachment.content_type = message_part.get_content_type()
attachment.size = len(file_data)
attachment.name = None
attachment.create_date = None
attachment.mod_date = None
attachment.read_date = None
if "filename" in dispo_dict:
attachment.name = dispo_dict['filename']
else:
content_type = message_part.get("Content-Type", None)
if content_type:
_, content_dict = self.parse_dispositions(content_type)
if 'name' in content_dict:
attachment.name = content_dict['name']
if "create-date" in dispo_dict:
attachment.create_date = dispo_dict['create-date'] # TODO: datetime
if "modification-date" in dispo_dict:
attachment.mod_date = dispo_dict['modification-date'] # TODO: datetime
if "read-date" in dispo_dict:
attachment.read_date = dispo_dict['read-date'] # TODO: datetime
return attachment
return None
def parse_attachment(self, message_part):
content_disposition = message_part.get("Content-Disposition", None)
if content_disposition:
dispo_type, dispo_dict = self.parse_dispositions(
content_disposition)
if dispo_type == "attachment" or (dispo_type == 'inline'
and 'filename' in dispo_dict):
file_data = message_part.get_payload(decode=True)
if file_data is None:
file_data = ""
attachment = StringIO(file_data)
attachment.content_type = message_part.get_content_type()
attachment.size = len(file_data)
attachment.name = None
attachment.create_date = None
attachment.mod_date = None
attachment.read_date = None
if "filename" in dispo_dict:
attachment.name = self.parse_header_field(
dispo_dict['filename'])
elif "create-date" in dispo_dict:
attachment.create_date = dispo_dict[
'create-date'] # TODO: datetime
elif "modification-date" in dispo_dict:
attachment.mod_date = dispo_dict[
'modification-date'] # TODO: datetime
elif "read-date" in dispo_dict:
attachment.read_date = dispo_dict[
'read-date'] # TODO: datetime
return attachment
return None
def test_should_put_data_into_existing_worksheet_with_offset_for_excel_and_auto(self):
for excel_encoding in [True, False]:
csv = StringIO()
csv.write('abc,123\n')
csv.write('def, \n')
csv.size = 10
csv.seek(0)
existing_worksheet = Worksheet()
for row in range(1, 6):
for col in range(1, 5):
existing_worksheet[col, row].formula = 'old'
worksheet = worksheet_from_csv(existing_worksheet, csv, 2, 3, excel_encoding)
self.assertEquals(worksheet.A1.formula, 'old')
self.assertEquals(worksheet.B1.formula, 'old')
self.assertEquals(worksheet.A2.formula, 'old')
self.assertEquals(worksheet.B3.formula, 'abc')
self.assertEquals(worksheet.C3.formula, '123')
self.assertEquals(worksheet.B4.formula, 'def')
self.assertEquals(worksheet.C4.formula, ' ')
self.assertEquals(worksheet.C5.formula, 'old')
self.assertEquals(worksheet.D3.formula, 'old')
self.assertEquals(worksheet.B5.formula, 'old')
def wallpaper_thumbnail(sender, **kwargs):
w = kwargs["instance"]
from os import path
try:
wallpath = w.wallpaper.path
thumbpath = w.thumbnail.path
except ValueError:
thumbpath = path.splitext(wallpath)[0] + ".thumb.jpg"
try:
if path.getmtime(wallpath) <= path.getmtime(thumbpath):
return
except EnvironmentError:
pass
from PIL import Image
image = Image.open(w.wallpaper.path)
if image.mode not in ('L', 'RGB'):
image = image.convert('RGB')
image.thumbnail((220, 220), Image.ANTIALIAS)
from StringIO import StringIO
data = StringIO()
image.save(data, "JPEG")
data.size = data.tell()
data.seek(0, 0)
#data.name = path.basename(thumbpath)
w.thumbnail.save(path.basename(thumbpath), File(data))
def parse_attachment(self, message_part):
content_disposition = message_part.get("Content-Disposition", None)
if content_disposition:
dispositions = content_disposition.strip().split(";")
if bool(content_disposition and dispositions[0].lower() == "attachment"):
file_data = message_part.get_payload(decode=True)
attachment = StringIO()
attachment.write(file_data)
attachment.content_type = message_part.get_content_type()
attachment.size = len(file_data)
attachment.name = None
attachment.create_date = None
attachment.mod_date = None
attachment.read_date = None
for param in dispositions[1:]:
name,value = param.split("=")
name = name.lower()
attachment.name = value.replace('"', '')
return attachment
return None
def parse_attachments_poptres(self, content_disposition, part):
dispositions = content_disposition.strip().split(";")
if bool(content_disposition and dispositions[0].lower() == "attachment"):
file_data = part.get_payload(decode=True)
attachment = StringIO(file_data)
attachment.content_type = part.get_content_type()
attachment.size = len(file_data)
attachment.name = None
attachment.create_date = None
attachment.mod_date = None
attachment.read_date = None
for param in dispositions[1:]:
name, value = param.split("=")
name = name.lower().strip()
value = value.replace('"', '').strip()
if name == "filename":
attachment.name = value
elif name == "create-date":
attachment.create_date = value
elif name == "modification-date":
attachment.mod_date = value
elif name == "read-date":
attachment.read_date = value
attachment.seek(0, 2)
f = InMemoryUploadedFile(attachment, "", attachment.name, attachment.content_type, attachment.tell(), None)
atch = Attachment()
atch.user = self.usr
atch.file.save(attachment.name, f)
atch.save()
return atch
def parse_attachment(message_part):
content_disposition = message_part.get("Content-Disposition", None)
if content_disposition:
dispositions = content_disposition.strip().split(";")
if bool(content_disposition and dispositions[0].lower() == "attachment"):
file_data = message_part.get_payload(decode=True)
attachment = StringIO(file_data)
attachment.content_type = message_part.get_content_type()
attachment.size = len(file_data)
attachment.name = None
attachment.create_date = None
attachment.mod_date = None
attachment.read_date = None
for param in dispositions[1:]:
name,value = param.split("=")
name = name.lower()
if name == "filename":
attachment.name = value
elif name == "create-date":
attachment.create_date = value #TODO: datetime
elif name == "modification-date":
attachment.mod_date = value #TODO: datetime
elif name == "read-date":
attachment.read_date = value #TODO: datetime
return attachment
return None
def get_attachment(self, msg, attach_id):
"Get and return an attachment"
num = 0
attach_id = int(attach_id)
for part in msg.walk():
attachment = part.get_param('attachment',
NOTFOUND, 'Content-Disposition')
if not attachment is NOTFOUND:
filename = part.get_filename(None)
if filename:
filename = re.sub(r"\s", "_", filename)
num += 1
if attach_id == num:
from StringIO import StringIO
if part.is_multipart():
data = part.as_string()
else:
data = part.get_payload(decode=True)
attachment = StringIO(data)
attachment.content_type = part.get_content_type()
attachment.size = len(data)
attachment.name = filename
return attachment
return None
def parse_attachment(message_part):
content_disposition = message_part.get("Content-Disposition", None)
if content_disposition:
dispositions = content_disposition.strip().split(";")
if bool(content_disposition
and dispositions[0].lower() == "attachment"):
file_data = message_part.get_payload(decode=True)
# Used a StringIO object since PIL didn't seem to recognize
# images using a custom file-like object
attachment = StringIO(file_data)
attachment.content_type = message_part.get_content_type()
attachment.size = len(file_data)
attachment.name = None
attachment.create_date = None
attachment.mod_date = None
attachment.read_date = None
# print dispositions
for param in dispositions[1:]:
name, value = param.split("=")
name = name.strip().lower()
if name == "filename":
attachment.name = value
elif name in ["create-date", "creation-date"]:
attachment.create_date = value #TODO: datetime
elif name == "modification-date":
attachment.mod_date = value #TODO: datetime
elif name == "read-date":
attachment.read_date = value #TODO: datetime
return attachment
return None
def read(self, f): f.seek(self.offset, 0) assert f.read(4) == self.tag assert read32(f) == self.size sio = StringIO(f.read(self.size)) sio.tag = self.tag sio.rid = self.rid sio.size = self.size return sio
def test_excel_csv_import_survives_japanes_utf8(self):
some_kanji = u'\u65b0\u4e16\u7d00\u30a8\u30f4\u30a1\u30f3\u30b2\u30ea\u30aa\u30f3'
japanese_file = StringIO()
japanese_file.write(some_kanji.encode('utf-8'))
japanese_file.name = 'filename'
japanese_file.size = 10
japanese_file.seek(0)
worksheet_from_csv(Worksheet(), japanese_file, 1, 1, True)
def test_excel_csv_import_survives_japanes_utf8(self):
some_kanji = u'\u65b0\u4e16\u7d00\u30a8\u30f4\u30a1\u30f3\u30b2\u30ea\u30aa\u30f3'
japanese_file = StringIO()
japanese_file.write(some_kanji.encode('utf-8'))
japanese_file.name = 'filename'
japanese_file.size = 10
japanese_file.seek(0)
worksheet_from_csv(Worksheet(), japanese_file, 1, 1, True)
def test_autodetect_import_csv_raises_on_null_bytes(self):
bin_file = StringIO()
bin_file.write("\xFF\x00\xFF")
bin_file.name = 'filename'
bin_file.size = 10
bin_file.seek(0)
self.assertRaises(
DirigibleImportError,
lambda: worksheet_from_csv(Worksheet(), bin_file, 1, 1, False))
def test_autodetect_import_csv_raises_on_null_bytes(self):
bin_file = StringIO()
bin_file.write("\xFF\x00\xFF")
bin_file.name = 'filename'
bin_file.size = 10
bin_file.seek(0)
self.assertRaises(DirigibleImportError,
lambda : worksheet_from_csv(Worksheet(), bin_file, 1, 1, False)
)
def exchange(self):
# Encode form data
form = {}
for field, value in self._form.items():
form[field] = str(value).encode("UTF-8")
# Compress and add payload to form
payload = open(self._report, "r").read()
compressed_payload = bz2.compress(payload)
file = StringIO(compressed_payload)
file.name = "%s.xml.bz2" % str(gethostname())
file.size = len(compressed_payload)
form["field.submission_data"] = file
if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
logging.debug("Uncompressed payload length: %d", len(payload))
transport = HTTPTransport(self.transport_url)
start_time = time.time()
response = transport.exchange(form, self._headers,
timeout=string_to_type(self.timeout))
end_time = time.time()
if not response:
self._manager.reactor.fire("exchange-error", _("""\
Failed to contact server. Please try
again or upload the following file name:
%s
directly to the system database:
https://launchpad.net/+hwdb/+submit""") % posixpath.abspath(self._report))
return
elif response.status != 200:
self._manager.reactor.fire("exchange-error", _("""\
Failed to upload to server,
please try again later."""))
return
if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
logging.debug("Response headers:\n%s",
pprint.pformat(response.getheaders()))
header = response.getheader("x-launchpad-hwdb-submission")
if not header:
self._manager.reactor.fire("exchange-error",
_("Information not posted to Launchpad."))
elif "Error" in header:
# HACK: this should return a useful error message
self._manager.reactor.fire("exchange-error", header)
logging.error(header)
else:
text = response.read()
logging.info("Sent %d bytes and received %d bytes in %s.",
file.size, len(text), format_delta(end_time - start_time))
def test_autodetect_can_handle_japanese_utf8(self):
some_kanji = u'\u65b0\u4e16\u7d00\u30a8\u30f4\u30a1\u30f3\u30b2\u30ea\u30aa\u30f3'
japanese_file = StringIO()
japanese_file.write(some_kanji.encode('utf-8'))
japanese_file.name = 'filename'
japanese_file.size = 10
japanese_file.seek(0)
worksheet = worksheet_from_csv(Worksheet(), japanese_file, 1, 1, False)
self.assertEquals(worksheet.A1.formula, some_kanji)
def test_autodetect_can_handle_japanese_utf8(self):
some_kanji = u'\u65b0\u4e16\u7d00\u30a8\u30f4\u30a1\u30f3\u30b2\u30ea\u30aa\u30f3'
japanese_file = StringIO()
japanese_file.write(some_kanji.encode('utf-8'))
japanese_file.name = 'filename'
japanese_file.size = 10
japanese_file.seek(0)
worksheet = worksheet_from_csv(Worksheet(), japanese_file, 1, 1, False)
self.assertEquals(worksheet.A1.formula, some_kanji)
def test_autodetect_csv_import_handles_carriage_returns_in_cells(self):
excel_csv = StringIO()
excel_csv.write(u'"carriage\nreturn!"\r\n'.encode('utf-8'))
excel_csv.write(u"normal line\r\n".encode('utf-8'))
excel_csv.name = 'filename'
excel_csv.size = 10
excel_csv.seek(0)
worksheet = worksheet_from_csv(Worksheet(), excel_csv, 2, 3, False)
self.assertEquals(worksheet.B3.formula, "carriage\nreturn!")
self.assertEquals(worksheet.B4.formula, "normal line")
def test_excel_csv_import_recognises_accents_and_currency_symbols(self):
excel_csv = StringIO()
excel_csv.write(u"\xe9".encode('windows-1252'))
excel_csv.write(u"\xa3".encode('windows-1252'))
excel_csv.write(u"\u20ac".encode('windows-1252'))
excel_csv.name = 'filename'
excel_csv.size = 10
excel_csv.seek(0)
worksheet = worksheet_from_csv(Worksheet(), excel_csv, 3, 4, True)
self.assertEquals(worksheet.C4.formula, u"\xe9\xa3\u20ac")
def test_excel_csv_import_recognises_accents_and_currency_symbols(self):
excel_csv = StringIO()
excel_csv.write(u"\xe9".encode('windows-1252'))
excel_csv.write(u"\xa3".encode('windows-1252'))
excel_csv.write(u"\u20ac".encode('windows-1252'))
excel_csv.name = 'filename'
excel_csv.size = 10
excel_csv.seek(0)
worksheet = worksheet_from_csv(Worksheet(), excel_csv, 3, 4, True)
self.assertEquals(worksheet.C4.formula, u"\xe9\xa3\u20ac")
def test_autodetect_csv_import_handles_carriage_returns_in_cells(self):
excel_csv = StringIO()
excel_csv.write(u'"carriage\nreturn!"\r\n'.encode('utf-8'))
excel_csv.write(u"normal line\r\n".encode('utf-8'))
excel_csv.name = 'filename'
excel_csv.size = 10
excel_csv.seek(0)
worksheet = worksheet_from_csv(Worksheet(), excel_csv, 2, 3, False)
self.assertEquals(worksheet.B3.formula, "carriage\nreturn!")
self.assertEquals(worksheet.B4.formula, "normal line")
def test_autodetect_import_csv_raises_on_failure_to_detect_encoding(
self, mock_UniversalDetector):
mock_detector = Mock()
mock_UniversalDetector.return_value = mock_detector
mock_detector.result = {'encoding': None}
mock_file = StringIO()
mock_file.write("\xFF\x00\xFF")
mock_file.name = 'filename'
mock_file.size = 10
self.assertRaises(
DirigibleImportError,
lambda: worksheet_from_csv(Worksheet(), mock_file, 1, 1, False))
def test_autodetect_import_csv_raises_on_failure_to_detect_encoding(
self, mock_UniversalDetector
):
mock_detector = Mock()
mock_UniversalDetector.return_value = mock_detector
mock_detector.result = {'encoding':None}
mock_file = StringIO()
mock_file.write("\xFF\x00\xFF")
mock_file.name = 'filename'
mock_file.size = 10
self.assertRaises(DirigibleImportError,
lambda : worksheet_from_csv(Worksheet(), mock_file, 1, 1, False)
)
def setUp(self):
self.user = User.objects.create_user(username='******')
#we need a uimodel
self.uimodel = UIModel.objects.create(name='curriculum1',
version='1')
unitset_view_model = ViewMap.objects.create(
uimodel=self.uimodel,
name='unitset_view_model',
parent=None)
# Create a new unit view model
ViewMap.objects.create(
uimodel=self.uimodel,
name='unit_view_model',
parent=unitset_view_model)
#add a content collection (draft and edition)
self.draft = Draft.objects.create_draft('draft1', self.user)
content = [
{
'fname': 'foo.html',
'description': 'some html',
'path': '/content/html',
'content': '<head></head><body>foo</body>',
},
{
'fname': 'bar.html',
'description': 'some more html',
'path': '/content/html',
'content': '<head></head><body>bar</body>',
}, ]
edit = Edit.objects.create(user=self.user, source='upload',
draft=self.draft)
for item in content:
file_like = StringIO(item['content'])
file_like.name = item['fname']
file_like.size = len(item['content'])
# Create a new imported item instance.
ImportItem.objects.create(
name=item['fname'],
description=item['description'],
requested_path=item['path'],
contentfile=DjangoFile(file_like, name=item['fname']),
edit=edit)
# Create a new edition
self.draft.make_edition('edition1')
def parse_attachment(message_part):
content_disposition = message_part.get("Content-Disposition", None)
if content_disposition:
#print content_disposition
dispositions = content_disposition.strip().split(";")
if bool(content_disposition
and dispositions[0].lower() == "attachment") or bool(
content_disposition
and dispositions[0].lower() == "inline"):
file_data = message_part.get_payload(decode=True)
# Used a StringIO object since PIL didn't seem to recognize
# images using a custom file-like object
attachment = StringIO(file_data)
attachment.content_type = message_part.get_content_type()
attachment.size = len(file_data)
attachment.name = None
attachment.create_date = None
attachment.mod_date = None
attachment.read_date = None
for param in dispositions[1:]:
param = param.strip()
name, value = param.split("=", 1)
name = name.lower()
if name == "filename":
h = email.Header.Header(value.lstrip('"').rstrip('"'))
dh = email.Header.decode_header(h)
fname = dh[0][0]
if dh[0][1] != None:
fname = fname.decode(dh[0][1])
attachment.name = fname
#print fname
elif name == "create-date":
attachment.create_date = value #TODO: datetime
elif name == "modification-date":
attachment.mod_date = value #TODO: datetime
elif name == "read-date":
attachment.read_date = value #TODO: datetime
return attachment
return None
def parse_attachment(message_part, attachments=None):
content_disposition = message_part.get("Content-Disposition", None)
if content_disposition:
try:
cd = parse_headers(content_disposition, relaxed=True)
if cd.disposition.lower() == "attachment":
if not cd.assocs.has_key("filename"):
#print error or warning?
return None
else:
file_data = message_part.get_payload(decode=True)
if not file_data:
payload = message_part.get_payload()
if isinstance(payload, list):
for msgobj in payload:
parse2(msgobj, attachments)
return None
print >>sys.stderr, message_part.get_payload()
print >>sys.stderr, message_part.get_content_charset()
attachment = StringIO(file_data)
attachment.content_type = message_part.get_content_type()
attachment.size = len(file_data)
attachment.name = cd.assocs['filename']
attachment.create_date = None
attachment.mod_date = None
attachment.read_date = None
for name, value in cd.assocs.iteritems():
if name == "create-date":
attachment.create_date = value #TODO: datetime
elif name == "modification-date":
attachment.mod_date = value #TODO: datetime
elif name == "read-date":
attachment.read_date = value #TODO: datetime
return attachment
except:
print >>sys.stderr, "content_disposition:", content_disposition
raise
return None
def render_as_pdf(self, data, xml_path):
template = self.render_as_xml(data)
template_string = StringIO(template.encode('utf-8'))
tmp_xml_name = 'document_%s.xml' % 'A'
tmp_pdf_name = 'document_%s.pdf' % 'A'
file_manager = xml_path
template_string.name = tmp_xml_name
template_string.size = template_string.len
#tmp_xml_name = file_manager.save(tmp_xml_name, File(template_string))
xsl_file_path = 'contract.xsl'
print 'fop'
#print '-xml %s' % file_manager.path(tmp_xml_name)
print '-xsl %s' % xsl_file_path
print '-pdf %s' % tmp_pdf_name
args = [
'fop',
'-xml', file_manager,
'-xsl', xsl_file_path,
'-pdf', tmp_pdf_name
]
process = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
retcode = process.wait()
#file_manager.delete(tmp_xml_name)
if retcode is 0:
pdf = file_manager.open(tmp_pdf_name)
response = pdf.read()
file_manager.delete(tmp_pdf_name)
return response
else:
return None
def __init__(self, relative_source, requested_size, opts=None,
quality=None, basedir=None, subdir=None, prefix=None,
relative_dest=None, processors=None, extension=None, storage_server=None):
if not storage_server:
storage_server = DefaultStorage()
relative_source = force_unicode(relative_source)
# Set the absolute filename for the source file
with self._get_data_as_tempfile(relative_source, storage_server) as source:
quality = get_thumbnail_setting('QUALITY', quality)
convert_path = get_thumbnail_setting('CONVERT')
wvps_path = get_thumbnail_setting('WVPS')
if processors is None:
processors = dynamic_import(get_thumbnail_setting('PROCESSORS'))
# Call super().__init__ now to set the opts attribute. generate() won't
# get called because we are not setting the dest attribute yet.
super(DjangoThumbnail, self).__init__(source, requested_size,
opts=opts, quality=quality, convert_path=convert_path,
wvps_path=wvps_path, processors=processors)
# Get the relative filename for the thumbnail image, then set the
# destination filename
if relative_dest is None:
relative_dest = \
self._get_relative_thumbnail(relative_source.name, basedir=basedir,
subdir=subdir, prefix=prefix,
extension=extension)
with NamedTemporaryFile() as dest:
self.dest = dest.name
self.generate()
dest.seek(0)
data = dest.read()
f = StringIO(data)
f.name = relative_dest
f.size = len(data)
self.relative_url = storage_server.save(relative_dest, File(f))
self.absolute_url = os.path.join(settings.MEDIA_URL, self.relative_url)
def read(self, f): if self.tag == "CFTC": return None f.seek(self.offset, 0) assert f.read(4).upper() == self.tag assert read32(f, True) == self.size assert read32(f, True) == self.rid name = None self.size = self.size - 4 name = readString(f) if len(name): print self.rid, self.tag, name self.size = self.size - len(name) - 1 if ((f.tell() - self.offset) & 1) != 0: f.read(1) # padding self.size = self.size - 1 sio = StringIO(f.read(self.size)) sio.tag = self.tag sio.rid = self.rid sio.name = name sio.size = self.size return sio
def parse_attachment(message_part, attachments=None):
content_disposition = message_part.get("Content-Disposition", None)
if content_disposition:
try:
cd = parse_headers(content_disposition, relaxed=True)
if cd.disposition.lower() == "attachment":
if not "filename" in cd.assocs:
#print error or warning?
return None
else:
file_data = message_part.get_payload(decode=True)
if not file_data:
payload = message_part.get_payload()
if isinstance(payload, list):
for msgobj in payload:
_parse2(msgobj, attachments)
return None # PSIPHON: fixed conditional return
attachment = StringIO(file_data)
attachment.content_type = message_part.get_content_type()
attachment.size = len(file_data)
attachment.name = cd.assocs['filename']
attachment.create_date = None
attachment.mod_date = None
attachment.read_date = None
for name, value in cd.assocs.iteritems():
if name == "create-date":
attachment.create_date = value # TODO: datetime
elif name == "modification-date":
attachment.mod_date = value # TODO: datetime
elif name == "read-date":
attachment.read_date = value # TODO: datetime
return attachment
except:
print >> sys.stderr, "content_disposition:", content_disposition
raise
return None
def fetch_avatar(self):
"""Fetches avatar from Twitter.
"""
for auth in self.user.social_auth.all():
avatar_url = None
username = None
if auth.provider == 'twitter':
profile_url = 'http://api.twitter.com/1/users/lookup.json?user_id={id}&include_entities=true'.format(
**auth.extra_data
)
response = requests.get(profile_url, timeout=3)
data = simplejson.loads(response.content)
if not data[0]['default_profile']:
# Download only customized profile images
username = data[0]['screen_name']
avatar_url = 'http://api.twitter.com/1/users/profile_image?screen_name={username}&size=original'.format(
username=username
)
elif auth.provider == 'facebook':
username = auth.extra_data['id']
avatar_url = 'https://graph.facebook.com/{id}/picture?type=large'.format(
**auth.extra_data
)
if username and avatar_url:
response = requests.get(avatar_url, timeout=3)
if response.status_code == 200:
fake_file = StringIO(response.content)
fake_file.size = len(response.content)
self.avatar = File(
fake_file,
name=username,
)
self.save()
break
def parse_attachments_poptres(self, content_disposition, part):
dispositions = content_disposition.strip().split(";")
if bool(content_disposition
and dispositions[0].lower() == "attachment"):
file_data = part.get_payload(decode=True)
attachment = StringIO(file_data)
attachment.content_type = part.get_content_type()
attachment.size = len(file_data)
attachment.name = None
attachment.create_date = None
attachment.mod_date = None
attachment.read_date = None
for param in dispositions[1:]:
name, value = param.split("=")
name = name.lower().strip()
value = value.replace('"', '').strip()
if name == "filename":
attachment.name = value
elif name == "create-date":
attachment.create_date = value
elif name == "modification-date":
attachment.mod_date = value
elif name == "read-date":
attachment.read_date = value
attachment.seek(0, 2)
f = InMemoryUploadedFile(attachment, "",
attachment.name, attachment.content_type,
attachment.tell(), None)
atch = Attachment()
atch.user = self.usr
atch.file.save(attachment.name, f)
atch.save()
return atch
def __init__(self, files):
"""FakeFilesData constructor.
This constructor expects a single argument, a sequence of (name,spec)
pairs specifying the fake upload files to be encoded.
"""
# Determine the MIME encoding boundary
try:
boundary = settings.FAKEUPLOAD_MIME_BOUNDARY
except AttributeError:
boundary = "----------thisisthemimeboundary"
# Construct each encoded file
self._files = [FakeFileData(f[0], f[1], boundary) for f in files]
# Add the end-of-request footer
footer = StringIO("--%s--\r\n" % (boundary, ))
footer.size = len(footer.getvalue())
self._files.append(footer)
# Construct the request headers
size = sum([f.size for f in self._files])
type = "multipart/form-data; boundary=%s" % (boundary, )
self.META = {"HTTP_CONTENT_LENGTH": size, "HTTP_CONTENT_TYPE": type}
# Internal read-ahead buffer
self._buffer = ""
def __init__(self,files):
"""FakeFilesData constructor.
This constructor expects a single argument, a sequence of (name,spec)
pairs specifying the fake upload files to be encoded.
"""
# Determine the MIME encoding boundary
try:
boundary = settings.FAKEUPLOAD_MIME_BOUNDARY
except AttributeError:
boundary = "----------thisisthemimeboundary"
# Construct each encoded file
self._files = [FakeFileData(f[0],f[1],boundary) for f in files]
# Add the end-of-request footer
footer = StringIO("--%s--\r\n" % (boundary,))
footer.size = len(footer.getvalue())
self._files.append(footer)
# Construct the request headers
size = sum([f.size for f in self._files])
type = "multipart/form-data; boundary=%s" % (boundary,)
self.META = {"HTTP_CONTENT_LENGTH":size,"HTTP_CONTENT_TYPE":type}
# Internal read-ahead buffer
self._buffer = ""
def launchpad_exchange(self):
# Maybe on the next exchange...
if not self._form["field.system"] \
or self._form["field.submission_data"]:
return
# Compress and add payload to form
payload = open(self._launchpad_report, "r").read()
compressed_payload = bz2.compress(payload)
file = StringIO(compressed_payload)
file.name = "%s.xml.bz2" % str(gethostname())
file.size = len(compressed_payload)
self._form["field.submission_data"] = file
if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
logging.debug("Uncompressed payload length: %d", len(payload))
# Encode form data
try:
form = FORM.coerce(self._form)
except ValueError, e:
self._manager.reactor.fire("exchange-error", _("""\
Failed to process form: %s""" % e))
return
def get_attachment(self, msg, attach_id):
"Get and return an attachment"
num = 0
attach_id = int(attach_id)
for part in msg.walk():
attachment = part.get_param('attachment',
NOTFOUND, 'Content-Disposition')
if not attachment is NOTFOUND:
filename = part.get_filename(None)
if filename:
filename = filename.replace(' ', '_')
num += 1
if attach_id == num:
if part.is_multipart():
data = part.as_string()
else:
data = part.get_payload(decode=True)
attachment = StringIO(data)
attachment.content_type = part.get_content_type()
attachment.size = len(data)
attachment.name = filename
return attachment
return None
ftp.cwd('OMSO2NRTb/')
filenames = ftp.nlst()
#print(filenames) # list directory contents
for filename in filenames:
if filename[-3:] == 'he5':
try:
nasa_data_file = NasaDataFile.objects.get(hdf='./' + filename)
except Exception:
nasa_data_file = NasaDataFile()
print(filename)
s = StringIO()
ftp.retrbinary("RETR " + filename, s.write)
s.size = s.tell()
nasa_data_file.hdf.save(filename, File(s))
if nasa_data_file.xml.name == '' or not nasa_data_file.xml.name:
xml_filename = filename + '.met'
print(xml_filename)
s = StringIO()
ftp.retrbinary("RETR " + xml_filename, s.write)
s.size = s.tell()
nasa_data_file.xml.save(xml_filename, File(s))
'''
# open xml file and read metadata
file_path = os.path.join(settings.PROJECT_ROOT + settings.MEDIA_URL, nasa_data_file.xml.name)
tree = ET.parse(file_path)
root = tree.getroot()
def parse_attachment(message_part, attachments=None):
content_disposition = message_part.get("Content-Disposition", None)
if content_disposition:
try:
try:
content_disposition = tweakContentDisposition(content_disposition)
dispositions = content_disposition.strip().split(";", 1)
except Exception as inst:
print type(inst)
print inst.args
print >>sys.stderr, "Error parsing file: {%s}%s" % (sharedVariablesAcrossModules.sourceFileUUID, sharedVariablesAcrossModules.sourceFilePath)
print >>sys.stderr, "Error parsing the content_disposition:", content_disposition
if "attachment" in content_disposition.lower() and "filename" in content_disposition.lower():
try:
filename = uuid.uuid4().__str__()
print >>sys.stderr, "Attempting extraction with random filename: %s" % (filename)
print >>sys.stderr
content_disposition = "attachment; filename=%s;" % (filename)
dispositions = content_disposition.strip().split(";")
except Exception as inst:
print >>sys.stderr, type(inst)
print >>sys.stderr, inst.args
print >>sys.stderr, "Failed"
print >>sys.stderr
return None
else:
print >>sys.stderr
return None
if content_disposition and dispositions[0].lower() == "attachment":
file_data = message_part.get_payload(decode=True)
if not file_data:
payload = message_part.get_payload()
if isinstance(payload, list):
for msgobj in payload:
parse2(msgobj, attachments)
return None
print >>sys.stderr, message_part.get_payload()
print >>sys.stderr, message_part.get_content_charset()
attachment = StringIO(file_data)
attachment.content_type = message_part.get_content_type()
attachment.size = len(file_data)
attachment.create_date = None
attachment.mod_date = None
attachment.read_date = None
attachment.name = ""
for param in dispositions[1:]:
name,value = param.split("=", 1)
name = name.lower().strip()
if name == "filename":
attachment.name = urllib2.unquote(value.strip()).strip('"')
if name == "filename*":
attachment.name = urllib2.unquote(value.strip())
try:
enc, name = attachment.name.split("''", 1)
attachment.name = name.decode(enc)
except Exception as inst:
print >>sys.stderr, type(inst)
print >>sys.stderr, inst.args
pass
elif name == "create-date":
attachment.create_date = value #TODO: datetime
elif name == "modification-date":
attachment.mod_date = value #TODO: datetime
elif name == "read-date":
attachment.read_date = value #TODO: datetime
if not attachment.name:
print >>sys.stderr, """Warning, no filename found in: [{%s}%s]%s""" % (sharedVariablesAcrossModules.sourceFileUUID, sharedVariablesAcrossModules.sourceFilePath, content_disposition)
filename = uuid.uuid4().__str__()
print >>sys.stderr, "Attempting extraction with random filename: %s" % (filename)
print >>sys.stderr
return attachment
except Exception as inst:
print >>sys.stderr, type(inst)
print >>sys.stderr, inst.args
print >>sys.stderr, "Error parsing file: {%s}%s" % (sharedVariablesAcrossModules.sourceFileUUID, sharedVariablesAcrossModules.sourceFilePath)
print >>sys.stderr, "Error parsing:", dispositions
print >>sys.stderr
sharedVariablesAcrossModules.errorCounter += 1
return None
ftp.cwd('OMSO2NRTb/')
filenames = ftp.nlst()
#print(filenames) # list directory contents
for filename in filenames:
if filename[-3:] == 'he5':
try:
nasa_data_file = NasaDataFile.objects.get(hdf='./' + filename)
except Exception:
nasa_data_file = NasaDataFile()
print(filename)
s = StringIO()
ftp.retrbinary("RETR " + filename, s.write)
s.size = s.tell()
nasa_data_file.hdf.save(filename, File(s))
if nasa_data_file.xml.name == '' or not nasa_data_file.xml.name:
xml_filename = filename + '.met'
print(xml_filename)
s = StringIO()
ftp.retrbinary("RETR " + xml_filename, s.write)
s.size = s.tell()
nasa_data_file.xml.save(xml_filename, File(s))
'''
# open xml file and read metadata
file_path = os.path.join(settings.PROJECT_ROOT + settings.MEDIA_URL, nasa_data_file.xml.name)
tree = ET.parse(file_path)
root = tree.getroot()
def parse_attachment(message_part, attachments=None):
""" Extract the attachment and metadata about it from the message.
Returns the content, content type, size, and create/modification/read dates
for the attachment.
"""
params = message_part.get_params(None, 'Content-Disposition')
if params:
# If a 'part' has a Content-Disposition, we assume it is an attachment
try:
params = dict(params)
print('\tContent-Disposition (for following email)', params)
if 'attachment' in params:
# Not sure what's going on here
# Why get payload with decode, then try again and reparse?
# See details at
# http://docs.python.org/2/library/email.message.html#email.message.Message.get_payload
file_data = message_part.get_payload(decode=True)
if not file_data:
payload = message_part.get_payload()
if isinstance(payload, list):
for msgobj in payload:
# TODO not sure this actually does anything
parse2(msgobj, attachments)
return None
print(message_part.get_payload(), file=sys.stderr)
print(message_part.get_content_charset(), file=sys.stderr)
attachment = StringIO(file_data)
attachment.content_type = message_part.get_content_type()
attachment.size = params.get('size', len(file_data))
attachment.create_date = params.get('create-date')
attachment.mod_date = params.get('modification-date')
attachment.read_date = params.get('read-date')
# TODO convert dates to datetime
filename = message_part.get_filename(None)
if filename:
# Filenames may be encoded with =?encoding?...
# If so, convert to unicode
name, encoding = email.header.decode_header(filename)[0]
if encoding:
print(
'\t{filename} encoded with {encoding}, converting to unicode'
.format(filename=filename, encoding=encoding))
filename = name.decode(encoding)
else: # filename not in Content-Disposition
print(
"""Warning, no filename found in: [{%s}%s] Content-Disposition: %s or Content-Type"""
%
(sharedVariablesAcrossModules.sourceFileUUID,
sharedVariablesAcrossModules.sourceFilePath, params),
file=sys.stderr)
filename = six.text_type(uuid.uuid4())
print("Attempting extraction with random filename: %s" %
(filename),
file=sys.stderr)
# Remove newlines from filename because that breaks everything
filename = filename.replace("\r", "").replace("\n", "")
attachment.name = filename
return attachment
except Exception as inst:
print(type(inst), file=sys.stderr)
print(inst.args, file=sys.stderr)
print("Error parsing: file: {%s}%s" %
(sharedVariablesAcrossModules.sourceFileUUID,
sharedVariablesAcrossModules.sourceFilePath),
file=sys.stderr)
print("Error parsing: Content-Disposition: ",
params,
file=sys.stderr)
print(file=sys.stderr)
sharedVariablesAcrossModules.errorCounter += 1
return None
def setUp(self):
self.user = User.objects.create_user(username='******')
#we need a uimodel
self.uimodel = UIModel.objects.create(name='curriculum1',
version ='1')
self.unitset_view_model = ViewMap.objects.create(
uimodel=self.uimodel,
name='unitset',
parent=None)
self.unit_view_model = ViewMap.objects.create(
uimodel=self.uimodel,
name='unit',
parent=self.unitset_view_model)
self.lesson_view_model = ViewMap.objects.create(
uimodel=self.uimodel,
name='lesson',
parent=self.unit_view_model)
self.slide_viewstate_model = Page.objects.create(
uimodel=self.uimodel,
viewmap=self.lesson_view_model,
name='slide')
self.device = Device.objects.create(name='device1')
self.uimodeldevice = UIModelDevice.objects.create(
uimodel=self.uimodel,
device=self.device)
#create a layout
self.layout1 = Layout.objects.create(name='layout1')
self.layout1_layer1 = LayoutLayer.objects.create(
name='layout1_layer1',
layout=self.layout1)
self.layerbox1 = LayerBox.objects.create(
name='layout1_box1',
ordinal=0,
layer=self.layout1_layer1)
self.layerbox2 = LayerBox.objects.create(
name='layout1_box2',
ordinal=1,
layer=self.layout1_layer1)
#hook the layout up to our uimodel
self.viewstatelayout1 = PageLayout.objects.create(
page=self.slide_viewstate_model,
device=self.uimodeldevice,
layout=self.layout1)
#add a content collection (draft and edition)
self.draft = Draft.objects.create_draft('draft1', self.user)
content_specs = [
{
'fname': 'foo.html',
'description': 'some html',
'path': '/content/html',
'content': '<head></head><body>foo</body>',
},
{
'fname': 'bar.html',
'description': 'some more html',
'path': '/content/html',
'content': '<head></head><body>bar</body>',
},
]
edit = Edit.objects.create(user=self.user, source='upload',
draft=self.draft)
for spec in content_specs:
file_like = StringIO(spec['content'])
file_like.name = spec['fname']
file_like.size = len(spec['content'])
imported = ImportItem.objects.create(
name=spec['fname'],
description=spec['description'],
requested_path=spec['path'],
contentfile=DjangoFile(file_like, name=spec['fname']),
edit=edit)
imported.detect_type()
item = CItem.objects.install_imported(self.user, imported)
spec['item'] = item
self.edition = self.draft.make_edition('edition1')
#create a profile
self.uiprofile = UIProfile.objects.create(uimodel=self.uimodel,
edition=self.edition,
is_frozen=False)
self.carton1 = PageState.objects.create(
uiprofile=self.uiprofile,
page_layout=self.viewstatelayout1)
self.contentforbox1 = ContentForBox.objects.create(
box=self.layerbox1,
citem=content_specs[0]['item'],
page_state=self.carton1)
