def parse_attachment(message_part):
content_disposition = message_part.get("Content-Disposition", None)
if not content_disposition:
return None
dispo_type, dispo_dict = parse_dispositions(content_disposition)
if not (dispo_type == "attachment" or
(dispo_type == 'inline' and 'filename' in dispo_dict)):
return None
content_type = message_part.get("Content-Type", None)
file_data = message_part.get_payload(decode=True)
if file_data is None:
payloads = message_part.get_payload()
file_data = '\n\n'.join([p.as_string()
for p in payloads]).encode('utf-8')
attachment = BytesIO(file_data)
attachment.content_type = message_part.get_content_type()
attachment.size = len(file_data)
attachment.name = None
attachment.create_date = None
attachment.mod_date = None
attachment.read_date = None
attachment.name = get_attachment_name(attachment,
dispo_dict,
content_type=content_type)
if "create-date" in dispo_dict:
attachment.create_date = dispo_dict['create-date'] # TODO: datetime
if "modification-date" in dispo_dict:
attachment.mod_date = dispo_dict['modification-date'] # TODO: datetime
if "read-date" in dispo_dict:
attachment.read_date = dispo_dict['read-date'] # TODO: datetime
return attachment
def parse_postmark(self, obj):
from_field = (obj['FromFull']['Name'], obj['FromFull']['Email'])
tos = [(o['Name'], o['Email']) for o in obj['ToFull']]
ccs = [(o['Name'], o['Email']) for o in obj['CcFull']]
attachments = []
for a in obj['Attachments']:
attachment = BytesIO(base64.b64decode(a['Content']))
attachment.content_type = a['ContentType']
attachment.size = a['ContentLength']
attachment.name = a['Name']
attachment.create_date = None
attachment.mod_date = None
attachment.read_date = None
attachments.append(attachment)
return ParsedEmail(None, **{
'postmark_msgobj': obj,
'date': parse_date(obj['Date']),
'subject': obj['Subject'],
'body': obj['TextBody'],
'html': obj['HtmlBody'],
'from_': from_field,
'to': tos,
'cc': ccs,
'resent_to': [],
'resent_cc': [],
'attachments': attachments
})
def parse_attachment(message_part):
content_disposition = message_part.get("Content-Disposition", None)
if not content_disposition:
return None
dispo_type, dispo_dict = parse_dispositions(content_disposition)
if not (dispo_type == "attachment" or (dispo_type == 'inline' and
'filename' in dispo_dict)):
return None
content_type = message_part.get("Content-Type", None)
file_data = message_part.get_payload(decode=True)
if file_data is None:
payloads = message_part.get_payload()
file_data = '\n\n'.join([p.as_string() for p in payloads]).encode('utf-8')
attachment = BytesIO(file_data)
attachment.content_type = message_part.get_content_type()
attachment.size = len(file_data)
attachment.name = None
attachment.create_date = None
attachment.mod_date = None
attachment.read_date = None
attachment.name = get_attachment_name(
attachment, dispo_dict, content_type=content_type
)
if "create-date" in dispo_dict:
attachment.create_date = dispo_dict['create-date'] # TODO: datetime
if "modification-date" in dispo_dict:
attachment.mod_date = dispo_dict['modification-date'] # TODO: datetime
if "read-date" in dispo_dict:
attachment.read_date = dispo_dict['read-date'] # TODO: datetime
return attachment
def _parse_attachment(message_part):
content_disposition = message_part.get("Content-Disposition", None)
if content_disposition:
dispositions = content_disposition.strip().split(";")
if content_disposition and dispositions[0].lower() == "attachment":
file_data = message_part.get_payload(decode=True)
attachment = BytesIO(file_data)
attachment.content_type = message_part.get_content_type()
attachment.size = len(file_data)
attachment.name = None
attachment.create_date = None
attachment.mod_date = None
attachment.read_date = None
for param in dispositions[1:]:
name, value = param.strip().split("=")
name = name.lower()
if name == "filename":
attachment.name = value
elif name == "create-date":
attachment.create_date = value # FIXME: datetime
elif name == "modification-date":
attachment.mod_date = value # FIXME: datetime
elif name == "read-date":
attachment.read_date = value # FIXME: datetime
attachment.checksum = _md5(attachment)
return attachment
return None
def parse_postmark(self, obj):
from_field = (obj['FromFull']['Name'], obj['FromFull']['Email'])
tos = [(o['Name'], o['Email']) for o in obj['ToFull']]
ccs = [(o['Name'], o['Email']) for o in obj['CcFull']]
attachments = []
for a in obj['Attachments']:
attachment = BytesIO(base64.b64decode(a['Content']))
attachment.content_type = a['ContentType']
attachment.size = a['ContentLength']
attachment.name = a['Name']
attachment.create_date = None
attachment.mod_date = None
attachment.read_date = None
attachments.append(attachment)
return ParsedEmail(
None, **{
'postmark_msgobj': obj,
'date': parse_date(obj['Date']),
'subject': obj['Subject'],
'body': obj['TextBody'],
'html': obj['HtmlBody'],
'from_': from_field,
'to': tos,
'cc': ccs,
'resent_to': [],
'resent_cc': [],
'attachments': attachments
})
def test_no_extension(user, course):
doc = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\n/\x00\x00\t\x81\x08\x06\x00\x00\x00'\x06\xfee\x00\x00\x00\tpHYs\x00\x00n\xba\x00\x00n\xba\x01\xd6\xde\xb1\x17\x00\x00\x00\x19tEXtSoftware\x00www.inkscape.org\x9b\xee<\x1a\x00\x00 \x00IDATx"
file = BytesIO(doc)
file.size = len(doc)
doc = logic.add_file_to_course(file, "My document", "", course, [], user)
assert doc
assert doc.file_type == ".png"
def create_stream(self, file_format, size):
""" Create temporary stream """
data = b'*' * size
data_io = BytesIO(data)
data_io.seek(0, os.SEEK_END)
data_io.size = data_io.tell()
data_io.name = 'video' + file_format
data_io.seek(0)
return data_io
def createSnapshotTar(slug: str,
name: str,
date: datetime,
padSize: int,
included_folders=None,
included_addons=None,
password=None) -> BytesIO:
snapshot_type = "full"
if included_folders:
folders = included_folders.copy()
else:
folders = all_folders.copy()
if included_addons:
snapshot_type = "partial"
addons = []
for addon in all_addons:
if addon['slug'] in included_addons:
addons.append(addon)
else:
addons = all_addons.copy()
snapshot_info = {
"slug": slug,
"name": name,
"date": date.isoformat(),
"type": snapshot_type,
"protected": password is not None,
"homeassistant": {
"ssl": True,
"watchdog": True,
"port": 8123,
"wait_boot": 600,
"boot": True,
"version": "0.92.2",
"refresh_token": "fake_token"
},
"folders": folders,
"addons": addons,
"repositories": ["https://github.com/hassio-addons/repository"]
}
stream = BytesIO()
tar = tarfile.open(fileobj=stream, mode="w")
add(tar, "snapshot.json", BytesIO(json.dumps(snapshot_info).encode()))
add(tar, "padding.dat", getTestStream(padSize))
tar.close()
stream.seek(0)
stream.size = lambda: len(stream.getbuffer())
return stream
def test_add_file_to_course(user, course):
Tag.objects.create(name="tag one")
tags = ["tag one", "tag two", Tag.objects.create(name="tag three")]
file = BytesIO(b"mybinarydocumentcontent")
file.size = len(b"mybinarydocumentcontent")
doc = logic.add_file_to_course(file, "My document", ".dll", course, tags,
user)
assert doc
assert doc in course.document_set.all()
assert doc.name == "My document"
assert doc.state == Document.DocumentState.READY_TO_QUEUE
assert Tag.objects.count() == 3
assert doc.tags.count() == 3
assert doc.file_type == ".dll"
def createBackupTar(slug: str,
name: str,
date: datetime,
padSize: int,
included_folders=None,
included_addons=None,
password=None) -> BytesIO:
backup_type = "full"
if included_folders is not None:
folders = included_folders.copy()
else:
folders = all_folders.copy()
if included_addons is not None:
backup_type = "partial"
addons = []
for addon in all_addons:
if addon['slug'] in included_addons:
addons.append(addon)
else:
addons = all_addons.copy()
backup_info = {
"slug": slug,
"name": name,
"date": date.isoformat(),
"type": backup_type,
"protected": password is not None,
"homeassistant": "0.92.2",
"folders": folders,
"addons": addons,
"repositories": ["https://github.com/hassio-addons/repository"]
}
stream = BytesIO()
tar = tarfile.open(fileobj=stream, mode="w")
add(tar, "backup.json", BytesIO(json.dumps(backup_info).encode()))
add(tar, "padding.dat", getTestStream(padSize))
tar.close()
stream.seek(0)
stream.size = lambda: len(stream.getbuffer())
return stream
def _open(self, name, mode="rb"):
"""Open file with filename `name` from the database."""
name = self.get_instance_name(name)
try:
# Load file from database.
f = models.File.objects.get_from_name(name)
content = f.content
size = f.size
if _settings.DB_FILES_AUTO_EXPORT_DB_TO_FS and not utils.is_fresh(
f.name, f.content_hash):
# Automatically write the file to the filesystem
# if it's missing and exists in the database.
# This happens if we're using multiple web servers connected
# to a common database behind a load balancer.
# One user might upload a file from one web server, and then
# another might access if from another server.
utils.write_file(name, f.content)
except models.File.DoesNotExist:
# If not yet in the database, check the local file system
# and load it into the database if present.
fqfn = self.path(name)
if os.path.isfile(fqfn):
# print('Loading file into database.')
self._save(name, open(fqfn, mode))
fh = super(DatabaseStorage, self)._open(name, mode)
content = fh.read()
size = fh.size
else:
# Otherwise we don't know where the file is so we return an
# empty file
size = 0
content = b""
# Normalize the content to a new file object.
fh = BytesIO(content)
fh.name = name
fh.mode = mode
fh.size = size
o = files.File(fh)
return o
def launchpad_exchange(self, interface):
# Maybe on the next exchange...
if not self._form["field.system"] \
or self._form["field.submission_data"]:
return
# Compress and add payload to form
payload = open(self._launchpad_report, "rb").read()
compressed_payload = bz2.compress(payload)
file = BytesIO(compressed_payload)
file.name = "%s.xml.bz2" % gethostname()
file.size = len(compressed_payload)
self._form["field.submission_data"] = file
if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
logging.debug("Uncompressed payload length: %d", len(payload))
# Encode form data
try:
form = FORM.coerce(self._form)
except ValueError as e:
self._manager.reactor.fire(
"exchange-error", _("""\
Failed to process form: %s""" % e))
return
transport = HTTPTransport(self.transport_url)
start_time = time.time()
try:
response = transport.exchange(form,
self._headers,
timeout=self.timeout)
except Exception as error:
self._manager.reactor.fire("exchange-error", str(error))
return
end_time = time.time()
if not response:
self._manager.reactor.fire(
"exchange-error",
_("""\
Failed to contact server. Please try
again or upload the following file name:
%s
directly to the system database:
https://launchpad.net/+hwdb/+submit""") %
posixpath.abspath(self._launchpad_report))
return
elif response.status != 200:
self._manager.reactor.fire(
"exchange-error",
_("""\
Failed to upload to server,
please try again later."""))
return
if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
logging.debug("Response headers:\n%s",
pprint.pformat(response.getheaders()))
header = response.getheader("x-launchpad-hwdb-submission")
if not header:
self._manager.reactor.fire(
"exchange-error", _("Information not posted to Launchpad."))
elif "Error" in header:
# HACK: this should return a useful error message
self._manager.reactor.fire("exchange-error", header)
logging.error(header)
else:
text = response.read()
self._manager.reactor.fire("exchange-success", text)
logging.info("Sent %d bytes and received %d bytes in %s.",
file.size, len(text),
format_delta(end_time - start_time))
def send(self, data, config=None, session_state=None):
""" Sends data to the specified server.
:param data:
Data containing the xml dump to be sent to the server. This can be
either bytes or a file-like object (BytesIO works fine too). If
this is a file-like object, it will be read and streamed "on the
fly".
:param config:
An optional PlainBoxConfig object. If http_proxy and https_proxy
values are set in this config object, they will be used to send
data via the specified protocols. Note that the transport also
honors the http_proxy and https_proxy environment variables.
Proxy string format is http://[user:password@]<proxy-ip>:port
:param session_state:
?
:returns:
A dictionary with responses from the server if submission was
successful and empty dictionary otherwise.
:raises ValueError:
If no session state was provided.
:raises TransportError:
- If sending timed out.
- If connection failed outright.
- If the server returned
a non-success result code
- If a required resource job is missing from the submission
or a resource job is missing a required attribute. The following
resource/attribute pairs are needed:
- dpkg: architecture
- lsb: distributor_id
- lsb: codename
- dmi: product
"""
proxies = None
if config and config.environment is not Unset:
proxies = {
proto[:-len("_proxy")]: config.environment[proto]
for proto in ['http_proxy', 'https_proxy']
if proto in config.environment
}
if session_state is None:
raise ValueError("LaunchpadTransport requires a session "
"state to be provided.")
logger.debug("Sending to %s, email is %s", self.url,
self.options['field.emailaddress'])
lp_headers = {"x-launchpad-hwdb-submission": ""}
form_fields = self._get_launchpad_form_fields(session_state)
form_fields['field.emailaddress'] = self.options['field.emailaddress']
compressed_payload = bz2.compress(data.encode('utf-8'))
file = BytesIO(compressed_payload)
file.name = "{}.xml.bz2".format(gethostname())
file.size = len(compressed_payload)
submission_data = {'field.submission_data': file}
try:
response = requests.post(self.url,
data=form_fields,
files=submission_data,
headers=lp_headers,
proxies=proxies)
except requests.exceptions.Timeout as exc:
raise TransportError(
_("Request to {0} timed out: {1}").format(self.url, exc))
except requests.exceptions.InvalidSchema as exc:
raise TransportError(_("Invalid destination URL: {0}").format(exc))
except requests.exceptions.ConnectionError as exc:
raise TransportError(
_("Unable to connect to {0}: {1}").format(self.url, exc))
if response is not None:
try:
# This will raise HTTPError for status != 20x
response.raise_for_status()
except requests.exceptions.RequestException as exc:
raise TransportError(str(exc))
logger.debug(_("Success! Server said %s"), response.text)
status = _('The submission was uploaded to Launchpad successfully')
if (response.headers['x-launchpad-hwdb-submission'] !=
('OK data stored')):
status = response.headers['x-launchpad-hwdb-submission']
return {'status': status}
# ISessionStateTransport.send must return dictionary
return {}
def create_file(self):
attachment = BytesIO(self.data)
attachment.content_type = self.content_type
attachment.size = self.size
attachment.name = "archivo.pdf"
return attachment
def _get_test_file(extension='txt'):
f = BytesIO(six.b('test'))
f.name = 'test.' + extension
f.size = len(f.getvalue())
return f
def launchpad_exchange(self):
# Maybe on the next exchange...
if not self._form["field.system"] \
or self._form["field.submission_data"]:
return
# Compress and add payload to form
payload = open(self._launchpad_report, "rb").read()
compressed_payload = bz2.compress(payload)
file = BytesIO(compressed_payload)
file.name = "%s.xml.bz2" % gethostname()
file.size = len(compressed_payload)
self._form["field.submission_data"] = file
if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
logging.debug("Uncompressed payload length: %d", len(payload))
# Encode form data
try:
form = FORM.coerce(self._form)
except ValueError as e:
self._manager.reactor.fire("exchange-error", _("""\
Failed to process form: %s""" % e))
return
transport = HTTPTransport(self.transport_url)
start_time = time.time()
try:
response = transport.exchange(form, self._headers,
timeout=self.timeout)
except Exception as error:
self._manager.reactor.fire("exchange-error", str(error))
return
end_time = time.time()
if not response:
self._manager.reactor.fire("exchange-error", _("""\
Failed to contact server. Please try
again or upload the following file name:
%s
directly to the system database:
https://launchpad.net/+hwdb/+submit""") % posixpath.abspath(self._launchpad_report))
return
elif response.status != 200:
self._manager.reactor.fire("exchange-error", _("""\
Failed to upload to server,
please try again later."""))
return
if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
logging.debug("Response headers:\n%s",
pprint.pformat(response.getheaders()))
header = response.getheader("x-launchpad-hwdb-submission")
if not header:
self._manager.reactor.fire("exchange-error",
_("Information not posted to Launchpad."))
elif "Error" in header:
# HACK: this should return a useful error message
self._manager.reactor.fire("exchange-error", header)
logging.error(header)
else:
text = response.read()
self._manager.reactor.fire("exchange-success", text)
logging.info("Sent %d bytes and received %d bytes in %s.",
file.size, len(text), format_delta(end_time - start_time))
def prep_data(length):
data = BytesIO(faker.binary(length=length))
data.name = faker.file_name(category='image')
data.content_type = faker.mime_type(category='image')
data.size = length
return data
