def generate_ssh_keypair(request):
if not request.user.is_staff:
raise PermissionDenied(_("only staff may use this tool"))
from paramiko import RSAKey
key_class = RSAKey
prv = key_class.generate(bits=2048)
import six
prv_bio = six.BytesIO()
prv.write_private_key(prv_bio)
prv_bio_read = six.BytesIO(prv_bio.getvalue())
pub = key_class.from_private_key(prv_bio_read)
pub_bio = six.BytesIO()
pub_bio.write("%s %s relate-course-key" %
(pub.get_name(), pub.get_base64()))
return render(
request, "course/keypair.html", {
"public_key": prv_bio.getvalue().decode(),
"private_key": pub_bio.getvalue().decode(),
})
def get_xlsx_export(self, context):
datas = self._get_datas(context)
output = six.BytesIO()
export_header = self._options_is_on('export_xlsx_header')
model_name = self.opts.verbose_name
book = xlsxwriter.Workbook(output)
sheet = book.add_worksheet(
u"%s %s" % (_(u'Sheet'), force_text(model_name)))
styles = {'datetime': book.add_format({'num_format': 'yyyy-mm-dd hh:mm:ss'}),
'date': book.add_format({'num_format': 'yyyy-mm-dd'}),
'time': book.add_format({'num_format': 'hh:mm:ss'}),
'header': book.add_format({'font': 'name Times New Roman', 'color': 'red', 'bold': 'on', 'num_format': '#,##0.00'}),
'default': book.add_format()}
if not export_header:
datas = datas[1:]
for rowx, row in enumerate(datas):
for colx, value in enumerate(row):
if export_header and rowx == 0:
cell_style = styles['header']
else:
if isinstance(value, datetime.datetime):
cell_style = styles['datetime']
elif isinstance(value, datetime.date):
cell_style = styles['date']
elif isinstance(value, datetime.time):
cell_style = styles['time']
else:
cell_style = styles['default']
sheet.write(rowx, colx, value, cell_style)
book.close()
output.seek(0)
return output.getvalue()
def get_xls_export(self, context):
datas = self._get_datas(context)
output = six.BytesIO()
export_header = self._options_is_on('export_xls_header')
model_name = self.opts.verbose_name
book = xlwt.Workbook(encoding=self.export_unicode_encoding)
sheet = book.add_sheet(u"%s %s" % (_(u'Sheet'), force_text(model_name)))
styles = {'datetime': xlwt.easyxf(num_format_str='yyyy-mm-dd hh:mm:ss'),
'date': xlwt.easyxf(num_format_str='yyyy-mm-dd'),
'time': xlwt.easyxf(num_format_str='hh:mm:ss'),
'header': xlwt.easyxf('font: name Times New Roman, color-index red, bold on', num_format_str='#,##0.00'),
'default': xlwt.Style.default_style}
if not export_header:
datas = datas[1:]
for rowx, row in enumerate(datas):
for colx, value in enumerate(row):
if export_header and rowx == 0:
cell_style = styles['header']
else:
if isinstance(value, datetime.datetime):
cell_style = styles['datetime']
elif isinstance(value, datetime.date):
cell_style = styles['date']
elif isinstance(value, datetime.time):
cell_style = styles['time']
else:
cell_style = styles['default']
sheet.write(rowx, colx, value, style=cell_style)
book.save(output)
output.seek(0)
return output.getvalue()
def _perform_form_overloading(self):
"""
If this is a form POST request, then we need to check if the method and
content/content_type have been overridden by setting them in hidden
form fields or not.
"""
USE_FORM_OVERLOADING = (self._METHOD_PARAM
or (self._CONTENT_PARAM
and self._CONTENTTYPE_PARAM))
# We only need to use form overloading on form POST requests.
if (self._request.method != 'POST' or not USE_FORM_OVERLOADING
or not is_form_media_type(self._content_type)):
return
# At this point we're committed to parsing the request as form data.
self._data = self._request.POST
self._files = self._request.FILES
self._full_data = self._data.copy()
self._full_data.update(self._files)
# Method overloading - change the method and remove the param from the content.
if (self._METHOD_PARAM and self._METHOD_PARAM in self._data):
self._method = self._data[self._METHOD_PARAM].upper()
# Content overloading - modify the content type, and force re-parse.
if (self._CONTENT_PARAM and self._CONTENTTYPE_PARAM
and self._CONTENT_PARAM in self._data
and self._CONTENTTYPE_PARAM in self._data):
self._content_type = self._data[self._CONTENTTYPE_PARAM]
self._stream = six.BytesIO(self._data[self._CONTENT_PARAM].encode(
self.parser_context['encoding']))
self._data, self._files, self._full_data = (Empty, Empty, Empty)
def create_thumbnail(self, size, quality=None):
# invalidate the cache of the thumbnail with the given size first
invalidate_cache(self.user, size)
try:
orig = self.avatar.storage.open(self.avatar.name, 'rb')
image = Image.open(orig)
image = self.transpose_image(image)
quality = quality or settings.AVATAR_THUMB_QUALITY
w, h = image.size
if w != size or h != size:
if w > h:
diff = int((w - h) / 2)
image = image.crop((diff, 0, w - diff, h))
else:
diff = int((h - w) / 2)
image = image.crop((0, diff, w, h - diff))
if image.mode not in ("RGB", "RGBA"):
image = image.convert("RGB")
image = image.resize((size, size),
settings.AVATAR_RESIZE_METHOD)
thumb = six.BytesIO()
image.save(thumb,
settings.AVATAR_THUMB_FORMAT,
quality=quality)
thumb_file = ContentFile(thumb.getvalue())
else:
thumb_file = File(orig)
thumb = self.avatar.storage.save(self.avatar_name(size),
thumb_file)
except IOError:
return # What should we do here? Render a "sorry, didn't work" img?
def create_inmemory_file(file_name='tmp.txt', content=b'', content_type=None):
stream = six.BytesIO()
stream.write(content)
file = InMemoryFile(stream, None, file_name, content_type, stream.tell(),
None)
file.seek(0)
return file
def process(self, value, scale=None, format=None, **kwargs):
cur_pos = value.tell()
value.seek(0)
stream = six.BytesIO(value.read())
stream_out = None
value.seek(cur_pos)
try:
image = self.get_image(stream,
scale=scale,
format=format,
**kwargs)
image = self.resize(image, scale=scale, format=format, **kwargs)
stream_out = self.convert(image,
scale=scale,
format=format,
**kwargs)
if stream_out is not None:
content = stream_out.getvalue()
else:
content = stream.getvalue()
except (IOError, OSError, Image.DecompressionBombWarning) as e:
raise ProcessingError(
"There was a problem with image conversion: %s" % e)
finally:
if stream_out is not None:
stream_out.close()
stream.close()
return ContentFile(content)
def generate_plot(process, plot_id, thumbnail, datafile_name):
try:
output = six.BytesIO()
if thumbnail:
figure = Figure(frameon=False, figsize=(4, 3))
canvas = FigureCanvasAgg(figure)
axes = figure.add_subplot(111)
axes.set_position((0.17, 0.16, 0.78, 0.78))
axes.grid(True)
process.draw_plot(axes, plot_id, datafile_name, for_thumbnail=True)
canvas.print_figure(output,
dpi=settings.THUMBNAIL_WIDTH / 4,
format="png")
else:
figure = Figure()
canvas = FigureCanvasAgg(figure)
axes = figure.add_subplot(111)
axes.grid(True)
axes.set_title(six.text_type(process))
process.draw_plot(axes,
plot_id,
datafile_name,
for_thumbnail=False)
# FixMe: Activate this line with Matplotlib 1.1.0.
# figure.tight_layout()
canvas.print_figure(output, format="pdf")
except PlotError as e:
raise Http404(six.text_type(e) or "Plot could not be generated.")
except ValueError as e:
raise Http404("Plot could not be generated: " + e.args[0])
else:
return output.getvalue()
def test_easting_northing_no_logging(self):
"""
Test easting-northing template download.
Important: Logging should not be necessary
"""
client = Client()
url = reverse('download:site-template-easting-northing')
resp = client.get(url)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.assertEqual(resp.get('content-type'),
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
content_disposition = resp.get('content-disposition')
# should be something like:
# 'attachment; filename=Sites_template_lat_long.xlsx
match = re.match('attachment; filename=(.+)', content_disposition)
self.assertIsNotNone(match)
filename, ext = path.splitext(match.group(1))
self.assertEqual(ext, '.xlsx')
self.assertEqual(filename, 'Sites_template_easting_northing')
# read content
wb = load_workbook(six.BytesIO(resp.content), read_only=True)
# one datasheet named 'Sites'
expected_sheet_name = 'Sites'
sheet_names = wb.sheetnames
self.assertEqual(1, len(sheet_names))
self.assertEqual(sheet_names[0], expected_sheet_name)
ws = wb[expected_sheet_name]
rows = list(ws.rows)
# only one row
self.assertEqual(len(rows), 1)
got_headers = [c.value for c in rows[0]]
expected_headers = ['Name', 'Code', 'Description', 'Easting', 'Northing', 'Datum', 'Zone']
self.assertEqual(got_headers, expected_headers)
def from_native(self, data):
"""
Checks that the file-upload field data contains a valid image (GIF, JPG,
PNG, possibly others -- whatever the Python Imaging Library supports).
"""
f = super(ImageField, self).from_native(data)
if f is None:
return None
# Try to import PIL in either of the two ways it can end up installed.
try:
from PIL import Image
except ImportError:
try:
import Image
except ImportError:
Image = None
assert (
Image is not None
), "Either Pillow or PIL must be installed for ImageField support."
# We need to get a file object for PIL. We might have a path or we might
# have to read the data into memory.
if hasattr(data, "temporary_file_path"):
file = data.temporary_file_path()
else:
if hasattr(data, "read"):
file = six.BytesIO(data.read())
else:
file = six.BytesIO(data["content"])
try:
# load() could spot a truncated JPEG, but it loads the entire
# image in memory, which is a DoS vector. See #3848 and #18520.
# verify() must be called immediately after the constructor.
Image.open(file).verify()
except ImportError:
# Under PyPy, it is possible to import PIL. However, the underlying
# _imaging C module isn't available, so an ImportError will be
# raised. Catch and re-raise.
raise
except Exception: # Python Imaging Library doesn't recognize it as an image
raise ValidationError(self.error_messages["invalid_image"])
if hasattr(f, "seek") and callable(f.seek):
f.seek(0)
return f
def read(stream):
if not isinstance(stream, six.BytesIO):
stream = six.BytesIO(stream)
if biplist.is_stream_binary_plist(stream):
stream.seek(0)
return biplist.readPlist(stream)
stream.seek(0)
return loads(stream)
def test_csv_new_import(self):
prev_count = PublicBody.objects.all().count()
csv = '''name,email,jurisdiction__slug,other_names,description,tags,url,parent__name,classification,contact,address,website_dump,request_note
Public Body X 76,[email protected],bund,,,,http://example.com,,Ministry,Some contact stuff,An address,,'''
imp = CSVImporter()
imp.import_from_file(six.BytesIO(csv.encode('utf-8')))
now_count = PublicBody.objects.all().count()
self.assertEqual(now_count - 1, prev_count)
def test_csv_export_import(self):
csv = export_csv_bytes(PublicBody.export_csv(PublicBody.objects.all()))
prev_count = PublicBody.objects.all().count()
imp = CSVImporter()
csv_file = six.BytesIO(csv)
imp.import_from_file(csv_file)
now_count = PublicBody.objects.all().count()
self.assertEqual(now_count, prev_count)
def get_thumbnail(cls, image, operations=None, timeout=None):
operations = operations or []
url = None
if isinstance(image, six.string_types):
if urlsplit(image).scheme in ('http', 'https'):
url = image
image = six.BytesIO()
image.path = url
op_id = cls.op_id(operations)
entries = cls.get_entries(image)
if entries is None:
entries = {}
cached_path = entries.get(op_id)
if cached_path is not None and not cls.storage.exists(cached_path):
# Something in cache but no file, drop entry
del entries[op_id]
cached_path = None
if not cached_path:
img_id = hashlib.md5(
force_bytes('{0}{1}'.format(image.path,
repr(operations)))).hexdigest()
# Open URL if needed
if url:
rsp = None
try:
rsp = urlopen(url)
image.write(rsp.read())
image.seek(0)
finally:
if rsp:
rsp.close()
# Create thumbnail
dest_file = ContentFile('')
if hasattr(image, 'closed') and image.closed:
image.open()
with cls.Processor(image) as p:
p.orientation()
p.operations(*operations).save(dest_file)
cached_path = '{0}.{1}'.format(
os.path.join(img_id[0:2], img_id[2:4], img_id), p.format)
cls.storage.save(cached_path, dest_file)
del dest_file
if hasattr(image, 'close'):
image.close()
entries[op_id] = cached_path
cls.set_entries(image, entries, timeout)
return FileWrapper(cached_path, cls.storage)
def _make_memfile(self, filename, content):
return InMemoryUploadedFile(
file=six.BytesIO(content),
field_name='test_field',
name='_save_new_file.txt',
content_type='text/plain',
size=0,
charset='utf8',
)
def test_save_contents(self):
release_file = factories.ReleaseFileFactory()
dummy_fh = six.BytesIO(six.b("release-file-contents"))
release_file.save_filecontent('dummy.txt', dummy_fh)
self.assertEqual(release_file.distribution.name,
'default/2.7/t/test-package/dummy.txt')
self.assertTrue(os.path.exists(release_file.distribution.path))
def request(self, host, handler, request_body, verbose=0):
self.verbose = verbose
response = self.client.post(handler,
request_body,
content_type="text/xml")
res = six.BytesIO(response.content)
setattr(res, 'getheader', lambda *args: '') # For Python >= 2.7
res.seek(0)
return self.parse_response(res)
def fake_urlopen(self, url):
"""Fake urlopen using client if domain
correspond to current_site else HTTPError"""
scheme, netloc, path, query, fragment = urlsplit(url)
if not netloc:
raise
if self.site.domain == netloc:
response = six.BytesIO(self.client.get(url).content)
return response
raise HTTPError(url, 404, 'unavailable url', {}, None)
def test_delete_file(self):
release_file = factories.ReleaseFileFactory()
dummy_fh = six.BytesIO(six.b("release-file-contents"))
release_file.save_filecontent('dummy.txt', dummy_fh)
self.assertTrue(os.path.exists(release_file.distribution.path))
utils.delete_files(models.ReleaseFile, instance=release_file)
self.assertFalse(os.path.exists(release_file.distribution.path))
def _download_file(self, url):
"""
Download file from URL using secure JIRA session.
:return: byte stream
:raises: requests.RequestException
"""
session = self.backend.manager._session
response = session.get(url)
response.raise_for_status()
return six.BytesIO(response.content)
def get_unicode_csv_export(self, context):
"""Exports the data in the configured encoding. Default utf8"""
if unicodecsv is None:
raise ImproperlyConfigured("Need to install module \"unicodecsv\" "
"in order to export csv as unicode.")
datas = self._get_datas(context)
stream = six.BytesIO()
writer = unicodecsv.writer(stream, encoding=self.export_unicode_encoding)
writer.writerows(datas)
return stream.getvalue()
def as_bytes(self, unixfrom=False):
"""Return the entire formatted message as bytes.
Optional `unixfrom' when True, means include the Unix From_ envelope
header.
This overrides the default as_bytes() implementation to not mangle
lines that begin with 'From '. See bug #13433 for details.
"""
fp = six.BytesIO()
g = generator.BytesGenerator(fp, mangle_from_=False)
g.flatten(self, unixfrom=unixfrom)
return fp.getvalue()
def test_invalid_json(self):
"""
Tests the error handling when there is invalid JSON.
"""
test_json = textwrap.dedent("""
{
"model": "config_models.ExampleDeserializeConfig",
"data": [{"name": "dino"
""")
if six.PY3:
test_json = test_json.encode('utf-8')
with six.assertRaisesRegex(self, Exception, "JSON parse error"):
deserialize_json(six.BytesIO(test_json), self.test_username)
def test_csv_existing_import(self):
factories.PublicBodyFactory.create(site=self.site, name='Public Body 76 X')
# reenable when django-taggit support atomic transaction wrapping
# factories.PublicBodyTagFactory.create(slug='public-body-topic-76-x', is_topic=True)
prev_count = PublicBody.objects.all().count()
# Existing entity via slug, no id reference
csv = '''name,email,jurisdiction__slug,other_names,description,tags,url,parent__name,classification,contact,address,website_dump,request_note
Public Body 76 X,[email protected],bund,,,public-body-topic-76-x,http://example.com,,Ministry,Some contact stuff,An address,,'''
imp = CSVImporter()
imp.import_from_file(six.BytesIO(csv.encode('utf-8')))
now_count = PublicBody.objects.all().count()
self.assertEqual(now_count, prev_count)
def request(self, host, handler, request_body, verbose=0):
self.verbose = verbose
response = self.client.post(handler,
request_body,
content_type="text/xml")
#I think(?) this is supposed to be bytes
res = six.BytesIO(response.content)
setattr(res, 'getheader', lambda *args: '') # For Python >= 2.7
res.seek(0)
#What's with the redundancy?
if not hasattr(res, 'getheader'):
setattr(res, 'getheader', lambda *args: "")
return self.parse_response(res)
def test_invalid_model(self):
"""
Tests the error handling when the configuration model specified does not exist.
"""
test_json = textwrap.dedent("""
{
"model": "xxx.yyy",
"data":[{"name": "dino"}]
}
""")
if six.PY3:
test_json = test_json.encode('utf-8')
with six.assertRaisesRegex(self, Exception, "No installed app"):
deserialize_json(six.BytesIO(test_json), self.test_username)
def get_xml_export(self, context):
results = self._get_objects(context)
stream = six.BytesIO()
xml = SimplerXMLGenerator(stream, self.export_unicode_encoding)
xml.startDocument()
xml.startElement("objects", {})
self._to_xml(xml, results)
xml.endElement("objects")
xml.endDocument()
return stream.getvalue().split((b'\n'))[1]
def test_bad_username(self):
"""
Tests the error handling when the specified user does not exist.
"""
test_json = textwrap.dedent("""
{
"model": "config_models.ExampleDeserializeConfig",
"data": [{"name": "dino"}]
}
""")
if six.PY3:
test_json = test_json.encode('utf-8')
with six.assertRaisesRegex(self, Exception,
"User matching query does not exist"):
deserialize_json(six.BytesIO(test_json), "unknown_username")
def flake8_code(code):
orig_stdin = sys.stdin
if six.PY2:
stdin = six.BytesIO(code)
else:
stdin = six.StringIO(code)
try:
sys.stdin = stdin
with captured_stdout() as stdout:
Flake8Application().run(['-'])
finally:
sys.stdin = orig_stdin
return [line for line in stdout.getvalue().split('\n') if line]
def create_inmemory_image(file_name='tmp.png', format=None, width=200, height=200, content_type=None):
from PIL import Image
if not format:
_, extension = os.path.splitext(file_name)
format = extension[1:].upper()
if not content_type:
content_type = 'image/{0}'.format(format)
stream = six.BytesIO()
size = (width, height)
color = (255, 0, 0, 0)
image = Image.new('RGBA', size, color)
image.save(stream, format=format)
image_file = InMemoryUploadedFile(stream, None, file_name, content_type, stream.tell(), None)
image_file.seek(0)
return image_file
