def dump_from_session_manager(self, manager, stream):
"""
Extract data from session manager and dump it into the stream.
:param session_manager:
SessionManager instance that manages session to be exported by
this exporter
:param stream:
Byte stream to write to.
"""
preset = None
mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
mem_mib = mem_bytes / (1024.**2)
# On systems with less than 1GiB of RAM, create the submission tarball
# without any compression level (i.e preset=0).
# See https://docs.python.org/3/library/lzma.html
# With preset 9 for example, the overhead for an LZMACompressor object
# can be as high as 800 MiB.
if mem_mib < 1200:
preset = 0
job_state_map = manager.default_device_context.state.job_state_map
with tarfile.TarFile.open(None, 'w:xz', stream, preset=preset) as tar:
for fmt in ('html', 'json', 'junit'):
unit = self._get_all_exporter_units()[
'com.canonical.plainbox::{}'.format(fmt)]
exporter = Jinja2SessionStateExporter(exporter_unit=unit)
with SpooledTemporaryFile(max_size=102400, mode='w+b') as _s:
exporter.dump_from_session_manager(manager, _s)
tarinfo = tarfile.TarInfo(name="submission.{}".format(fmt))
tarinfo.size = _s.tell()
tarinfo.mtime = time.time()
_s.seek(0) # Need to rewind the file, puagh
tar.addfile(tarinfo, _s)
for job_id in manager.default_device_context.state.job_state_map:
job_state = job_state_map[job_id]
try:
recordname = job_state.result.io_log_filename
except AttributeError:
continue
for stdstream in ('stdout', 'stderr'):
filename = recordname.replace('record.gz', stdstream)
folder = 'test_output'
if job_state.job.plugin == 'attachment':
folder = 'attachment_files'
if os.path.exists(filename) and os.path.getsize(filename):
arcname = os.path.basename(filename)
if stdstream == 'stdout':
arcname = os.path.splitext(arcname)[0]
tar.add(filename,
os.path.join(folder, arcname),
recursive=False)
def _get_file(self):
if self._file is None:
self._file = SpooledTemporaryFile(
max_size=self._storage.max_memory_size,
suffix=".GSStorageFile",
dir=setting("FILE_UPLOAD_TEMP_DIR", None)
)
if 'r' in self._mode:
self._is_dirty = False
self.blob.download_to_file(self._file)
self._file.seek(0)
return self._file
def store_filelike(self, filelike):
copy = SpooledTemporaryFile() # boto3 now closes the file.
copy.write(filelike.read())
copy.seek(0)
details = {'ACL': self.acl}
if self.content_type:
details['ContentType'] = self.content_type
self._key.upload_fileobj(copy, ExtraArgs=details)
return self.url()
def test_render_tile_with_data(tile_generator: TileGenerator,
tile_test_cases: Dict[str, dict]):
"""
test render tiles for tile_test_cases with using ohdm test data
Arguments:
tile_generator {TileGenerator} -- default TileGenerator
"""
# cleanup data
clear_mapnik_tables()
# fill database
run_import(
file_path="/niue-latest.osm.pbf",
db_cache_size=10000,
cache2file=False,
)
tile_generator.request_date = timezone.now()
for test_case in tile_test_cases:
# contine if test case has not test data
if not tile_test_cases[test_case]["has_date_data"]:
continue
print("test: {}".format(test_case))
tile_generator.zoom = tile_test_cases[test_case]["zoom"]
tile_generator.x_pixel = tile_test_cases[test_case]["x"]
tile_generator.y_pixel = tile_test_cases[test_case]["y"]
# generate new tile into a tmp file
new_tile: SpooledTemporaryFile = SpooledTemporaryFile()
new_tile.write(tile_generator.render_tile())
new_tile.seek(0)
# open new tile as image
new_tile_image: Image = Image.open(new_tile)
# check if the tile is a PNG file
if new_tile_image.format != "PNG":
raise AssertionError
# monochrome & resize images to better compare them
reference_tile = Image.open(
"/app/compose/local/django/test_tile/{}".format(
tile_test_cases[test_case]["tile_png"]))
diff: bool = ImageChops.difference(reference_tile,
new_tile_image).getbbox() is None
assert diff is False
# cleanup data
clear_mapnik_tables()
def pipestring_process(cmd_string, stdin_string=''):
"""Pipe a python string to standard input for cmd_string
>>> pipestring_process('grep 2', '1\\n2\\n3\\n')
(0, '2\\n', '')
"""
f=SpooledTemporaryFile()
f.write(stdin_string)
f.seek(0)
results=process(cmd_string, stdin=f)
f.close()
return results
def run_command(self, command, stdin=None, env=None):
"""
Launch a shell command line.
:param command: Command line to launch
:type command: str
:param stdin: Standard input of command
:type stdin: file
:param env: Environment variable used in command
:type env: dict
:return: Standard output of command
:rtype: file
"""
cmd = shlex.split(command)
stdout = SpooledTemporaryFile(max_size=settings.TMP_FILE_MAX_SIZE,
dir=settings.TMP_DIR)
stderr = SpooledTemporaryFile(max_size=settings.TMP_FILE_MAX_SIZE,
dir=settings.TMP_DIR)
full_env = os.environ.copy() if self.use_parent_env else {}
full_env.update(self.env)
full_env.update(env or {})
try:
if isinstance(stdin, File):
process = Popen(
cmd, stdin=stdin.open("rb"), stdout=stdout, stderr=stderr,
env=full_env
)
else:
process = Popen(cmd, stdin=stdin, stdout=stdout, stderr=stderr, env=full_env)
process.wait()
if process.poll():
stderr.seek(0)
raise exceptions.CommandConnectorError(
"Error running: {}\n{}".format(command, stderr.read().decode('utf-8')))
stdout.seek(0)
stderr.seek(0)
return stdout, stderr
except OSError as err:
raise exceptions.CommandConnectorError(
"Error running: {}\n{}".format(command, str(err)))
def _send_boto3(self, boto3, resp, request):
from botocore.exceptions import BotoCoreError, ClientError
bucket_name, key_string = url_to_s3_info(request.url)
# Get the key without validation that it exists and that we have
# permissions to list its contents.
key = boto3.resource('s3').Object(bucket_name, key_string[1:])
try:
response = key.get()
except (BotoCoreError, ClientError) as exc:
# This exception will occur if the bucket does not exist or if the
# user does not have permission to list its contents.
resp.status_code = 404
message = {
"error": "error downloading file from s3",
"path": request.url,
"exception": repr(exc),
}
fh = SpooledTemporaryFile()
fh.write(ensure_binary(json.dumps(message)))
fh.seek(0)
resp.raw = fh
resp.close = resp.raw.close
return resp
key_headers = response['ResponseMetadata']['HTTPHeaders']
resp.headers = CaseInsensitiveDict({
"Content-Type": key_headers.get('content-type', "text/plain"),
"Content-Length": key_headers['content-length'],
"Last-Modified": key_headers['last-modified'],
})
f = SpooledTemporaryFile()
key.download_fileobj(f)
f.seek(0)
resp.raw = f
resp.close = resp.raw.close
return resp
def image(type, spec=' ', ext='png'):
# Parameters for `suml`.
import suml.common
import optparse
options = optparse.Values(({
'scruffy': True,
'png': ext == 'png',
'svg': ext == 'svg' or ext == 'pdf',
'font': suml.common.defaultScruffyFont(),
'shadow': False,
}))
from tempfile import SpooledTemporaryFile
fout = SpooledTemporaryFile()
# Execute Scruffy `suml`.
if type == 'class':
suml.yuml2dot.transform(spec, fout, options)
elif type == 'sequence':
suml.suml2pic.transform(spec, fout, options)
else:
return HTTPError(404, 'Unhandled diagram type.')
# Retrieve the data generated.
fout.seek(0)
data = fout.read()
fout.close()
# Convert SVG to PDF?
if ext == 'pdf':
# Load SVG file.
doc = xml.dom.expatbuilder.parseString(data)
# Convert to a RLG drawing
svg_renderer = svglib.svglib.SvgRenderer()
svg_renderer.render(doc.documentElement)
drawing = svg_renderer.finish()
# Generate PDF.
data = reportlab.graphics.renderPDF.drawToString(drawing)
# Server the generated image.
if ext == 'png':
response.content_type = 'image/png'
elif ext == 'svg':
response.content_type = 'image/svg+xml'
elif ext == 'pdf':
response.content_type = 'application/pdf'
else:
return HTTPError(500, 'Unhandled extension type.')
return data
def retrieve_shard_file(self, url, shard_index):
farmer_tries = 0
self.__logger.debug('Downloading shard at index %s from farmer: %s',
shard_index, url)
tries_download_from_same_farmer = 0
while MAX_RETRIES_DOWNLOAD_FROM_SAME_FARMER > \
tries_download_from_same_farmer:
tries_download_from_same_farmer += 1
farmer_tries += 1
try:
# data is spooled in memory until the file size exceeds max_size
shard = SpooledTemporaryFile(self.max_spooled, 'wb')
# Request the shard
r = requests.get(url, stream=True, timeout=self.client.timeout)
if r.status_code != 200 and r.status_code != 304:
raise StorjFarmerError()
# Write the file
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
shard.write(chunk)
# Everything ok
# move file read pointer at beginning
shard.seek(0)
return shard
except StorjFarmerError as e:
self.__logger.error(e)
# Update shard download state
self.__logger.error('First try failed. Retrying... (%s)' %
str(farmer_tries))
except requests.exceptions.Timeout as ret:
self.__logger.error('Request number %s for shard %s timed out.\
Took too much.' % (farmer_tries, shard_index))
self.__logger.error(ret)
except Exception as e:
self.__logger.error(e)
self.__logger.error('Unhandled error')
self.__logger.error('Error occured while downloading shard at '
'index %s. Retrying... (%s)' %
(shard_index, farmer_tries))
self.__logger.error('Shard download at index %s failed' % shard_index)
raise ClientError()
def _get_file(self):
if self._file is None:
self._file = SpooledTemporaryFile(
max_size=self._storage.max_memory_size,
suffix='.S3BotoStorageFile',
dir=setting('FILE_UPLOAD_TEMP_DIR', None))
if 'r' in self._mode:
self._is_dirty = False
self.key.get_contents_to_file(self._file)
self._file.seek(0)
if self._storage.gzip and self.key.content_encoding == 'gzip':
self._file = GzipFile(mode=self._mode, fileobj=self._file)
return self._file
def _get_file(self):
if self._file is None:
self._file = SpooledTemporaryFile(
max_size=self._storage.max_memory_size,
suffix=".S3Boto3StorageFile",
dir=setting("FILE_UPLOAD_TEMP_DIR", None))
if 'r' in self._mode:
self._is_dirty = False
self._file.write(self.obj.get()['Body'].read())
self._file.seek(0)
if self._storage.gzip and self.obj.content_encoding == 'gzip':
self._file = GzipFile(mode=self._mode, fileobj=self._file)
return self._file
def _write_logs(filename, logs, log_archive):
with SpooledTemporaryFile(MEMORY_TEMPFILE_SIZE) as tempfile:
with GzipFile("temp_action_log_rotate",
fileobj=tempfile,
compresslevel=1) as zipstream:
for chunk in StreamingJSONEncoder().iterencode(logs):
zipstream.write(chunk.encode("utf-8"))
tempfile.seek(0)
log_archive.store_file(tempfile,
JSON_MIMETYPE,
content_encoding="gzip",
file_id=filename)
def _get_file(self):
if self._file is None:
with metrics.timer('filestore.read', instance='gcs'):
self._file = SpooledTemporaryFile(
max_size=self._storage.max_memory_size,
suffix=".GSStorageFile",
dir=None,
)
if 'r' in self._mode:
self._is_dirty = False
self.blob.download_to_file(self._file)
self._file.seek(0)
return self._file
def getAsFile(self, chunk_size=512):
"""
Return the attachment as a TempFile
:param chunk_size:
:return: SpooledTemporaryFile
"""
tf = SpooledTemporaryFile(max_size=1024 * 1024, mode='w+b')
r = self._client.attachment._get_file(self.sys_id)
for chunk in r.iter_content(chunk_size=chunk_size):
tf.write(chunk)
tf.seek(0)
return tf
def __init__(self, blob, maxsize=1000):
"""
:type blob: google.cloud.storage.blob.Blob
"""
self._dirty = False
self._tmpfile = SpooledTemporaryFile(
max_size=maxsize,
prefix="django_gcloud_storage_"
)
self._blob = blob
super(GCloudFile, self).__init__(self._tmpfile)
def open(self, name, spool_size=5 * 1024 * 1024):
if isinstance(name, LocalHeader):
name = name.filename
offset, header = self._get_file_info(name)
self.stream.seek(offset)
dest = SpooledTemporaryFile(max_size=spool_size)
if header.compression_method == ZIP_STORED:
copy_stored_file(self.stream, header.compressed_size, dest)
else:
copy_compressed_file(self.stream, header.compressed_size, dest)
dest.seek(0)
return dest
def generate_thumbnail(self, content):
content = file_from_content(content)
try:
uploaded_image = Image.open(content)
except:
flask.abort(
400,
ValidationError({
'message': 'File Format',
'object': {
"error": "Format Incorrect"
},
}))
if max(uploaded_image.size) >= self.max_size:
uploaded_image.thumbnail((self.max_size, self.max_size),
Image.BILINEAR)
content = SpooledTemporaryFile(INMEMORY_FILESIZE)
uploaded_image.save(content, uploaded_image.format)
content.seek(0)
thumbnail = uploaded_image.copy()
thumbnail.thumbnail(self.thumbnail_size, Image.ANTIALIAS)
thumbnail = thumbnail.convert('RGBA')
thumbnail.format = self.thumbnail_format
output = SpooledTemporaryFile(INMEMORY_FILESIZE)
thumbnail.save(output, self.thumbnail_format)
output.seek(0)
thumb_path, thumb_id = self.store_content(
output, 'thumb.%s' % self.thumbnail_format.lower())
self['thumb_id'] = thumb_id
self['thumb_path'] = thumb_path
thumbnail_file = self.thumb_file
self['_thumb_public_url'] = thumbnail_file.public_url
content.close()
def default_stream_factory(
total_content_length: int,
content_type: t.Optional[str],
filename: t.Optional[str],
content_length: t.Optional[int] = None,
) -> t.BinaryIO:
max_size = 1024 * 500
if SpooledTemporaryFile is not None:
return t.cast(t.BinaryIO, SpooledTemporaryFile(max_size=max_size, mode="rb+"))
elif total_content_length is None or total_content_length > max_size:
return t.cast(t.BinaryIO, TemporaryFile("rb+"))
return BytesIO()
def _stream_factory(self,
total_content_length,
filename,
content_type,
content_length=None):
if SpooledTemporaryFile is not None:
return SpooledTemporaryFile(
max_size=self.options.max_file_memory_size, mode='wb+')
if (total_content_length is None
or total_content_length > self.options.max_file_memory_size):
return TemporaryFile('wb+')
return BytesIO()
def _buffer_response(status_headers, iterator):
out = SpooledTemporaryFile(ProxyRouter.BUFF_RESPONSE_MEM_SIZE)
size = 0
for buff in iterator:
size += len(buff)
out.write(buff)
content_length_str = str(size)
# remove existing content length
status_headers.replace_header('Content-Length', content_length_str)
out.seek(0)
return RewriteContent.stream_to_gen(out)
def export_survey_tables(request):
"""Export a zip file of CSVs of all types of beekeepers surveys."""
# prepare queries to the database
db_options = 'dbname={} user={} host={} password={}'.format(
DB['NAME'], DB['USER'], DB['HOST'], DB['PASSWORD'])
connection = psycopg2.connect(db_options)
cur = connection.cursor()
tables = dict(
apiary=None,
novembersurvey=None,
aprilsurvey=None,
monthlysurvey=None,
usersurvey="""
SELECT auth_user.username, auth_user.email,
beekeepers_usersurvey.*
FROM beekeepers_usersurvey
INNER JOIN auth_user ON beekeepers_usersurvey.user_id=auth_user.id
""",
survey="""
SELECT beekeepers_survey.*, beekeepers_apiary.lat,
beekeepers_apiary.lng, beekeepers_apiary.user_id
FROM beekeepers_survey
INNER JOIN beekeepers_apiary
ON beekeepers_survey.apiary_id=beekeepers_apiary.id
""",
)
# the zipped CSVs are written in memory
date_stamp = now().strftime('%Y-%m-%d_%H-%M-%S')
zip_dir = 'beekeepers_exports_{}'.format(date_stamp)
stream = StringIO()
with ZipFile(stream, 'w') as zf:
for table, query in tables.iteritems():
if query is None:
query = 'SELECT * FROM beekeepers_{}'.format(table)
filename = '{}/{}_{}.csv'.format(zip_dir, table, date_stamp)
full_query = 'COPY ({0}) TO STDOUT WITH CSV HEADER'.format(query)
tempfile = SpooledTemporaryFile()
cur.copy_expert(full_query, tempfile)
tempfile.seek(0)
zf.writestr(filename, tempfile.read())
zf.close()
resp = HttpResponse(stream.getvalue(), content_type='application/zip')
resp['Content-Disposition'] = 'attachment; filename={}.zip'.format(zip_dir)
connection.close()
return resp
def _get_file(self):
if self._file is None:
self._file = SpooledTemporaryFile(
max_size=self._storage.max_memory_size,
suffix='.S3Boto3StorageFile',
dir=setting('FILE_UPLOAD_TEMP_DIR')
)
if 'r' in self._mode:
self._is_dirty = False
self.obj.download_fileobj(self._file)
self._file.seek(0)
if self._storage.gzip and self.obj.content_encoding == 'gzip':
self._file = self._decompress_file(mode=self._mode, file=self._file)
return self._file
def _get_file(self):
def _try_download():
self.blob.download_to_file(self._file)
self._file.seek(0)
if self._file is None:
with metrics.timer("filestore.read", instance="gcs"):
self._file = SpooledTemporaryFile(
max_size=self._storage.max_memory_size, suffix=".GSStorageFile", dir=None
)
if "r" in self._mode:
self._is_dirty = False
try_repeated(_try_download)
return self._file
def _get_file(self):
if self._file is None:
self._file = SpooledTemporaryFile(
max_size=self._storage.max_memory_size,
suffix=".S3Boto3StorageFile",
dir=get_setting("FILE_UPLOAD_TEMP_DIR")
)
if 'r' in self._mode:
self._is_dirty = False
self.obj.download_fileobj(self._file)
self._file.seek(0)
if self._storage.gzip and self.obj.content_encoding == 'gzip':
self._file = GzipFile(mode=self._mode, fileobj=self._file, mtime=0.0)
return self._file
def _get_file(self):
if self._file is None:
self._file = SpooledTemporaryFile(
max_size=self._storage.max_memory_size,
suffix=".GSStorageFile",
dir=setting("FILE_UPLOAD_TEMP_DIR")
)
if 'r' in self._mode:
self._is_dirty = False
self.blob.download_to_file(self._file)
self._file.seek(0)
if self._storage.gzip and self.blob.content_encoding == 'gzip':
self._file = self._decompress_file(mode=self._mode, file=self._file)
return self._file
def buffer_iter(status_headers, iterator, buff_size=BUFF_SIZE * 4):
out = SpooledTemporaryFile(buff_size)
size = 0
for buff in iterator:
size += len(buff)
out.write(buff)
content_length_str = str(size)
# remove existing content length
status_headers.replace_header('Content-Length', content_length_str)
out.seek(0)
return StreamIter(out)
def download_model(request):
"""
Sends model.pickle as a download response to the end-user if it exists in the filesystem,
if not, sends an empty response.
:param request:
:return:
"""
if 'model_id' in request.GET:
model_id = request.GET['model_id']
task_object = Task.objects.get(pk=model_id)
if task_object.status == "completed":
unique_id = task_object.unique_id
task_type = task_object.task_type
task_xml_name = "task_{}.xml".format(unique_id)
model_name = "model_{}".format(unique_id)
model_file_path = os.path.join(MODELS_DIR, task_type, model_name)
media_path = os.path.join(PROTECTED_MEDIA, "task_manager/", task_type, model_name)
model_files = get_wildcard_files(model_file_path)
media_files = get_wildcard_files(media_path)
zip_path = "zipped_model_{}.zip".format(model_id)
# Make temporary Zip file
with SpooledTemporaryFile() as tmp:
with ZipFile(tmp, 'w', zipfile.ZIP_DEFLATED) as archive:
# Write Task model object as xml
task_xml_data = serializers.serialize("xml", [task_object])
archive.writestr(task_xml_name, task_xml_data)
# Write model files
for path, name in model_files:
archive.write(path, "model/"+name)
for path, name in media_files:
archive.write(path, "media/"+name)
# Reset file pointer
tmp.seek(0)
# Write file data to response
response = HttpResponse(tmp.read())
# Download file
response['Content-Disposition'] = 'attachment; filename=' + os.path.basename(zip_path)
return response
return HttpResponse()
def upload_product(self):
decoded_base64 = base64.b64decode(self.images)
res = []
with SpooledTemporaryFile() as tmp:
tmp.write(decoded_base64)
archive = ZipFile(tmp, 'r')
for file in archive.filelist:
content = archive.read(file.filename)
if content:
product_name = file.filename.split("/", 1)[1]
name = product_name.split(".", 1)[0]
product_obj = self.env['product.template']
product_rec = product_obj.search([('name', '=', name)])
if len(product_rec) != 0:
encoded = base64.b64encode(content)
product_rec.write({"image_medium": encoded})
else:
res.append(name)
if len(res) != 0:
product_not_found = ','.join(res)
return {
'name': 'Message',
'type': 'ir.actions.act_window',
'view_type': 'form',
'view_mode': 'form',
'res_model': 'custom.pop.message',
'target': 'new',
'context': {
'default_name':
"Please correct following image with product name %s is not avalible."
% (product_not_found)
}
}
else:
return {
'name': 'Message',
'type': 'ir.actions.act_window',
'view_type': 'form',
'view_mode': 'form',
'res_model': 'custom.pop.message',
'target': 'new',
'context': {
'default_name': "All Product image updated "
}
}
def _get_file(self):
if self._file is None:
self._file = SpooledTemporaryFile()
# get from redis
content = self._storage.client.get(self.name)
# stored as base64 .. decode
content = base64.b64decode(content)
with io.BytesIO(content) as file_content:
copyfileobj(file_content, self._file)
self._file.seek(0)
return self._file
def __init__(self, name, req_t, session):
logger.debug("creating window %s", name)
self.name = name
self.status = RUNNING
self.req_t = req_t
self.sid = req_t[1]
self.rid = req_t[2]
self.plg_id = req_t[4]
self.wid = RecvWindow.ID
self.session = session
self.is_used = False
self.closed = False
self._buffer = SpooledTemporaryFile(max_size="8096", mode="wr+b")
RecvWindow.ID += 1
