def test_job_state_from_processing_to_success(self, api_client,
job_factory, mocker,
lambda_auth_token,
default_storage):
job = job_factory(job_state=ds_constants.ProcessingState.PROCESSING,
result=None,
error="")
default_storage.save(name="path/to/result.csv",
content=base.ContentFile("test,1,2".encode()))
payload = dict(
job_state=ds_constants.ProcessingState.SUCCESS,
result="path/to/result.csv",
result_parquet="path/to/result.csv",
error="test",
)
set_active_job_mock = mocker.patch(
"schemacms.datasources.models.DataSource.set_active_job")
response = api_client.post(
self.get_url(job.pk),
payload,
HTTP_AUTHORIZATION="Token {}".format(lambda_auth_token),
format="json",
)
job.refresh_from_db()
assert response.status_code == status.HTTP_204_NO_CONTENT, response.content
assert job.job_state == payload["job_state"]
assert job.result == payload["result"]
assert job.error == ""
set_active_job_mock.assert_called_with(job)
def save(self, *args, **kwargs):
""" Override the model's saving function to resize the screenshot
"""
# Code inspired by:
# https://github.com/ericflo/django-avatar/blob/master/avatar/models.py
# Create image that are no more than 700 pixels wide
# for use in the website
self.img_file_raw.seek(0)
orig = self.img_file_raw.read()
image = Image.open(StringIO(orig))
if image.format in IMG_ACCEPTABLE_FORMATS:
format_used = image.format
else:
format_used = IMG_DEFAULT_FORMAT
(w, h) = image.size
if w > IMG_MAX_WIDTH:
h = int(h * IMG_MAX_WIDTH / (w + 0.0))
w = IMG_MAX_WIDTH
if image.mode != "RGB" and image.mode != "RGBA":
image = image.convert("RGB")
image = image.resize((w, h), IMG_RESIZE_METHOD)
thumb = StringIO()
image.save(thumb, format_used, quality=IMG_QUALITY)
thumb_file = base.ContentFile(thumb.getvalue())
if h > IMG_MAX_HEIGHT:
w = int(w * IMG_MAX_HEIGHT / (h + 0.0))
h = IMG_MAX_HEIGHT
if image.mode != "RGB" and image.mode != "RGBA":
image = image.convert("RGB")
image = image.resize((w, h), IMG_RESIZE_METHOD)
thumb = StringIO()
image.save(thumb, format_used, quality=IMG_QUALITY)
thumb_file = base.ContentFile(thumb.getvalue())
else:
thumb_file = base.ContentFile(orig)
dirname = force_unicode(datetime.datetime.now().strftime(
smart_str(settings.SPC['resized_image_dir'])))
location = os.path.normpath(
os.path.join(dirname, self.img_file_raw.name))
self.img_file = self.img_file.storage.save(location, thumb_file)
super(Screenshot, self).save(*args, **kwargs)
def __call__(self, resolution, image_format='JPEG', name=None):
filename = name or 'default.{}'.format(image_format.lower())
color = 'blue'
image = Image.new('RGB', resolution, color)
image_data = BytesIO()
image.save(image_data, image_format)
image_content = base.ContentFile(image_data.getvalue())
return images.ImageFile(image_content.file, filename)
def init_script(name, inline, person):
spath = str(settings.MEDIA_ROOT) + str(
get_folder_name("scripts", name, None))
if not os.path.isdir(spath):
os.makedirs(spath)
dbitem = breeze.models.Rscripts(
name=name,
category=breeze.models.Script_categories.objects.get(
category="general"),
inln=inline,
author=person,
details="empty",
order=0)
# create empty files for header, code and xml
dbitem.header.save('name.txt',
base.ContentFile('# write your header here...'))
dbitem.code.save(
'name.r', base.ContentFile('# copy and paste main code here...'))
dbitem.save()
root = xml.Element('rScript')
root.attrib['ID'] = str(dbitem.id)
input_array = xml.Element('inputArray')
input_array.text = "empty"
root.append(input_array)
newxml = open(
str(settings.TEMP_FOLDER) + 'script_%s.xml' % (person), 'w')
xml.ElementTree(root).write(newxml)
newxml.close()
dbitem.docxml.save(
'script.xml',
File(open(str(settings.TEMP_FOLDER) + 'script_%s.xml' % (person))))
dbitem.save()
os.remove(str(settings.TEMP_FOLDER) + 'script_%s.xml' % (person))
# dbitem.docxml.save('name.xml', base.ContentFile(''))
return spath
return False
def generate_image(self):
svg = self.achievement.svg_file
svg.open(mode='rb')
try:
image_data = svg_to_png(svg.read(), self.width, self.height)
self.image.save("thumb.png",
files.ContentFile(image_data),
save=False)
finally:
svg.close()
def _open(self, name, mode='rb'):
if mode != 'rb':
# In future when allowed to operate locally (self.local_storage)
# all modes can be allowed. However this will require executing
# distribution upon closing file opened for updates. This worth
# evaluating.
raise IOError('Illegal mode "%s". Only "rb" is supported.')
if self.local_storage:
return self.local_storage.open(name, mode)
host = random.choice(self.hosts)
return base.ContentFile(self.transport.get(host, name))
def init_pipeline(form, user):
"""
Initiates a new RetortType item in the DB.
Creates a folder for initial pipeline data.
"""
# First Save the data that comes with a form:
# 'type', 'description', 'search', 'access'
new_pipeline = form.save(commit=False)
new_pipeline.author = user
# Add configuration file
new_pipeline.config.save(
'config.txt', base.ContentFile('# Configuration Module \n'))
new_pipeline.save()
return True
def save_freeform_annotation(request, cur_course_user, submission_id,
assessment_page_number):
""" Save the freeform annotation image for the specified submission page. """
if request.method != 'POST':
return http.HttpResponse(status=404)
submission_page = shortcuts.get_object_or_404(
models.SubmissionPage,
submission=submission_id,
page_number=int(assessment_page_number))
folder_name = 'freeform-annotation-png'
try:
annotation = submission_page.freeformannotation
except models.FreeformAnnotation.DoesNotExist:
annotation = models.FreeformAnnotation(submission_page=submission_page)
else: # If there is an annotation for the given `SubmissionPage`, delete old image from S3
old_url = annotation.annotation_image.url
# Get path starting from the folder i.e. remove the domain from the URL
storage.default_storage.delete(old_url[old_url.index(folder_name):])
# The image is encoded as a B64 string. Decode it, and save it.
b64_img = request.POST['annotation_image']
header = 'data:image/png;base64,'
# The header must be in the b64_image
if b64_img[0:len(header)] != header:
return http.HttpResponse(status=422)
try:
img_bytestream = base64.b64decode(b64_img[len(header):])
except TypeError:
return http.HttpResponse(status=422)
# Must be a PNG image
if imghdr.what(None, img_bytestream) != 'png':
return http.HttpResponse(status=422)
img = base.ContentFile(img_bytestream, 'tmp')
annotation.annotation_image.save(folder_name, img)
annotation.save()
return http.HttpResponse(status=204)
def handle_upload(request, func, *args, **kwargs):
"""
Uploading files via Ajax
"""
if request.method == "POST":
if not request.FILES:
upload = base.ContentFile(request.read())
if 'qqfile' in request.GET:
filename = request.GET['qqfile']
else:
filename = ""
else:
if len(request.FILES) == 1:
upload = next(request.FILES.values())
else:
raise http.Http404('Bad Upload')
filename = upload.name
res = func(request, upload, filename, *args, **kwargs)
# let Ajax Upload know whether we saved it or not
ret_json = {'success': True}
if res:
ret_json = res
res['success'] = True
return http.HttpResponse(json.dumps(ret_json))
def _open(self, name, mode='rb'):
return base.ContentFile(self.container.open(name).read())
def image_upload_file(self, filename="test.png"):
return base.ContentFile(content=make_image().read(), name=filename)
def python_upload_file(self, code, filename="test.py"):
return base.ContentFile(content=make_script(code).getvalue(),
name=filename)
def csv_upload_file(self, filename="test.csv", cols_num=3, rows_num=1):
return base.ContentFile(content=make_csv(cols_num=cols_num,
rows_num=rows_num).getvalue(),
name=filename)
def build_report(report_data, request_data, report_property, sections):
""" Assembles report home folder, configures DRMAA and R related files
and spawns a new process for reports DRMAA job on cluster.
Arguments:
report_data -- report info dictionary
request_data -- a copy of request object
report_property -- report property form
sections -- a list of 'Rscripts' db objects
"""
# 'report_name' - report's headline
rt = breeze.models.ReportType.objects.get(type=report_data['report_type'])
report_name = report_data['report_type'] + ' Report' + ' :: ' + report_data[
'instance_name'] + ' <br> ' + str(rt.description)
# This trick is to extract users's names from the form
# buddies = list()
# for e in list(report_property.cleaned_data['share']):
# buddies.append( str(e) )
# shared_users = breeze.models.User.objects.filter(username__in=buddies)
shared_users = aux.extract_users(request_data.POST.get('Groups'),
request_data.POST.get('Individuals'))
insti = breeze.models.UserProfile.objects.get(
user=request_data.user).institute_info
# create initial instance so that we can use its db id
dbitem = breeze.models.Report(
type=breeze.models.ReportType.objects.get(
type=report_data['report_type']),
name=str(report_data['instance_name']),
author=request_data.user,
progress=0,
project=breeze.models.Project.objects.get(
id=request_data.POST.get('project')),
institute=insti
# project=breeze.models.Project.objects.get(name=report_property.cleaned_data['Project'])
)
dbitem.save()
if shared_users:
dbitem.shared = shared_users
# define location: that is report's folder name
path = slugify(
str(dbitem.id) + '_' + dbitem.name + '_' + dbitem.author.username)
loc = str(settings.MEDIA_ROOT) + str("reports/") + path
dochtml = loc + '/report'
dbitem.home = str("reports/") + path
dbitem.save()
# BUILD R-File
script_string = 'setwd(\"%s\")\n' % loc
script_string += 'require( Nozzle.R1 )\n\n'
script_string += 'path <- \"%s\"\n' % loc
script_string += 'report_name <- \"%s\"\n' % report_name
# define a function for exception handler
script_string += 'failed_fun_print <- function(section_name, error_report){\n'
script_string += ' Error_report_par <- newParagraph("<br>", asStrong( "Error Log Details: " ),"<br><br>",asCode(paste(error_report,collapse=""))); \n'
script_string += ' section_name <- addTo( section_name, newParagraph( "This section FAILED! Contact the development team... " ), Error_report_par )\n'
script_string += ' return (section_name)\n}\n\n'
script_string += dump_project_parameters(dbitem.project, dbitem)
script_string += dump_pipeline_config(rt, report_data['instance_id'])
script_string += 'REPORT <- newCustomReport(report_name)\n'
dummy_flag = False
for tag in sections:
secID = 'Section_dbID_' + str(tag.id)
if secID in request_data.POST and request_data.POST[secID] == '1':
tree = xml.parse(str(settings.MEDIA_ROOT) + str(tag.docxml))
script_string += '##### TAG: %s #####\n' % tag.name
if tag.name == "Import to FileMaker":
dummy_flag = True
# source main code segment
code_path = str(settings.MEDIA_ROOT) + str(tag.code)
script_string += '# <---------- body ----------> \n' + open(
code_path, 'r').read() + '\n'
script_string += '# <------- end of body --------> \n'
# input parameters definition
script_string += '# <---------- parameters ----------> \n'
script_string += gen_params_string(
tree, request_data.POST,
str(settings.MEDIA_ROOT) + dbitem.home, request_data.FILES)
script_string += '# <------- end of parameters --------> \n'
# final step - fire header
header_path = str(settings.MEDIA_ROOT) + str(tag.header)
script_string += '# <---------- header ----------> \n' + open(
header_path, 'r').read() + '\n\n'
script_string += 'new_section <- newSection( section_name )\n'
script_string += 'tag_section <- tryCatch({section_body(new_section)}, error = function(e){ failed_fun_print(new_section,e) })\n'
script_string += 'REPORT <- addTo( REPORT, tag_section )\n'
script_string += '# <------- end of header --------> \n'
script_string += '##### END OF TAG #####\n\n\n'
script_string += 'setwd(\"%s\")\n' % loc
else: # if tag disabled - do nothing
pass
# render report to file
script_string += '# Render the report to a file\n' + 'writeReport( REPORT, filename=toString(\"%s\"))\n' % dochtml
script_string += 'system("chmod -R 770 .")'
# save r-file
dbitem.rexec.save('script.r', base.ContentFile(script_string))
dbitem.save()
# configure shell-file
config_path = loc + '/sgeconfig.sh'
config = open(config_path, 'w')
# config should be executble
st = os.stat(config_path)
os.chmod(config_path, st.st_mode | stat.S_IEXEC)
command = '#!/bin/bash \n' + str(
settings.R_ENGINE_PATH) + 'CMD BATCH --no-save ' + str(
settings.MEDIA_ROOT) + str(dbitem.rexec)
config.write(command)
config.close()
# open report's folder for others
st = os.stat(loc)
os.chmod(loc, st.st_mode | stat.S_IRWXG)
# submit r-code
p = Process(target=run_report, args=(dbitem, dummy_flag))
#print(dbitem)
#run_report(dbitem,dummy_flag)
p.start()
return True
def converter_arquivo(self):
"""Converte um arquivo binário csv em um dicionário de dados.""" # noqa: D401
arquivo = self.validated_data['arquivo']
arquivo = base.ContentFile(arquivo.file.read().decode('utf-8'))
return list(csv.DictReader(arquivo))
def save(self, *args, **kwargs):
if self._state.adding:
self.audiofile.save('name', base.ContentFile(b''), save=False)
self.stmfile.save('name', base.ContentFile(b''), save=False)
super().save(*args, **kwargs)