def test_job_state_from_processing_to_success(self, api_client, job_factory, mocker, lambda_auth_token, default_storage): job = job_factory(job_state=ds_constants.ProcessingState.PROCESSING, result=None, error="") default_storage.save(name="path/to/result.csv", content=base.ContentFile("test,1,2".encode())) payload = dict( job_state=ds_constants.ProcessingState.SUCCESS, result="path/to/result.csv", result_parquet="path/to/result.csv", error="test", ) set_active_job_mock = mocker.patch( "schemacms.datasources.models.DataSource.set_active_job") response = api_client.post( self.get_url(job.pk), payload, HTTP_AUTHORIZATION="Token {}".format(lambda_auth_token), format="json", ) job.refresh_from_db() assert response.status_code == status.HTTP_204_NO_CONTENT, response.content assert job.job_state == payload["job_state"] assert job.result == payload["result"] assert job.error == "" set_active_job_mock.assert_called_with(job)
def save(self, *args, **kwargs): """ Override the model's saving function to resize the screenshot """ # Code inspired by: # https://github.com/ericflo/django-avatar/blob/master/avatar/models.py # Create image that are no more than 700 pixels wide # for use in the website self.img_file_raw.seek(0) orig = self.img_file_raw.read() image = Image.open(StringIO(orig)) if image.format in IMG_ACCEPTABLE_FORMATS: format_used = image.format else: format_used = IMG_DEFAULT_FORMAT (w, h) = image.size if w > IMG_MAX_WIDTH: h = int(h * IMG_MAX_WIDTH / (w + 0.0)) w = IMG_MAX_WIDTH if image.mode != "RGB" and image.mode != "RGBA": image = image.convert("RGB") image = image.resize((w, h), IMG_RESIZE_METHOD) thumb = StringIO() image.save(thumb, format_used, quality=IMG_QUALITY) thumb_file = base.ContentFile(thumb.getvalue()) if h > IMG_MAX_HEIGHT: w = int(w * IMG_MAX_HEIGHT / (h + 0.0)) h = IMG_MAX_HEIGHT if image.mode != "RGB" and image.mode != "RGBA": image = image.convert("RGB") image = image.resize((w, h), IMG_RESIZE_METHOD) thumb = StringIO() image.save(thumb, format_used, quality=IMG_QUALITY) thumb_file = base.ContentFile(thumb.getvalue()) else: thumb_file = base.ContentFile(orig) dirname = force_unicode(datetime.datetime.now().strftime( smart_str(settings.SPC['resized_image_dir']))) location = os.path.normpath( os.path.join(dirname, self.img_file_raw.name)) self.img_file = self.img_file.storage.save(location, thumb_file) super(Screenshot, self).save(*args, **kwargs)
def __call__(self, resolution, image_format='JPEG', name=None): filename = name or 'default.{}'.format(image_format.lower()) color = 'blue' image = Image.new('RGB', resolution, color) image_data = BytesIO() image.save(image_data, image_format) image_content = base.ContentFile(image_data.getvalue()) return images.ImageFile(image_content.file, filename)
def init_script(name, inline, person): spath = str(settings.MEDIA_ROOT) + str( get_folder_name("scripts", name, None)) if not os.path.isdir(spath): os.makedirs(spath) dbitem = breeze.models.Rscripts( name=name, category=breeze.models.Script_categories.objects.get( category="general"), inln=inline, author=person, details="empty", order=0) # create empty files for header, code and xml dbitem.header.save('name.txt', base.ContentFile('# write your header here...')) dbitem.code.save( 'name.r', base.ContentFile('# copy and paste main code here...')) dbitem.save() root = xml.Element('rScript') root.attrib['ID'] = str(dbitem.id) input_array = xml.Element('inputArray') input_array.text = "empty" root.append(input_array) newxml = open( str(settings.TEMP_FOLDER) + 'script_%s.xml' % (person), 'w') xml.ElementTree(root).write(newxml) newxml.close() dbitem.docxml.save( 'script.xml', File(open(str(settings.TEMP_FOLDER) + 'script_%s.xml' % (person)))) dbitem.save() os.remove(str(settings.TEMP_FOLDER) + 'script_%s.xml' % (person)) # dbitem.docxml.save('name.xml', base.ContentFile('')) return spath return False
def generate_image(self): svg = self.achievement.svg_file svg.open(mode='rb') try: image_data = svg_to_png(svg.read(), self.width, self.height) self.image.save("thumb.png", files.ContentFile(image_data), save=False) finally: svg.close()
def _open(self, name, mode='rb'): if mode != 'rb': # In future when allowed to operate locally (self.local_storage) # all modes can be allowed. However this will require executing # distribution upon closing file opened for updates. This worth # evaluating. raise IOError('Illegal mode "%s". Only "rb" is supported.') if self.local_storage: return self.local_storage.open(name, mode) host = random.choice(self.hosts) return base.ContentFile(self.transport.get(host, name))
def init_pipeline(form, user): """ Initiates a new RetortType item in the DB. Creates a folder for initial pipeline data. """ # First Save the data that comes with a form: # 'type', 'description', 'search', 'access' new_pipeline = form.save(commit=False) new_pipeline.author = user # Add configuration file new_pipeline.config.save( 'config.txt', base.ContentFile('# Configuration Module \n')) new_pipeline.save() return True
def save_freeform_annotation(request, cur_course_user, submission_id, assessment_page_number): """ Save the freeform annotation image for the specified submission page. """ if request.method != 'POST': return http.HttpResponse(status=404) submission_page = shortcuts.get_object_or_404( models.SubmissionPage, submission=submission_id, page_number=int(assessment_page_number)) folder_name = 'freeform-annotation-png' try: annotation = submission_page.freeformannotation except models.FreeformAnnotation.DoesNotExist: annotation = models.FreeformAnnotation(submission_page=submission_page) else: # If there is an annotation for the given `SubmissionPage`, delete old image from S3 old_url = annotation.annotation_image.url # Get path starting from the folder i.e. remove the domain from the URL storage.default_storage.delete(old_url[old_url.index(folder_name):]) # The image is encoded as a B64 string. Decode it, and save it. b64_img = request.POST['annotation_image'] header = 'data:image/png;base64,' # The header must be in the b64_image if b64_img[0:len(header)] != header: return http.HttpResponse(status=422) try: img_bytestream = base64.b64decode(b64_img[len(header):]) except TypeError: return http.HttpResponse(status=422) # Must be a PNG image if imghdr.what(None, img_bytestream) != 'png': return http.HttpResponse(status=422) img = base.ContentFile(img_bytestream, 'tmp') annotation.annotation_image.save(folder_name, img) annotation.save() return http.HttpResponse(status=204)
def handle_upload(request, func, *args, **kwargs): """ Uploading files via Ajax """ if request.method == "POST": if not request.FILES: upload = base.ContentFile(request.read()) if 'qqfile' in request.GET: filename = request.GET['qqfile'] else: filename = "" else: if len(request.FILES) == 1: upload = next(request.FILES.values()) else: raise http.Http404('Bad Upload') filename = upload.name res = func(request, upload, filename, *args, **kwargs) # let Ajax Upload know whether we saved it or not ret_json = {'success': True} if res: ret_json = res res['success'] = True return http.HttpResponse(json.dumps(ret_json))
def _open(self, name, mode='rb'): return base.ContentFile(self.container.open(name).read())
def image_upload_file(self, filename="test.png"): return base.ContentFile(content=make_image().read(), name=filename)
def python_upload_file(self, code, filename="test.py"): return base.ContentFile(content=make_script(code).getvalue(), name=filename)
def csv_upload_file(self, filename="test.csv", cols_num=3, rows_num=1): return base.ContentFile(content=make_csv(cols_num=cols_num, rows_num=rows_num).getvalue(), name=filename)
def build_report(report_data, request_data, report_property, sections): """ Assembles report home folder, configures DRMAA and R related files and spawns a new process for reports DRMAA job on cluster. Arguments: report_data -- report info dictionary request_data -- a copy of request object report_property -- report property form sections -- a list of 'Rscripts' db objects """ # 'report_name' - report's headline rt = breeze.models.ReportType.objects.get(type=report_data['report_type']) report_name = report_data['report_type'] + ' Report' + ' :: ' + report_data[ 'instance_name'] + ' <br> ' + str(rt.description) # This trick is to extract users's names from the form # buddies = list() # for e in list(report_property.cleaned_data['share']): # buddies.append( str(e) ) # shared_users = breeze.models.User.objects.filter(username__in=buddies) shared_users = aux.extract_users(request_data.POST.get('Groups'), request_data.POST.get('Individuals')) insti = breeze.models.UserProfile.objects.get( user=request_data.user).institute_info # create initial instance so that we can use its db id dbitem = breeze.models.Report( type=breeze.models.ReportType.objects.get( type=report_data['report_type']), name=str(report_data['instance_name']), author=request_data.user, progress=0, project=breeze.models.Project.objects.get( id=request_data.POST.get('project')), institute=insti # project=breeze.models.Project.objects.get(name=report_property.cleaned_data['Project']) ) dbitem.save() if shared_users: dbitem.shared = shared_users # define location: that is report's folder name path = slugify( str(dbitem.id) + '_' + dbitem.name + '_' + dbitem.author.username) loc = str(settings.MEDIA_ROOT) + str("reports/") + path dochtml = loc + '/report' dbitem.home = str("reports/") + path dbitem.save() # BUILD R-File script_string = 'setwd(\"%s\")\n' % loc script_string += 'require( Nozzle.R1 )\n\n' script_string += 'path <- \"%s\"\n' % loc script_string += 'report_name <- \"%s\"\n' % report_name # define a function for exception handler script_string += 'failed_fun_print <- function(section_name, error_report){\n' script_string += ' Error_report_par <- newParagraph("<br>", asStrong( "Error Log Details: " ),"<br><br>",asCode(paste(error_report,collapse=""))); \n' script_string += ' section_name <- addTo( section_name, newParagraph( "This section FAILED! Contact the development team... " ), Error_report_par )\n' script_string += ' return (section_name)\n}\n\n' script_string += dump_project_parameters(dbitem.project, dbitem) script_string += dump_pipeline_config(rt, report_data['instance_id']) script_string += 'REPORT <- newCustomReport(report_name)\n' dummy_flag = False for tag in sections: secID = 'Section_dbID_' + str(tag.id) if secID in request_data.POST and request_data.POST[secID] == '1': tree = xml.parse(str(settings.MEDIA_ROOT) + str(tag.docxml)) script_string += '##### TAG: %s #####\n' % tag.name if tag.name == "Import to FileMaker": dummy_flag = True # source main code segment code_path = str(settings.MEDIA_ROOT) + str(tag.code) script_string += '# <---------- body ----------> \n' + open( code_path, 'r').read() + '\n' script_string += '# <------- end of body --------> \n' # input parameters definition script_string += '# <---------- parameters ----------> \n' script_string += gen_params_string( tree, request_data.POST, str(settings.MEDIA_ROOT) + dbitem.home, request_data.FILES) script_string += '# <------- end of parameters --------> \n' # final step - fire header header_path = str(settings.MEDIA_ROOT) + str(tag.header) script_string += '# <---------- header ----------> \n' + open( header_path, 'r').read() + '\n\n' script_string += 'new_section <- newSection( section_name )\n' script_string += 'tag_section <- tryCatch({section_body(new_section)}, error = function(e){ failed_fun_print(new_section,e) })\n' script_string += 'REPORT <- addTo( REPORT, tag_section )\n' script_string += '# <------- end of header --------> \n' script_string += '##### END OF TAG #####\n\n\n' script_string += 'setwd(\"%s\")\n' % loc else: # if tag disabled - do nothing pass # render report to file script_string += '# Render the report to a file\n' + 'writeReport( REPORT, filename=toString(\"%s\"))\n' % dochtml script_string += 'system("chmod -R 770 .")' # save r-file dbitem.rexec.save('script.r', base.ContentFile(script_string)) dbitem.save() # configure shell-file config_path = loc + '/sgeconfig.sh' config = open(config_path, 'w') # config should be executble st = os.stat(config_path) os.chmod(config_path, st.st_mode | stat.S_IEXEC) command = '#!/bin/bash \n' + str( settings.R_ENGINE_PATH) + 'CMD BATCH --no-save ' + str( settings.MEDIA_ROOT) + str(dbitem.rexec) config.write(command) config.close() # open report's folder for others st = os.stat(loc) os.chmod(loc, st.st_mode | stat.S_IRWXG) # submit r-code p = Process(target=run_report, args=(dbitem, dummy_flag)) #print(dbitem) #run_report(dbitem,dummy_flag) p.start() return True
def converter_arquivo(self): """Converte um arquivo binĂ¡rio csv em um dicionĂ¡rio de dados.""" # noqa: D401 arquivo = self.validated_data['arquivo'] arquivo = base.ContentFile(arquivo.file.read().decode('utf-8')) return list(csv.DictReader(arquivo))
def save(self, *args, **kwargs): if self._state.adding: self.audiofile.save('name', base.ContentFile(b''), save=False) self.stmfile.save('name', base.ContentFile(b''), save=False) super().save(*args, **kwargs)