def create_function(self): try: response = self.client.create_function(self.get_property("name"), self.get_property("runtime"), self.get_property("iam", "role"), self.get_property("handler"), self.get_property("code"), self.get_property("environment"), self.get_property("description"), self.get_property("time"), self.get_property("memory"), self.get_property("tags")) if response and 'FunctionArn' in response: self.properties["function_arn"] = response['FunctionArn'] response_parser.parse_lambda_function_creation_response(response, self.get_function_name(), self.client.get_access_key(), self.get_output_type()) except ClientError as ce: error_msg = "Error initializing lambda function." logger.error(error_msg, error_msg + ": %s" % ce) utils.finish_failed_execution() finally: # Remove the files created in the operation utils.delete_file(self.properties["zip_file_path"])
def upload_file_to_S3_bucket(image_file, deployment_bucket, file_key): if(utils.get_tree_size(scar_temporal_folder) > MAX_S3_PAYLOAD_SIZE): error_msg = "Uncompressed image size greater than 250MB.\nPlease reduce the uncompressed image and try again." logger.error(error_msg) utils.delete_file(zip_file_path) exit(1) logger.info("Uploading '%s' to the '%s' S3 bucket" % (image_file, deployment_bucket)) file_data = utils.read_file(image_file, 'rb') S3().upload_file(deployment_bucket, file_key, file_data)
def face_detection(): filename = upload_file(request, UPLOAD_FOLDER) if filename: try: result = find_faces(filename) delete_file(filename, UPLOAD_FOLDER) return result except: delete_file(filename, UPLOAD_FOLDER) return {'message': "Bad request"}, status.HTTP_400_BAD_REQUEST else: return {'message': "Bad request"}, status.HTTP_400_BAD_REQUEST
def create_function(self): try: response = self.client.create_function(**self.get_creations_args()) if response and 'FunctionArn' in response: self.properties["function_arn"] = response['FunctionArn'] response_parser.parse_lambda_function_creation_response( response, self.get_function_name(), self.client.get_access_key(), self.get_output_type()) except ClientError as ce: error_msg = "Error initializing lambda function." logger.error(error_msg, error_msg + ": %s" % ce) finally: # Remove the files created in the operation utils.delete_file(self.properties["zip_file_path"])
def put(self, data): UPLOADS_FOLDER = current_app.config['UPLOADS_FOLDER'] user = current_user if 'picture' in req.files: file = req.files['picture'] filename = save_file(file, UPLOADS_FOLDER) data['avatar'] = filename if user.avatar: delete_file(UPLOADS_FOLDER, user.avatar) user.update(**data) return user
async def upload_custom_pois( *, db: AsyncSession = Depends(deps.get_db), file: UploadFile, poi_category=Body(..., example=request_examples["poi_category"]), current_user: models.User = Depends(deps.get_current_active_user), ) -> Any: """Handle uploaded custom pois.""" defined_uuid = uuid.uuid4().hex file_name = defined_uuid + os.path.splitext(file.filename)[1] file_dir = f"/tmp/{file_name}" real_file_size = 0 temp: IO = NamedTemporaryFile(delete=False) for chunk in file.file: real_file_size += len(chunk) if real_file_size > MaxUploadFileSize.max_upload_poi_file_size.value: temp.close() delete_file(temp.name) raise HTTPException( status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail= "The uploaded file size is to big the largest allowd size is %s MB." % round(MaxUploadFileSize.max_upload_poi_file_size / 1024.0**2, 2), ) temp.write(chunk) temp.close() # Write file to file system shutil.move(temp.name, file_dir) await crud.upload.upload_custom_pois( db=db, poi_category=poi_category, file=file, file_dir=file_dir, file_name=file_name, current_user=current_user, ) updated_settings = await dynamic_customization.build_main_setting_json( db=db, current_user=current_user) return updated_settings
async def upload_custom_pois( self, *, db: AsyncSession, file: UploadFile, file_dir: str, file_name: str, poi_category: str, current_user: models.User, ): """Handle uploaded custom pois.""" # Check if poi_category is already uploaded for study area try: query_poi_features = (select(models.PoiUser.category).join( models.DataUpload).where( and_( models.DataUpload.user_id == current_user.id, models.DataUpload.study_area_id == current_user.active_study_area_id, models.PoiUser.data_upload_id == models.DataUpload.id, models.PoiUser.category == poi_category, )).limit(1)) poi_features = await db.execute(query_poi_features) poi_features = poi_features.first() except: delete_file(file_dir) raise HTTPException( status_code=400, detail="Failed reading the file.", ) if poi_features is not None: delete_file(file_dir) raise HTTPException( status_code=400, detail= "The chosen custom poi category already exists. Please delete the old data-set first in case you want to replace it with the new one", ) required_attributes = ["geometry"] optional_attributes = [ "opening_hours", "name", "street", "housenumber", "zipcode", "opening_hours", "wheelchair", ] # Get active study area study_area_obj = await crud.study_area.get( db=db, id=current_user.active_study_area_id, extra_fields=["geom"]) study_area_geom = to_shape(study_area_obj.geom) if UploadFileTypes.geojson.value in file_name: try: gdf = gpd_read_file(file_dir, driver="GeoJSON") delete_file(file_dir) except: delete_file(file_dir) raise HTTPException( status_code=400, detail="Failed reading the file in GeodataFrame", ) elif UploadFileTypes.zip.value in file_name: unzipped_file_dir = ( os.path.splitext(file_dir)[0] + "/" + file.filename.replace(UploadFileTypes.zip.value, "")) # Create directory try: shutil.unpack_archive(file_dir, os.path.splitext(file_dir)[0], "zip") except: clean_unpacked_zip(zip_path=file_dir, dir_path=file_dir.replace( UploadFileTypes.zip.value, "")) raise HTTPException(status_code=400, detail="Could not read or process file.") # List shapefiles try: available_shapefiles = [ f for f in os.listdir(unzipped_file_dir) if f.endswith(".shp") ] except: clean_unpacked_zip(zip_path=file_dir, dir_path=file_dir.replace( UploadFileTypes.zip.value, "")) raise HTTPException(status_code=400, detail="No shapefiles inside folder.") # Read shapefiles and append to GeoDataFrame if len(available_shapefiles) == 1: gdf = gpd_read_file( f"{unzipped_file_dir}/{available_shapefiles[0]}") elif len(available_shapefiles) > 1: clean_unpacked_zip(zip_path=file_dir, dir_path=file_dir.replace( UploadFileTypes.zip.value, "")) raise HTTPException( status_code=400, detail="More then one shapefiles inside folder.") else: raise HTTPException(status_code=400, detail="No shapefiles inside folder.") clean_unpacked_zip(zip_path=file_dir, dir_path=file_dir.replace( UploadFileTypes.zip.value, "")) else: raise HTTPException(status_code=400, detail="Invalid file type") # Convert to EPSG 4326 gdf_schema = dict(gdf.dtypes) if gdf.crs.name == "unknown": raise HTTPException(status_code=400, detail="Invalid CRS") else: gdf.to_crs(epsg=4326, inplace=True) gdf.set_crs(epsg=4326) gdf = gdf.clip(study_area_geom) # Drop not needed columns columns_to_drop = [] for attribute in gdf_schema: if attribute not in optional_attributes + required_attributes: columns_to_drop.append(attribute) gdf = gdf.drop(columns_to_drop, axis=1) if len(gdf) == 0: raise HTTPException( status_code=400, detail="No valid data in file or data outside the study area.") # Assign specified category to all points gdf["category"] = poi_category # Create entry in upload table upload_obj = models.DataUpload( data_type=file.content_type, upload_type=models.PoiUser.__table__.name, user_id=current_user.id, upload_size=int(file.file.tell() / 1000), study_area_id=current_user.active_study_area_id, ) upload_obj = await data_upload.create(db=db, obj_in=upload_obj) # Write to database try: gdf["uid"] = (gdf.centroid.map( lambda p: str(format(round(p.x, 4), ".4f")).replace(".", "") + "_" + str(format(round(p.y, 4), ".4f")).replace(".", "")) + "_" + str(poi_category)) gdf["count_uid"] = gdf.groupby(["uid"]).cumcount() + 1 gdf["uid"] = (gdf["uid"] + "_" + gdf["count_uid"].astype(str) + "_u" + str(upload_obj.id)) gdf["data_upload_id"] = upload_obj.id gdf.rename_geometry("geom", inplace=True) gdf.drop(["count_uid"], axis=1, inplace=True) gdf.to_postgis( name="poi_user", schema="customer", con=legacy_engine, if_exists="append", chunksize=1000, ) except: await db.execute( """DELETE FROM customer.data_upload WHERE id = :data_upload_id""", {"data_upload_id": upload_obj.id}, ) await db.commit() raise HTTPException( status_code=400, detail= "An error happened when writing the data into the database.", ) try: default_poi_categories = ( await crud.dynamic_customization.get_all_default_poi_categories(db)) if poi_category not in default_poi_categories: hex_color = "#%06x" % random.randint(0, 0xFFFFFF) new_setting = { poi_category: { "icon": "fas fa-question", "color": [hex_color] } } if check_dict_schema(PoiCategory, new_setting) == False: raise HTTPException(status_code=400, detail="Invalid JSON-schema") await crud.dynamic_customization.insert_opportunity_setting( db=db, current_user=current_user, insert_settings=new_setting, data_upload_id=upload_obj.id) except: await db.execute( """DELETE FROM customer.data_upload WHERE id = :data_upload_id""", {"data_upload_id": upload_obj.id}, ) await db.commit() raise HTTPException( status_code=400, detail= "An error happened when writing new settings to the database.", ) return {"msg": "Upload successful"}
def clean_tmp_folders(self): if os.path.isfile(self.properties['ZipFilePath']): utils.delete_file(self.properties['ZipFilePath']) # Delete created temporal files if os.path.isdir(self.scar_temporal_folder): shutil.rmtree(self.scar_temporal_folder, ignore_errors=True)
def payload_size_error(zip_file_path, message): logger.error(message) utils.delete_file(zip_file_path) exit(1)
def payload_size_error(zip_file_path, message): logger.error(message) utils.delete_file(zip_file_path) utils.finish_failed_execution()
def main(input_img_path, ann_path, model_type, metric): """ Runs training process and save model. """ logger = logging.getLogger(__name__) # collect images path input_img_dir = Path(input_img_path) imgs_path_train = sorted([ i.absolute() for i in (input_img_dir / 'train').glob("*.png") if i.is_file() ]) imgs_path_val = sorted([ i.absolute() for i in (input_img_dir / 'val').glob("*.png") if i.is_file() ]) imgs_path_test = sorted([ i.absolute() for i in (input_img_dir / 'test').glob("*.png") if i.is_file() ]) # Annotations path ann_path_dir = Path(ann_path) ann_train_path = ann_path_dir / 'train' / ann_file_name ann_val_path = ann_path_dir / 'val' / ann_file_name ann_test_path = ann_path_dir / 'test' / ann_file_name # Add randomly more paths imgs_path_train = create_random_list_of_size( imgs_path_train, len(imgs_path_train) * multiply_by) imgs_path_val = create_random_list_of_size( imgs_path_val, len(imgs_path_val) * multiply_by) imgs_path_test = create_random_list_of_size( imgs_path_test, len(imgs_path_test) * multiply_by) # Generate Data on the fly for train and validation data_generator_train = GDXrayDataGenerator(imgs_path_train, ann_train_path, labels, n_classes, batch_size=batch_size, dim=dim) data_generator_val = GDXrayDataGenerator(imgs_path_val, ann_val_path, labels, n_classes, batch_size=batch_size, dim=dim) data_generator_test = GDXrayDataGenerator(imgs_path_test, ann_test_path, labels, n_classes, batch_size=batch_size, dim=dim) # Model Path model_path = Path('models') / (model_name % model_type) delete_file(model_path) # Set-up model selected model = Unet(dim, n_classes, n_filters) conf, call_backs = get_model_configures(metric, str(model_path)) model.build_model(**conf) # Fit the model TODO add timer history = model.fit(x=data_generator_train, steps_per_epoch=len(data_generator_train), validation_data=data_generator_val, epochs=50, verbose=1, callbacks=call_backs) # Evaluate model result = model.evaluate(data_generator_test) print(result) print(dict(zip(model.metrics_names, result))) model.save_model(str(model_path)) # Save History save_history(metric, history)