def download_detection_csv(task_code): detection_task = services.detection.get_task_by_code(task_code) detection_result = services.detection.get_result_by_code(task_code) if not detection_result: return handle_error(404, "No detection found!") if not detection_result.company_id == g.user.company_id: return handle_error(403, "Unauthorised") k, x0, anomaly_prior = services.detection.load_calibration_values( detection_task) detection_result = services.detection.create_watson_detection_result_from_dictionary( detection_result.result) data_view = DataView.create_from_detection_result(detection_result, k, x0, anomaly_prior) data_view.wrap_columns_name('Sensor {}') return Response(data_view.to_csv(DEFAULT_VIEW_TIME_RESAMPLE_RULE, normalize=True), mimetype='text/csv', headers={ "Content-disposition": "attachment; filename={}.csv".format( detection_task.name) })
def login(): email = g.json.get('email') password = g.json.get('password') user = services.user.get_by_email(email) if not user: logging.debug(f"No user found for {email}") return handle_error(401, 'Incorrect user or password') if not services.user.verify_password(user, password): logging.warning(f"Incorrect password for {email}") return handle_error(401, 'Incorrect user or password') if not user.confirmed: logging.warning(f"User {user.email} hasn't been confirmed!") return handle_error(401, f'Please confirm user {user.email} first') token = services.user.generate_auth_token(user, expiration=TOKEN_EXPIRATION) ascii_token = token.decode('ascii') response = ApiResponse( content_type=request.accept_mimetypes.best, next=url_for('customer.dashboard'), context={'token': ascii_token} ) response.set_cookie( 'token', ascii_token, expires=datetime.datetime.now() + datetime.timedelta(minutes=TOKEN_EXPIRATION) ) return response()
def get_configuration(datasource_configuration_id): company_id = g.user.company_id datasource_configuration = DataSourceConfigurationEntity.get_for_id( datasource_configuration_id) if not datasource_configuration: return handle_error(404, 'No datasource configuration found') if not datasource_configuration.company_id == company_id: return handle_error(403, 'Unauthorised') response = ApiResponse(content_type=request.accept_mimetypes.best, context=datasource_configuration) return response()
def result(detection_task_code): diagnostic_task = services.diagnostic.get_task_by_code(detection_task_code) if not diagnostic_task: return handle_error(404, "No diagnostics found!") response = ApiResponse(content_type=request.accept_mimetypes.best, context=diagnostic_task.diagnostic_result.result) return response()
def detail(task_code): detection = services.detection.get_task_by_code(task_code) diagnostic_task = services.diagnostic.get_task_by_code(task_code) training_task = app.services.training.get_training_for_id( detection.training_task_id) if not detection: logging.debug(f"No task found for code {task_code}") return handle_error(404, 'No task found!') if not detection.company_id == g.user.company_id: return handle_error(403, "Unauthorised") response = ApiResponse(content_type=request.accept_mimetypes.best, context={ 'detection': detection, 'diagnostic': diagnostic_task, 'training_task': training_task }, template='detection/detail.html') return response()
def detail(detection_task_code): diagnostic_task = services.diagnostic.get_task_by_code(detection_task_code) if not diagnostic_task: return handle_error(404, "No diagnostics found!") response = ApiResponse( content_type=request.accept_mimetypes.best, context={'diagnostic': diagnostic_task}, template='diagnostic/detail.html', ) return response()
def save_label(): upload_code = g.json.get('upload_code') datasource_label = g.json.get('datasource_label') datasource_entity = DataSourceEntity.query.filter( DataSourceEntity.upload_code == upload_code).one_or_none() if not datasource_entity: logging.debug(f"No datasource was found for code {upload_code}") return handle_error(404, 'No data source found!') if not datasource_entity.company_id == g.user.company_id: return handle_error(403, "Unauthorised") if datasource_label: try: label_type = LabelTypes[datasource_label] datasource_entity.label = label_type except KeyError: return handle_error( 404, f'Invalid label {datasource_label} for datasource') else: datasource_entity.label = None datasource_entity.update() datasource = services.datasource.get_by_upload_code(upload_code) flash(f"Flight label has been set to '{datasource_label}'", category='success') response = ApiResponse(content_type=request.accept_mimetypes.best, next=url_for('datasource.detail', upload_code=upload_code), context={'datasource': datasource}, status_code=201) return response()
def detail(upload_code): datasource = services.datasource.get_by_upload_code(upload_code) if not datasource: logging.debug(f"No flight was found for code {upload_code}") return handle_error(404, 'No flight found!') if not datasource.company_id == g.user.company_id: return handle_error(403, "Unauthorised") query = TrainingTaskEntity.query query = services.training.filter_by_datasource_configuration_id( query, datasource.datasource_configuration_id) query = services.training.filter_by_company_id(query, datasource.company_id) query = services.training.filter_by_status(query, TaskStatusTypes.successful) query.order_by(TrainingTaskEntity.id.desc()) training_task_list = query.all() response = ApiResponse( content_type=request.accept_mimetypes.best, context={ 'datasource': datasource, 'label_types': {member.name: member.value for member in LabelTypes}, 'detection_task_list': services.detection.get_task_for_datasource_id(datasource.id), 'training_task_list': training_task_list }, template='datasource/detail.html') return response()
def delete(datasource_id): datasource = services.datasource.get_by_upload_code(datasource_id) if datasource.is_original: message = f"Tried to delete original ingestion datasource: {datasource_id}" logging.debug(message) return handle_error(400, message) services.datasource.delete(datasource) response = ApiResponse(content_type=request.accept_mimetypes.best, next=url_for('customer.list_datasources'), status_code=200) flash(f"Historical data version {datasource_id} has been deleted", category='success') return response()
def delete(training_task_code): training_task_entity = services.training.get_training_for_task_code(training_task_code) if not training_task_entity: return handle_error(404, f'No training found for code {training_task_code}') # Terminate the running task AsyncResult(training_task_code).revoke(terminate=True) training_task_entity.delete() response = ApiResponse( content_type=request.accept_mimetypes.best, next=url_for('training.list'), status_code=200 ) flash(f"Training task {training_task_code} has been deleted", category='success') return response()
def detail(training_task_code): company_id = g.user.company_id training_task_entity = TrainingTaskEntity.get_for_company_id( company_id).filter(TrainingTaskEntity.task_code == training_task_code).one_or_none() if not training_task_entity: return handle_error(404, 'No training task found!') context = { 'training_task': TrainingTask.from_model(training_task_entity) } response = ApiResponse( content_type=request.accept_mimetypes.best, context=context, template='training/detail.html' ) return response()
def sensor_detail(detection_task_code, chunk_index, sensor_id): diagnostic_task = services.diagnostic.get_task_by_code(detection_task_code) if not diagnostic_task: return handle_error(404, "No diagnostics found!") training_task = diagnostic_task._model.detection_task.training_task transformer = services.watson.create_transformer_from_configuration( training_task) company = g.user.company datasource_class = services.watson.get_datasource_class_from_company_configuration( company.current_configuration) uploaded_file = services.datasource.get_by_upload_code( diagnostic_task.upload_code) datasource = datasource_class(uploaded_file.location, transformer) sensor_result = {} for resampling_method in [ ResampleMethod.MEAN, ResampleMethod.MAX, ResampleMethod.MIN ]: transformer = SimpleTransformer.create_from_original_transformer( transformer, resampling_method) datasource._transformer = transformer samples = list(datasource.get_test_data('NORMAL')) sample = samples[0] chunk = sample.get_chunk(chunk_index) chunk_timedelta = sample.get_timedelta_for_chunk(chunk_index) if len(chunk.shape) == 1: chunk = np.expand_dims(chunk, axis=0) sensor_result[resampling_method.value] = chunk[sensor_id] final_dataframe = _build_chunk_data_for_time_plot( diagnostic_task, chunk_timedelta, pd.DataFrame.from_dict(sensor_result)) return Response(final_dataframe.to_csv(), mimetype='text/plain')
def delete(datasource_id): upload_manager = services.company.get_upload_manager( g.user.company.current_configuration) datasource = services.datasource.get_by_upload_code(datasource_id) if datasource.is_part_of_training_set: return handle_error( 400, f"Flight {datasource.name} is part of training set, cannot be deleted" ) services.datasource.delete(datasource) upload_manager.cleanup(datasource.location) response = ApiResponse(content_type=request.accept_mimetypes.best, next=url_for('datasource.list'), status_code=200) flash(f"Flight {datasource.name} version {datasource_id} has been deleted", category='success') return response()
def configuration_update(company_id): company = services.company.get_by_id(company_id) if not company: return handle_error(404, "No company could be found!") configuration_request = g.json data, errors = ModelConfigurationSchema().load(configuration_request) if errors or not data: logging.debug(f"Invalid configuration supplied: {str(errors)}") return abort(400, f"Invalid configuration: {str(errors)}") configuration = CompanyConfiguration(company_id=company_id, user_id=g.user.id, configuration=json_reload(data)) configuration = services.company.insert_configuration(configuration) response = ApiResponse(content_type=request.accept_mimetypes.best, context=configuration.configuration, status_code=201) return response()
def for_plot(detection_task_code): diagnostic_task = services.diagnostic.get_task_by_code(detection_task_code) if not diagnostic_task: return handle_error(404, "No diagnostics found!") detection_task = diagnostic_task._model.detection_task detection_result = services.detection.create_watson_detection_result_from_dictionary( detection_task.detection_result.result) chunk_duration_in_seconds = detection_result.get_chunk_duration() / 1000 transformer_configuration = detection_task.training_task.configuration[ 'transformer']['configuration'] downsample_factor = transformer_configuration.get('downsample_factor', 4) diagnostic_result = diagnostic_task.diagnostic_result.result is_frequency_domain = diagnostic_task.is_frequency_domain data_keys = ['diagnostic', 'original', 'synthetic'] for single_chunk in diagnostic_result: for key in data_keys: whole_signal_df = pd.DataFrame(single_chunk[key]).T if is_frequency_domain: positive_spectrum = _calculate_meaningful_fft_values( chunk_duration_in_seconds, whole_signal_df, downsample_factor) positive_spectrum = positive_spectrum.rename( columns=lambda column: "Sensor {}".format(column)) single_chunk[key] = positive_spectrum.to_dict() else: data_frame = _build_chunk_data_for_time_plot( diagnostic_task, single_chunk['chunk_timedelta'], whole_signal_df) data_frame = data_frame.rename( columns=lambda column: "Sensor {}".format(column)) single_chunk[key] = data_frame.to_dict() return jsonify({'diagnostic': diagnostic_task})
def submit_configuration(): company_id = g.user.company_id configuration_name = g.json.get('name') # TODO: use a schema for this configuration_metadata = g.json.get('meta') if not configuration_metadata or not configuration_name: return handle_error(400, 'Error: You must specify name and meta values') new_datasource_configuration = DataSourceConfigurationEntity( company_id=company_id, name=configuration_name, meta=configuration_metadata) new_datasource_configuration.save() response = ApiResponse( content_type=request.accept_mimetypes.best, next=url_for('datasource.list'), context={'datasource_config': new_datasource_configuration}, status_code=201) return response()
def list(): PER_PAGE = 10 company_id = g.user.company_id if not g.user.company.current_configuration: return handle_error( 404, 'No company configuration found, please provide one.') upload_manager = services.company.get_upload_manager( g.user.company.current_configuration) current_page = int(request.args.get('page', 1)) query = services.datasource.filter_by_company_id(DataSourceEntity.query, company_id) pagination = paginate(query, current_page, PER_PAGE) response = ApiResponse( content_type=request.accept_mimetypes.best, context={ 'datasources': pagination.items, 'pagination': pagination, 'current_page': current_page, 'allowed_extensions': ','.join([ ".{}".format(ext) for ext in upload_manager.allowed_extensions ]), 'max_file_size': 2147483648, 'datasource_types': DataSourceConfigurationEntity.get_for_company_id(company_id).all() }, template='datasource/list.html') return response()
def submit(): user_id = g.user.id company_id = g.user.company_id name = g.json.get('name') enable_fft = g.json.get('enable_fft', None) downsample_factor = int(g.json.get('downsample_factor')) train_iters = int(g.json.get('train_iters')) if enable_fft: enable_fft = bool(int(enable_fft)) datasource_configuration_id = g.json.get('datasource_configuration_id') parent_training_id = g.json.get('parent_training_id', None) if TrainingTaskEntity.query.filter(TrainingTaskEntity.name == name, TrainingTaskEntity.company_id == company_id).all(): return handle_error(400, f'Training with name {name} already exists') latest_company_configuration = g.user.company.current_configuration datasource_configuration = DataSourceConfigurationEntity.query.filter( DataSourceConfigurationEntity.company_id == company_id ).filter( DataSourceConfigurationEntity.id == datasource_configuration_id ).one_or_none() if not datasource_configuration: return handle_error(404, 'No datasource type found!') number_of_sensors = datasource_configuration.meta.get('number_of_sensors') if not number_of_sensors: return handle_error(400, 'No number of sensors specified! Check datasource configuration') number_of_timesteps = 784 // number_of_sensors * downsample_factor datasources_for_train = DataSourceEntity.get_for_datasource_configuration(datasource_configuration) if not datasources_for_train: return handle_error(400, f'No valid datasources available for type {datasource_configuration.name}') training_task_code = services.detection.generate_task_code() training_configuration = services.training.create_training_configuration( training_task_code, company_id, latest_company_configuration.configuration, datasource_configuration.meta, enable_fft, parent_training_id ) updated_training_configuration = add_training_parameters_to_configuration( downsample_factor, number_of_timesteps, train_iters, training_configuration ) logging.info(f"created training configuration: {updated_training_configuration}") training_task_entity = TrainingTaskEntity( company_id=company_id, user_id=user_id, task_code=training_task_code, company_configuration_id=latest_company_configuration.id, datasource_configuration_id=datasource_configuration_id, datasources=datasources_for_train, configuration=updated_training_configuration, name=name, parent_training_id=int(parent_training_id) if parent_training_id else None) training_task_entity.save() os.makedirs(training_task_entity.train_data_dir, exist_ok=True) services.training.set_task_status(training_task_entity, status=TaskStatusTypes.queued, message='Enqueued') train_celery_task.apply_async( args=(training_task_entity.task_code,), task_id=training_task_code ) response = ApiResponse( content_type=request.accept_mimetypes.best, context=TrainingTask.from_model(training_task_entity), status_code=201, next=url_for('training.list') ) return response()
def upload(): user = g.user company = user.company datasource_name = g.json.get('name') if not datasource_name: return handle_error(400, "Please provide a name for the data upload") if services.datasource.datasource_name_exists(datasource_name): return handle_error( 400, f"A data source for {datasource_name} already exists") datasource_type_id = g.json.get('datasource_type_id') if not datasource_type_id: return handle_error(400, "Please provide a datasource type id!") datasource_type = DataSourceConfigurationEntity.get_for_id( datasource_type_id) if not datasource_type: return handle_error( 404, f"No datasource type found for {datasource_type_id}") if not len(request.files): logging.debug("No file was uploaded") return handle_error(400, "No file Provided!") company_configuration = company.current_configuration uploaded_file = request.files['upload'] if not company_configuration: uploaded_file.close() return handle_error( 400, f"{company.name} cannot upload data yet, please contact support.") upload_manager = services.company.get_upload_manager(company_configuration) try: uploaded_dataframe = upload_manager.validate(uploaded_file, datasource_type_id) except Exception as e: message = f"Data validation has failed: {str(e)}" return handle_error(400, message) upload_code = generate_upload_code() try: datasource = services.datasource.save_datasource( datasource_name, company.id, datasource_type_id, upload_code, upload_manager, uploaded_dataframe, user) except Exception as e: upload_manager.cleanup(upload_code) return handle_error(400, str(e)) flash("Flight uploaded successfully.", category='success') response = ApiResponse(content_type=request.accept_mimetypes.best, next=url_for('datasource.detail', upload_code=upload_code), context={'datasource': datasource}, status_code=201) return response()
def upload(): user = g.user company = user.company company_configuration = company.current_configuration if not company_configuration: return handle_error( 400, f"{company.name} cannot upload historical data yet, please contact support." ) if not len(request.files): logging.debug("No file was uploaded") return handle_error(400, "No file Provided!") uploaded_file = request.files['upload'] if not allowed_extension(uploaded_file.filename): logging.debug(f"Invalid extension for upload {uploaded_file.filename}") return handle_error( 400, f'File extension for {uploaded_file.filename} not allowed!') upload_code = generate_upload_code() filename = services.datasource.generate_filename( upload_code, secure_filename(uploaded_file.filename)) interpreter = services.company.get_datasource_interpreter( company_configuration) target_feature = company_configuration.configuration.target_feature data_frame, errors = interpreter.from_csv_to_dataframe(uploaded_file) if errors: logging.debug(f"Invalid file uploaded: {', '.join(errors)}") return handle_error(400, ', '.join(errors)) if not target_feature in list(data_frame.columns): return handle_error( 400, f"Required feature {target_feature} not in {uploaded_file.filename}" ) features = list(data_frame.columns) if user.company.current_datasource: data_source = services.datasource.get_by_upload_code( user.company.current_datasource.upload_code) existing_data_frame = data_source._model.get_file() data_frame = pd.concat([existing_data_frame, data_frame]) saved_path = os.path.join(current_app.config['UPLOAD_FOLDER'], filename + '.hdf5') data_frame = data_frame.reset_index() data_frame = data_frame.drop_duplicates() data_frame = data_frame.set_index(interpreter.INDEX_COLUMN) data_frame = data_frame.sort_index(ascending=True) data_frame.to_hdf(saved_path, key=current_app.config['HDF5_STORE_INDEX']) original = True if len(company.data_sources) == 0 else False upload = DataSource( user_id=user.id, company_id=company.id, upload_code=upload_code, type=UploadTypes.FILESYSTEM, location=saved_path, filename=filename, start_date=data_frame.index[0].to_pydatetime(), end_date=data_frame.index[-1].to_pydatetime(), is_original=original, features=', '.join(features), target_feature=target_feature, ) datasource = services.datasource.insert(upload) upload_strategy_class = company_configuration.configuration.upload_strategy upload_strategy = services.strategies.get_upload_strategy( upload_strategy_class) upload_strategy.run(datasource=datasource, company_configuration=company_configuration) response = ApiResponse(content_type=request.accept_mimetypes.best, context=datasource, next=url_for('customer.list_datasources'), status_code=201) return response()