def download_detection_csv(task_code):
detection_task = services.detection.get_task_by_code(task_code)
detection_result = services.detection.get_result_by_code(task_code)
if not detection_result:
return handle_error(404, "No detection found!")
if not detection_result.company_id == g.user.company_id:
return handle_error(403, "Unauthorised")
k, x0, anomaly_prior = services.detection.load_calibration_values(
detection_task)
detection_result = services.detection.create_watson_detection_result_from_dictionary(
detection_result.result)
data_view = DataView.create_from_detection_result(detection_result, k, x0,
anomaly_prior)
data_view.wrap_columns_name('Sensor {}')
return Response(data_view.to_csv(DEFAULT_VIEW_TIME_RESAMPLE_RULE,
normalize=True),
mimetype='text/csv',
headers={
"Content-disposition":
"attachment; filename={}.csv".format(
detection_task.name)
})
def login():
email = g.json.get('email')
password = g.json.get('password')
user = services.user.get_by_email(email)
if not user:
logging.debug(f"No user found for {email}")
return handle_error(401, 'Incorrect user or password')
if not services.user.verify_password(user, password):
logging.warning(f"Incorrect password for {email}")
return handle_error(401, 'Incorrect user or password')
if not user.confirmed:
logging.warning(f"User {user.email} hasn't been confirmed!")
return handle_error(401, f'Please confirm user {user.email} first')
token = services.user.generate_auth_token(user, expiration=TOKEN_EXPIRATION)
ascii_token = token.decode('ascii')
response = ApiResponse(
content_type=request.accept_mimetypes.best,
next=url_for('customer.dashboard'),
context={'token': ascii_token}
)
response.set_cookie(
'token', ascii_token,
expires=datetime.datetime.now() + datetime.timedelta(minutes=TOKEN_EXPIRATION)
)
return response()
def get_configuration(datasource_configuration_id):
company_id = g.user.company_id
datasource_configuration = DataSourceConfigurationEntity.get_for_id(
datasource_configuration_id)
if not datasource_configuration:
return handle_error(404, 'No datasource configuration found')
if not datasource_configuration.company_id == company_id:
return handle_error(403, 'Unauthorised')
response = ApiResponse(content_type=request.accept_mimetypes.best,
context=datasource_configuration)
return response()
def result(detection_task_code):
diagnostic_task = services.diagnostic.get_task_by_code(detection_task_code)
if not diagnostic_task:
return handle_error(404, "No diagnostics found!")
response = ApiResponse(content_type=request.accept_mimetypes.best,
context=diagnostic_task.diagnostic_result.result)
return response()
def detail(task_code):
detection = services.detection.get_task_by_code(task_code)
diagnostic_task = services.diagnostic.get_task_by_code(task_code)
training_task = app.services.training.get_training_for_id(
detection.training_task_id)
if not detection:
logging.debug(f"No task found for code {task_code}")
return handle_error(404, 'No task found!')
if not detection.company_id == g.user.company_id:
return handle_error(403, "Unauthorised")
response = ApiResponse(content_type=request.accept_mimetypes.best,
context={
'detection': detection,
'diagnostic': diagnostic_task,
'training_task': training_task
},
template='detection/detail.html')
return response()
def detail(detection_task_code):
diagnostic_task = services.diagnostic.get_task_by_code(detection_task_code)
if not diagnostic_task:
return handle_error(404, "No diagnostics found!")
response = ApiResponse(
content_type=request.accept_mimetypes.best,
context={'diagnostic': diagnostic_task},
template='diagnostic/detail.html',
)
return response()
def save_label():
upload_code = g.json.get('upload_code')
datasource_label = g.json.get('datasource_label')
datasource_entity = DataSourceEntity.query.filter(
DataSourceEntity.upload_code == upload_code).one_or_none()
if not datasource_entity:
logging.debug(f"No datasource was found for code {upload_code}")
return handle_error(404, 'No data source found!')
if not datasource_entity.company_id == g.user.company_id:
return handle_error(403, "Unauthorised")
if datasource_label:
try:
label_type = LabelTypes[datasource_label]
datasource_entity.label = label_type
except KeyError:
return handle_error(
404, f'Invalid label {datasource_label} for datasource')
else:
datasource_entity.label = None
datasource_entity.update()
datasource = services.datasource.get_by_upload_code(upload_code)
flash(f"Flight label has been set to '{datasource_label}'",
category='success')
response = ApiResponse(content_type=request.accept_mimetypes.best,
next=url_for('datasource.detail',
upload_code=upload_code),
context={'datasource': datasource},
status_code=201)
return response()
def detail(upload_code):
datasource = services.datasource.get_by_upload_code(upload_code)
if not datasource:
logging.debug(f"No flight was found for code {upload_code}")
return handle_error(404, 'No flight found!')
if not datasource.company_id == g.user.company_id:
return handle_error(403, "Unauthorised")
query = TrainingTaskEntity.query
query = services.training.filter_by_datasource_configuration_id(
query, datasource.datasource_configuration_id)
query = services.training.filter_by_company_id(query,
datasource.company_id)
query = services.training.filter_by_status(query,
TaskStatusTypes.successful)
query.order_by(TrainingTaskEntity.id.desc())
training_task_list = query.all()
response = ApiResponse(
content_type=request.accept_mimetypes.best,
context={
'datasource':
datasource,
'label_types':
{member.name: member.value
for member in LabelTypes},
'detection_task_list':
services.detection.get_task_for_datasource_id(datasource.id),
'training_task_list':
training_task_list
},
template='datasource/detail.html')
return response()
def delete(datasource_id):
datasource = services.datasource.get_by_upload_code(datasource_id)
if datasource.is_original:
message = f"Tried to delete original ingestion datasource: {datasource_id}"
logging.debug(message)
return handle_error(400, message)
services.datasource.delete(datasource)
response = ApiResponse(content_type=request.accept_mimetypes.best,
next=url_for('customer.list_datasources'),
status_code=200)
flash(f"Historical data version {datasource_id} has been deleted",
category='success')
return response()
def delete(training_task_code):
training_task_entity = services.training.get_training_for_task_code(training_task_code)
if not training_task_entity:
return handle_error(404, f'No training found for code {training_task_code}')
# Terminate the running task
AsyncResult(training_task_code).revoke(terminate=True)
training_task_entity.delete()
response = ApiResponse(
content_type=request.accept_mimetypes.best,
next=url_for('training.list'),
status_code=200
)
flash(f"Training task {training_task_code} has been deleted", category='success')
return response()
def detail(training_task_code):
company_id = g.user.company_id
training_task_entity = TrainingTaskEntity.get_for_company_id(
company_id).filter(TrainingTaskEntity.task_code == training_task_code).one_or_none()
if not training_task_entity:
return handle_error(404, 'No training task found!')
context = {
'training_task': TrainingTask.from_model(training_task_entity)
}
response = ApiResponse(
content_type=request.accept_mimetypes.best,
context=context,
template='training/detail.html'
)
return response()
def sensor_detail(detection_task_code, chunk_index, sensor_id):
diagnostic_task = services.diagnostic.get_task_by_code(detection_task_code)
if not diagnostic_task:
return handle_error(404, "No diagnostics found!")
training_task = diagnostic_task._model.detection_task.training_task
transformer = services.watson.create_transformer_from_configuration(
training_task)
company = g.user.company
datasource_class = services.watson.get_datasource_class_from_company_configuration(
company.current_configuration)
uploaded_file = services.datasource.get_by_upload_code(
diagnostic_task.upload_code)
datasource = datasource_class(uploaded_file.location, transformer)
sensor_result = {}
for resampling_method in [
ResampleMethod.MEAN, ResampleMethod.MAX, ResampleMethod.MIN
]:
transformer = SimpleTransformer.create_from_original_transformer(
transformer, resampling_method)
datasource._transformer = transformer
samples = list(datasource.get_test_data('NORMAL'))
sample = samples[0]
chunk = sample.get_chunk(chunk_index)
chunk_timedelta = sample.get_timedelta_for_chunk(chunk_index)
if len(chunk.shape) == 1:
chunk = np.expand_dims(chunk, axis=0)
sensor_result[resampling_method.value] = chunk[sensor_id]
final_dataframe = _build_chunk_data_for_time_plot(
diagnostic_task, chunk_timedelta,
pd.DataFrame.from_dict(sensor_result))
return Response(final_dataframe.to_csv(), mimetype='text/plain')
def delete(datasource_id):
upload_manager = services.company.get_upload_manager(
g.user.company.current_configuration)
datasource = services.datasource.get_by_upload_code(datasource_id)
if datasource.is_part_of_training_set:
return handle_error(
400,
f"Flight {datasource.name} is part of training set, cannot be deleted"
)
services.datasource.delete(datasource)
upload_manager.cleanup(datasource.location)
response = ApiResponse(content_type=request.accept_mimetypes.best,
next=url_for('datasource.list'),
status_code=200)
flash(f"Flight {datasource.name} version {datasource_id} has been deleted",
category='success')
return response()
def configuration_update(company_id):
company = services.company.get_by_id(company_id)
if not company:
return handle_error(404, "No company could be found!")
configuration_request = g.json
data, errors = ModelConfigurationSchema().load(configuration_request)
if errors or not data:
logging.debug(f"Invalid configuration supplied: {str(errors)}")
return abort(400, f"Invalid configuration: {str(errors)}")
configuration = CompanyConfiguration(company_id=company_id,
user_id=g.user.id,
configuration=json_reload(data))
configuration = services.company.insert_configuration(configuration)
response = ApiResponse(content_type=request.accept_mimetypes.best,
context=configuration.configuration,
status_code=201)
return response()
def for_plot(detection_task_code):
diagnostic_task = services.diagnostic.get_task_by_code(detection_task_code)
if not diagnostic_task:
return handle_error(404, "No diagnostics found!")
detection_task = diagnostic_task._model.detection_task
detection_result = services.detection.create_watson_detection_result_from_dictionary(
detection_task.detection_result.result)
chunk_duration_in_seconds = detection_result.get_chunk_duration() / 1000
transformer_configuration = detection_task.training_task.configuration[
'transformer']['configuration']
downsample_factor = transformer_configuration.get('downsample_factor', 4)
diagnostic_result = diagnostic_task.diagnostic_result.result
is_frequency_domain = diagnostic_task.is_frequency_domain
data_keys = ['diagnostic', 'original', 'synthetic']
for single_chunk in diagnostic_result:
for key in data_keys:
whole_signal_df = pd.DataFrame(single_chunk[key]).T
if is_frequency_domain:
positive_spectrum = _calculate_meaningful_fft_values(
chunk_duration_in_seconds, whole_signal_df,
downsample_factor)
positive_spectrum = positive_spectrum.rename(
columns=lambda column: "Sensor {}".format(column))
single_chunk[key] = positive_spectrum.to_dict()
else:
data_frame = _build_chunk_data_for_time_plot(
diagnostic_task, single_chunk['chunk_timedelta'],
whole_signal_df)
data_frame = data_frame.rename(
columns=lambda column: "Sensor {}".format(column))
single_chunk[key] = data_frame.to_dict()
return jsonify({'diagnostic': diagnostic_task})
def submit_configuration():
company_id = g.user.company_id
configuration_name = g.json.get('name') # TODO: use a schema for this
configuration_metadata = g.json.get('meta')
if not configuration_metadata or not configuration_name:
return handle_error(400,
'Error: You must specify name and meta values')
new_datasource_configuration = DataSourceConfigurationEntity(
company_id=company_id,
name=configuration_name,
meta=configuration_metadata)
new_datasource_configuration.save()
response = ApiResponse(
content_type=request.accept_mimetypes.best,
next=url_for('datasource.list'),
context={'datasource_config': new_datasource_configuration},
status_code=201)
return response()
def list():
PER_PAGE = 10
company_id = g.user.company_id
if not g.user.company.current_configuration:
return handle_error(
404, 'No company configuration found, please provide one.')
upload_manager = services.company.get_upload_manager(
g.user.company.current_configuration)
current_page = int(request.args.get('page', 1))
query = services.datasource.filter_by_company_id(DataSourceEntity.query,
company_id)
pagination = paginate(query, current_page, PER_PAGE)
response = ApiResponse(
content_type=request.accept_mimetypes.best,
context={
'datasources':
pagination.items,
'pagination':
pagination,
'current_page':
current_page,
'allowed_extensions':
','.join([
".{}".format(ext) for ext in upload_manager.allowed_extensions
]),
'max_file_size':
2147483648,
'datasource_types':
DataSourceConfigurationEntity.get_for_company_id(company_id).all()
},
template='datasource/list.html')
return response()
def submit():
user_id = g.user.id
company_id = g.user.company_id
name = g.json.get('name')
enable_fft = g.json.get('enable_fft', None)
downsample_factor = int(g.json.get('downsample_factor'))
train_iters = int(g.json.get('train_iters'))
if enable_fft:
enable_fft = bool(int(enable_fft))
datasource_configuration_id = g.json.get('datasource_configuration_id')
parent_training_id = g.json.get('parent_training_id', None)
if TrainingTaskEntity.query.filter(TrainingTaskEntity.name == name,
TrainingTaskEntity.company_id == company_id).all():
return handle_error(400, f'Training with name {name} already exists')
latest_company_configuration = g.user.company.current_configuration
datasource_configuration = DataSourceConfigurationEntity.query.filter(
DataSourceConfigurationEntity.company_id == company_id
).filter(
DataSourceConfigurationEntity.id == datasource_configuration_id
).one_or_none()
if not datasource_configuration:
return handle_error(404, 'No datasource type found!')
number_of_sensors = datasource_configuration.meta.get('number_of_sensors')
if not number_of_sensors:
return handle_error(400, 'No number of sensors specified! Check datasource configuration')
number_of_timesteps = 784 // number_of_sensors * downsample_factor
datasources_for_train = DataSourceEntity.get_for_datasource_configuration(datasource_configuration)
if not datasources_for_train:
return handle_error(400, f'No valid datasources available for type {datasource_configuration.name}')
training_task_code = services.detection.generate_task_code()
training_configuration = services.training.create_training_configuration(
training_task_code,
company_id,
latest_company_configuration.configuration,
datasource_configuration.meta,
enable_fft,
parent_training_id
)
updated_training_configuration = add_training_parameters_to_configuration(
downsample_factor, number_of_timesteps, train_iters, training_configuration
)
logging.info(f"created training configuration: {updated_training_configuration}")
training_task_entity = TrainingTaskEntity(
company_id=company_id,
user_id=user_id,
task_code=training_task_code,
company_configuration_id=latest_company_configuration.id,
datasource_configuration_id=datasource_configuration_id,
datasources=datasources_for_train,
configuration=updated_training_configuration,
name=name,
parent_training_id=int(parent_training_id) if parent_training_id else None)
training_task_entity.save()
os.makedirs(training_task_entity.train_data_dir, exist_ok=True)
services.training.set_task_status(training_task_entity, status=TaskStatusTypes.queued, message='Enqueued')
train_celery_task.apply_async(
args=(training_task_entity.task_code,), task_id=training_task_code
)
response = ApiResponse(
content_type=request.accept_mimetypes.best,
context=TrainingTask.from_model(training_task_entity),
status_code=201,
next=url_for('training.list')
)
return response()
def upload():
user = g.user
company = user.company
datasource_name = g.json.get('name')
if not datasource_name:
return handle_error(400, "Please provide a name for the data upload")
if services.datasource.datasource_name_exists(datasource_name):
return handle_error(
400, f"A data source for {datasource_name} already exists")
datasource_type_id = g.json.get('datasource_type_id')
if not datasource_type_id:
return handle_error(400, "Please provide a datasource type id!")
datasource_type = DataSourceConfigurationEntity.get_for_id(
datasource_type_id)
if not datasource_type:
return handle_error(
404, f"No datasource type found for {datasource_type_id}")
if not len(request.files):
logging.debug("No file was uploaded")
return handle_error(400, "No file Provided!")
company_configuration = company.current_configuration
uploaded_file = request.files['upload']
if not company_configuration:
uploaded_file.close()
return handle_error(
400,
f"{company.name} cannot upload data yet, please contact support.")
upload_manager = services.company.get_upload_manager(company_configuration)
try:
uploaded_dataframe = upload_manager.validate(uploaded_file,
datasource_type_id)
except Exception as e:
message = f"Data validation has failed: {str(e)}"
return handle_error(400, message)
upload_code = generate_upload_code()
try:
datasource = services.datasource.save_datasource(
datasource_name, company.id, datasource_type_id, upload_code,
upload_manager, uploaded_dataframe, user)
except Exception as e:
upload_manager.cleanup(upload_code)
return handle_error(400, str(e))
flash("Flight uploaded successfully.", category='success')
response = ApiResponse(content_type=request.accept_mimetypes.best,
next=url_for('datasource.detail',
upload_code=upload_code),
context={'datasource': datasource},
status_code=201)
return response()
def upload():
user = g.user
company = user.company
company_configuration = company.current_configuration
if not company_configuration:
return handle_error(
400,
f"{company.name} cannot upload historical data yet, please contact support."
)
if not len(request.files):
logging.debug("No file was uploaded")
return handle_error(400, "No file Provided!")
uploaded_file = request.files['upload']
if not allowed_extension(uploaded_file.filename):
logging.debug(f"Invalid extension for upload {uploaded_file.filename}")
return handle_error(
400, f'File extension for {uploaded_file.filename} not allowed!')
upload_code = generate_upload_code()
filename = services.datasource.generate_filename(
upload_code, secure_filename(uploaded_file.filename))
interpreter = services.company.get_datasource_interpreter(
company_configuration)
target_feature = company_configuration.configuration.target_feature
data_frame, errors = interpreter.from_csv_to_dataframe(uploaded_file)
if errors:
logging.debug(f"Invalid file uploaded: {', '.join(errors)}")
return handle_error(400, ', '.join(errors))
if not target_feature in list(data_frame.columns):
return handle_error(
400,
f"Required feature {target_feature} not in {uploaded_file.filename}"
)
features = list(data_frame.columns)
if user.company.current_datasource:
data_source = services.datasource.get_by_upload_code(
user.company.current_datasource.upload_code)
existing_data_frame = data_source._model.get_file()
data_frame = pd.concat([existing_data_frame, data_frame])
saved_path = os.path.join(current_app.config['UPLOAD_FOLDER'],
filename + '.hdf5')
data_frame = data_frame.reset_index()
data_frame = data_frame.drop_duplicates()
data_frame = data_frame.set_index(interpreter.INDEX_COLUMN)
data_frame = data_frame.sort_index(ascending=True)
data_frame.to_hdf(saved_path, key=current_app.config['HDF5_STORE_INDEX'])
original = True if len(company.data_sources) == 0 else False
upload = DataSource(
user_id=user.id,
company_id=company.id,
upload_code=upload_code,
type=UploadTypes.FILESYSTEM,
location=saved_path,
filename=filename,
start_date=data_frame.index[0].to_pydatetime(),
end_date=data_frame.index[-1].to_pydatetime(),
is_original=original,
features=', '.join(features),
target_feature=target_feature,
)
datasource = services.datasource.insert(upload)
upload_strategy_class = company_configuration.configuration.upload_strategy
upload_strategy = services.strategies.get_upload_strategy(
upload_strategy_class)
upload_strategy.run(datasource=datasource,
company_configuration=company_configuration)
response = ApiResponse(content_type=request.accept_mimetypes.best,
context=datasource,
next=url_for('customer.list_datasources'),
status_code=201)
return response()