def _generate_transformation_record(self, submitted_request, status):
request_id = submitted_request.request_id
count = TransformationResult.count(request_id)
time = int(datetime.now(tz=timezone.utc).timestamp() * 1000)
current_stats = TransformationResult.statistics(request_id)
events_transformed = 0 if not current_stats else current_stats[
'total-events']
return {
"name": 'Transformation Request',
"description": 'Transformation Request',
"dataset": submitted_request.did,
"dataset_size": int(submitted_request.total_bytes or 0),
"dataset_files": count,
"dataset_events": int(submitted_request.total_events or 0),
"columns": submitted_request.columns,
"events": 0,
"events_transformed": events_transformed,
"events_served": 0,
"events_processed": 0,
"created_at": submitted_request.submit_time,
"modified_at": time,
"status": status,
"info": ' '
}
def get(self, request_id):
is_auth, auth_reject_message = self._validate_user()
if not is_auth:
return {'message': f'Authentication Failed: {str(auth_reject_message)}'}, 401
submitted_request = TransformRequest.return_request(request_id)
if not submitted_request:
return "Transform Not Found", "404"
status_request = status_request_parser.parse_args()
count = TransformationResult.count(request_id)
stats = TransformationResult.statistics(request_id)
failures = TransformationResult.failed_files(request_id)
print(count, stats)
print(TransformRequest.files_remaining(request_id))
result_dict = {
"status": submitted_request.status,
"request-id": request_id,
"files-processed": count - failures,
"files-skipped": failures,
"files-remaining": TransformRequest.files_remaining(request_id),
"stats": stats
}
if status_request.details:
result_dict['details'] = TransformationResult.to_json_list(
TransformationResult.get_all_status(request_id))
return jsonify(result_dict)
def put(self, request_id):
info = request.get_json()
submitted_request = TransformRequest.return_request(request_id)
dataset_file = DatasetFile.get_by_id(info['file-id'])
rec = TransformationResult(
did=submitted_request.did,
file_id=dataset_file.id,
request_id=request_id,
file_path=info['file-path'],
transform_status=info['status'],
transform_time=info['total-time'],
total_bytes=info['total-bytes'],
total_events=info['total-events'],
avg_rate=info['avg-rate'],
messages=info['num-messages']
)
rec.save_to_db()
if self.elasticsearch_adapter:
self.elasticsearch_adapter.create_update_path(
dataset_file.get_path_id(),
self._generate_file_status_record(dataset_file, info['status']))
self.elasticsearch_adapter.create_update_request(
request_id,
self._generate_transformation_record(submitted_request, 'transforming'))
files_remaining = TransformRequest.files_remaining(request_id)
if files_remaining is not None and files_remaining <= 0:
namespace = current_app.config['TRANSFORMER_NAMESPACE']
print("Job is all done... shutting down transformers")
self.transformer_manager.shutdown_transformer_job(request_id, namespace)
submitted_request.status = "Complete"
submitted_request.save_to_db()
if self.elasticsearch_adapter:
self.elasticsearch_adapter.create_update_request(
request_id,
self._generate_transformation_record(submitted_request, 'complete'))
print(info)
db.session.commit()
return "Ok"
def get(self, request_id):
status_request = status_request_parser.parse_args()
count = TransformationResult.count(request_id)
stats = TransformationResult.statistics(request_id)
failures = TransformationResult.failed_files(request_id)
print(count, stats)
print(TransformRequest.files_remaining(request_id))
result_dict = {
"request-id": request_id,
"files-processed": count - failures,
"files-skipped": failures,
"files-remaining": TransformRequest.files_remaining(request_id),
"stats": stats
}
if status_request.details:
result_dict['details'] = TransformationResult.to_json_list(
TransformationResult.get_all_status(request_id))
return jsonify(result_dict)