def unlink_instrument_from_exercise(self, instrument_id, exercise_id, session=None):
"""
Unlink a collection instrument and a collection exercise. If there is a link between the exercise
and a business, this will also be removed.
:param instrument_id: A collection instrument id (UUID)
:param exercise_id: A collection exercise id (UUID)
:param session: database session
:return: True if instrument has been successfully unlinked to exercise
"""
bound_logger = log.bind(instrument_id=instrument_id, exercise_id=exercise_id)
bound_logger.info('Unlinking instrument and exercise')
instrument = self.get_instrument_by_id(instrument_id, session)
exercise = self.get_exercise_by_id(exercise_id, session)
if not instrument or not exercise:
bound_logger.info('Failed to unlink, unable to find instrument or exercise')
raise RasError('Unable to find instrument or exercise', 404)
instrument.exercises.remove(exercise)
for business in instrument.businesses:
bound_logger.info("Removing business/exercise link", business_id=business.id, ru_ref=business.ru_ref)
business.instruments.remove(instrument)
if not self.publish_remove_collection_instrument(exercise_id, instrument.instrument_id):
raise RasError('Failed to publish upload message', 500)
bound_logger.info('Successfully unlinked instrument to exercise')
return True
def upload_instrument_with_no_collection_exercise(self, survey_id, classifiers=None, session=None):
"""
Upload a collection instrument to the db without a collection exercise
:param classifiers: Classifiers associated with the instrument
:param session: database session
:param survey_id: database session
:return: a collection instrument instance
"""
log.info('Upload instrument', survey_id=survey_id)
validate_uuid(survey_id)
instrument = InstrumentModel(ci_type='EQ')
survey = self._find_or_create_survey_from_survey_id(survey_id, session)
instrument.survey = survey
if classifiers:
deserialized_classifiers = loads(classifiers)
instruments = self._get_instruments_by_classifier(deserialized_classifiers, None, session)
for instrument in instruments:
if instrument.classifiers == deserialized_classifiers:
raise RasError("Cannot upload an instrument with an identical set of classifiers", 400)
instrument.classifiers = deserialized_classifiers
session.add(instrument)
return instrument
def service_request(service, endpoint, search_value):
"""
Makes a request to a different micro service
:param service: The micro service to call to
:param endpoint: The end point of the micro service
:param search_value: The value to search on
:return: response
"""
auth = (current_app.config.get('SECURITY_USER_NAME'),
current_app.config.get('SECURITY_USER_PASSWORD'))
try:
service = {
'survey-service':
current_app.config['SURVEY_URL'],
'collectionexercise-service':
current_app.config['COLLECTION_EXERCISE_URL'],
'case-service':
current_app.config['CASE_URL'],
'party-service':
current_app.config['PARTY_URL']
}[service]
service_url = f'{service}/{endpoint}/{search_value}'
log.info(f'Making request to {service_url}')
except KeyError:
raise RasError(f"service '{service}' not configured", 500)
response = requests.get(service_url, auth=auth)
response.raise_for_status()
return response
def patch_collection_instrument(instrument_id):
file = request.files["file"]
if file.filename == "":
raise RasError("Missing filename", 400)
CollectionInstrument().patch_seft_instrument(instrument_id, file)
return make_response(PATCH_SUCCESSFUL, 200)
def validate_non_duplicate_instrument(file, exercise_id, session):
exercise = query_exercise_by_id(exercise_id, session)
if exercise:
for i in exercise.instruments:
if i.seft_file.file_name == file.filename:
log.error("Collection instrument file already uploaded for this collection exercise")
raise RasError("Collection instrument file already uploaded for this collection exercise", 400)
return
def patch_seft_instrument(self, instrument_id: str, file, session):
"""
Replaces the the seft_file for an instrument with the one provided.
:param instrument_id: The top level instrument id that needs changing
:param file: A FileStorage object with the new file
:param session: A database session
:raises RasError: Raised when instrument id is invalid, instrument not found, or instrument isn't of type SEFT
"""
validate_uuid(instrument_id)
instrument = self.get_instrument_by_id(instrument_id, session)
if instrument is None:
log.error('Instrument not found')
raise RasError('Instrument not found', 400)
if instrument.type != 'SEFT':
log.error('Not a SEFT instrument')
raise RasError('Not a SEFT instrument', 400)
seft_model = self._update_seft_file(instrument.seft_file, file)
session.add(seft_model)
def link_collection_instrument(instrument_id, exercise_id):
CollectionInstrument().link_instrument_to_exercise(instrument_id,
exercise_id)
response = publish_uploaded_collection_instrument(exercise_id,
instrument_id)
if response.status_code != 200:
log.error("Failed to publish upload message",
instrument_id=instrument_id,
collection_exercise_id=exercise_id)
raise RasError("Failed to publish upload message", 500)
return make_response(LINK_SUCCESSFUL, 200)
def validate_uuid(*values):
"""
validate value is a uuid
:param values: array of values to check
:return: boolean
"""
for value in values:
try:
UUID(value)
except ValueError:
raise RasError(f"Value is not a valid UUID ({value})", 400)
return True
def patch_seft_instrument(self, instrument_id: str, file, session):
"""
Replaces the the seft_file for an instrument with the one provided.
:param instrument_id: The top level instrument id that needs changing
:param file: A FileStorage object with the new file
:param session: A database session
:raises RasError: Raised when instrument id is invalid, instrument not found, or instrument isn't of type SEFT
"""
validate_uuid(instrument_id)
instrument = self.get_instrument_by_id(instrument_id, session)
if instrument is None:
log.error("Instrument not found")
raise RasError("Instrument not found", 400)
if instrument.type != "SEFT":
log.error("Not a SEFT instrument")
raise RasError("Not a SEFT instrument", 400)
survey_ref = get_survey_ref(instrument.survey.survey_id)
exercise_id = str(instrument.exids[0])
seft_model = self._update_seft_file(instrument.seft_file, file, survey_ref, exercise_id)
session.add(seft_model)
def upload_collection_instrument(exercise_id, ru_ref=None):
file = request.files["file"]
classifiers = request.args.get("classifiers")
instrument = CollectionInstrument().upload_to_bucket(
exercise_id, file, ru_ref=ru_ref, classifiers=classifiers)
if not publish_uploaded_collection_instrument(exercise_id,
instrument.instrument_id):
log.error(
"Failed to publish upload message",
instrument_id=instrument.instrument_id,
collection_exercise_id=exercise_id,
ru_ref=ru_ref,
)
raise RasError("Failed to publish upload message", 500)
return make_response(UPLOAD_SUCCESSFUL, 200)
def upload_file_to_bucket(self, file):
path = file.filename
if self.prefix != "":
path = self.prefix + "/" + path
log.info("Uploading SEFT CI to GCP bucket: " + path)
key = current_app.config.get("ONS_CRYPTOKEY", None)
if key is None:
log.error("Customer defined encryption key is missing.")
raise RasError(
"can't find customer defined encryption, hence can't perform this task",
500)
customer_supplied_encryption_key = sha256(key.encode("utf-8")).digest()
blob = self.bucket.blob(
blob_name=path, encryption_key=customer_supplied_encryption_key)
blob.upload_from_file(file_obj=file.stream, rewind=True)
log.info("Successfully put SEFT CI in bucket")
return
def _update_seft_file(seft_model, file):
"""
Updates a seft_file with a new version of the data
:param file: A file object from which we can read the file contents
:return: instrument
"""
log.info('Updating instrument seft file')
file_contents = file.read()
file_size = len(file_contents)
if file_size == 0:
raise RasError('File is empty', 400)
cryptographer = Cryptographer()
encrypted_file = cryptographer.encrypt(file_contents)
seft_model.data = encrypted_file
seft_model.length = file_size
seft_model.file_name = file.filename
return seft_model
def download_file_from_bucket(self, file_location: str):
if self.prefix != "":
path = self.prefix + "/" + file_location
else:
path = file_location
log.info("Downloading SEFT CI from GCP bucket: " + path)
key = current_app.config.get("ONS_CRYPTOKEY", None)
if key is None:
log.error("Customer defined encryption key is missing.")
raise RasError(
"can't find customer defined encryption, hence can't perform this task",
500)
customer_supplied_encryption_key = sha256(key.encode("utf-8")).digest()
blob = self.bucket.blob(
blob_name=path, encryption_key=customer_supplied_encryption_key)
file = blob.download_as_bytes()
log.info("Successfully downloaded SEFT CI from GCP bucket")
return file
def test_ras_error_in_session(self):
# Given an upload file and a patched survey_id response which returns a RasError
data = {'file': (BytesIO(b'test data'), 'test.xls')}
mock_survey_service = RasError('The service raised an error')
with patch(
'application.controllers.collection_instrument.service_request',
side_effect=mock_survey_service):
# When a post is made to the upload end point
response = self.client.post(
'/collection-instrument-api/1.0.2/upload/cb0711c3-0ac8-41d3-ae0e-567e5ea1ef87'
'?classifiers={"form_type": "001"}',
headers=self.get_auth_headers(),
data=data,
content_type='multipart/form-data')
# Then a error is reported
self.assertIn('The service raised an error', response.data.decode())
def validate_one_instrument_for_ru_specific_upload(exercise, business, session):
"""
Checks there hasn't been an instrument loaded for this reporting unit in this collection exercise already.
The algorithm for this is as follows:
- Check if this business_id is related to other instrument_id's.
- If true, check each the exercise_id for each of those instruments to see if it matches the exercise we're
attempting to add to.
- If any match, then we're trying to add a second collection instrument for a reporting unit for this
collection exercise and an exception will be raised.
:param exercise: A db object representing the collection exercise
:param business: A db object representing the business data
:param session: A database session
:raises RasError: Raised when a duplicate is found
"""
bound_logger = log.bind(ru_ref=business.ru_ref)
bound_logger.info("Validating only one instrument per reporting unit per exercise")
business = query_business_by_ru(business.ru_ref, session)
if business:
business_id = str(business.id)
for instrument in business.instruments:
instrument_id = str(instrument.id)
bound_logger.bind(id_of_instrument=instrument_id, business_id=business_id)
bound_logger.info("Reporting unit has had collection instruments uploaded for it in the past")
for related_exercise in instrument.exercises:
related_exercise_id = related_exercise.exercise_id
exercise_id = exercise.exercise_id
bound_logger.bind(exercise_id=exercise_id, related_exercise_id=related_exercise_id)
bound_logger.info("About to check exercise for match")
if related_exercise_id == exercise_id:
bound_logger.info(
"Was about to add a second instrument for a reporting unit for a " "collection exercise"
)
ru_ref = business.ru_ref
error_text = (
f"Reporting unit {ru_ref} already has an instrument "
f"uploaded for this collection exercise"
)
raise RasError(error_text, 400)
def _update_seft_file(seft_model, file, survey_ref, exercise_id):
"""
Updates a seft_file with a new version of the data
:param file: A file object from which we can read the file contents
:return: instrument
"""
log.info("Updating instrument seft file")
file_contents = file.read()
file_size = len(file_contents)
if file_size == 0:
raise RasError("File is empty", 400)
seft_model.length = file_size
old_filename = seft_model.file_name
seft_model.file_name = file.filename
file.filename = survey_ref + "/" + exercise_id + "/" + file.filename
old_filename = survey_ref + "/" + exercise_id + "/" + old_filename
seft_ci_bucket = GoogleCloudSEFTCIBucket(current_app.config)
seft_ci_bucket.delete_file_from_bucket(old_filename)
seft_ci_bucket.upload_file_to_bucket(file=file)
seft_model.gcs = True
return seft_model