def init_tests(self):
self.maps = load_json_file('data/attributeMaps.json', encoding='utf8')
self.datasets = []
datasets = load_json_file('data/testDatasets.json')
for dt in datasets:
d = Dataset()
self.datasets.append(d.unmarshall(dt))
def test_4(self):
maps = load_json_file('data/attributeMaps.json', encoding='utf8')
reqj = load_json_file('data/testLinkRequest4.json', encoding='utf8')
lreq = LinkRequest()
lreq.unmarshall(reqj)
r = Reconciliation()
r.set_mappings(maps)
with self.assertRaises(NoMatchingRules) as context:
sim = r.similarity(lreq.datasetA, lreq.datasetB)
def test_3(self):
maps = load_json_file('data/attributeMaps.json', encoding='utf8')
reqj = load_json_file('data/testLinkRequest3.json', encoding='utf8')
lreq = LinkRequest()
lreq.unmarshall(reqj)
r = Reconciliation()
r.set_mappings(maps)
sim = r.similarity(lreq.datasetA, lreq.datasetB)
print(reqj)
print(sim)
self.assertEqual(sim, 1.0)
def init_tests(self):
# Test data sets
self.datasets = []
datasets = load_json_file('data/testDatasets.json')
for dt in datasets:
d = Dataset()
self.datasets.append(d.unmarshall(dt))
self.mappings = []
mappings = load_json_file('data/attributeMaps.json')
for mt in mappings:
m = AttributeMap()
self.mappings.append(m.unmarshall(mt))
def test_unmarshall_marshall_null_field(self):
lreq = load_json_file('data/testLinkRequestWithNulls.json')
d = LinkRequest()
d.unmarshall(lreq)
print(d)
d.json_marshall()
print(d.json_marshall())
def test_LinkRequest(self):
lreq = load_json_file('data/testLinkRequest.json')
r = LinkRequest()
r.unmarshall(lreq)
self.assertEqual(r.datasetA.properties['test'],
lreq['datasetA']['properties']['test'])
def test_Dataset(self):
datasets = load_json_file('data/testDatasets.json')
d = Dataset()
d.unmarshall(datasets[0])
self.assertEqual(d.properties['test'],
datasets[0]['properties']['test'])
def test_AttributeMap(self):
mappings = load_json_file('data/attributeMaps.json')
m = AttributeMap()
m.unmarshall(mappings[0])
self.assertEqual(m.pairings[0].attributes,
mappings[0]['pairings'][0]['attributes'])
def test_datastore_api(self):
cm = CMHandler('data/')
sm = cm.get_microservice_by_api('SM')
smh = SMHandler(sm,
key='data/httpsig_key_esmo.pem',
retries=5,
validate=False)
lreq = load_json_file('data/testLinkRequest.json')
storeEnt = StoreEntry()
storeEnt.id = 'testID'
storeEnt.type = 'linkRequest'
storeEnt.data = lreq # json.dumps(lreq)
# Start session
sessionID = smh.startSession()
self.assertIsNotNone(sessionID)
smh.resetDatastore()
smh.addDatastoreEntry(storeEnt.id, storeEnt.type, storeEnt.data)
read_req = smh.getDatastoreEntry(storeEnt.id)
self.assertEqual(storeEnt.id, read_req.id)
self.assertEqual(storeEnt.type, read_req.type)
self.assertEqual(storeEnt.data['datasetA']['id'],
read_req.data.datasetA.id)
print("storeEnt.data:" + json.dumps(storeEnt.data))
print("read_req.data:" + json.dumps(read_req.data.marshall()))
read_reqs = smh.searchDatastoreEntries(storeEnt.type)
read_req = read_reqs[0]
self.assertEqual(storeEnt.id, read_req.id)
self.assertEqual(storeEnt.type, read_req.type)
self.assertEqual(storeEnt.data['datasetA']['id'],
read_req.data.datasetA.id)
smh.deleteDatastoreEntry(storeEnt.id)
with self.assertRaises(SessionManagerError):
smh.getDatastoreEntry(storeEnt.id)
def test_client_submit_linking_request():
smh = sm_handler()
smh.startSession()
print("Started session: " + smh.sessId + "<br/>\n")
smh.writeSessionVar(cburl, 'ClientCallbackAddr')
token = smh.generateToken("SAMLms_0001", "SAMLms_0001")
print("Generated msToken: " + token + "<br/>\n")
streq = load_json_file(test_link_req)
treq = LinkRequest()
treq.unmarshall(streq)
smh.writeSessionVar(treq.marshall(), 'linkRequest')
print("Wrote linkRequest in Session: " + treq.json_marshall() + "<br/>\n")
template = {
'url': '/link/request/submit',
'token': token,
}
return render_template('msToken.html', template=template)
def submit_linking_request():
clean_expired()
# Check msToken against SM
if 'msToken' in request.form:
msToken = request.form['msToken']
elif 'msToken' in request.args:
msToken = request.args['msToken']
else:
return "missing msToken POST/GET parameter or bad content-type", 404
smh = get_session_manager()
try:
smh.validateToken(msToken)
except SessionManagerError as err:
return "Error validating msToken: " + str(err), 403
try:
dest_url = smh.getSessionVar('ClientCallbackAddr')
except SessionManagerError as err:
return "Error retrieving ClientCallbackAddr from SM: " + str(err), 403
# Load mappings to apply
mappings_dicts = load_json_file(data_dir + 'attributeMaps.json')
reconciliation = Reconciliation()
reconciliation.set_mappings(mappings_dicts)
# Parse input request
try:
link_req = smh.getSessionVar('linkRequest')
except SessionManagerError as err:
# return "Error retrieving linkRequest from SM: " + str(err), 403
return redirect_return(
smh, dest_url, 'ERROR', msID, apigwID,
"Error retrieving linkRequest from SM: " + str(err))
req = LinkRequest()
req.json_unmarshall(link_req)
now = datetime.now()
request_id = str(uuid.uuid4())
# Create a database row object
db_req = database.Request(request_id=request_id,
request_date=now,
status=StatusCodes.PENDING,
dataset_a=req.datasetA.json_marshall(),
dataset_b=req.datasetB.json_marshall())
# Store it on DB
session = database.DbSession()
session.add(db_req)
session.commit()
# Calculate similarity
try:
similarity = reconciliation.similarity(req.datasetA, req.datasetB)
except (NoMatchingRules, MissingOrBadParams) as err:
return redirect_return(
smh, dest_url, 'ERROR', msID, apigwID,
"Error performing the similarity check: " + str(err))
db_req.similarity = similarity
# Update request with the LLoA and the acceptance status
if similarity >= acceptance_threshold:
db_req.status = StatusCodes.ACCEPTED
else:
db_req.status = StatusCodes.REJECTED
session.add(db_req)
session.commit()
session.close()
# Fill in the requestID and store the request object in the dataStore
req.id = request_id
request_json = req.json_marshall()
try:
# Build the unique persistent storeEntry identifier # TODO: parametrise module name (search more incidences)
entry_id = build_store_id_from_req("seal-autorecon", issuer,
req.datasetA, req.datasetB)
smh.addDatastoreEntry(entry_id, "linkRequest",
req.marshall()) # TODO: parametrise type
except SessionManagerError as err:
return redirect_return(
smh, dest_url, 'ERROR', msID, apigwID,
"Error writing updated linkRequest to dataStore: " + str(err))
except StoreIDBuildError as err:
return redirect_return(smh, dest_url, 'ERROR', msID, apigwID,
"Error building store ID: " + str(err))
# We also overwrite it in SM session
try:
smh.writeSessionVar(req.marshall(),
'linkRequest') # TODO: expected variable?
except SessionManagerError as err:
# return "Error writing updated linkRequest to SM: " + str(err), 403
return redirect_return(
smh, dest_url, 'ERROR', msID, apigwID,
"Error writing updated linkRequest to SM: " + str(err))
# return Response(request_json,
# status=200,
# mimetype='application/json')
return redirect_return(smh, dest_url, 'OK', msID, apigwID)
def _get(self, file_path, remote_url=None):
ret_dict = None
logging.debug(
f"Try to get CM metadata set from local file cache in {file_path}")
# First, try to read locally
if os.path.isfile(file_path) \
and os.path.getsize(file_path) > 0:
try:
logging.debug("There is a file and has size")
ret_dict = load_json_file(file_path)
logging.debug(f"Read from file: {ret_dict}")
except (OSError, JSONDecodeError, TypeError) as err:
logging.warning(f"Error accessing file {file_path}: {err}")
else:
logging.debug("Not a file or empty")
# If cache is outdated, fetch file (if we have a url and key)
last_write = datetime.fromtimestamp(os.path.getmtime(file_path))
threshold = datetime.now() - timedelta(seconds=self.lifetime)
logging.debug(f"is cache outdated? {last_write} < {threshold} ?")
if self.httpsig and last_write < threshold:
res = None
try:
logging.debug(f"Fetching CM metadata set from {remote_url}")
res = self.httpsig.get(remote_url)
logging.debug(f"Fetched")
except HttpError as err:
logging.warning(
f"Error fetching remote config file {remote_url}: {err}")
except HttpConnectError as err:
logging.error(
f"Error fetching remote config file {remote_url}: {err}")
# Parse the fetched file.
if res and res.text:
logging.debug(f"Fetched data is not empty: {res.text}")
fetched_dict = None
try:
logging.debug(f"Parsing data into json")
fetched_dict = json.loads(res.text)
except (JSONDecodeError, TypeError) as err:
logging.warning(
f"Error decoding fetched file {file_path} from {remote_url}: {err}"
)
# If successful, overwrite
if fetched_dict:
logging.debug(f"Successfully parsed")
try:
logging.debug(f"Overwriting cache in {file_path}")
with open(file_path, 'w') as f:
f.write(res.text)
except (OSError, IOError) as err:
logging.warning(
f"Error overwriting file {file_path}: {err}")
# Regardless of we were able to overwrite or not, return the fetched data
logging.debug(f"Overwritten")
ret_dict = fetched_dict
# Return config set contents, or error if empty
if not ret_dict:
raise CMHandlerError(
"Requested config set not found or empty and could not be fetched remotely"
)
return ret_dict
def test_unmarshall_marshall(self):
datasets = load_json_file('data/testDatasets.json')
d = Dataset()
d.unmarshall(datasets[0])
d.json_marshall()
def test_caster_works(self):
mappings = load_json_file('data/attributeMaps.json')
for mt in mappings:
m = cast_from_dict(mt, AttributeMap)
self.assertEqual(m.pairings[0].attributes,
mappings[0]['pairings'][0]['attributes'])
