def fetch():
"""
Main function to get all remote resources for each service and write to database.
"""
LOGGER.info("Getting database session")
session = db.get_db_session_from_settings(echo=False)
for service_type in SYNC_SERVICES_TYPES:
LOGGER.info("Fetching data for service type: %s" % service_type)
fetch_all_services_by_type(service_type, session)
session.commit()
session.close()
def fetch(settings=None):
"""
Main function to get all remote resources for each service and write to database.
"""
with transaction.manager:
LOGGER.info("Getting database session")
session = db.get_db_session_from_settings(settings=settings,
echo=False)
for service_type in SYNC_SERVICES_TYPES:
LOGGER.info("Fetching data for service type: %s", service_type)
fetch_all_services_by_type(service_type, session)
transaction.commit()
def test_register_permissions_missing_group_create_new_entries(self):
utils.TestSetup.create_TestService(
self,
override_service_name=self.test_perm_svc_name,
override_service_type=ServiceAPI.service_type)
utils.TestSetup.delete_TestGroup(
self, override_group_name=self.test_perm_grp_name)
session = get_db_session_from_settings(self.app.app.registry.settings)
svc_perm = Permission.READ_MATCH
res1_perm = Permission.READ
res2_perm = Permission.WRITE
res1_name = "test-resource"
res2_name = "sub-test-resource"
perm_config = {
"permissions": [
{
"service": self.test_perm_svc_name,
"permission": svc_perm.value,
"action": "create",
"group": self.test_perm_grp_name,
},
{
"service": self.test_perm_svc_name,
"resource": res1_name,
"permission": res1_perm.value,
"action": "create",
"group": self.test_perm_grp_name,
},
{
"service": self.test_perm_svc_name,
"resource": res1_name + "/" + res2_name,
"permission": res2_perm.value,
"action": "create",
"group": self.test_perm_grp_name,
},
]
}
utils.check_no_raise(
lambda: register.magpie_register_permissions_from_config(
perm_config, db_session=session))
# check that all service, resources, group are created correctly
services = utils.TestSetup.get_RegisteredServicesList(self)
utils.check_val_is_in(self.test_perm_svc_name,
[s["service_name"] for s in services])
groups = utils.TestSetup.get_RegisteredGroupsList(self)
utils.check_val_is_in(self.test_perm_grp_name, groups)
resp = utils.test_request(
self.app, "GET",
"/services/{}/resources".format(self.test_perm_svc_name))
body = utils.check_response_basic_info(resp)
svc_res = body[self.test_perm_svc_name]["resources"]
svc_res_id = body[self.test_perm_svc_name]["resource_id"]
utils.check_val_is_in(res1_name,
[svc_res[r]["resource_name"] for r in svc_res])
res1_id = [
svc_res[r]["resource_id"] for r in svc_res
if svc_res[r]["resource_name"] == res1_name
][0]
res1_sub = svc_res[str(res1_id)]["children"]
utils.check_val_is_in(res2_name,
[res1_sub[r]["resource_name"] for r in res1_sub])
res2_id = [
res1_sub[r]["resource_id"] for r in res1_sub
if res1_sub[r]["resource_name"] == res2_name
][0]
# check that all permissions are created correctly
path = "/groups/{}/resources/{}/permissions".format(
self.test_perm_grp_name, svc_res_id)
resp = utils.test_request(self.app, "GET", path)
body = utils.check_response_basic_info(resp)
utils.check_val_is_in(svc_perm.value, body["permission_names"])
path = "/groups/{}/resources/{}/permissions".format(
self.test_perm_grp_name, res1_id)
resp = utils.test_request(self.app, "GET", path)
body = utils.check_response_basic_info(resp)
utils.check_val_is_in(res1_perm.value, body["permission_names"])
path = "/groups/{}/resources/{}/permissions".format(
self.test_perm_grp_name, res2_id)
resp = utils.test_request(self.app, "GET", path)
body = utils.check_response_basic_info(resp)
utils.check_val_is_in(res2_perm.value, body["permission_names"])
def test_register_permissions_existing_group_without_intermediate_entries(
self):
utils.TestSetup.create_TestService(
self,
override_service_name=self.test_perm_svc_name,
override_service_type=ServiceAPI.service_type)
utils.TestSetup.create_TestGroup(
self, override_group_name=self.test_perm_grp_name)
session = get_db_session_from_settings(self.app.app.registry.settings)
res1_name = "test-resource"
res2_name = "sub-test-resource"
res3_name = "sub-sub-test-resource"
res3_perm = Permission.WRITE_MATCH
perm_config = {
"permissions": [
{
"service": self.test_perm_svc_name, # exists
"resource": res1_name + "/" + res2_name + "/" +
res3_name, # none exist, all created for perm
"permission": res3_perm.value, # perm only to lowest child
"action": "create",
"group": self.test_perm_grp_name,
},
]
}
utils.check_no_raise(
lambda: register.magpie_register_permissions_from_config(
perm_config, db_session=session))
# check that all service, resources, group are created correctly
services = utils.TestSetup.get_RegisteredServicesList(self)
utils.check_val_is_in(self.test_perm_svc_name,
[s["service_name"] for s in services])
groups = utils.TestSetup.get_RegisteredGroupsList(self)
utils.check_val_is_in(self.test_perm_grp_name, groups)
resp = utils.test_request(
self.app, "GET",
"/services/{}/resources".format(self.test_perm_svc_name))
body = utils.check_response_basic_info(resp)
svc_res = body[self.test_perm_svc_name]["resources"]
svc_res_id = body[self.test_perm_svc_name]["resource_id"]
utils.check_val_is_in(res1_name,
[svc_res[r]["resource_name"] for r in svc_res])
res1_id = [
svc_res[r]["resource_id"] for r in svc_res
if svc_res[r]["resource_name"] == res1_name
][0]
res1_sub = svc_res[str(res1_id)]["children"]
utils.check_val_is_in(res2_name,
[res1_sub[r]["resource_name"] for r in res1_sub])
res2_id = [
res1_sub[r]["resource_id"] for r in res1_sub
if res1_sub[r]["resource_name"] == res2_name
][0]
res2_sub = res1_sub[str(res2_id)]["children"]
utils.check_val_is_in(res3_name,
[res2_sub[r]["resource_name"] for r in res2_sub])
res3_id = [
res2_sub[r]["resource_id"] for r in res2_sub
if res2_sub[r]["resource_name"] == res3_name
][0]
# check that all permissions are created correctly (only for final item)
path = "/groups/{}/resources/{}/permissions".format(
self.test_perm_grp_name, svc_res_id)
resp = utils.test_request(self.app, "GET", path)
body = utils.check_response_basic_info(resp)
utils.check_val_equal(body["permission_names"], [])
path = "/groups/{}/resources/{}/permissions".format(
self.test_perm_grp_name, res1_id)
resp = utils.test_request(self.app, "GET", path)
body = utils.check_response_basic_info(resp)
utils.check_val_equal(body["permission_names"], [])
path = "/groups/{}/resources/{}/permissions".format(
self.test_perm_grp_name, res2_id)
resp = utils.test_request(self.app, "GET", path)
body = utils.check_response_basic_info(resp)
utils.check_val_equal(body["permission_names"], [])
path = "/groups/{}/resources/{}/permissions".format(
self.test_perm_grp_name, res3_id)
resp = utils.test_request(self.app, "GET", path)
body = utils.check_response_basic_info(resp)
utils.check_all_equal(body["permission_names"], [res3_perm.value])
def test_register_permissions_multiple_resource_type_possible(self):
"""
Verify that service that allows multiple resource-types children retrieves ``type`` field for their creation.
"""
svc_type = ServiceTHREDDS.service_type
svc_name = "unittest-service-thredds-register-children-resources"
utils.TestSetup.create_TestService(self,
override_service_name=svc_name,
override_service_type=svc_type)
utils.TestSetup.create_TestGroup(
self, override_group_name=self.test_perm_grp_name)
session = get_db_session_from_settings(self.app.app.registry.settings)
res1_name = "test-resource"
res2_name = "sub-test-resource"
res3_name = "sub-sub-test-resource"
res3_perm = "write-deny-recursive"
res3_path = res1_name + "/" + res2_name + "/" + res3_name # none exist, all created for final permission
perm_config = {
"permissions": [
{
"service": svc_name, # exists
"resource": res3_path,
"permission": res3_perm, # perm only on lowest child
"type": Directory.
resource_type_name, # without this, fails because cannot infer Directory/File
"action": "create",
"group": self.test_perm_grp_name,
},
]
}
utils.check_no_raise(
lambda: register.magpie_register_permissions_from_config(
perm_config, db_session=session))
# check that all service, resources, group are created correctly
services = utils.TestSetup.get_RegisteredServicesList(self)
utils.check_val_is_in(svc_name, [s["service_name"] for s in services])
groups = utils.TestSetup.get_RegisteredGroupsList(self)
utils.check_val_is_in(self.test_perm_grp_name, groups)
resp = utils.test_request(self.app, "GET",
"/services/{}/resources".format(svc_name))
body = utils.check_response_basic_info(resp)
svc_res = body[svc_name]["resources"] # type: JSON
utils.check_val_is_in(res1_name,
[svc_res[r]["resource_name"] for r in svc_res])
res1_id = [
svc_res[r]["resource_id"] for r in svc_res
if svc_res[r]["resource_name"] == res1_name
][0]
res1_sub = svc_res[str(res1_id)]["children"] # type: JSON
utils.check_val_is_in(res2_name,
[res1_sub[r]["resource_name"] for r in res1_sub])
res2_id = [
res1_sub[r]["resource_id"] for r in res1_sub
if res1_sub[r]["resource_name"] == res2_name
][0]
res2_sub = res1_sub[str(res2_id)]["children"] # type: JSON
utils.check_val_is_in(res3_name,
[res2_sub[r]["resource_name"] for r in res2_sub])
res3_id = [
res2_sub[r]["resource_id"] for r in res2_sub
if res2_sub[r]["resource_name"] == res3_name
][0]
path = "/groups/{}/resources/{}/permissions".format(
self.test_perm_grp_name, res3_id)
resp = utils.test_request(self.app, "GET", path)
body = utils.check_response_basic_info(resp)
utils.check_val_is_in(res3_perm, body["permission_names"])
# validate that without the 'type', similar resource would not be created due to missing information
res_missing_type_name = "missing-type"
res_missing_type_path = res3_path.replace("/" + res3_name,
"/" + res_missing_type_name)
missing_type_perm_config = copy.deepcopy(perm_config) # type: JSON
missing_type_perm_config["permissions"][0].pop("type")
missing_type_perm_config["permissions"][0][
"resource"] = res_missing_type_path
utils.check_no_raise(
lambda: register.magpie_register_permissions_from_config(
missing_type_perm_config, db_session=session))
resp = utils.test_request(self.app, "GET",
"/resources/{}".format(res2_id))
body = utils.check_response_basic_info(resp)
res_children = body["resource"]["children"]
res_found = [
res for _, res in res_children.items()
if res["resource_name"] == res_missing_type_name
]
utils.check_val_equal(res_found, [])