def verify_spec(rendered_template, spec, cursor, verbose, attributes,
memberships, ownerships, privileges):
assert isinstance(spec, dict)
dbcontext = context.DatabaseContext(cursor, verbose)
error_messages = []
# Having all roles represented exactly once is critical for all submodules
# so we check this regardless of which submodules are being used
error_messages += ensure_no_duplicate_roles(rendered_template)
error_messages += ensure_no_undocumented_roles(spec, dbcontext)
error_messages += ensure_no_except_on_schema(spec)
if ownerships:
for objkind in context.PRIVILEGE_MAP.keys():
if objkind == 'schemas':
error_messages += ensure_no_unowned_schemas(spec, dbcontext)
error_messages += ensure_no_schema_owned_twice(spec)
else:
# We run each of these functions once per object kind as it is possible that
# two objects of different kinds could have the same name in the same schema
error_messages += ensure_no_missing_objects(
spec, dbcontext, objkind)
error_messages += ensure_no_object_owned_twice(
spec, dbcontext, objkind)
error_messages += ensure_no_dependent_object_is_owned(
spec, dbcontext, objkind)
if privileges:
error_messages += ensure_no_redundant_privileges(spec)
if error_messages:
common.fail('\n'.join(error_messages))
def verify_spec(rendered_template, spec):
assert isinstance(spec, dict)
error_messages = []
error_messages += detect_multiple_role_definitions(rendered_template)
verification_functions = (verify_schema, check_for_multi_schema_owners,
check_read_write_obj_references)
for fn in verification_functions:
error_messages += fn(spec)
if error_messages:
common.fail('\n'.join(error_messages))
def identify_desired_objects(self):
"""
Create the sets of desired privileges. The sets will look like the following:
self.desired_nondefaults:
{(ObjectName(schema, unqualified_name), priv_name), ...}
Example: {('myschema.mytable', 'SELECT'), ...}
self.desired_defaults:
{(grantor, schema, priv_name), ...}
Example: {('svc-hr-etl', 'hr_schema', 'SELECT'), ...}
"""
desired_nondefault_objs = set()
schemas = []
for objname in self.desired_items:
if objname == common.ObjectName(
'personal_schemas') and self.object_kind == 'schemas':
desired_nondefault_objs.update(self.personal_schemas)
elif objname == common.ObjectName(
'personal_schemas') and self.object_kind != 'schemas':
# The end-user is asking something impossible
common.fail(
PERSONAL_SCHEMAS_ERROR_MSG.format(self.rolename,
self.object_kind,
self.access))
elif objname == common.ObjectName('personal_schemas', '*'):
schemas.extend(self.personal_schemas)
elif objname.unqualified_name != '*':
# This is a single non-default privilege ask
owner = self.get_object_owner(objname)
if owner != self.rolename:
desired_nondefault_objs.add(objname)
else:
# We were given a schema.*; we'll process those below
schemas.append(objname.only_schema())
for schema in schemas:
# For schemas, we wish to have privileges for all existing objects, so get all
# existing objects not owned by this role and add them to self.desired_nondefaults
schema_objects = self.get_schema_objects(schema.qualified_name)
desired_nondefault_objs.update(schema_objects)
#Remove excepted elements
desired_nondefault_objs.difference_update(self.excepted_items)
# Cross our desired objects with the desired privileges
priv_types = PRIVILEGE_MAP[self.object_kind][self.access]
self.desired_nondefaults = set(
itertools.product(desired_nondefault_objs, priv_types))
if self.default_acl_possible:
self.determine_desired_defaults(schemas)
def get_object_owner(self, objname, objkind=None):
objkind = objkind or self.object_kind
object_owners = self.all_object_attrs.get(objkind, dict()).get(
objname.schema, dict())
owner = object_owners.get(objname, dict()).get('owner', None)
if owner:
return owner
else:
obj_kind_singular = objkind[:-1]
common.fail(
OBJECT_DOES_NOT_EXIST_ERROR_MSG.format(obj_kind_singular,
objname.qualified_name,
self.rolename))
def get_object_owner(self, item, objkind=None):
objkind = objkind or self.object_kind
schema = item.split('.', 1)[0]
object_owners = self.all_object_owners.get(objkind,
dict()).get(schema, dict())
owner = object_owners.get(item, None)
if owner:
return owner
else:
obj_kind_singular = objkind[:-1]
common.fail(
OBJECT_DOES_NOT_EXIST_ERROR_MSG.format(obj_kind_singular, item,
self.rolename))
def print_spec(spec_path):
""" Validate a spec passes various checks and, if so, return the loaded spec. """
rendered_template = render_template(spec_path)
unconverted_spec = yaml.safe_load(rendered_template)
# Validate the schema before verifying anything else about the spec. If the spec is invalid
# then other checks may fail in erratic ways, so it is better to error out here
error_messages = ensure_valid_schema(unconverted_spec)
if error_messages:
common.fail('\n'.join(error_messages))
spec = convert_spec_to_objectnames(unconverted_spec)
return spec
def render_template(path):
""" Load a spec. There may be templated password variables, which we render using Jinja. """
try:
dir_path, filename = os.path.split(path)
environment = jinja2.Environment(loader=jinja2.FileSystemLoader(dir_path),
undefined=jinja2.StrictUndefined)
loaded = environment.get_template(filename)
rendered = loaded.render(env=os.environ)
except jinja2.exceptions.TemplateNotFound as err:
common.fail(FILE_OPEN_ERROR_MSG.format(path, err))
except jinja2.exceptions.UndefinedError as err:
common.fail(MISSING_ENVVAR_MSG.format(err))
else:
return rendered
def identify_desired_objects(self):
"""
Create the sets of desired privileges. The sets will look like the following:
self.desired_nondefaults:
{(objname, priv_name), ...}
Example: {('myschema.mytable', 'SELECT'), ...}
self.desired_defaults:
{(grantor, schema, priv_name), ...}
Example: {('svc-hr-etl', 'hr_schema', 'SELECT'), ...}
"""
desired_nondefault_objs = set()
schemas = []
for item in self.desired_items:
if item == 'personal_schemas' and self.object_kind == 'schemas':
desired_nondefault_objs.update(self.personal_schemas)
elif item == 'personal_schemas' and self.object_kind != 'schemas':
# The end-user is asking something impossible
common.fail(
PERSONAL_SCHEMAS_ERROR_MSG.format(self.rolename,
self.object_kind,
self.access))
elif item == 'personal_schemas.*':
schemas.extend(list(self.personal_schemas))
elif not item.endswith('.*'):
# This is a single non-default privilege ask
quoted_item = ensure_quoted_identifier(item)
owner = self.get_object_owner(quoted_item)
if owner != self.rolename:
desired_nondefault_objs.add(quoted_item)
else:
# We were given a schema.*; we'll process those below
schemas.append(item[:-2])
for schema in schemas:
# For schemas, we wish to have privileges for all existing objects, so get all
# existing objects not owned by this role and add them to self.desired_nondefaults
schema_objects = self.get_schema_objects(schema)
desired_nondefault_objs.update(schema_objects)
# Cross our desired objects with the desired privileges
priv_types = PRIVILEGE_MAP[self.object_kind][self.access]
self.desired_nondefaults = set(
itertools.product(desired_nondefault_objs, priv_types))
if self.default_acl_possible:
self.determine_desired_defaults(schemas)
def load_spec(spec_path, cursor, verbose, attributes, memberships, ownerships,
privileges, attributes_source_table):
""" Validate a spec passes various checks and, if so, return the loaded spec. """
rendered_template = render_template(spec_path)
unconverted_spec = yaml.safe_load(rendered_template)
# Validate the schema before verifying anything else about the spec. If the spec is invalid
# then other checks may fail in erratic ways, so it is better to error out here
error_messages = ensure_valid_schema(unconverted_spec)
if error_messages:
common.fail('\n'.join(error_messages))
spec = convert_spec_to_objectnames(unconverted_spec)
verify_spec(rendered_template, spec, cursor, verbose, attributes,
memberships, ownerships, privileges, attributes_source_table)
return spec
def fail_if_undocumented_schemas(spec, dbcontext):
"""
Refuse to continue if schemas are in the database but are not documented in spec. This is done
(vs. just deleting the schemas programmatically) because the schema likely contains tables,
those tables may contain permissions, etc. There's enough going on that if the user just made
a mistake by forgetting to add a schema to their spec we've caused serious damage; better to
ask them to manually resolve this
"""
current_schemas_and_owners = dbcontext.get_all_schemas_and_owners()
current_schemas = set(current_schemas_and_owners.keys())
spec_schemas = get_spec_schemas(spec)
undocumented_schemas = current_schemas.difference(spec_schemas)
if undocumented_schemas:
undocumented_schemas_fmtd = '"' + '", "'.join(
sorted(undocumented_schemas)) + '"'
common.fail(
msg=UNDOCUMENTED_SCHEMAS_MSG.format(undocumented_schemas_fmtd))
def fail_if_undocumented_roles(spec, dbcontext):
"""
Refuse to continue if roles are in the database cluster but are not documented in spec. This
is done (vs. just deleting the roles programmatically) because the roles may own schemas,
tables, functions, etc. There's enough going on that if the user just made a mistake by
forgetting to add a role to their spec then we've caused serious damage; it's safer to ask
them to manually resolve this.
"""
current_role_attributes = dbcontext.get_all_role_attributes()
spec_roles = set(spec.keys())
current_roles = set(current_role_attributes.keys())
undocumented_roles = current_roles.difference(spec_roles)
if undocumented_roles:
undocumented_roles_fmtd = '"' + '", "'.join(
sorted(undocumented_roles)) + '"'
common.fail(msg=UNDOCUMENTED_ROLES_MSG.format(undocumented_roles_fmtd))
def run_password_sql(cursor, all_password_sql_to_run):
"""
Run one or more SQL statements that contains a password. We do this outside of the
common.run_query() framework for two reasons:
1) If verbose mode is requested then common.run_query() will show the password in its
reporting of the queries that are executed
2) The input to common.run_query() is the module output. This output is faithfully rendered
as-is to STDOUT upon pgbedrock's completion, so we would leak the password there as well.
By running password-containing queries outside of the common.run_query() approach we can avoid
these issues
"""
query = '\n'.join(all_password_sql_to_run)
try:
cursor.execute(query)
except Exception as e:
common.fail(msg=common.FAILED_QUERY_MSG.format(query, e))
def converted_attributes(self):
""" Convert the list of attributes provided in the spec to postgres-compatible
keywords and values.
"""
converted_attributes = {}
for spec_attribute in self.spec_attributes:
# We do spec_attribute.upper() in each spot in order to leave the original
# spec_attribute unchanged in case it is a password, in which case we don't want to
# change the case
if spec_attribute.upper().startswith('CONNECTION LIMIT'):
val = spec_attribute[17:].strip()
converted_attributes['rolconnlimit'] = int(val)
elif spec_attribute.upper().startswith('VALID UNTIL'):
val = spec_attribute[12:].strip()
converted_attributes['rolvaliduntil'] = val
elif spec_attribute.upper().startswith('IGNORE PASSWORD'):
# This is the case where we don't want to check the password, so
# set the 'rollpassword' attribute to False, which will cause
# the password comparison check to be skipped
converted_attributes['rolpassword'] = False
elif 'PASSWORD' in spec_attribute.upper():
# Regardless whether the spec specified ENCRYPTED or UNENCRYPTED for the password,
# we throw this away as we will be storing the password in encrypted form
val = spec_attribute.split('PASSWORD ', 1)[-1]
# Trim leading and ending quotes, if there are any
if val[0] == '"' or val[0] == "'":
val = val[1:]
if val[-1] == '"' or val[-1] == "'":
val = val[:-1]
if "'" in val or '"' in val:
common.fail(msg=UNSUPPORTED_CHAR_MSG.format(self.rolename))
converted_attributes['rolpassword'] = val
elif spec_attribute.upper().startswith('NO'):
keyword = spec_attribute.upper()[2:]
colname = PG_COLUMN_NAME.get(keyword)
if not colname:
common.fail(UNKNOWN_ATTRIBUTE_MSG.format(spec_attribute))
converted_attributes[colname] = False
else:
keyword = spec_attribute.upper()
colname = PG_COLUMN_NAME.get(keyword)
if not colname:
common.fail(UNKNOWN_ATTRIBUTE_MSG.format(spec_attribute))
converted_attributes[colname] = True
return converted_attributes
