def data_from_ldif(self, filename):
with open(filename, "rb") as fd:
parser = LDIFParser(fd)
for dn, entry in parser.parse():
doc_id = doc_id_from_dn(dn)
oc = entry.get("objectClass") or entry.get("objectclass")
if oc:
if "top" in oc:
oc.remove("top")
if len(oc) == 1 and oc[0].lower() in ("organizationalunit",
"organization"):
continue
table_name = oc[-1]
if "objectClass" in entry:
entry.pop("objectClass")
elif "objectclass" in entry:
entry.pop("objectclass")
attr_mapping = OrderedDict({
"doc_id": doc_id,
"objectClass": table_name,
"dn": dn,
})
for attr in entry:
value = self.transform_value(attr, entry[attr])
attr_mapping[attr] = value
yield table_name, attr_mapping
def ldap_import(ldif_file):
print(f"Importing LDAP environment from {ldif_file}")
try:
parser = LDIFParser(open(ldif_file, "rb"))
for dn, record in parser.parse():
print(f"Importing {dn}")
# print(record) # Uncomment this file to print record if you want to debug
tuple_list_as_bytes = []
for attribute_name, attribute_values in record.items():
value_as_bytes = []
for value in attribute_values:
value_as_bytes.append(value.encode("utf-8")) # str to bytes
tuple_list_as_bytes.append((attribute_name, value_as_bytes))
try:
conn.add_s(dn, tuple_list_as_bytes)
print("Import successful")
except ldap.ALREADY_EXISTS:
print("Entry already existing in your LDAP, ignoring ...")
except Exception as err:
print(f"Unable to import record due to {err}")
except FileNotFoundError:
print(f"Unable to locate {ldif_file}. Make sure the path is correct")
sys.exit(1)
except ValueError as err:
print(f"Unable to read ldif. Make sure the LDIF was created with this utility. Error {err}")
sys.exit(1)
except Exception as err:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(f"Unable to import LDAP environment due to {err}. {exc_type}, {fname}, {exc_tb.tb_lineno}")
sys.exit(1)
def import_ldif(self, bucket_mappings):
ctx = prepare_template_ctx(self.manager)
attr_processor = AttrProcessor()
for _, mapping in bucket_mappings.items():
for file_ in mapping["files"]:
logger.info(f"Importing {file_} file")
src = f"/app/templates/{file_}"
dst = f"/app/tmp/{file_}"
os.makedirs(os.path.dirname(dst), exist_ok=True)
render_ldif(src, dst, ctx)
with open(dst, "rb") as fd:
parser = LDIFParser(fd)
for dn, entry in parser.parse():
if len(entry) <= 2:
continue
key = id_from_dn(dn)
entry["dn"] = [dn]
entry = transform_entry(entry, attr_processor)
data = json.dumps(entry)
# using INSERT will cause duplication error, but the data is left intact
query = 'INSERT INTO `%s` (KEY, VALUE) VALUES ("%s", %s)' % (
mapping["bucket"], key, data)
req = self.client.exec_query(query)
if not req.ok:
logger.warning(
"Failed to execute query, reason={}".format(
req.json()))
def parse_ldif_file(ldif_file: str) -> list[tuple[str, dict[str, Any]]]:
logger.info(f"### Parsing LDIF file {ldif_file}")
orig_ldif_fd = open(ldif_file, "rb")
ldif_fd = BytesIO()
for line in orig_ldif_fd.readlines():
if line.startswith(b"# search result"):
break
ldif_fd.write(line)
ldif_fd.seek(0)
parser = LDIFParser(ldif_fd)
return list(parser.parse())
def import_ldif(self):
optional_scopes = json.loads(self.manager.config.get("optional_scopes", "[]"))
ldif_mappings = get_ldif_mappings(optional_scopes)
# hybrid means only a subsets of ldif are needed
persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap")
ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default")
if persistence_type == "hybrid":
mapping = ldap_mapping
ldif_mappings = {mapping: ldif_mappings[mapping]}
# # these mappings require `base.ldif`
# opt_mappings = ("user", "token",)
# `user` mapping requires `o=gluu` which available in `base.ldif`
# if mapping in opt_mappings and "base.ldif" not in ldif_mappings[mapping]:
if "base.ldif" not in ldif_mappings[mapping]:
ldif_mappings[mapping].insert(0, "base.ldif")
ctx = prepare_template_ctx(self.manager)
for mapping, files in ldif_mappings.items():
self.check_indexes(mapping)
for file_ in files:
logger.info(f"Importing {file_} file")
src = f"/app/templates/{file_}"
dst = f"/app/tmp/{file_}"
os.makedirs(os.path.dirname(dst), exist_ok=True)
render_ldif(src, dst, ctx)
with open(dst, "rb") as fd:
parser = LDIFParser(fd)
for dn, entry in parser.parse():
self.add_entry(dn, entry)
in_file_ext = in_file_path.suffix
if in_file_ext not in valid_ext:
fatal_error("Error: Invalid input file!")
if not in_file_path.exists() or in_file_path.is_dir():
fatal_error("Error: Invalid input file path!")
with in_file_path.open('rb') as f:
if in_file_ext == '.ml':
ml_bytes = f.read()
ml = CscaMasterList.load(ml_bytes)
verify_and_extract_masterlist(ml,
default_out_dir_csca.joinpath(get_ml_out_dir_name(ml))
)
else:
parser = LDIFParser(f)
print("Note: DSC and CRL won't be verified against issuing CSCA!")
for dn, entry in parser.parse():
# ML
if 'CscaMasterListData' in entry:
ml = entry['CscaMasterListData'][0]
ml = CscaMasterList.load(ml)
verify_and_extract_masterlist(ml,
default_out_dir_csca.joinpath(get_ml_out_dir_name(ml))
)
# DSC
elif 'userCertificate' in entry or 'userCertificate;binary' in entry:
dn = parse_dn(dn)
dsc = entry['userCertificate;binary'][0]
def get_backup_data(self):
print('Checking backup file {0}.'.format(self.args.backup_file))
with gzip.open(self.args.backup_file, 'rb') as f:
self.ldif_parser = LDIFParser(f)
self.ldif_parser.handle = self.ldap_parser_handle
self.ldif_parser.parse()