def __init__( self, file1, file2):
"""
Parameters:
file1
filename of first file to read
file2
filename of second file to read
"""
self.src1 = LDIFRecordList(open(file1))
self.src2 = LDIFRecordList(open(file2))
def parse_ldif(ldif):
'''
:param ldif: string in LDIF format to parse
:returns: list of tuples where the first item of the tuple is DN and the
second one is a hash of attributes
'''
parser = LDIFRecordList(StringIO(ldif))
parser.parse()
return parser.all_records
def parse_crls(crls: BinaryIO, store: X509Store) -> None:
"""
Parse CRLs from the ICAO PKD DSC-CRL ldif
"""
parser = LDIFRecordList(crls)
parser.parse_entry_records()
for record in parser.all_records:
if "certificateRevocationList;binary" in record[1]:
CRL = load_crl(FILETYPE_ASN1, record[1]["certificateRevocationList;binary"][0])
print(f"\t[+] Loaded CRL: {record[1]['cn'][0]}")
store.add_crl(CRL)
def from_ldif(cls, data):
"""Convert LDIF data to dictionary."""
try:
ldif_record_list = LDIFRecordList(StringIO(data))
ldif_record_list.parse()
returned = []
for entry in ldif_record_list.all_records:
returned.append([entry[0], cls.decode_values(entry[1])])
return returned
except Exception:
raise AnsibleFilterError(
'Invalid LDIF data for LDIFRecordList (%s)' % data)
def fetchCRL(crlURI):
crlLocation = urlparse(crlURI)
if 'ldap' == crlLocation.scheme:
buffer = StringIO()
c = pycurl.Curl()
c.setopt(c.URL, crlURI)
c.setopt(c.WRITEDATA, buffer)
c.perform()
c.close()
body = buffer.getvalue()
buffer = StringIO(body)
parser = LDIFRecordList(buffer)
parser.parse()
dn, entry = parser.all_records[0]
crl = entry[entry.keys()[-1]][0]
return crl
else:
return urllib.urlopen(crlURI).read()
def parse_csca_certs(master_list: BinaryIO, store: X509Store) -> None:
"""
Parse CSCA certificates from the ICAO PKD ML ldif
"""
parser = LDIFRecordList(master_list)
parser.parse_entry_records()
unique_certs: List[X509] = []
for record in parser.all_records:
if "CscaMasterListData" not in record[1]:
continue
print(f"\t[i] Reading {record[1]['cn'][0]}")
cmd = "openssl cms -inform der -noverify -verify"
(signed_data, err) = execute(cmd, record[1]["CscaMasterListData"][0])
if err.decode("utf8").strip() != "Verification successful":
# print(f"\t[-] [{err.decode('utf8')}]")
print("\t[-] Verification of Masterlist data failed\n")
continue
print("\t[+] MasterList Verification successful")
cert_list = extract_certificates(signed_data)
print("\t[i] Removing duplicates")
unique_certs_from_ml = [x for x in cert_list if unique_hash(x)]
print(f"\t[i] Removed {len(cert_list)-len(unique_certs_from_ml)} duplicate certificates\n")
unique_certs = unique_certs + unique_certs_from_ml
print(f"\t[i] Total unique entries: {len(unique_certs)}\n")
for cert in unique_certs:
if is_self_signed(cert):
print(f"\t[+] Loaded certificate: {cert.get_subject().countryName}")
print_valid_time("\t\t", cert)
store.add_cert(cert)
class LDIFVerifier():
"""
Check that two LDIF files contain equivalent LDAP information. If they
don't, emit a summary of the differences
"""
def __init__( self, file1, file2):
"""
Parameters:
file1
filename of first file to read
file2
filename of second file to read
"""
self.src1 = LDIFRecordList(open(file1))
self.src2 = LDIFRecordList(open(file2))
def emitDifferingValues( self, attributes1, attributes2):
"""
Emit a description of the differences between two dictionaries of attribute values
"""
for attributeName, attributeValues1 in attributes1.iteritems():
if attributeName in attributes2:
attributeValues2 = attributes2[attributeName]
if attributeValues1 != attributeValues2:
print " " + attributeName + ": " + str(attributeValues1) + " != " + str(attributeValues2)
else:
print " " + attributeName + ": missing in second file"
def emitDifferences( self, summary1, summary2):
"""
Emit all differences between the two LDAP objects. The
supplied parameters are dictionaries between the object DN and
a list of attributes
"""
count = 0
for dnLower,wrappedObject1 in summary1.iteritems():
(dn,attributes1) = wrappedObject1
if dnLower in summary2:
wrappedObject2 = summary2 [dnLower]
(dn2,attributes2) = wrappedObject2
if( attributes1 != attributes2):
count += 1
print "\n dn: " + dn
print " [difference in attribute values]\n"
self.emitDifferingValues( attributes1, attributes2)
else:
count += 1
print "\n dn: " + dn
print " [object missing in second file]\n"
self.printSummary( dn, attributes1)
for dnLower,wrappedObject2 in summary2.iteritems():
(dn,attributes2) = wrappedObject2
if not dnLower in summary1:
count += 1
print "\n dn: " + dn
print " [object missing in first file]\n"
self.printSummary( dn, attributes2)
return count
def printSummary( self, dn, attributes):
"""
Print a complete LDAP object
"""
for attributeName, attributeValues in attributes.iteritems():
for attributeValue in attributeValues:
print " " + attributeName + ": " + attributeValue
def buildSummary( self, records):
"""
Build
"""
summary = {}
for record in records:
dn,attributes = record
summary [dn.lower()] = (dn,attributes)
return summary
def compare( self):
"""
Check whether the two named files are equal.
"""
self.src1.parse()
summary1 = self.buildSummary( self.src1.all_records)
self.src2.parse()
summary2 = self.buildSummary( self.src2.all_records)
count = self.emitDifferences( summary1, summary2)
if( count > 0):
exit(1)
def perform_ldif_operation(filename, binddn, rootpassword):
parser = LDIFRecordList(open(filename, 'r'))
parser.parse()
for dn, entry in parser.all_records:
add_record = list(entry.items())
BaseConfig.add_attribute(binddn, dn, add_record, rootpassword)
from os.path import isfile, join
from ldif import LDIFParser, LDIFRecordList
from tri_core.common.storetokens import storetokens
import common.credentials.ldap as _ldap
ldap_path = '/srv/api/ldap/'
onlyfiles = [f for f in listdir(ldap_path) if isfile(join(ldap_path, f))]
groups = dict()
# collect all the groups
for filename in onlyfiles:
file = open(ldap_path + filename, 'rb')
records = LDIFRecordList(file)
records.parse()
file.close()
for dn, record in records.all_records:
groups[dn] = []
if not 'esiRefreshToken' in record:
continue
for authgroup in record['esiRefreshToken']:
group = authgroup.decode('utf-8')
groups[dn].append(group)
print('closing file {0}'.format(filename))
def main():
def pretty_print(data):
"""
This function prints the output
:param data:
:return:
"""
object_count: int = 0
def log(text: str, level=1):
if (level <= VERBOSITY):
sys.stderr.write(str(text) + "\n")
def central_print(
text,
level=1,
space="-",
size=max(20, int(os.popen('stty size', 'r').read().split()[1]))):
log(
space * ((size - len(text) - 2) // 2) + " " + text + " " +
space * ((size - len(text) - 2) // 2), level)
def run_watchdog(reverse, regexp, value, attribute):
log(
"watchdog debug: Reverse: {}, Regexp: {}, Value: {}, Attribute: {}"
.format(reverse, regexp, value, attribute), 9)
result = True
try:
if type(value).__name__ == 'list':
for item in value:
result = result and run_watchdog(
reverse, regexp, item, attribute)
else:
if regexp.match(value.decode()):
log(
"Matched the regexp '{}' in attribute {}!".format(
regexp.pattern, value), 7)
if (reverse):
log("WATCHDOG SPOTTED: {}".format(value))
return False
else:
if (not (reverse)):
log(
"WATCHDOG NOT MATCHING: {} on {}".format(
value, regexp), 3)
return False
except TypeError:
log("Unsupported watched object type on attribute {}, got {}".
format(attribute,
type(value).__name__))
return result
def output(cn, content, object_type="Object"):
central_print(
"Object " + " '" + cn.split(",")[0].split("=")[1] + "'", 3)
to_be_removed = []
for banned in FILTERED_CLASSES:
pattern = re.compile("^" + str(banned) + "$")
for value in content["objectClass"]:
if pattern.match(str(value)) is not None:
to_be_removed.append(value)
else:
log(
"{} not matched for removal with {}".format(
value, pattern.pattern), 9)
for remove in to_be_removed:
content["objectClass"].remove(remove)
log("Removing objectClass: {}".format(remove), 4)
to_be_removed = []
watchdog_result = True
for key in content.keys():
if len(ATTRIBUTES_LIST) and key not in ATTRIBUTES_LIST:
log(
"Based on attributes list removing attribute {} ".
format(key), 3)
to_be_removed.append(key)
for banned in FILTERED_ATTRIBUTES:
pattern = re.compile("^" + str(banned) + "$")
if pattern.match(key) is not None:
to_be_removed.append(key)
for (attribute, old_value, new_value) in SUBSTITUTE:
if (key == attribute):
log(
"Found matching attribute {} for substitution".
format(key), 9)
if type(content[key]).__name__ == 'list':
_to_be_removed = []
_to_be_added = []
for item in content[key]:
if item.decode() == old_value:
_to_be_removed.append(item)
_to_be_added.append(new_value.encode())
log(
"Replacing on key {} value {} for {} ".
format(key, old_value, new_value), 5)
else:
log(
"Array replace: {} is not {}".format(
item, old_value), 9)
for remove in _to_be_removed:
content[key].remove(remove)
for append in _to_be_added:
content[key].append(append)
elif content[key] == old_value:
content[key] = new_value
log(
"Replacing on key {} value {} for {} ".format(
key, old_value, new_value), 5)
else:
log(
"Replace condition {}:{} not met key {}".format(
attribute, old_value, key), 9)
for (reverse, attribute, regexp) in WATCHER:
if key != attribute:
continue
else:
watchdog_result = run_watchdog(reverse, regexp,
content[key], attribute)
if watchdog_result:
log(
"Removing {} due to watchdog results".format(
cn), 1)
for remove in to_be_removed:
try:
content.pop(remove)
log("Removing attribute: {}".format(remove), 4)
except KeyError:
log("Failed to remove {}, already removed?".format(remove),
9)
if watchdog_result and WATCHDOG_STRICT:
log("Watchdog blocked")
return
for (attribute, value) in NEW_ATTRIBUTES:
value = value.encode()
log(
"Adding attribute {} with value {}".format(
attribute, value), 4)
try:
content.get(attribute)
if (type(content[attribute]).__name__ == 'list'):
content[attribute].append(value)
else:
content[attribute] = value
except KeyError:
raise Exception(
"Attribute {} is already present! For rewriting attributes use another function."
.format(attribute))
if not len(content.keys()):
if REMOVE_WHEN_ZERO_ATTRS:
log(
'Removing {} because it has no attribute left. Rest in piece(s)'
.format(cn), 6)
return
log('Please notice, that {} has no attributes!'.format(cn), 2)
log(content, 9)
OUTPUT_FILE.unparse(cn, content)
log("Written the object {}".format(cn), 6)
for (cn, content) in data:
if cn.endswith(RESTRICT):
object_count += 1
output(cn, content)
else:
log("{} restricted from output".format(cn), 10)
log("", 2)
central_print("FINISHED", 2)
log("Object count: {}".format(object_count), 2)
from ldif import LDIFRecordList, LDIFWriter
import argparse
import re
import sys
import os
parser = argparse.ArgumentParser(
description='LDIF manipulation program. Please check full in')
parser.add_argument('-i',
'--input',
metavar='input',
type=str,
required=True,
help='Ldif file to be proceed')
parser.add_argument('-o',
'--output',
metavar='output',
type=str,
required=False,
default=None,
help='Ldif file to be outputed')
parser.add_argument(
'-r',
'--restrict',
metavar='restrict',
type=str,
default="",
help=
'Restrict the output only to CN ending with this string. \nIf you wan\'t the object itself to be immitted prepend with ","'
)
parser.add_argument(
'-s',
'--substitute',
metavar='substitute',
type=str,
nargs="+",
required=False,
default=[],
help=
'Substitutes matched attribute:key values for desired value. Syntax "attribute:old_value|new_value"'
)
parser.add_argument(
'-f',
'--filter',
metavar='filter',
type=str,
nargs="+",
required=False,
default=[],
help='ObjectClass filter which will be removed from the user')
parser.add_argument(
'-a',
'--attributes',
metavar='attributes',
type=str,
nargs="+",
required=False,
default=[],
help='Attributes filter which will be removed from the user')
parser.add_argument(
'-n',
'--new',
metavar='new',
type=str,
nargs="+",
required=False,
default=[],
help=
'Attributes which needs to be added, appends to existings lists. Syntax "attribute:value"'
)
parser.add_argument(
'-v',
'--verbosity',
metavar='verbosity',
type=int,
required=False,
default=1,
help='Configure the verbosity level, 0 is nothing, 10 is a lot')
parser.add_argument(
'-w',
'--watch',
metavar='watch',
type=str,
nargs="+",
required=False,
default=[],
help=
'Watch attributes with values matching regexp. Prepending the expression with \'^\' checks if the string does not match.\nUsage: \nuserClass:[Tt]roll\n^userPassword:{CRYPT}'
)
parser.add_argument('-ww',
metavar="watchdog_strict",
type=bool,
required=False,
default=None,
help="Delete after watchdog match? True/False")
parser.add_argument(
'-l',
'--list',
metavar='list',
type=str,
nargs="+",
required=False,
default=[],
help=
'List of allowed attributes. Other attributes will be therefore removed'
)
parser.add_argument('-z',
'--zero',
metavar='zero',
type=bool,
required=False,
default=False,
help='Remove object with zero attributes?')
args = parser.parse_args()
input_file = open(args.input, "r")
if args.output is not None:
OUTPUT_FILE = LDIFWriter(open(args.output, "w"))
else:
OUTPUT_FILE = LDIFWriter(sys.stdout)
VERBOSITY = args.verbosity
RESTRICT = args.restrict
FILTERED_CLASSES = [x.encode() for x in args.filter]
FILTERED_ATTRIBUTES = args.attributes
WATCHER = []
WATCHDOG_STRICT = args.ww
for watcher in args.watch:
try:
if watcher.startswith("^"):
reverse = True
watcher = watcher[1:]
else:
reverse = False
attribute = watcher.split(":")[0]
regexp = ":".join(watcher.split(":")[1:])
WATCHER.append(
(reverse, attribute, re.compile("^" + regexp + "$")))
except KeyError:
log(
"Watcher string not matching our expectations! Check the help -h!",
0)
SUBSTITUTE = []
for substitute in args.substitute:
try:
attribute = substitute.split(":")[0]
old_value, new_value = ":".join(
substitute.split(":")[1:]).split("|")
SUBSTITUTE.append((attribute, old_value, new_value))
except KeyError:
log(
"Substitute string not matching expectations! Check the help -h",
0)
NEW_ATTRIBUTES = []
try:
NEW_ATTRIBUTES = [item.split(":", 1) for item in args.new]
except KeyError:
log("Attribute creation error! Check the help -h", 0)
try:
ATTRIBUTES_LIST = args.list
except KeyError:
log("Error when parsing allowed attributes list, check your syntax")
REMOVE_WHEN_ZERO_ATTRS = True
record_list = LDIFRecordList(input_file, max_entries=0)
record_list.parse()
pretty_print(record_list.all_records)
print >> ofh, "%s: %s" % (key, value)
else:
print >> ofh, "%s: " % key
ofh.close()
mtime_before = os.stat(filename)[stat.ST_MTIME]
#print "mtime before:", mtime_before
editfile.edit_file(filename)
mtime_after = os.stat(filename)[stat.ST_MTIME]
#print "mtime after:", mtime_after
# Parse file
ifh = open(filename, "r")
ldif_parser = LDIFRecordList(ifh)
ldif_parser.parse()
ifh.close()
record = ldif_parser.all_records[0]
#print record
mod = modlist.addModlist(record[1])
#print mod
# Strip off stars
clean_mod = []
for pair in mod:
name, values = pair
if name[0] == "*":
name = name[1:]
clean_mod.append((name, values))
