def test_compoundWithMultipleExplicitRecordTypes(self):
expression = CompoundExpression([
CompoundExpression([
MatchExpression(self.service.fieldName.fullNames,
u"be",
matchType=MatchType.contains),
MatchExpression(self.service.fieldName.emailAddresses,
u"be",
matchType=MatchType.startsWith),
], Operand.OR),
CompoundExpression([
MatchExpression(self.service.fieldName.fullNames,
u"test",
matchType=MatchType.contains),
MatchExpression(self.service.fieldName.emailAddresses,
u"test",
matchType=MatchType.startsWith),
], Operand.OR),
], Operand.AND)
records = yield self.service.recordsFromExpression(
expression,
recordTypes=[
self.service.recordType.user, self.service.recordType.group
])
# We should get back users and groups:
self.verifyResults(records, [
"odtestbetty", "odtestalbert", "anotherodtestalbert",
"odtestgroupbetty", "odtestgroupalbert"
], ["odtestamanda", "odtestbill", "odtestgroupa", "odtestgroupb"])
def test_compoundExpressionAsJSON_expressions(
self, serialize=compoundExpressionAsJSON):
"""
L{compoundExpressionAsJSON} with 0, 1 and 2 sub-expressions.
"""
for uids in (
(),
(u"UID1", ),
(u"UID1", u"UID2"),
):
subExpressions = [
MatchExpression(FieldName.uid, uid) for uid in uids
]
subExpressionsText = [
matchExpressionAsJSON(e) for e in subExpressions
]
expression = CompoundExpression(subExpressions, Operand.AND)
json = compoundExpressionAsJSON(expression)
expected = {
u"type": u"CompoundExpression",
u"expressions": subExpressionsText,
u"operand": u"AND",
}
self.assertEquals(json, expected)
def test_matchExpressionFromJSON_types(self):
"""
L{matchExpressionFromJSON} with various match types.
"""
service = self.service()
uid = u"Some UID"
for matchType, matchText in (
(MatchType.equals, u"equals"),
(MatchType.endsWith, u"endsWith"),
(MatchType.lessThanOrEqualTo, u"lessThanOrEqualTo"),
):
jsonText = (u"""
{{
"type": "MatchExpression",
"field": "uid",
"match": "{matchType}",
"value": "{uid}",
"flags": "{{}}"
}}
""").format(uid=uid, matchType=matchText)
json = from_json_text(jsonText)
expected = MatchExpression(FieldName.uid, uid, matchType=matchType)
expression = matchExpressionFromJSON(service, json)
self.assertEquals(expression, expected)
def test_matchExpressionFromJSON_flags(self):
"""
L{matchExpressionFromJSON} with various flags.
"""
service = self.service()
uid = u"Some UID"
for flags, flagsText, in (
(MatchFlags.none, u"{}"),
(MatchFlags.NOT, u"NOT"),
(MatchFlags.caseInsensitive, u"caseInsensitive"),
(MatchFlags.NOT | MatchFlags.caseInsensitive,
u"{NOT,caseInsensitive}"),
):
jsonText = (u"""
{{
"type": "MatchExpression",
"field": "uid",
"match": "equals",
"value": "{uid}",
"flags": "{flagsText}"
}}
""").format(uid=uid, flagsText=flagsText)
json = from_json_text(jsonText)
expected = MatchExpression(FieldName.uid, uid, flags=flags)
expression = matchExpressionFromJSON(service, json)
self.assertEquals(expression, expected)
def recordsMatchingTokens(self,
tokens,
context=None,
limitResults=50,
timeoutSeconds=10):
fields = [
("fullNames", MatchType.contains),
("emailAddresses", MatchType.startsWith),
]
outer = []
for token in tokens:
inner = []
for name, matchType in fields:
inner.append(
MatchExpression(self.fieldName.lookupByName(name), token,
matchType, MatchFlags.caseInsensitive))
outer.append(CompoundExpression(inner, Operand.OR))
if len(outer) == 1:
expression = outer[0]
else:
expression = CompoundExpression(outer, Operand.AND)
if context is not None:
recordTypes = self.recordTypesForSearchContext(context)
else:
recordTypes = None
results = yield self.recordsFromExpression(expression,
recordTypes=recordTypes)
log.debug("Tokens ({t}) matched {n} records", t=tokens, n=len(results))
returnValue(results)
def recordsMatchingFields(self,
fields,
operand=Operand.OR,
recordType=None):
"""
@param fields: a iterable of tuples, each tuple consisting of:
directory field name (C{unicode})
search term (C{unicode})
match flags (L{twext.who.expression.MatchFlags})
match type (L{twext.who.expression.MatchType})
"""
subExpressions = []
for fieldName, searchTerm, matchFlags, matchType in fields:
try:
field = self.fieldName.lookupByName(fieldName)
except ValueError:
log.debug("Unsupported field name: {fieldName}",
fieldName=fieldName)
continue
subExpression = MatchExpression(field, searchTerm, matchType,
matchFlags)
subExpressions.append(subExpression)
if len(subExpressions) == 1:
expression = subExpressions[0]
else:
expression = CompoundExpression(subExpressions, operand)
if recordType is not None:
recordTypes = [recordType]
else:
recordTypes = None
return self.recordsFromExpression(expression, recordTypes=recordTypes)
def test_matchExpressionAsJSON_types(self,
serialize=matchExpressionAsJSON):
"""
L{matchExpressionAsJSON} with various match types.
"""
uid = u"Some UID"
for matchType, matchText in (
(MatchType.equals, u"equals"),
(MatchType.endsWith, u"endsWith"),
(MatchType.lessThanOrEqualTo, u"lessThanOrEqualTo"),
):
expression = MatchExpression(FieldName.uid,
uid,
matchType=matchType)
json = serialize(expression)
expected = {
u"type": u"MatchExpression",
u"field": u"uid",
u"match": matchText,
u"value": uid,
u"flags": u"{}",
}
self.assertEquals(json, expected)
def test_matchExpressionAsJSON_flags(self,
serialize=matchExpressionAsJSON):
"""
L{matchExpressionAsJSON} with various flags.
"""
uid = u"Some UID"
for flags, flagsText, in (
(MatchFlags.none, u"{}"),
(MatchFlags.NOT, u"NOT"),
(MatchFlags.caseInsensitive, u"caseInsensitive"),
(MatchFlags.NOT | MatchFlags.caseInsensitive,
u"{NOT,caseInsensitive}"),
):
expression = MatchExpression(FieldName.uid, uid, flags=flags)
json = serialize(expression)
expected = {
u"type": u"MatchExpression",
u"field": u"uid",
u"match": u"equals",
u"value": uid,
u"flags": flagsText,
}
self.assertEquals(json, expected)
def test_compoundWithEmbeddedSingleRecordType(self):
expression = CompoundExpression(
[
CompoundExpression(
[
CompoundExpression(
[
MatchExpression(
self.service.fieldName.fullNames, u"be",
matchType=MatchType.contains
),
MatchExpression(
self.service.fieldName.emailAddresses, u"be",
matchType=MatchType.startsWith
),
],
Operand.OR
),
CompoundExpression(
[
MatchExpression(
self.service.fieldName.fullNames, u"test",
matchType=MatchType.contains
),
MatchExpression(
self.service.fieldName.emailAddresses, u"test",
matchType=MatchType.startsWith
),
],
Operand.OR
),
],
Operand.AND
),
MatchExpression(
self.service.fieldName.recordType, self.service.recordType.user,
),
],
Operand.AND
)
try:
yield self.service.recordsFromExpression(expression)
except QueryNotSupportedError:
pass
else:
self.fail("This should have raised")
def lookup(shortNames):
service = DirectoryService()
print(
"Service = {service}\n"
"Session = {service.session}\n"
"Node = {service.node}\n"
# "Local node = {service.localNode}\n"
.format(service=service)
)
print("-" * 80)
for shortName in shortNames:
print("Looking up short name: {0}".format(shortName))
record = yield service.recordWithShortName(service.recordType.user, shortName)
if record:
print(record.description())
continue
matchExpression = MatchExpression(
service.fieldName.shortNames, shortName,
matchType=MatchType.equals,
)
records = yield service.recordsFromExpression(matchExpression)
for record in records:
print(record.description())
compoundExpression = CompoundExpression(
[
MatchExpression(
service.fieldName.shortNames, shortName,
matchType=MatchType.contains
),
MatchExpression(
service.fieldName.emailAddresses, shortName,
matchType=MatchType.contains
),
],
Operand.OR
)
records = yield service.recordsFromExpression(compoundExpression)
for record in records:
print(record.description())
def test_shortNameStartsWith(self):
records = yield self.service.recordsFromExpression(
MatchExpression(self.service.fieldName.shortNames,
u"odtest",
matchType=MatchType.startsWith))
self.verifyResults(records,
chain(LOCAL_SHORTNAMES, NETWORK_SHORTNAMES),
["anotherodtestamanda", "anotherodtestalbert"])
def recordsMatchingTokens(self, tokens, context=None, limitResults=None,
timeoutSeconds=None):
fields = [
("fullNames", MatchType.contains),
("emailAddresses", MatchType.startsWith),
]
outer = []
for token in tokens:
if token:
token = token.strip()
inner = []
for name, matchType in fields:
inner.append(
MatchExpression(
self.fieldName.lookupByName(name),
token,
matchType,
MatchFlags.caseInsensitive
)
)
outer.append(
CompoundExpression(
inner,
Operand.OR
)
)
if len(outer) == 1:
expression = outer[0]
else:
expression = CompoundExpression(outer, Operand.AND)
if context is not None:
recordTypes = self.recordTypesForSearchContext(context)
else:
recordTypes = None
# If a filter has been set, pass self.recordsFromExpression to it for
# result processing
if getattr(self, "_resultFilter", None):
results = yield self._resultFilter(
self.recordsFromExpression, tokens, expression,
recordTypes=recordTypes, limitResults=limitResults,
timeoutSeconds=timeoutSeconds
)
else:
results = yield self.recordsFromExpression(
expression, recordTypes=recordTypes, limitResults=limitResults,
timeoutSeconds=timeoutSeconds
)
log.debug(
"Tokens ({t}) matched {n} records",
t=tokens, n=len(results)
)
returnValue(results)
def test_recordsFromMatchExpressionNonUnicode(self):
expression = MatchExpression(
FieldName.guid,
UUID("6423F94A-6B76-4A3A-815B-D52CFD77935D"),
MatchType.equals,
MatchFlags.caseInsensitive
)
records = yield self.directory.recordsFromExpression(expression)
self.assertEquals(len(records), 1)
def test_recordsFromMatchExpression(self):
expression = MatchExpression(
FieldName.uid,
u"6423F94A-6B76-4A3A-815B-D52CFD77935D",
MatchType.equals,
MatchFlags.none
)
records = yield self.directory.recordsFromExpression(expression)
self.assertEquals(len(records), 1)
def matchExpression(fieldName, matchString, matchType, matchFlags):
# special case recordType field
if fieldName == FieldName.recordType:
# change kind to record type
matchValue = vCardKindToRecordTypeMap.get(matchString.lower())
if matchValue is None:
matchValue = NamedConstant()
matchValue.description = u""
# change types and flags
matchFlags &= ~MatchFlags.caseInsensitive
matchType = MatchType.equals
else:
matchValue = matchString.decode("utf-8")
return MatchExpression(fieldName, matchValue, matchType, matchFlags)
def test_expressionAsJSONText(self):
"""
L{expressionAsJSON} with compound expression.
"""
uid = u"Some UID"
expression = MatchExpression(FieldName.uid, uid)
jsonText = expressionAsJSONText(expression)
expected = {
"type": "MatchExpression",
"field": "uid",
"value": uid,
"match": "equals",
"flags": "{}",
}
self.assertEquals(json.loads(jsonText), expected)
def test_matchExpressionAsJSON_basic(self,
serialize=matchExpressionAsJSON):
"""
L{matchExpressionAsJSON} with default matching and flags.
"""
uid = u"Some UID"
expression = MatchExpression(FieldName.uid, uid)
json = serialize(expression)
expected = {
u"type": u"MatchExpression",
u"field": u"uid",
u"match": u"equals",
u"value": uid,
u"flags": u"{}",
}
self.assertEquals(json, expected)
def matchExpressionFromJSON(service, json):
try:
jsonField = json["field"]
jsonValue = json["value"]
except KeyError as e:
raise ValueError("JSON match expression must have {!r} key.".format(
e[0]))
fieldName = service.fieldName.lookupByName(jsonField)
fieldValue = unicode(jsonValue)
matchType = matchTypeFromJSON(json.get("match", "equals"))
flags = matchFlagsFromJSON(json.get("flags", "{}"))
return MatchExpression(
fieldName,
fieldValue,
matchType=matchType,
flags=flags,
)
def test_matchExpressionFromJSON_basic(self):
"""
L{test_matchExpressionFromJSON_basic} with default matching and flags.
"""
service = self.service()
uid = u"Some UID"
jsonText = (u"""
{{
"type": "MatchExpression",
"field": "uid",
"value": "{uid}"
}}
""").format(uid=uid)
json = from_json_text(jsonText)
expected = MatchExpression(FieldName.uid, uid)
expression = matchExpressionFromJSON(service, json)
self.assertEquals(expression, expected)
def test_expressionAsJSONText(self):
"""
L{expressionAsJSON} with compound expression.
"""
uid = u"Some UID"
expression = MatchExpression(FieldName.uid, uid)
jsonText = expressionAsJSONText(expression)
expected = (u"""
{{
"field": "uid",
"flags": "{{}}",
"match": "equals",
"value": "{uid}",
"type": "MatchExpression"
}}
""").replace(" ", "").replace("\n", "").format(uid=uid)
self.assertEquals(jsonText, expected)
def copyAssignments(assignments, pod, directory, store):
"""
Go through the list of assignments from the old db, and selectively copy them
into the new store.
@param assignments: the assignments from the old db
@type assignments: a list of (delegator group, delegate) tuples
@param pod: the name of the pod you want to migrate assignments for; assignments
for delegators who don't reside on this pod will be ignored. Set this
to None to copy all assignments.
@param directory: the directory service
@param store: the store
"""
delegatorsMissingPodInfo = set()
numCopied = 0
numOtherPod = 0
numDirectoryBased = 0
numExamined = 0
# If locations and resources' delegate assignments come from the directory,
# then we're only interested in copying assignments where the delegator is a
# user.
if config.GroupCaching.Enabled and config.GroupCaching.UseDirectoryBasedDelegates:
delegatorRecordTypes = (RecordType.user, )
else:
delegatorRecordTypes = None
# When faulting in delegates, only worry about users and groups.
delegateRecordTypes = (RecordType.user, RecordType.group)
total = len(assignments)
for groupname, delegateUID in assignments:
numExamined += 1
if numExamined % 100 == 0:
print("Processed: {} of {}...".format(numExamined, total))
if "#" in groupname:
delegatorUID, permission = groupname.split("#")
try:
delegatorRecords = yield directory.recordsFromExpression(
MatchExpression(FieldName.uid,
delegatorUID,
matchType=MatchType.equals),
recordTypes=delegatorRecordTypes)
delegatorRecord = uniqueResult(delegatorRecords)
except Exception, e:
print("Failed to look up record for {}: {}".format(
delegatorUID, str(e)))
continue
if delegatorRecord is None:
continue
if config.GroupCaching.Enabled and config.GroupCaching.UseDirectoryBasedDelegates:
if delegatorRecord.recordType != RecordType.user:
print("Skipping non-user")
numDirectoryBased += 1
continue
if pod:
try:
if delegatorRecord.serviceNodeUID != pod:
numOtherPod += 1
continue
except AttributeError:
print("Record missing serviceNodeUID",
delegatorRecord.fullNames)
delegatorsMissingPodInfo.add(delegatorUID)
continue
try:
delegateRecords = yield directory.recordsFromExpression(
MatchExpression(FieldName.uid,
delegateUID,
matchType=MatchType.equals),
recordTypes=delegateRecordTypes)
delegateRecord = uniqueResult(delegateRecords)
except Exception, e:
print("Failed to look up record for {}: {}".format(
delegateUID, str(e)))
continue
if delegateRecord is None:
continue
txn = store.newTransaction(label="DelegatesMigrationService")
yield Delegates.addDelegate(txn, delegatorRecord, delegateRecord,
(permission == "calendar-proxy-write"))
numCopied += 1
yield txn.commit()
def test_recordsFromMatchExpression(self):
expression = MatchExpression(FieldName.uid, testUID, MatchType.equals,
MatchFlags.none)
records = yield self.directory.recordsFromExpression(expression)
self.assertEquals(len(records), 1)
def test_recordsFromMatchExpression(self):
expression = MatchExpression(FieldName.uid, u"wsanchez",
MatchType.equals, MatchFlags.none)
records = yield self.client.recordsFromExpression(expression)
self.assertEquals(len(records), 1)