def checkForJob(self,workDir):
jobFile = os.path.join(workDir,'sentinel')
if os.path.exists(jobFile):
partId = int(os.path.basename(workDir))
mc = ManagementContainer.getInstance()
pm = mc.getPartitionManager()
partition = pm.getPartition(partId)
log('Found a purge job for partition',partId)
if self.processJob(partition,workDir,jobFile):
log('Purge succeeded. Clearing jobs directory (',workDir,')to signal dispatcher that a partition is completed')
self.clearJobDirectory(workDir)
pm.refreshFreeSpaceInfo(Collections.singletonList(partition))
else:
log('Purge failed. Not clearing job directory(',workDir,'). Will retry with current batch later')
count = 1
if partId in self.failuresByPartId:
count = self.failuresByPartId[partId]
count = count + 1
if count >= self.maxFailures:
log('Purge: failed purge',count,'times for partition',partId,'. Clearing job')
self.clearJobDirectory(workDir)
count = 0
self.failuresByPartId[partId] = count
return True
return False
def getPreselectionIDPParams(self):
param = { "saml" : None, "social": None }
acrs = [self.getAcrFor(True), self.getAcrFor(False)]
custScriptService = CdiUtil.bean(CustomScriptService)
scriptsList = custScriptService.findCustomScripts(Collections.singletonList(CustomScriptType.PERSON_AUTHENTICATION), "oxConfigurationProperty", "displayName", "gluuStatus")
for customScript in scriptsList:
if customScript.isEnabled() and customScript.getName() in acrs:
for prop in customScript.getConfigurationProperties():
if prop.getValue1() == "authz_req_param_provider" and StringHelper.isNotEmpty(prop.getValue2()):
param["saml" if customScript.getName() == "passport_saml" else "social"] = prop.getValue2()
break
if param["saml"] != None:
print "Casa. getPreselectionIDPParams. Found oxAuth cust param for SAML IDPs authz requests '%s'" % param["saml"]
else:
print "Casa. getPreselectionIDPParams. oxAuth cust param for SAML IDPs authz requests not found. IDPs won't be available"
if param["social"] != None:
print "Casa. getPreselectionIDPParams. Found oxAuth cust param for OAuth/OIDC providers' authz requests '%s'" % param["social"]
else:
print "Casa. getPreselectionIDPParams. oxAuth cust param for for OAuth/OIDC providers' authz requests not found. OPs won't be available"
return param
def test_ops_on_same_key(self):
"""
1. Set key
2. Delete a key
3. Set the same key
4. Validate the rev_id for the key is maintained
Ref: MB-48179
"""
if self.durability_level in ["", Bucket.DurabilityLevel.NONE]:
self.fail("Test supported only for sync_write scenarios")
crud_pattern = self.input.param("crud_pattern", "async:sync:async")
crud_pattern = crud_pattern.split(":")
rev_ids = dict()
client = SDKClient([self.cluster.master], self.cluster.buckets[0])
# Async create of keys
for i in range(self.num_items):
key = self.key + str(i)
durability = ""
if crud_pattern[0] == "sync":
durability = self.durability_level
client.crud(DocLoading.Bucket.DocOps.CREATE,
key, {},
durability=durability)
# Sync delete of keys
for i in range(self.num_items):
key = self.key + str(i)
durability = ""
if crud_pattern[1] == "sync":
durability = self.durability_level
client.crud(DocLoading.Bucket.DocOps.DELETE,
key,
durability=durability)
# Async create of keys
for i in range(self.num_items):
key = self.key + str(i)
durability = ""
if crud_pattern[2] == "sync":
durability = self.durability_level
client.crud(DocLoading.Bucket.DocOps.CREATE,
key, {},
durability=durability)
result = client.collection.lookupIn(
key,
Collections.singletonList(
LookupInSpec.get(LookupInMacro.REV_ID).xattr()))
rev_ids[key] = int(result.contentAs(0, String))
client.close()
# Rev_id validation
for i in range(self.num_items):
key = self.key + str(i)
if rev_ids[key] != 3:
self.fail("Rev id mismatch for key '%s'. RevId: %s" %
(key, rev_ids[key]))
def init(self, configurationAttributes):
print "Casa. init called"
self.authenticators = {}
self.configFileLocation = "/etc/gluu/conf/casa.json"
self.uid_attr = self.getLocalPrimaryKey()
custScriptService = CdiUtil.bean(CustomScriptService)
self.scriptsList = custScriptService.findCustomScripts(
Collections.singletonList(CustomScriptType.PERSON_AUTHENTICATION),
"oxConfigurationProperty", "displayName", "oxEnabled", "oxLevel")
dynamicMethods = self.computeMethods(self.scriptsList)
if len(dynamicMethods) > 0:
print "Casa. init. Loading scripts for dynamic modules: %s" % dynamicMethods
for acr in dynamicMethods:
moduleName = self.modulePrefix + acr
try:
external = __import__(moduleName, globals(), locals(),
["PersonAuthentication"], -1)
module = external.PersonAuthentication(
self.currentTimeMillis)
print "Casa. init. Got dynamic module for acr %s" % acr
configAttrs = self.getConfigurationAttributes(
acr, self.scriptsList)
if acr == self.ACR_U2F:
u2f_application_id = configurationAttributes.get(
"u2f_app_id").getValue2()
configAttrs.put(
"u2f_application_id",
SimpleCustomProperty("u2f_application_id",
u2f_application_id))
elif acr == self.ACR_SG:
application_id = configurationAttributes.get(
"supergluu_app_id").getValue2()
configAttrs.put(
"application_id",
SimpleCustomProperty("application_id",
application_id))
if module.init(configAttrs):
module.configAttrs = configAttrs
self.authenticators[acr] = module
else:
print "Casa. init. Call to init in module '%s' returned False" % moduleName
except:
print "Casa. init. Failed to load module %s" % moduleName
print "Exception: ", sys.exc_info()[1]
mobile_methods = configurationAttributes.get("mobile_methods")
self.mobile_methods = [] if mobile_methods == None else StringHelper.split(
mobile_methods.getValue2(), ",")
print "Casa. init. Initialized successfully"
return True
def refeedMessage(mc,clusLocId,partId,messageId,customerId,senderId,recipIds):
mm = mc.getMessageMatchStoreManager()
rmmd = RemoteMatchDescriptor(customerId)
rmmd.setPartitionID(int(partId))
rmmd.setStorageID(long(messageId))
rmmd.setSenderId(int(senderId))
rmmd.setRecipientIds(recipIds)
msgData = Collections.singletonList(rmmd)
mm.addFastUpdateReference(msgData,int(clusLocId),1)
def setupExportCustomer(numMessages, islandId, cloudDomain, stage1Mta):
print time.asctime(), 'Start setting '
userMsgs = [0] * len(users)
mc = ManagementContainer.getInstance()
custId = setupCustomer(mc, str(islandId),domain=domainName, name=custname, isCloud=True,userAccounts=users, recvDate="now")
print time.asctime(), 'customerID', custId
propagateMtaConfig()
custList = mc.getCustomerManager().findCustomers([SearchConstraint(ICustomerManager.PROP_NAME, SearchConstraintOperator.CONSTRAINT_EQUALS, custname)])
cust = custList[0]
office365Guid = cust.getGuids(CloudService.OFFICE365)[0].getGuid()
# stage cloud messages
for i in range(numMessages):
if (i % 5) == 0:
sendJournalMessage(office365Guid, users[0], ["invaliduser"], None, None, domainName, cloudDomain, stage1Mta)
userMsgs[0] += 1
elif (i % 4) == 0:
sendJournalMessage(office365Guid, "invaliduser", [users[1]], None, None, domainName, cloudDomain, stage1Mta)
userMsgs[1] += 1
elif (i % 3) == 0:
sendJournalMessage(office365Guid, "invaliduser", None, [users[2]], None, domainName, cloudDomain, stage1Mta)
userMsgs[2] += 1
elif (i % 2) == 0:
sendJournalMessage(office365Guid, "invaliduser", None, None, [users[3]], domainName, cloudDomain, stage1Mta)
userMsgs[3] += 1
else :
sendJournalMessage(office365Guid, users[0], [users[1]], [users[2]], [users[3]], domainName, cloudDomain, stage1Mta)
for j in range(len(users)):
userMsgs[j] += 1
sleep(1)
# create reviewer group
print time.asctime(), 'creating reviewer group...'
allUsers = InternalUserSets.getAllUsersSet(cust.getCustID())
mc.getUserManager().saveUserSet(allUsers)
reviewer = mc.getUserManager().findUserForEmail(users[0] + '@' + domainName)
reviewerSet = SavedUserSet(cust.getCustID())
reviewerSet.addUsers(Collections.singletonList(SearchConstraint(IUserManager.PROP_USERID, SearchConstraintOperator.CONSTRAINT_EQUALS, reviewer.getUserID())))
mc.getUserManager().saveUserSet(reviewerSet)
mc.getReviewerGroupManager().createReviewerGroup(cust.getCustID(), REVIEWER_GROUP_NAME, reviewerSet, allUsers, None)
sys.exit(0)
def set_azure_container_permission(self, container_public_access=False):
if container_public_access:
identifier = BlobSignedIdentifier().setId(
self.blob_container_client.getBlobContainerName(
)).setAccessPolicy(BlobAccessPolicy().setStartsOn(
OffsetDateTime.now()).setExpiresOn(
OffsetDateTime.now().plusMinutes(1)).setPermissions(
"racwdl"))
self.blob_container_client.setAccessPolicy(
PublicAccessType.CONTAINER,
Collections.singletonList(identifier))
else:
self.blob_container_client.setAccessPolicy(
PublicAccessType.CONTAINER, None)
def init(self, configurationAttributes):
print "Casa. init called"
self.authenticators = {}
self.configFileLocation = "/etc/gluu/conf/casa.json"
self.uid_attr = self.getLocalPrimaryKey()
custScriptService = CdiUtil.bean(CustomScriptService)
self.scriptsList = custScriptService.findCustomScripts(Collections.singletonList(CustomScriptType.PERSON_AUTHENTICATION), "oxConfigurationProperty", "displayName", "gluuStatus", "oxLevel")
dynamicMethods = self.computeMethods(self.scriptsList)
if len(dynamicMethods) > 0:
print "Casa. init. Loading scripts for dynamic modules: %s" % dynamicMethods
for acr in dynamicMethods:
moduleName = self.modulePrefix + acr
try:
external = __import__(moduleName, globals(), locals(), ["PersonAuthentication"], -1)
module = external.PersonAuthentication(self.currentTimeMillis)
print "Casa. init. Got dynamic module for acr %s" % acr
configAttrs = self.getConfigurationAttributes(acr, self.scriptsList)
if acr == self.ACR_U2F:
u2f_application_id = configurationAttributes.get("u2f_app_id").getValue2()
configAttrs.put("u2f_application_id", SimpleCustomProperty("u2f_application_id", u2f_application_id))
elif acr == self.ACR_SG:
client_redirect_uri = configurationAttributes.get("supergluu_app_id").getValue2()
configAttrs.put("client_redirect_uri", SimpleCustomProperty("client_redirect_uri", client_redirect_uri))
if module.init(configAttrs):
module.configAttrs = configAttrs
self.authenticators[acr] = module
else:
print "Casa. init. Call to init in module '%s' returned False" % moduleName
except:
print "Casa. init. Failed to load module %s" % moduleName
print "Exception: ", sys.exc_info()[1]
mobile_methods = configurationAttributes.get("mobile_methods")
self.mobile_methods = [] if mobile_methods == None else StringHelper.split(mobile_methods.getValue2(), ",")
print "Casa. init. Initialized successfully"
return True
def _create_scene(self, config):
Options.v().set_process_dir(
Collections.singletonList(config["input_file"]))
if config["input_format"] == "apk":
Options.v().set_android_jars(config["android_sdk"])
Options.v().set_process_multiple_dex(True)
Options.v().set_src_prec(Options.src_prec_apk)
elif config["input_format"] == "jar":
Options.v().set_soot_classpath(config["soot_classpath"])
else:
raise Exception("invalid input type")
if config["ir_format"] == "jimple":
Options.v().set_output_format(Options.output_format_jimple)
elif config["ir_format"] == "shimple":
Options.v().set_output_format(Options.output_format_shimple)
else:
raise Exception("invalid ir format")
Options.v().set_allow_phantom_refs(True)
# this options may or may not work
Options.v().setPhaseOption("cg", "all-reachable:true")
Options.v().setPhaseOption("jb.dae", "enabled:false")
Options.v().setPhaseOption("jb.uce", "enabled:false")
Options.v().setPhaseOption("jj.dae", "enabled:false")
Options.v().setPhaseOption("jj.uce", "enabled:false")
# this avoids an exception in some apks
Options.v().set_wrong_staticness(Options.wrong_staticness_ignore)
Scene.v().loadNecessaryClasses()
PackManager.v().runPacks()
l.debug("Soot is done!")
self.scene = Scene.v()
self.raw_classes = self.scene.getClasses()
self._init_class_hierarchy()
l.debug("Soot init is done!")
def getFunctionAddedChildren(self, analyticFunctionOrdering):
""" generated source for method getFunctionAddedChildren """
# We can't just add those functions that
# are "ready" to be added. We should be adding all those variables
# "leading up to" the functions and then applying the functions.
# We can even take this one step further by only adding one child
# per remaining constant function; we choose as our function output the
# variable that is a candidate for functionhood that has the
# largest domain, or one that is tied for largest.
# New criterion: Must also NOT be in preassignment.
children = ArrayList()
# It would be really nice here to just analytically choose
# the set of functions we're going to use.
# Here's one approach for doing that:
# For each variable, get a list of the functions that could
# potentially produce it.
# For all the variables with no functions, add them.
# Then repeatedly find the function with the fewest
# number of additional variables (hopefully 0!) needed to
# specify it and add it as a function.
# The goal here is not to be optimal, but to be efficient!
# Certain games (e.g. Pentago) break the old complete search method!
# TODO: Eventual possible optimization here:
# If something is dependent on a connected component that it is
# not part of, wait until the connected component is resolved
# (or something like that...)
if analyticFunctionOrdering and len(self.functionalSentencesInfo) > 8:
# For each variable, a list of functions
# (refer to functions by their indices)
# and the set of outstanding vars they depend on...
# We start by adding to the varOrdering the vars not produced by functions
# First, we have to find them
while i < len(self.functionalSentencesInfo):
for producibleVar in producibleVars:
if not functionsProducingVars.containsKey(producibleVar):
functionsProducingVars.put(producibleVar, HashSet())
functionsProducingVars.get(producibleVar).add(i)
i += 1
# Non-producible vars get iterated over before we start
# deciding which functions to add
for var in varsToAssign:
if not self.varOrdering.contains(var):
if not functionsProducingVars.containsKey(var):
# Add var to the ordering
self.varOrdering.add(var)
self.functionalConjunctIndices.add(-1)
self.varSources.add(-1)
# Map is from potential set of dependencies to function indices
# Create this map...
while i < len(self.functionalSentencesInfo):
# Variables already in varOrdering don't go in dependents list
producibleVars.removeAll(self.varOrdering)
allVars.removeAll(self.varOrdering)
for producibleVar in producibleVars:
dependencies.addAll(allVars)
dependencies.remove(producibleVar)
if not functionsHavingDependencies.containsKey(dependencies):
functionsHavingDependencies.put(dependencies, HashSet())
functionsHavingDependencies.get(dependencies).add(i)
i += 1
# Now, we can keep creating functions to generate the remaining variables
while len(self.varOrdering) < len(self.varsToAssign):
if functionsHavingDependencies.isEmpty():
raise RuntimeException("We should not run out of functions we could use")
# Find the smallest set of dependencies
if functionsHavingDependencies.containsKey(Collections.emptySet()):
dependencySetToUse = Collections.emptySet()
else:
for dependencySet in functionsHavingDependencies.keySet():
if len(dependencySet) < smallestSize:
smallestSize = len(dependencySet)
dependencySetToUse = dependencySet
# See if any of the functions are applicable
for function_ in functions:
producibleVars.removeAll(dependencySetToUse)
producibleVars.removeAll(self.varOrdering)
if not producibleVars.isEmpty():
functionToUse = function_
varProduced = producibleVars.iterator().next()
break
if functionToUse == -1:
# None of these functions were actually useful now?
# Dump the dependency set
functionsHavingDependencies.remove(dependencySetToUse)
else:
# Apply the function
# 1) Add the remaining dependencies as iterated variables
for var in dependencySetToUse:
self.varOrdering.add(var)
self.functionalConjunctIndices.add(-1)
self.varSources.add(-1)
# 2) Add the function's produced variable (varProduced)
self.varOrdering.add(varProduced)
self.functionalConjunctIndices.add(functionToUse)
self.varSources.add(-1)
# 3) Remove all vars added this way from all dependency sets
addedVars.addAll(dependencySetToUse)
addedVars.add(varProduced)
# Tricky, because we have to merge sets
# Easier to use a new map
for entry in functionsHavingDependencies.entrySet():
newKey.removeAll(addedVars)
if not newFunctionsHavingDependencies.containsKey(newKey):
newFunctionsHavingDependencies.put(newKey, HashSet())
newFunctionsHavingDependencies.get(newKey).addAll(entry.getValue())
functionsHavingDependencies = newFunctionsHavingDependencies
# 4) Remove this function from the lists?
for functionSet in functionsHavingDependencies.values():
functionSet.remove(functionToUse)
# Now we need to actually return the ordering in a list
# Here's the quick way to do that...
# (since we've added all the new stuff to ourself already)
return Collections.singletonList(IterationOrderCandidate(self))
else:
# Let's try a new technique for restricting the space of possibilities...
# We already have an ordering on the functions
# Let's try to constrain things to that order
# Namely, if i<j and constant form j is already used as a function,
# we cannot use constant form i UNLESS constant form j supplies
# as its variable something used by constant form i.
# We might also try requiring that c.f. i NOT provide a variable
# used by c.f. j, though there may be multiple possibilities as
# to what it could provide.
if not self.functionalConjunctIndices.isEmpty():
lastFunctionUsedIndex = Collections.max(self.functionalConjunctIndices)
while i < len(self.functionalConjunctIndices):
if self.functionalConjunctIndices.get(i) != -1:
varsProducedByFunctions.add(self.varOrdering.get(i))
i += 1
while i < len(self.functionalSentencesInfo):
if i < lastFunctionUsedIndex:
# We need to figure out whether i could use any of the
# vars we're producing with functions
# TODO: Try this with a finer grain
# i.e., see if i needs a var from a function that is after
# it, not one that might be before it
if Collections.disjoint(varsInSentence, varsProducedByFunctions):
continue
# What is the best variable to grab from this form, if there are any?
if bestVariable == None:
continue
children.add(newCandidate)
i += 1
# If there are no more functions to add, add the completed version
if children.isEmpty():
children.add(IterationOrderCandidate(self))
return children
