def getAnObjectStoreFromUser(
prompt, giveUserOptionToReturnTwoObjectStoresIfItIsMigrateType=False):
selectedConfig = userSelectOptionFromList(prompt, configs, "shortName")
f = open(configs[selectedConfig]["full"], "r")
objectStoreConfigDict = json.loads(f.read())
f.close()
if giveUserOptionToReturnTwoObjectStoresIfItIsMigrateType:
if objectStoreConfigDict["Type"] == "Migrating":
objectStoreFromDict = objectStoreConfigDict["From"]
objectStoreToDict = objectStoreConfigDict["To"]
print("Creating FROM object store...")
objectStoreFrom = createObjectStoreInstance(
objectStoreConfigDict["From"], fns, detailLogging=False)
print("Creating TO object store...")
objectStoreTo = createObjectStoreInstance(
objectStoreConfigDict["To"], fns, detailLogging=False)
return objectStoreFrom, objectStoreTo
print("Creating object store...")
objectStore = createObjectStoreInstance(objectStoreConfigDict,
fns,
detailLogging=False)
return objectStore
def test_creationWithMissingBaseLocationFails(self):
ConfigDictI = {"Type": "SimpleFileStore"}
with self.assertRaises(Exception) as context:
undertest.createObjectStoreInstance(
ConfigDictI, self.getObjectStoreExternalFns())
self.checkGotRightExceptionType(context,
undertest.ObjectStoreConfigError)
self.assertEqual(
str(context.exception),
"APIAPP_OBJECTSTORECONFIG SimpleFileStore ERROR - BaseLocation param Missing",
msg="Wrong error message")
def test_simpleCreationBasePAthWithTrailingBackSlash(self):
ConfigDictI = copy.deepcopy(ConfigDict)
ConfigDictI["BaseLocation"] = ConfigDictI["BaseLocation"] + "\\"
with self.assertRaises(Exception) as context:
undertest.createObjectStoreInstance(
ConfigDictI, self.getObjectStoreExternalFns())
self.checkGotRightExceptionType(context,
undertest.ObjectStoreConfigError)
self.assertEqual(
str(context.exception),
"APIAPP_OBJECTSTORECONFIG SimpleFileStore ERROR - BaseLocation should not end with backslash",
msg="Wrong error message")
def test_creationWithInvalidBaseLocationFails(self):
ConfigDictI = {
"Type": "SimpleFileStore",
"BaseLocation": "./tests/TestHelperSuperClass.py"
}
with self.assertRaises(Exception) as context:
undertest.createObjectStoreInstance(
ConfigDictI, self.getObjectStoreExternalFns())
self.checkGotRightExceptionType(context,
undertest.ObjectStoreConfigError)
self.assertEqual(
str(context.exception),
"APIAPP_OBJECTSTORECONFIG SimpleFileStore ERROR - BaseLocation is not a directory",
msg="Wrong error message")
def test_memoryCreation(self):
a = "{\"Type\":\"Memory\"}"
objectStoreConfigDict = json.loads(a)
a = undertest.createObjectStoreInstance(
objectStoreConfigDict, self.getObjectStoreExternalFns())
if not isinstance(a, undertest.ObjectStore_Memory):
self.assertTrue(False, msg='Wrong type of object store created')
def test_remove_objectversionmismatch(self):
objectType = "chartnames"
keyToTest = "usr::linkvisAutoconfigTestUser/:_/untitle dsdsWITHPACES"
someDict = {"d": "d"}
storeConnection = undertest.createObjectStoreInstance(
ConfigDict, self.getObjectStoreExternalFns())
storeConnection.resetDataForTest()
def someFn(connectionContext):
#print(self.jobs[jobGUID]._caculatedDict(self.appObj))
return connectionContext.saveJSONObject(objectType,
keyToTest,
someDict,
objectVersion=None)
newObjectVersion = storeConnection.executeInsideTransaction(someFn)
def removeButWithWrongObjectVersion(connectionContext):
#print(self.jobs[jobGUID]._caculatedDict(self.appObj))
return connectionContext.removeJSONObject(
objectType=objectType,
objectKey=keyToTest,
objectVersion=123,
ignoreMissingObject=False)
with self.assertRaises(Exception) as context:
newObjectVersion = storeConnection.executeInsideTransaction(
removeButWithWrongObjectVersion)
self.checkGotRightExceptionType(context,
undertest.WrongObjectVersionException)
def test_dictWithUnknownTypePassedToCreation(self):
objectStoreConfigDict = {'Type': 'SomeInvalidObjectStoreType'}
with self.assertRaises(Exception) as context:
a = undertest.createObjectStoreInstance(
objectStoreConfigDict, self.getObjectStoreExternalFns())
self.checkGotRightExceptionType(
context, undertest.InvalidObjectStoreConfigUnknownTypeClass)
def test_ObjectRemovalInNewInstance(self):
objectType = "chartnames"
keyToTest = "usr::linkvisAutoconfigTestUser/:_/untitle dsdsWITHPACES"
someDict = {"d": "d"}
storeConnection = undertest.createObjectStoreInstance(
ConfigDict, self.getObjectStoreExternalFns())
storeConnection.resetDataForTest()
def someFn(connectionContext):
#print(self.jobs[jobGUID]._caculatedDict(self.appObj))
newObjectVersion = connectionContext.saveJSONObject(
objectType, keyToTest, someDict, objectVersion=None)
self.assertEqual(newObjectVersion, 1)
storeConnection.executeInsideTransaction(someFn)
def someFn(connectionContext):
#print(self.jobs[jobGUID]._caculatedDict(self.appObj))
obj, objVersion, creationDateTime, lastUpdateDateTime, objKey = connectionContext.getObjectJSON(
objectType, keyToTest)
self.assertEqual(obj, someDict)
return objVersion
objVersion = storeConnection.executeInsideTransaction(someFn)
storeConnection2 = undertest.createObjectStoreInstance(
ConfigDict, self.getObjectStoreExternalFns())
def someFn(connectionContext):
#print(self.jobs[jobGUID]._caculatedDict(self.appObj))
newObjectVersion = connectionContext.removeJSONObject(
objectType=objectType,
objectKey=keyToTest,
objectVersion=objVersion,
ignoreMissingObject=False)
self.assertEqual(newObjectVersion, None)
storeConnection2.executeInsideTransaction(someFn)
def someFn(connectionContext):
#print(self.jobs[jobGUID]._caculatedDict(self.appObj))
obj, objVersion, creationDateTime, lastUpdateDateTime, objKey = connectionContext.getObjectJSON(
objectType, keyToTest)
self.assertEqual(obj, None)
storeConnection2.executeInsideTransaction(someFn)
def test_sqlAlchemyCreation(self):
if SKIPSQLALCHEMYTESTS:
print("Skipping SQLAlchemyTests")
return
a = "{\"Type\":\"SQLAlchemy\", \"connectionString\":\"mysql+pymysql://saas_user_man_user:[email protected]:10103/saas_user_man_rad\"}"
objectStoreConfigDict = json.loads(a)
a = undertest.createObjectStoreInstance(
objectStoreConfigDict, self.getObjectStoreExternalFns())
if not isinstance(a, undertest.ObjectStore_SQLAlchemy):
self.assertTrue(False, msg='Wrong type of object store created')
def pre_setUpHook(self):
if SKIPSQLALCHEMYTESTS:
print("Skipping SQLAlchemyTests")
return
#App object isn't instalised so we need to make an object to reset the data for the test
fns = {'getCurDateTime': appObj.getCurDateTime}
objectStore = createObjectStoreInstance(
json.loads(self._getEnvironment()['APIAPP_OBJECTSTORECONFIG']),
fns)
objectStore.resetDataForTest()
def init(self, env, serverStartTime, testingMode = False, objectStoreTestingPopulationHookFn = None):
try:
self.minutesBeforeMostRecentCompletionStatusBecomesUnknown = 49 * 60
self.curDateTimeOverrideForTesting = None
self.serverStartTime = serverStartTime
if self.jobExecutor is not None:
#for testing we init multiple times. We need to stop the thread running in this case
self.jobExecutor.stopThreadRunning()
if self.jobExecutor.isAlive():
self.jobExecutor.join()
self.jobExecutor = None
super(appObjClass, self).init(env, serverStartTime, testingMode, serverinfoapiprefix='')
resetJobsData(self)
self.userforjobs = readFromEnviroment(env, 'APIAPP_USERFORJOBS', None, None)
self.groupforjobs = readFromEnviroment(env, 'APIAPP_GROUPFORJOBS', None, None)
skipUserCheck = readFromEnviroment(env, 'APIAPP_SKIPUSERCHECK', False, [False, True])
self.jobExecutor = JobExecutorClass(self, skipUserCheck)
#When we are testing we will launch the loop iterations manually
if not testingMode:
self.jobExecutor.start()
objectStoreConfigJSON = readFromEnviroment(env, 'APIAPP_OBJECTSTORECONFIG', '{}', None)
objectStoreConfigDict = None
try:
if objectStoreConfigJSON != '{}':
objectStoreConfigDict = json.loads(objectStoreConfigJSON)
except Exception as err:
print(err) # for the repr
print(str(err)) # for just the message
print(err.args) # the arguments that the exception has been called with.
raise(InvalidObjectStoreConfigInvalidJSONException)
fns = {
'getCurDateTime': self.getCurDateTime
}
self.objectStore = createObjectStoreInstance(objectStoreConfigDict, fns)
if testingMode:
if objectStoreTestingPopulationHookFn is not None:
# Give our tests the chance to inject some base data
objectStoreTestingPopulationHookFn(objectStore = self.objectStore)
appObj.appData['jobsData'].loadFromObjectStore()
except Exception as a:
self.stopThread()
raise a
def test_dockJobBug(self):
objectType = "jobsData"
storeConnection = undertest.createObjectStoreInstance(
ConfigDict, self.getObjectStoreExternalFns())
class jobClass():
objectVersion = None
def _caculatedDict(self, appObj):
return {
"aa": "fff",
"bb": "fff22",
}
jobGUID = 'abc123'
jobs = {}
jobs[jobGUID] = jobClass()
def someFn(connectionContext):
#print(self.jobs[jobGUID]._caculatedDict(self.appObj))
newObjectVersion = connectionContext.saveJSONObject(
objectType,
jobGUID,
jobs[jobGUID]._caculatedDict(appObj=None),
objectVersion=jobs[jobGUID].objectVersion)
jobs[jobGUID].objectVersion = newObjectVersion
storeConnection.executeInsideTransaction(someFn)
def someFn(connectionContext):
paginatedParamValues = {
'offset': 0,
'pagesize': 100000,
'query': None,
'sort': None,
}
loadedData = connectionContext.getPaginatedResult(
objectType,
paginatedParamValues=paginatedParamValues,
outputFN=None)
##print(loadedData)
print("Found " + str(len(loadedData["result"])) +
" jobs in datastore")
storeConnection.executeInsideTransaction(someFn)
def init(self, env, serverStartTime, testingMode = False):
##self.setupLogging() Comment in when debugging
super(appObjClass, self).init(env, serverStartTime, testingMode, serverinfoapiprefix='public/info')
#This app always needs a JWT key
if self.APIAPP_JWTSECRET is None:
print("ERROR - APIAPP_JWTSECRET should always be set")
raise invalidConfigurationException
objectStoreConfigJSON = readFromEnviroment(env, 'APIAPP_OBJECTSTORECONFIG', '{}', None)
objectStoreConfigDict = None
try:
if objectStoreConfigJSON != '{}':
objectStoreConfigDict = json.loads(objectStoreConfigJSON)
except Exception as err:
print(err) # for the repr
print(str(err)) # for just the message
print(err.args) # the arguments that the exception has been called with.
raise(InvalidObjectStoreConfigInvalidJSONException)
self.APIAPP_OBJECTSTOREDETAILLOGGING = readFromEnviroment(
env=env,
envVarName='APIAPP_OBJECTSTOREDETAILLOGGING',
defaultValue='N',
acceptableValues=['Y', 'N'],
nullValueAllowed=True
).strip()
if (self.APIAPP_OBJECTSTOREDETAILLOGGING=='Y'):
print("APIAPP_OBJECTSTOREDETAILLOGGING set to Y - statement logging enabled")
fns = {
'getCurDateTime': self.getCurDateTime
}
self.objectStore = createObjectStoreInstance(
objectStoreConfigDict,
fns,
detailLogging=(self.APIAPP_OBJECTSTOREDETAILLOGGING=='Y')
)
def fileIsObjectStoreConfig(fil):
statinfo = os.stat(fil)
if statinfo.st_size < (1024 * 500): #less than 500 kb in size
print("Checking config for ", fil)
try:
f = open(fil, "r")
objectStoreConfigDict = json.loads(f.read())
f.close()
print(" JSON OK - now creating object")
objectStoreFrom = createObjectStoreInstance(objectStoreConfigDict,
fns,
detailLogging=False)
if objectStoreFrom is None:
return False
except Exception as err:
print(" Objectstore creation errored:")
print(err) # for the repr
print(str(err)) # for just the message
print(err.args
) # the arguments that the exception has been called with.
print("---------")
return False #If something went wrong ignore the file
return True
return False
def test_nonDictPassedToCreation(self):
with self.assertRaises(Exception) as context:
a = undertest.createObjectStoreInstance(
"Not A Dict", self.getObjectStoreExternalFns())
self.checkGotRightExceptionType(
context, undertest.ObjectStoreConfigNotDictObjectExceptionClass)
def init(self, env, serverStartTime, testingMode=False):
##self.setupLogging() Comment in when debugging
self.TicketManager = ticketManager.ticketManagerClass(appObj=self)
self.ApiKeyManager = apiKeyManager.apiKeyManagerClass(appObj=self)
authProviders_resetStaticData()
self.scheduler = BackgroundScheduler(timezone="UTC")
self.defaultUserGUID = str(uuid.uuid4())
if testingMode:
self.defaultUserGUID = conDefaultUserGUID
self.testingDefaultPersonGUID = conTestingDefaultPersonGUID
super(appObjClass, self).init(env,
serverStartTime,
testingMode,
serverinfoapiprefix='public/info')
#This app always needs a JWT key
if self.APIAPP_JWTSECRET is None:
print("ERROR - APIAPP_JWTSECRET should always be set")
raise invalidConfigurationException
self.APIAPP_MASTERPASSWORDFORPASSHASH = readFromEnviroment(
env, 'APIAPP_MASTERPASSWORDFORPASSHASH', None, None).strip()
self.APIAPP_DEFAULTHOMEADMINUSERNAME = readFromEnviroment(
env, 'APIAPP_DEFAULTHOMEADMINUSERNAME', 'Admin', None).strip()
self.APIAPP_DEFAULTHOMEADMINPASSWORD = readFromEnviroment(
env, 'APIAPP_DEFAULTHOMEADMINPASSWORD', None,
None).strip() #no default must be read in
self.APIAPP_JWT_TOKEN_TIMEOUT = int(
readFromEnviroment(env, 'APIAPP_JWT_TOKEN_TIMEOUT', 60 * 10,
None)) #default to 10 minutes
self.APIAPP_REFRESH_TOKEN_TIMEOUT = int(
readFromEnviroment(env, 'APIAPP_REFRESH_TOKEN_TIMEOUT',
60 * 60 * 2, None)) #default to 2 hours
self.APIAPP_REFRESH_SESSION_TIMEOUT = int(
readFromEnviroment(env, 'APIAPP_REFRESH_SESSION_TIMEOUT',
60 * 60 * 12, None)) #default to 12 hours
autoconfigraw = readFromEnviroment(env=env,
envVarName='APIAPP_AUTOCONFIG',
defaultValue='',
acceptableValues=None,
nullValueAllowed=False).strip()
if autoconfigraw == '':
self.APIAPP_AUTOCONFIG = None
else:
try:
self.APIAPP_AUTOCONFIG = json.loads(autoconfigraw)
except:
print("ERROR - APIAPP_AUTOCONFIG is not valid json")
raise invalidConfigurationException
if self.APIAPP_REFRESH_TOKEN_TIMEOUT < self.APIAPP_JWT_TOKEN_TIMEOUT:
print(
"ERROR - APIAPP_REFRESH_TOKEN_TIMEOUT should never be less than APIAPP_JWT_TOKEN_TIMEOUT"
)
raise invalidConfigurationException
if self.APIAPP_REFRESH_SESSION_TIMEOUT < self.APIAPP_REFRESH_TOKEN_TIMEOUT:
print(
"ERROR - APIAPP_REFRESH_SESSION_TIMEOUT should never be less than APIAPP_REFRESH_SESSION_TIMEOUT"
)
raise invalidConfigurationException
self.APIAPP_DEFAULTMASTERTENANTJWTCOLLECTIONALLOWEDORIGINFIELD = readFromEnviroment(
env, 'APIAPP_DEFAULTMASTERTENANTJWTCOLLECTIONALLOWEDORIGINFIELD',
'http://localhost', None)
# add to the baseapp default so allowed origns are in this list and extra vals
self.accessControlAllowOriginObj = uniqueCommaSeperatedListClass(
self.APIAPP_DEFAULTMASTERTENANTJWTCOLLECTIONALLOWEDORIGINFIELD +
", " + self.accessControlAllowOriginObj.toString())
# print('uniqueCommaSeperatedListClass:', self.accessControlAllowOriginObj.toString())
print('APIAPP_JWT_TOKEN_TIMEOUT:' +
str(self.APIAPP_JWT_TOKEN_TIMEOUT) + ' seconds')
print('APIAPP_REFRESH_TOKEN_TIMEOUT:' +
str(self.APIAPP_REFRESH_TOKEN_TIMEOUT) + ' seconds')
print('APIAPP_REFRESH_SESSION_TIMEOUT:' +
str(self.APIAPP_REFRESH_SESSION_TIMEOUT) + ' seconds')
objectStoreConfigJSON = readFromEnviroment(env,
'APIAPP_OBJECTSTORECONFIG',
'{}', None)
objectStoreConfigDict = None
try:
if objectStoreConfigJSON != '{}':
objectStoreConfigDict = json.loads(objectStoreConfigJSON)
except Exception as err:
print(err) # for the repr
print(str(err)) # for just the message
print(err.args
) # the arguments that the exception has been called with.
raise (InvalidObjectStoreConfigInvalidJSONException)
self.APIAPP_OBJECTSTOREDETAILLOGGING = readFromEnviroment(
env=env,
envVarName='APIAPP_OBJECTSTOREDETAILLOGGING',
defaultValue='N',
acceptableValues=['Y', 'N'],
nullValueAllowed=True).strip()
if (self.APIAPP_OBJECTSTOREDETAILLOGGING == 'Y'):
print(
"APIAPP_OBJECTSTOREDETAILLOGGING set to Y - statement logging enabled"
)
fns = {'getCurDateTime': self.getCurDateTime}
self.objectStore = createObjectStoreInstance(
objectStoreConfigDict,
fns,
detailLogging=(self.APIAPP_OBJECTSTOREDETAILLOGGING == 'Y'))
def dbChangingFn(storeConnection):
if GetTenant(masterTenantName, storeConnection,
appObj=self) is None:
print("********No master tenant found - creating********")
def someFn(connectionContext):
CreateMasterTenant(self, testingMode, storeConnection)
storeConnection.executeInsideTransaction(someFn)
self.objectStore.executeInsideConnectionContext(dbChangingFn)
self.gateway = getGatewayInterface(env, self)
self.refreshTokenManager = RefreshTokenManager(self)
self.scheduler.start()
# Load origins from tenants
def dbfn(storeConnection):
for curTenant in storeConnection.getAllRowsForObjectType(
"tenants", None, None, ""):
print("Loading allowed origins for " + curTenant[0]["Name"] +
":" +
str(curTenant[0]["JWTCollectionAllowedOriginList"]))
appObj.accessControlAllowOriginObj.addList(
curTenant[0]["JWTCollectionAllowedOriginList"])
##return storeConnection.getPaginatedResult("tenants", paginatedParamValues, outputFN)
self.objectStore.executeInsideConnectionContext(dbfn)
if self.APIAPP_AUTOCONFIG != None:
autoConfigRunner = autoConfig.AutoConfigRunner(
self.APIAPP_AUTOCONFIG)
def configRunnerFn(storeConnection):
autoConfigRunner.run(self, storeConnection)
self.objectStore.executeInsideTransaction(configRunnerFn)
tenantOnAppInit(self)
def test_defaultCreation(self):
objectStoreConfigDict = None
a = undertest.createObjectStoreInstance(
objectStoreConfigDict, self.getObjectStoreExternalFns())
if not isinstance(a, undertest.ObjectStore_Memory):
self.assertTrue(False, msg='Wrong type of object store created')
def init(self, env, serverStartTime, testingMode = False):
##self.setupLogging() Comment in when debugging
super(appObjClass, self).init(env, serverStartTime, testingMode, serverinfoapiprefix='public/info')
##print("appOBj init")
self.APIAPP_REDIRECTPREFIX = readFromEnviroment(
env=env,
envVarName='APIAPP_REDIRECTPREFIX',
defaultValue=None,
acceptableValues=None,
nullValueAllowed=False
)
print("APIAPP_REDIRECTPREFIX:", self.APIAPP_REDIRECTPREFIX)
if self.APIAPP_REDIRECTPREFIX.endswith("/"):
raise InvalidRedirectPrefixException
if not self.APIAPP_REDIRECTPREFIX.startswith("http://"):
if not self.APIAPP_REDIRECTPREFIX.startswith("https://"):
raise InvalidRedirectPrefixException
self.APIAPP_URLEXPIREDAYS = readFromEnviroment(
env=env,
envVarName='APIAPP_URLEXPIREDAYS',
defaultValue=None,
acceptableValues=None,
nullValueAllowed=False
)
print("APIAPP_URLEXPIREDAYS:", self.APIAPP_URLEXPIREDAYS)
self.APIAPP_URLEXPIREDAYS = int(self.APIAPP_URLEXPIREDAYS)
if (self.APIAPP_URLEXPIREDAYS < 0):
print("ERROR - Expire days less than 0")
raise invalidConfigurationException
objectStoreConfigDict = readDictFromEnviroment(
env=env,
envVarName='APIAPP_OBJECTSTORECONFIG',
defaultValue='{}'
)
self.APIAPP_OBJECTSTOREDETAILLOGGING = readFromEnviroment(
env=env,
envVarName='APIAPP_OBJECTSTOREDETAILLOGGING',
defaultValue='N',
acceptableValues=['Y', 'N'],
nullValueAllowed=True
).strip()
if (self.APIAPP_OBJECTSTOREDETAILLOGGING=='Y'):
print("APIAPP_OBJECTSTOREDETAILLOGGING set to Y - statement logging enabled")
fns = {
'getCurDateTime': self.getCurDateTime
}
self.objectStore = createObjectStoreInstance(
objectStoreConfigDict,
fns,
detailLogging=(self.APIAPP_OBJECTSTOREDETAILLOGGING == 'Y')
)
self.shortUrlFunctions = Logic.ShortUrlFunctionClass(appObj=self)
self.APIAPP_DESTWHITELIST = readDictFromEnviroment(
env=env,
envVarName='APIAPP_DESTWHITELIST',
defaultValue='{}',
safeToOutput=True
)
if self.APIAPP_DESTWHITELIST is None:
self.APIAPP_DESTWHITELIST = {}
print("APIAPP_DESTWHITELIST:", self.APIAPP_DESTWHITELIST)
for curTenant in self.APIAPP_DESTWHITELIST:
if not isinstance(self.APIAPP_DESTWHITELIST[curTenant], list):
print("ERROR - APIAPP_DESTWHITELIST tenant values must be lists of string")
raise invalidConfigurationException
for ite in self.APIAPP_DESTWHITELIST[curTenant]:
if not isinstance(ite, str):
print("ERROR - APIAPP_DESTWHITELIST tenant value lists can only contain strings")
raise invalidConfigurationException
objectStoreConfigJSON = readFromEnviroment(os.environ, 'APIAPP_OBJECTSTORECONFIG', '{}', None)
objectStoreConfigDict = None
try:
if objectStoreConfigJSON != '{}':
objectStoreConfigDict = json.loads(objectStoreConfigJSON)
except Exception as err:
raise err
def getCurDateTime():
return datetime.datetime.now(pytz.timezone("UTC"))
fns = {
'getCurDateTime': getCurDateTime
}
objectStoreInstance = createObjectStoreInstance(objectStoreConfigDict, fns)
print("objectStoreInstance:", objectStoreInstance)
def someFn(connectionContext):
paginatedParamValues = {}
#paginatedParamValues['query'] = ""
#paginatedParamValues['sort'] = ""
paginatedParamValues['query'] = None
paginatedParamValues['sort'] = None
paginatedParamValues['offset'] = 0
paginatedParamValues['pagesize'] = 100
print("---")
print("-A call with NO query-")
print("---")
def test_simpleCreationWithNoOperation(self):
undertest.createObjectStoreInstance(ConfigDict,
self.getObjectStoreExternalFns())
