def test_config_load_good_source_bad_name(self):
conf = PrototypeConfig()
scan = Scan(conf.ioc)
conf.load(reloadConf=True)
configList = [
{
"name": "test1",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test2",
"job": "fakeJob",
"exe_env": "windows",
"source": "Google",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test3",
"job": "fakeJob",
"exe_env": "windows",
"source": "stackOverflow",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
],
"exists": [
{
"field": "var"
}
]
}
}
]
self.assertTrue(isinstance(conf.load(ConfigurationList=configList), dict))
configs = scan.generate_config_set(config='test7', source='swapi')
self.assertTrue(isinstance(configs, list))
self.assertTrue(len(configs) == 0)
conf.load(reloadConf=True)
def test_validation_list_good(self):
conf = PrototypeConfig()
conf.load(reloadConf=True)
configList = [
{
"name": "test1",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test2",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test3",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
],
"exists": [
{
"field": "var"
}
]
}
}
]
self.assertEqual(configList, conf.validate_config_list(configList))
conf.load(reloadConf=True)
def test_empty_conf(self):
ioc = GreaseContainer()
# clean up
for root, dirnames, filenames in os.walk(ioc.getConfig().get('Configuration', 'dir')):
for filename in fnmatch.filter(filenames, '*.config.json'):
self.assertIsNone(os.remove(os.path.join(root, filename)))
conf = PrototypeConfig()
conf.load(reloadConf=True)
self.assertTrue(conf.getConfiguration())
self.assertListEqual(conf.getConfiguration().get('configuration').get('pkg'), [])
self.assertListEqual(conf.getConfiguration().get('configuration').get('fs'), [])
self.assertListEqual(conf.getConfiguration().get('configuration').get('mongo'), [])
conf.load(reloadConf=True)
def test_get_sources_all_good(self):
conf = PrototypeConfig()
conf.load(reloadConf=True)
configList = [
{
"name": "test1",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test2",
"job": "fakeJob",
"exe_env": "windows",
"source": "stackOverflow",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test3",
"job": "fakeJob",
"exe_env": "windows",
"source": "Google",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
],
"exists": [
{
"field": "var"
}
]
}
}
]
conf.load(ConfigurationList=configList)
self.assertEqual(['swapi', 'stackOverflow', 'Google'], conf.get_sources())
conf.load(reloadConf=True)
def test_scheduleScheduling(self):
d = Detect()
p = PrototypeConfig(d.ioc)
source = {
'key': 'var',
'ver': 'key',
'greg': 'old',
'created': '2017-11-24'
}
configuration = {
'name': 'demoConfig',
'job': 'otherThing',
'exe_env': 'general',
'source': 'Google',
'logic': {
'DateRange': [{
'field': 'created',
'format': '%Y-%m-%d',
'min': '2017-11-23',
'max': '2017-11-25'
}],
'Regex': [{
'field': 'key',
'pattern': '.*',
'variable': True,
'variable_name': 'field'
}, {
'field': 'ver',
'pattern': '.*'
}],
'Exists': [{
'field': 'greg',
'variable': True,
'variable_name': 'greg'
}]
}
}
p.load(True, [configuration])
sourceId = d.ioc.getCollection('SourceData').insert_one({
'grease_data': {
'sourcing': {
'server': ObjectId(d.ioc.getConfig().NodeIdentity)
},
'detection': {
'server': ObjectId(d.ioc.getConfig().NodeIdentity),
'detectionStart': None,
'detectionEnd': None,
'detection': {}
},
'scheduling': {
'schedulingServer': None,
'schedulingStart': None,
'schedulingEnd': None
},
'execution': {
'server': None,
'assignmentTime': None,
'executionStart': None,
'executionEnd': None,
'context': {},
'executionSuccess': False,
'commandSuccess': False,
'failures': 0,
'retryTime': datetime.datetime.utcnow()
}
},
'source':
str('test').encode('utf-8'),
'configuration':
configuration.get('name'),
'data':
source,
'createTime':
datetime.datetime.utcnow(),
'expiry':
Deduplication.generate_max_expiry_time(1)
}).inserted_id
scheduleServer = d.ioc.getCollection('JobServer').insert_one({
'jobs':
0,
'os':
platform.system().lower(),
'roles': ["general"],
'prototypes': ["schedule"],
'active':
True,
'activationTime':
datetime.datetime.utcnow()
}).inserted_id
self.assertTrue(d.detectSource())
self.assertFalse(d.getScheduledSource())
self.assertTrue(
len(
d.ioc.getCollection('SourceData').find_one(
{
'grease_data.scheduling.server':
ObjectId(scheduleServer),
'grease_data.scheduling.start': None,
'grease_data.scheduling.end': None
},
sort=[('createTime', pymongo.DESCENDING)])))
d.ioc.getCollection('JobServer').delete_one(
{'_id': ObjectId(scheduleServer)})
d.ioc.getCollection('SourceData').delete_one(
{'_id': ObjectId(sourceId)})
def test_mongo_load_bad(self):
ioc = GreaseContainer()
# clean up
for root, dirnames, filenames in os.walk(ioc.getConfig().get('Configuration', 'dir')):
for filename in fnmatch.filter(filenames, '*.config.json'):
self.assertIsNone(os.remove(os.path.join(root, filename)))
configList = [
{
"name": "test1",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "badtest1",
"exe_env": "windows",
"source": "stackOverflow",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test3",
"job": "fakeJob",
"exe_env": "windows",
"source": "Google",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
],
"exists": [
{
"field": "var"
}
]
}
}
]
GoodConfigList = [
{
"name": "test1",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test3",
"job": "fakeJob",
"exe_env": "windows",
"source": "Google",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
],
"exists": [
{
"field": "var"
}
]
}
}
]
for conf in configList:
ioc.getCollection('Configuration').insert_one(conf)
ioc.getCollection('Configuration').update_many({}, {'$set': {'active': True, 'type': 'prototype_config'}})
# sleep because travis is slow sometimes
time.sleep(1.5)
conf = PrototypeConfig(ioc)
conf.load(reloadConf=True)
self.assertEqual(len(conf.getConfiguration().get('configuration').get('mongo')), len(GoodConfigList))
self.assertEqual(len(conf.getConfiguration().get('raw')), len(GoodConfigList))
self.assertEqual(len(conf.getConfiguration().get('source').get('swapi')), 1)
self.assertEqual(len(conf.getConfiguration().get('source').get('Google')), 1)
self.assertEqual(2, len(conf.get_sources()))
self.assertEqual(2, len(conf.get_names()))
self.assertEqual(len(conf.get_source('Google')), 1)
self.assertTrue(isinstance(conf.get_config('test1'), dict))
self.assertTrue(conf.get_config('test1'))
# clean up
ioc.getCollection('Configuration').drop()
# clear the config
conf.load(reloadConf=True)
def test_pkg_load_bad(self):
ioc = GreaseContainer()
# clean up
for root, dirnames, filenames in os.walk(pkg_resources.resource_filename('tgt_grease.enterprise.Model', 'config/')):
for filename in fnmatch.filter(filenames, '*.config.json'):
self.assertIsNone(os.remove(os.path.join(root, filename)))
configList = [
{
"name": "test1",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "badtest1",
"exe_env": "windows",
"source": "stackOverflow",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test3",
"job": "fakeJob",
"exe_env": "windows",
"source": "Google",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
],
"exists": [
{
"field": "var"
}
]
}
}
]
GoodConfigList = [
{
"name": "test1",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test3",
"job": "fakeJob",
"exe_env": "windows",
"source": "Google",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
],
"exists": [
{
"field": "var"
}
]
}
}
]
i = 0
for conf in configList:
with open(pkg_resources.resource_filename('tgt_grease.enterprise.Model', 'config/') + 'conf{0}.config.json'.format(i), 'w') as fil:
fil.write(json.dumps(conf, indent=4))
i += 1
conf = PrototypeConfig(ioc)
conf.load(reloadConf=True)
self.assertEqual(len(conf.getConfiguration().get('configuration').get('pkg')), len(GoodConfigList))
self.assertEqual(len(conf.getConfiguration().get('raw')), len(GoodConfigList))
self.assertEqual(len(conf.getConfiguration().get('source').get('swapi')), 1)
self.assertEqual(len(conf.getConfiguration().get('source').get('Google')), 1)
self.assertEqual(2, len(conf.get_sources()))
self.assertEqual(2, len(conf.get_names()))
self.assertEqual(len(conf.get_source('Google')), 1)
self.assertTrue(isinstance(conf.get_config('test1'), dict))
self.assertTrue(conf.get_config('test1'))
# clean up
for root, dirnames, filenames in os.walk(pkg_resources.resource_filename('tgt_grease.enterprise.Model', 'config/')):
for filename in fnmatch.filter(filenames, '*.config.json'):
self.assertIsNone(os.remove(os.path.join(root, filename)))
# clear the config
conf.load(reloadConf=True)
def test_load_bad(self):
conf = PrototypeConfig()
conf.load(reloadConf=True)
configList = [
{
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {}
},
{
"name": "test1",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test2",
"job": "fakeJob",
"exe_env": "windows",
},
{
"name": "test3",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": []
},
{
"name": "test4",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
}
},
{
"name": "test5",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "var",
"pattern": "ver.*"
}
]
}
}
]
CompareList = [
{
"name": "test5",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "var",
"pattern": "ver.*"
}
]
}
}
]
self.assertDictEqual(
{
'raw': CompareList,
'configuration': {
'ConfigurationList': CompareList
},
'source': {
'swapi': CompareList
},
'sources': ['swapi'],
'names': ['test5'],
'name': {
'test5': CompareList[0]
}
},
conf.load(ConfigurationList=configList)
)
conf.load(reloadConf=True)
def test_all_load_bad(self):
ioc = GreaseContainer()
# clean up
for root, dirnames, filenames in os.walk(ioc.getConfig().get('Configuration', 'dir')):
for filename in fnmatch.filter(filenames, '*.config.json'):
self.assertIsNone(os.remove(os.path.join(root, filename)))
# clean up
for root, dirnames, filenames in os.walk(pkg_resources.resource_filename('tgt_grease.enterprise.Model', 'config/')):
for filename in fnmatch.filter(filenames, '*.config.json'):
self.assertIsNone(os.remove(os.path.join(root, filename)))
configList = [
{
"name": "test1",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "badtest1",
"exe_env": "windows",
"source": "stackOverflow",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test3",
"job": "fakeJob",
"exe_env": "windows",
"source": "Google",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
],
"exists": [
{
"field": "var"
}
]
}
}
]
GoodConfigList = [
{
"name": "test1",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test3",
"job": "fakeJob",
"exe_env": "windows",
"source": "Google",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
],
"exists": [
{
"field": "var"
}
]
}
}
]
i = 0
length = len(configList) - 1
while i <= length:
if i == 0:
with open(ioc.getConfig().get('Configuration', 'dir') + 'conf{0}.config.json'.format(i), 'w') as fil:
fil.write(json.dumps(configList[i], indent=4))
if i == 1:
with open(pkg_resources.resource_filename('tgt_grease.enterprise.Model',
'config/') + 'conf{0}.config.json'.format(i), 'w') as fil:
fil.write(json.dumps(configList[i], indent=4))
if i == 2:
ioc.getCollection('Configuration').insert_one(configList[i])
i += 1
ioc.getCollection('Configuration').update_many({}, {'$set': {'active': True, 'type': 'prototype_config'}})
# sleep because travis is slow
time.sleep(1.5)
conf = PrototypeConfig(ioc)
conf.load(reloadConf=True)
self.assertEqual(len(conf.getConfiguration().get('configuration').get('mongo')), 1)
self.assertEqual(len(conf.getConfiguration().get('configuration').get('pkg')), 0)
self.assertEqual(len(conf.getConfiguration().get('configuration').get('fs')), 1)
self.assertEqual(len(conf.getConfiguration().get('raw')), len(GoodConfigList))
self.assertEqual(len(conf.getConfiguration().get('source').get('swapi')), 1)
self.assertEqual(len(conf.getConfiguration().get('source').get('Google')), 1)
self.assertEqual(2, len(conf.get_names()))
self.assertEqual(len(conf.get_source('Google')), 1)
self.assertTrue(isinstance(conf.get_config('test1'), dict))
self.assertTrue(conf.get_config('test1'))
# clean up
ioc.getCollection('Configuration').drop()
for root, dirnames, filenames in os.walk(ioc.getConfig().get('Configuration', 'dir')):
for filename in fnmatch.filter(filenames, '*.config.json'):
self.assertIsNone(os.remove(os.path.join(root, filename)))
# clean up
for root, dirnames, filenames in os.walk(pkg_resources.resource_filename('tgt_grease.enterprise.Model', 'config/')):
for filename in fnmatch.filter(filenames, '*.config.json'):
self.assertIsNone(os.remove(os.path.join(root, filename)))
# clear the config
conf.load(reloadConf=True)
def test_scan(self):
# setup
configList = [
{
"name": "test1",
"job": "fakeJob",
"exe_env": "windows",
"source": "TestSource",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
}
]
ioc = GreaseContainer()
ioc.ensureRegistration()
ioc.getConfig().set('trace', True, 'Logging')
ioc.getConfig().set('verbose', True, 'Logging')
fil = open(ioc.getConfig().greaseConfigFile, 'r')
data = json.loads(fil.read())
fil.close()
fil = open(ioc.getConfig().greaseConfigFile, 'w')
data['Import']['searchPath'].append('tgt_grease.enterprise.Model.tests')
fil.write(json.dumps(data, sort_keys=True, indent=4))
fil.close()
Configuration.ReloadConfig()
jServer = ioc.getCollection('JobServer')
jID1 = jServer.insert_one({
'jobs': 0,
'os': platform.system().lower(),
'roles': ["general"],
'prototypes': ["detect"],
'active': True,
'activationTime': datetime.utcnow()
}).inserted_id
time.sleep(1)
jID2 = jServer.insert_one({
'jobs': 0,
'os': platform.system().lower(),
'roles': ["general"],
'prototypes': ["detect"],
'active': True,
'activationTime': datetime.utcnow()
}).inserted_id
# Begin Test
conf = PrototypeConfig(ioc)
conf.load(reloadConf=True, ConfigurationList=configList)
scanner = Scan(ioc)
# Scan Environment
self.assertTrue(scanner.Parse())
# Begin ensuring environment is how we expect
# we assert less or equal because sometimes uuid's are close :p
self.assertLessEqual(ioc.getCollection('SourceData').find({
'detectionServer': ObjectId(jID1)
}).count(), 3)
self.assertLessEqual(ioc.getCollection('SourceData').find({
'detectionServer': ObjectId(jID2)
}).count(), 3)
self.assertLessEqual(ioc.getCollection('JobServer').find_one({
'_id': ObjectId(jID1)
})['jobs'], 3)
self.assertLessEqual(ioc.getCollection('JobServer').find_one({
'_id': ObjectId(jID2)
})['jobs'], 3)
# clean up
fil = open(ioc.getConfig().greaseConfigFile, 'r')
data = json.loads(fil.read())
fil.close()
# remove collection
ioc.getCollection('TestProtoType').drop()
# remove prototypes
data['NodeInformation']['ProtoTypes'] = []
# pop search path
trash = data['Import']['searchPath'].pop()
# close out
fil = open(ioc.getConfig().greaseConfigFile, 'w')
fil.write(json.dumps(data, sort_keys=True, indent=4))
fil.close()
jServer.delete_one({'_id': ObjectId(jID1)})
jServer.delete_one({'_id': ObjectId(jID2)})
ioc.getCollection('SourceData').drop()
ioc.getCollection('Dedup_Sourcing').drop()
ioc.getConfig().set('trace', False, 'Logging')
ioc.getConfig().set('verbose', False, 'Logging')
Configuration.ReloadConfig()
def test_scheduling_fail(self):
d = Detect()
p = PrototypeConfig(d.ioc)
s = Scheduler(d.ioc)
source = {
'key': 'var',
'ver': 'key',
'greg': 'old',
'created': '2017-11-24'
}
configuration = {
'name': 'demoConfig',
'job': 'otherThing',
'exe_env': 'minix',
'source': 'Google',
'logic': {
'DateRange': [
{
'field': 'created',
'format': '%Y-%m-%d',
'min': '2017-11-23',
'max': '2017-11-25'
}
],
'Regex': [
{
'field': 'key',
'pattern': '.*',
'variable': True,
'variable_name': 'field'
},
{
'field': 'ver',
'pattern': '.*'
}
],
'Exists': [
{
'field': 'greg',
'variable': True,
'variable_name': 'greg'
}
]
}
}
p.load(True, [configuration])
sourceId = d.ioc.getCollection('SourceData').insert_one({
'grease_data': {
'sourcing': {
'server': ObjectId(d.ioc.getConfig().NodeIdentity)
},
'detection': {
'server': ObjectId(d.ioc.getConfig().NodeIdentity),
'detectionStart': None,
'detectionEnd': None,
'detection': {}
},
'scheduling': {
'schedulingServer': None,
'schedulingStart': None,
'schedulingEnd': None
},
'execution': {
'server': None,
'assignmentTime': None,
'executionStart': None,
'executionEnd': None,
'context': {},
'executionSuccess': False,
'commandSuccess': False,
'failures': 0,
'retryTime': datetime.datetime.utcnow()
}
},
'source': str('test').encode('utf-8'),
'configuration': configuration.get('name'),
'data': source,
'createTime': datetime.datetime.utcnow(),
'expiry': Deduplication.generate_max_expiry_time(1)
}).inserted_id
d.ioc.getCollection('JobServer').update_one(
{'_id': ObjectId(d.ioc.getConfig().NodeIdentity)},
{
'$set': {
'prototypes': ["scan", "detect", "schedule"]
}
}
)
self.assertTrue(d.detectSource())
self.assertFalse(d.getScheduledSource())
self.assertTrue(s.getDetectedSource())
self.assertFalse(s.scheduleExecution())
self.assertTrue(d.ioc.getCollection('JobServer').update_one(
{'_id': ObjectId(d.ioc.getConfig().NodeIdentity)},
{
'$set': {
'prototypes': []
}
}
))
d.ioc.getCollection('SourceData').delete_one({'_id': ObjectId(sourceId)})
def test_rescheduleJobsFailed(self):
n = NodeMonitoring()
p = PrototypeConfig(n.ioc)
server1 = n.ioc.getCollection('JobServer').insert_one({
'jobs':
9,
'os':
platform.system().lower(),
'roles': [],
'prototypes': [],
'active':
True,
'activationTime':
datetime.datetime.utcnow()
}).inserted_id
server2 = n.ioc.getCollection('JobServer').insert_one({
'jobs':
9,
'os':
platform.system().lower(),
'roles': ['test1'],
'prototypes': ['detect', 'schedule'],
'active':
True,
'activationTime':
datetime.datetime.utcnow()
}).inserted_id
config = n.ioc.getCollection('Configuration').insert_one({
'active': True,
'type': 'prototype_config',
"name": "test",
"job": "help",
"exe_env": "test",
"source": "test",
"logic": {
"Regex": [{
"field": "url",
"pattern": ".*",
'variable': True,
'variable_name': 'url'
}],
'Range': [{
'field': 'status_code',
'min': 199,
'max': 201
}]
},
'constants': {
'test': 'ver'
}
}).inserted_id
p.load(reloadConf=True)
source = n.ioc.getCollection('SourceData').insert_one({
'grease_data': {
'sourcing': {
'server': server1
},
'detection': {
'server': server1,
'start': datetime.datetime.utcnow(),
'end': datetime.datetime.utcnow(),
'detection': {}
},
'scheduling': {
'server': server1,
'start': datetime.datetime.utcnow(),
'end': datetime.datetime.utcnow(),
},
'execution': {
'server': server1,
'assignmentTime': datetime.datetime.utcnow(),
'completeTime': None,
'returnData': {},
'executionSuccess': False,
'commandSuccess': False,
'failures': 0
}
},
'source':
'test',
'configuration':
'test',
'createTime':
datetime.datetime.utcnow(),
'expiry':
Deduplication.generate_max_expiry_time(1)
}).inserted_id
self.assertTrue(n.deactivateServer(str(server1)))
self.assertFalse(n.rescheduleJobs(str(server1)))
self.assertFalse(
n.ioc.getCollection('SourceData').find({
'_id':
source,
'grease_data.execution.server':
server2
}).count())
self.assertEqual(
n.ioc.getCollection('JobServer').delete_one({
'_id': server1
}).deleted_count, 1)
self.assertEqual(
n.ioc.getCollection('JobServer').delete_one({
'_id': server2
}).deleted_count, 1)
n.ioc.getCollection('SourceData').delete_one({'_id': source})
n.ioc.getCollection('Configuration').drop()
n.ioc.getCollection('ServerHealth').drop()
p.load(reloadConf=True)
def test_job_execution(self):
ioc = GreaseContainer()
cmd = DaemonProcess(ioc)
proto = PrototypeConfig(ioc)
ioc.getCollection('Configuration').insert_one(
{
'active': True,
'type': 'prototype_config',
"name": "exe_test",
"job": "help",
"exe_env": "general",
"source": "url_source",
"logic": {
"Regex": [
{
"field": "url",
"pattern": ".*",
'variable': True,
'variable_name': 'url'
}
],
'Range': [
{
'field': 'status_code',
'min': 199,
'max': 201
}
]
},
'constants': {
'test': 'ver'
}
}
)
proto.load(reloadConf=True)
jobid = ioc.getCollection('SourceData').insert_one({
'grease_data': {
'sourcing': {
'server': ObjectId(ioc.getConfig().NodeIdentity)
},
'detection': {
'server': ObjectId(ioc.getConfig().NodeIdentity),
'start': datetime.datetime.utcnow(),
'end': datetime.datetime.utcnow(),
'detection': {}
},
'scheduling': {
'server': ObjectId(ioc.getConfig().NodeIdentity),
'start': datetime.datetime.utcnow(),
'end': datetime.datetime.utcnow(),
},
'execution': {
'server': ObjectId(ioc.getConfig().NodeIdentity),
'assignmentTime': datetime.datetime.utcnow(),
'completeTime': None,
'returnData': {},
'executionSuccess': False,
'commandSuccess': False,
'failures': 0
}
},
'source': 'dev',
'configuration': 'exe_test',
'data': {},
'createTime': datetime.datetime.utcnow(),
'expiry': Deduplication.generate_max_expiry_time(1)
}).inserted_id
# Run for a bit
self.assertTrue(cmd.server())
self.assertTrue(cmd.drain_jobs(ioc.getCollection('SourceData')))
result = ioc.getCollection('SourceData').find_one({'_id': ObjectId(jobid)})
self.assertTrue(result)
self.assertTrue(result.get('grease_data').get('execution').get('executionSuccess'))
self.assertTrue(result.get('grease_data').get('execution').get('commandSuccess'))
ioc.getCollection('SourceData').drop()
ioc.getCollection('Configuration').drop()
def test_real(self):
#############################################
# SETUP UP TIME
#############################################
ioc = GreaseContainer()
pConf = PrototypeConfig(ioc)
ioc.ensureRegistration()
ioc.getCollection('JobServer').update_one(
{'_id': ObjectId(ioc.getConfig().NodeIdentity)},
{'$set': {
'prototypes': ['scan', 'detect', 'schedule']
}})
ioc.getCollection('Configuration').insert_one({
'active':
True,
'type':
'prototype_config',
"name":
"full_stack_test",
"job":
"help",
"exe_env":
"general",
"source":
"url_source",
"url": ['http://google.com'],
"logic": {
"Regex": [{
"field": "url",
"pattern": ".*",
'variable': True,
'variable_name': 'url'
}],
'Range': [{
'field': 'status_code',
'min': 199,
'max': 201
}]
},
'constants': {
'test': 'ver'
}
})
pConf.load(reloadConf=True)
#############################################
# EXECUTE SCANNING
#############################################
Scanner = scan()
Scanner.ioc.getLogger().getConfig().set('verbose', True, 'Logging')
Scanner.ioc.getLogger().getConfig().set('trace', True, 'Logging')
Scanner.ioc.getLogger().getConfig().set('config', 'full_stack_test',
'Sourcing')
self.assertTrue(Scanner.execute({'loop': 1}))
#############################################
# ASSERT SCANNING
#############################################
self.assertTrue(
ioc.getCollection('SourceData').find_one({
'grease_data.sourcing.server':
ObjectId(ioc.getConfig().NodeIdentity),
'grease_data.detection.server':
ObjectId(ioc.getConfig().NodeIdentity),
'grease_data.detection.start':
None,
'grease_data.detection.end':
None
}))
#############################################
# EXECUTE DETECTION
#############################################
Detect = detect()
Detect.ioc.getLogger().getConfig().set('verbose', True, 'Logging')
Detect.ioc.getLogger().getConfig().set('trace', True, 'Logging')
Detect.ioc.getLogger().getConfig().set('config', 'full_stack_test',
'Sourcing')
self.assertTrue(Detect.execute({'loop': 1}))
#############################################
# ASSERT DETECTION
#############################################
self.assertTrue(
ioc.getCollection('SourceData').find_one({
'grease_data.sourcing.server':
ObjectId(ioc.getConfig().NodeIdentity),
'grease_data.detection.server':
ObjectId(ioc.getConfig().NodeIdentity),
'grease_data.scheduling.server':
ObjectId(ioc.getConfig().NodeIdentity),
'grease_data.scheduling.start':
None,
'grease_data.scheduling.end':
None
}))
#############################################
# EXECUTE SCHEDULING
#############################################
Scheduling = schedule()
Scheduling.ioc.getLogger().getConfig().set('verbose', True, 'Logging')
Scheduling.ioc.getLogger().getConfig().set('trace', True, 'Logging')
Scheduling.ioc.getLogger().getConfig().set('config', 'full_stack_test',
'Sourcing')
self.assertTrue(Scheduling.execute({'loop': 1}))
#############################################
# ASSERT SCHEDULING
#############################################
self.assertTrue(
ioc.getCollection('SourceData').find_one({
'grease_data.sourcing.server':
ObjectId(ioc.getConfig().NodeIdentity),
'grease_data.detection.server':
ObjectId(ioc.getConfig().NodeIdentity),
'grease_data.scheduling.server':
ObjectId(ioc.getConfig().NodeIdentity),
'grease_data.execution.server':
ObjectId(ioc.getConfig().NodeIdentity),
'grease_data.execution.start':
None,
'grease_data.execution.end':
None
}))
#############################################
# EXECUTE JOBS
#############################################
ioc.getCollection('JobServer').update_one(
{'_id': ObjectId(ioc.getConfig().NodeIdentity)},
{'$set': {
'prototypes': []
}})
Daemon = DaemonProcess(ioc)
Daemon.ioc.getLogger().getConfig().set('verbose', True, 'Logging')
Daemon.ioc.getLogger().getConfig().set('trace', True, 'Logging')
Daemon.ioc.getLogger().getConfig().set('config', 'full_stack_test',
'Sourcing')
self.assertTrue(Daemon.server())
self.assertTrue(Daemon.drain_jobs(ioc.getCollection('SourceData')))
#############################################
# ASSERT JOB EXECUTION
#############################################
# sleep a few for seconds to let help complete
time.sleep(5)
self.assertTrue(
ioc.getCollection('SourceData').find_one({
'grease_data.sourcing.server':
ObjectId(ioc.getConfig().NodeIdentity),
'grease_data.detection.server':
ObjectId(ioc.getConfig().NodeIdentity),
'grease_data.scheduling.server':
ObjectId(ioc.getConfig().NodeIdentity),
'grease_data.execution.server':
ObjectId(ioc.getConfig().NodeIdentity),
'grease_data.execution.commandSuccess':
True,
'grease_data.execution.executionSuccess':
True
}))
#############################################
# CLEAN UP TIME
#############################################
ioc.getCollection('JobServer').update_one(
{'_id': ObjectId(ioc.getConfig().NodeIdentity)},
{'$set': {
'prototypes': []
}})
ioc.getCollection('Configuration').drop()
ioc.getCollection('SourceData').drop()
ioc.getCollection('DeDup_Sourcing').drop()
pConf.load(reloadConf=True)
