def test_config_load_good_source_bad_name(self): conf = PrototypeConfig() scan = Scan(conf.ioc) conf.load(reloadConf=True) configList = [ { "name": "test1", "job": "fakeJob", "exe_env": "windows", "source": "swapi", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ] } }, { "name": "test2", "job": "fakeJob", "exe_env": "windows", "source": "Google", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ] } }, { "name": "test3", "job": "fakeJob", "exe_env": "windows", "source": "stackOverflow", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ], "exists": [ { "field": "var" } ] } } ] self.assertTrue(isinstance(conf.load(ConfigurationList=configList), dict)) configs = scan.generate_config_set(config='test7', source='swapi') self.assertTrue(isinstance(configs, list)) self.assertTrue(len(configs) == 0) conf.load(reloadConf=True)
def test_validation_list_good(self): conf = PrototypeConfig() conf.load(reloadConf=True) configList = [ { "name": "test1", "job": "fakeJob", "exe_env": "windows", "source": "swapi", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ] } }, { "name": "test2", "job": "fakeJob", "exe_env": "windows", "source": "swapi", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ] } }, { "name": "test3", "job": "fakeJob", "exe_env": "windows", "source": "swapi", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ], "exists": [ { "field": "var" } ] } } ] self.assertEqual(configList, conf.validate_config_list(configList)) conf.load(reloadConf=True)
def test_empty_conf(self): ioc = GreaseContainer() # clean up for root, dirnames, filenames in os.walk(ioc.getConfig().get('Configuration', 'dir')): for filename in fnmatch.filter(filenames, '*.config.json'): self.assertIsNone(os.remove(os.path.join(root, filename))) conf = PrototypeConfig() conf.load(reloadConf=True) self.assertTrue(conf.getConfiguration()) self.assertListEqual(conf.getConfiguration().get('configuration').get('pkg'), []) self.assertListEqual(conf.getConfiguration().get('configuration').get('fs'), []) self.assertListEqual(conf.getConfiguration().get('configuration').get('mongo'), []) conf.load(reloadConf=True)
def test_get_sources_all_good(self): conf = PrototypeConfig() conf.load(reloadConf=True) configList = [ { "name": "test1", "job": "fakeJob", "exe_env": "windows", "source": "swapi", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ] } }, { "name": "test2", "job": "fakeJob", "exe_env": "windows", "source": "stackOverflow", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ] } }, { "name": "test3", "job": "fakeJob", "exe_env": "windows", "source": "Google", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ], "exists": [ { "field": "var" } ] } } ] conf.load(ConfigurationList=configList) self.assertEqual(['swapi', 'stackOverflow', 'Google'], conf.get_sources()) conf.load(reloadConf=True)
def test_scheduleScheduling(self): d = Detect() p = PrototypeConfig(d.ioc) source = { 'key': 'var', 'ver': 'key', 'greg': 'old', 'created': '2017-11-24' } configuration = { 'name': 'demoConfig', 'job': 'otherThing', 'exe_env': 'general', 'source': 'Google', 'logic': { 'DateRange': [{ 'field': 'created', 'format': '%Y-%m-%d', 'min': '2017-11-23', 'max': '2017-11-25' }], 'Regex': [{ 'field': 'key', 'pattern': '.*', 'variable': True, 'variable_name': 'field' }, { 'field': 'ver', 'pattern': '.*' }], 'Exists': [{ 'field': 'greg', 'variable': True, 'variable_name': 'greg' }] } } p.load(True, [configuration]) sourceId = d.ioc.getCollection('SourceData').insert_one({ 'grease_data': { 'sourcing': { 'server': ObjectId(d.ioc.getConfig().NodeIdentity) }, 'detection': { 'server': ObjectId(d.ioc.getConfig().NodeIdentity), 'detectionStart': None, 'detectionEnd': None, 'detection': {} }, 'scheduling': { 'schedulingServer': None, 'schedulingStart': None, 'schedulingEnd': None }, 'execution': { 'server': None, 'assignmentTime': None, 'executionStart': None, 'executionEnd': None, 'context': {}, 'executionSuccess': False, 'commandSuccess': False, 'failures': 0, 'retryTime': datetime.datetime.utcnow() } }, 'source': str('test').encode('utf-8'), 'configuration': configuration.get('name'), 'data': source, 'createTime': datetime.datetime.utcnow(), 'expiry': Deduplication.generate_max_expiry_time(1) }).inserted_id scheduleServer = d.ioc.getCollection('JobServer').insert_one({ 'jobs': 0, 'os': platform.system().lower(), 'roles': ["general"], 'prototypes': ["schedule"], 'active': True, 'activationTime': datetime.datetime.utcnow() }).inserted_id self.assertTrue(d.detectSource()) self.assertFalse(d.getScheduledSource()) self.assertTrue( len( d.ioc.getCollection('SourceData').find_one( { 'grease_data.scheduling.server': ObjectId(scheduleServer), 'grease_data.scheduling.start': None, 'grease_data.scheduling.end': None }, sort=[('createTime', pymongo.DESCENDING)]))) d.ioc.getCollection('JobServer').delete_one( {'_id': ObjectId(scheduleServer)}) d.ioc.getCollection('SourceData').delete_one( {'_id': ObjectId(sourceId)})
def test_mongo_load_bad(self): ioc = GreaseContainer() # clean up for root, dirnames, filenames in os.walk(ioc.getConfig().get('Configuration', 'dir')): for filename in fnmatch.filter(filenames, '*.config.json'): self.assertIsNone(os.remove(os.path.join(root, filename))) configList = [ { "name": "test1", "job": "fakeJob", "exe_env": "windows", "source": "swapi", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ] } }, { "name": "badtest1", "exe_env": "windows", "source": "stackOverflow", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ] } }, { "name": "test3", "job": "fakeJob", "exe_env": "windows", "source": "Google", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ], "exists": [ { "field": "var" } ] } } ] GoodConfigList = [ { "name": "test1", "job": "fakeJob", "exe_env": "windows", "source": "swapi", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ] } }, { "name": "test3", "job": "fakeJob", "exe_env": "windows", "source": "Google", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ], "exists": [ { "field": "var" } ] } } ] for conf in configList: ioc.getCollection('Configuration').insert_one(conf) ioc.getCollection('Configuration').update_many({}, {'$set': {'active': True, 'type': 'prototype_config'}}) # sleep because travis is slow sometimes time.sleep(1.5) conf = PrototypeConfig(ioc) conf.load(reloadConf=True) self.assertEqual(len(conf.getConfiguration().get('configuration').get('mongo')), len(GoodConfigList)) self.assertEqual(len(conf.getConfiguration().get('raw')), len(GoodConfigList)) self.assertEqual(len(conf.getConfiguration().get('source').get('swapi')), 1) self.assertEqual(len(conf.getConfiguration().get('source').get('Google')), 1) self.assertEqual(2, len(conf.get_sources())) self.assertEqual(2, len(conf.get_names())) self.assertEqual(len(conf.get_source('Google')), 1) self.assertTrue(isinstance(conf.get_config('test1'), dict)) self.assertTrue(conf.get_config('test1')) # clean up ioc.getCollection('Configuration').drop() # clear the config conf.load(reloadConf=True)
def test_pkg_load_bad(self): ioc = GreaseContainer() # clean up for root, dirnames, filenames in os.walk(pkg_resources.resource_filename('tgt_grease.enterprise.Model', 'config/')): for filename in fnmatch.filter(filenames, '*.config.json'): self.assertIsNone(os.remove(os.path.join(root, filename))) configList = [ { "name": "test1", "job": "fakeJob", "exe_env": "windows", "source": "swapi", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ] } }, { "name": "badtest1", "exe_env": "windows", "source": "stackOverflow", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ] } }, { "name": "test3", "job": "fakeJob", "exe_env": "windows", "source": "Google", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ], "exists": [ { "field": "var" } ] } } ] GoodConfigList = [ { "name": "test1", "job": "fakeJob", "exe_env": "windows", "source": "swapi", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ] } }, { "name": "test3", "job": "fakeJob", "exe_env": "windows", "source": "Google", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ], "exists": [ { "field": "var" } ] } } ] i = 0 for conf in configList: with open(pkg_resources.resource_filename('tgt_grease.enterprise.Model', 'config/') + 'conf{0}.config.json'.format(i), 'w') as fil: fil.write(json.dumps(conf, indent=4)) i += 1 conf = PrototypeConfig(ioc) conf.load(reloadConf=True) self.assertEqual(len(conf.getConfiguration().get('configuration').get('pkg')), len(GoodConfigList)) self.assertEqual(len(conf.getConfiguration().get('raw')), len(GoodConfigList)) self.assertEqual(len(conf.getConfiguration().get('source').get('swapi')), 1) self.assertEqual(len(conf.getConfiguration().get('source').get('Google')), 1) self.assertEqual(2, len(conf.get_sources())) self.assertEqual(2, len(conf.get_names())) self.assertEqual(len(conf.get_source('Google')), 1) self.assertTrue(isinstance(conf.get_config('test1'), dict)) self.assertTrue(conf.get_config('test1')) # clean up for root, dirnames, filenames in os.walk(pkg_resources.resource_filename('tgt_grease.enterprise.Model', 'config/')): for filename in fnmatch.filter(filenames, '*.config.json'): self.assertIsNone(os.remove(os.path.join(root, filename))) # clear the config conf.load(reloadConf=True)
def test_load_bad(self): conf = PrototypeConfig() conf.load(reloadConf=True) configList = [ { "job": "fakeJob", "exe_env": "windows", "source": "swapi", "logic": {} }, { "name": "test1", "exe_env": "windows", "source": "swapi", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ] } }, { "name": "test2", "job": "fakeJob", "exe_env": "windows", }, { "name": "test3", "job": "fakeJob", "exe_env": "windows", "source": "swapi", "logic": [] }, { "name": "test4", "job": "fakeJob", "exe_env": "windows", "source": "swapi", "logic": { } }, { "name": "test5", "job": "fakeJob", "exe_env": "windows", "source": "swapi", "logic": { "regex": [ { "field": "var", "pattern": "ver.*" } ] } } ] CompareList = [ { "name": "test5", "job": "fakeJob", "exe_env": "windows", "source": "swapi", "logic": { "regex": [ { "field": "var", "pattern": "ver.*" } ] } } ] self.assertDictEqual( { 'raw': CompareList, 'configuration': { 'ConfigurationList': CompareList }, 'source': { 'swapi': CompareList }, 'sources': ['swapi'], 'names': ['test5'], 'name': { 'test5': CompareList[0] } }, conf.load(ConfigurationList=configList) ) conf.load(reloadConf=True)
def test_all_load_bad(self): ioc = GreaseContainer() # clean up for root, dirnames, filenames in os.walk(ioc.getConfig().get('Configuration', 'dir')): for filename in fnmatch.filter(filenames, '*.config.json'): self.assertIsNone(os.remove(os.path.join(root, filename))) # clean up for root, dirnames, filenames in os.walk(pkg_resources.resource_filename('tgt_grease.enterprise.Model', 'config/')): for filename in fnmatch.filter(filenames, '*.config.json'): self.assertIsNone(os.remove(os.path.join(root, filename))) configList = [ { "name": "test1", "job": "fakeJob", "exe_env": "windows", "source": "swapi", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ] } }, { "name": "badtest1", "exe_env": "windows", "source": "stackOverflow", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ] } }, { "name": "test3", "job": "fakeJob", "exe_env": "windows", "source": "Google", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ], "exists": [ { "field": "var" } ] } } ] GoodConfigList = [ { "name": "test1", "job": "fakeJob", "exe_env": "windows", "source": "swapi", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ] } }, { "name": "test3", "job": "fakeJob", "exe_env": "windows", "source": "Google", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ], "exists": [ { "field": "var" } ] } } ] i = 0 length = len(configList) - 1 while i <= length: if i == 0: with open(ioc.getConfig().get('Configuration', 'dir') + 'conf{0}.config.json'.format(i), 'w') as fil: fil.write(json.dumps(configList[i], indent=4)) if i == 1: with open(pkg_resources.resource_filename('tgt_grease.enterprise.Model', 'config/') + 'conf{0}.config.json'.format(i), 'w') as fil: fil.write(json.dumps(configList[i], indent=4)) if i == 2: ioc.getCollection('Configuration').insert_one(configList[i]) i += 1 ioc.getCollection('Configuration').update_many({}, {'$set': {'active': True, 'type': 'prototype_config'}}) # sleep because travis is slow time.sleep(1.5) conf = PrototypeConfig(ioc) conf.load(reloadConf=True) self.assertEqual(len(conf.getConfiguration().get('configuration').get('mongo')), 1) self.assertEqual(len(conf.getConfiguration().get('configuration').get('pkg')), 0) self.assertEqual(len(conf.getConfiguration().get('configuration').get('fs')), 1) self.assertEqual(len(conf.getConfiguration().get('raw')), len(GoodConfigList)) self.assertEqual(len(conf.getConfiguration().get('source').get('swapi')), 1) self.assertEqual(len(conf.getConfiguration().get('source').get('Google')), 1) self.assertEqual(2, len(conf.get_names())) self.assertEqual(len(conf.get_source('Google')), 1) self.assertTrue(isinstance(conf.get_config('test1'), dict)) self.assertTrue(conf.get_config('test1')) # clean up ioc.getCollection('Configuration').drop() for root, dirnames, filenames in os.walk(ioc.getConfig().get('Configuration', 'dir')): for filename in fnmatch.filter(filenames, '*.config.json'): self.assertIsNone(os.remove(os.path.join(root, filename))) # clean up for root, dirnames, filenames in os.walk(pkg_resources.resource_filename('tgt_grease.enterprise.Model', 'config/')): for filename in fnmatch.filter(filenames, '*.config.json'): self.assertIsNone(os.remove(os.path.join(root, filename))) # clear the config conf.load(reloadConf=True)
def test_scan(self): # setup configList = [ { "name": "test1", "job": "fakeJob", "exe_env": "windows", "source": "TestSource", "logic": { "regex": [ { "field": "character", "pattern": ".*skywalker.*" } ] } } ] ioc = GreaseContainer() ioc.ensureRegistration() ioc.getConfig().set('trace', True, 'Logging') ioc.getConfig().set('verbose', True, 'Logging') fil = open(ioc.getConfig().greaseConfigFile, 'r') data = json.loads(fil.read()) fil.close() fil = open(ioc.getConfig().greaseConfigFile, 'w') data['Import']['searchPath'].append('tgt_grease.enterprise.Model.tests') fil.write(json.dumps(data, sort_keys=True, indent=4)) fil.close() Configuration.ReloadConfig() jServer = ioc.getCollection('JobServer') jID1 = jServer.insert_one({ 'jobs': 0, 'os': platform.system().lower(), 'roles': ["general"], 'prototypes': ["detect"], 'active': True, 'activationTime': datetime.utcnow() }).inserted_id time.sleep(1) jID2 = jServer.insert_one({ 'jobs': 0, 'os': platform.system().lower(), 'roles': ["general"], 'prototypes': ["detect"], 'active': True, 'activationTime': datetime.utcnow() }).inserted_id # Begin Test conf = PrototypeConfig(ioc) conf.load(reloadConf=True, ConfigurationList=configList) scanner = Scan(ioc) # Scan Environment self.assertTrue(scanner.Parse()) # Begin ensuring environment is how we expect # we assert less or equal because sometimes uuid's are close :p self.assertLessEqual(ioc.getCollection('SourceData').find({ 'detectionServer': ObjectId(jID1) }).count(), 3) self.assertLessEqual(ioc.getCollection('SourceData').find({ 'detectionServer': ObjectId(jID2) }).count(), 3) self.assertLessEqual(ioc.getCollection('JobServer').find_one({ '_id': ObjectId(jID1) })['jobs'], 3) self.assertLessEqual(ioc.getCollection('JobServer').find_one({ '_id': ObjectId(jID2) })['jobs'], 3) # clean up fil = open(ioc.getConfig().greaseConfigFile, 'r') data = json.loads(fil.read()) fil.close() # remove collection ioc.getCollection('TestProtoType').drop() # remove prototypes data['NodeInformation']['ProtoTypes'] = [] # pop search path trash = data['Import']['searchPath'].pop() # close out fil = open(ioc.getConfig().greaseConfigFile, 'w') fil.write(json.dumps(data, sort_keys=True, indent=4)) fil.close() jServer.delete_one({'_id': ObjectId(jID1)}) jServer.delete_one({'_id': ObjectId(jID2)}) ioc.getCollection('SourceData').drop() ioc.getCollection('Dedup_Sourcing').drop() ioc.getConfig().set('trace', False, 'Logging') ioc.getConfig().set('verbose', False, 'Logging') Configuration.ReloadConfig()
def test_scheduling_fail(self): d = Detect() p = PrototypeConfig(d.ioc) s = Scheduler(d.ioc) source = { 'key': 'var', 'ver': 'key', 'greg': 'old', 'created': '2017-11-24' } configuration = { 'name': 'demoConfig', 'job': 'otherThing', 'exe_env': 'minix', 'source': 'Google', 'logic': { 'DateRange': [ { 'field': 'created', 'format': '%Y-%m-%d', 'min': '2017-11-23', 'max': '2017-11-25' } ], 'Regex': [ { 'field': 'key', 'pattern': '.*', 'variable': True, 'variable_name': 'field' }, { 'field': 'ver', 'pattern': '.*' } ], 'Exists': [ { 'field': 'greg', 'variable': True, 'variable_name': 'greg' } ] } } p.load(True, [configuration]) sourceId = d.ioc.getCollection('SourceData').insert_one({ 'grease_data': { 'sourcing': { 'server': ObjectId(d.ioc.getConfig().NodeIdentity) }, 'detection': { 'server': ObjectId(d.ioc.getConfig().NodeIdentity), 'detectionStart': None, 'detectionEnd': None, 'detection': {} }, 'scheduling': { 'schedulingServer': None, 'schedulingStart': None, 'schedulingEnd': None }, 'execution': { 'server': None, 'assignmentTime': None, 'executionStart': None, 'executionEnd': None, 'context': {}, 'executionSuccess': False, 'commandSuccess': False, 'failures': 0, 'retryTime': datetime.datetime.utcnow() } }, 'source': str('test').encode('utf-8'), 'configuration': configuration.get('name'), 'data': source, 'createTime': datetime.datetime.utcnow(), 'expiry': Deduplication.generate_max_expiry_time(1) }).inserted_id d.ioc.getCollection('JobServer').update_one( {'_id': ObjectId(d.ioc.getConfig().NodeIdentity)}, { '$set': { 'prototypes': ["scan", "detect", "schedule"] } } ) self.assertTrue(d.detectSource()) self.assertFalse(d.getScheduledSource()) self.assertTrue(s.getDetectedSource()) self.assertFalse(s.scheduleExecution()) self.assertTrue(d.ioc.getCollection('JobServer').update_one( {'_id': ObjectId(d.ioc.getConfig().NodeIdentity)}, { '$set': { 'prototypes': [] } } )) d.ioc.getCollection('SourceData').delete_one({'_id': ObjectId(sourceId)})
def test_rescheduleJobsFailed(self): n = NodeMonitoring() p = PrototypeConfig(n.ioc) server1 = n.ioc.getCollection('JobServer').insert_one({ 'jobs': 9, 'os': platform.system().lower(), 'roles': [], 'prototypes': [], 'active': True, 'activationTime': datetime.datetime.utcnow() }).inserted_id server2 = n.ioc.getCollection('JobServer').insert_one({ 'jobs': 9, 'os': platform.system().lower(), 'roles': ['test1'], 'prototypes': ['detect', 'schedule'], 'active': True, 'activationTime': datetime.datetime.utcnow() }).inserted_id config = n.ioc.getCollection('Configuration').insert_one({ 'active': True, 'type': 'prototype_config', "name": "test", "job": "help", "exe_env": "test", "source": "test", "logic": { "Regex": [{ "field": "url", "pattern": ".*", 'variable': True, 'variable_name': 'url' }], 'Range': [{ 'field': 'status_code', 'min': 199, 'max': 201 }] }, 'constants': { 'test': 'ver' } }).inserted_id p.load(reloadConf=True) source = n.ioc.getCollection('SourceData').insert_one({ 'grease_data': { 'sourcing': { 'server': server1 }, 'detection': { 'server': server1, 'start': datetime.datetime.utcnow(), 'end': datetime.datetime.utcnow(), 'detection': {} }, 'scheduling': { 'server': server1, 'start': datetime.datetime.utcnow(), 'end': datetime.datetime.utcnow(), }, 'execution': { 'server': server1, 'assignmentTime': datetime.datetime.utcnow(), 'completeTime': None, 'returnData': {}, 'executionSuccess': False, 'commandSuccess': False, 'failures': 0 } }, 'source': 'test', 'configuration': 'test', 'createTime': datetime.datetime.utcnow(), 'expiry': Deduplication.generate_max_expiry_time(1) }).inserted_id self.assertTrue(n.deactivateServer(str(server1))) self.assertFalse(n.rescheduleJobs(str(server1))) self.assertFalse( n.ioc.getCollection('SourceData').find({ '_id': source, 'grease_data.execution.server': server2 }).count()) self.assertEqual( n.ioc.getCollection('JobServer').delete_one({ '_id': server1 }).deleted_count, 1) self.assertEqual( n.ioc.getCollection('JobServer').delete_one({ '_id': server2 }).deleted_count, 1) n.ioc.getCollection('SourceData').delete_one({'_id': source}) n.ioc.getCollection('Configuration').drop() n.ioc.getCollection('ServerHealth').drop() p.load(reloadConf=True)
def test_job_execution(self): ioc = GreaseContainer() cmd = DaemonProcess(ioc) proto = PrototypeConfig(ioc) ioc.getCollection('Configuration').insert_one( { 'active': True, 'type': 'prototype_config', "name": "exe_test", "job": "help", "exe_env": "general", "source": "url_source", "logic": { "Regex": [ { "field": "url", "pattern": ".*", 'variable': True, 'variable_name': 'url' } ], 'Range': [ { 'field': 'status_code', 'min': 199, 'max': 201 } ] }, 'constants': { 'test': 'ver' } } ) proto.load(reloadConf=True) jobid = ioc.getCollection('SourceData').insert_one({ 'grease_data': { 'sourcing': { 'server': ObjectId(ioc.getConfig().NodeIdentity) }, 'detection': { 'server': ObjectId(ioc.getConfig().NodeIdentity), 'start': datetime.datetime.utcnow(), 'end': datetime.datetime.utcnow(), 'detection': {} }, 'scheduling': { 'server': ObjectId(ioc.getConfig().NodeIdentity), 'start': datetime.datetime.utcnow(), 'end': datetime.datetime.utcnow(), }, 'execution': { 'server': ObjectId(ioc.getConfig().NodeIdentity), 'assignmentTime': datetime.datetime.utcnow(), 'completeTime': None, 'returnData': {}, 'executionSuccess': False, 'commandSuccess': False, 'failures': 0 } }, 'source': 'dev', 'configuration': 'exe_test', 'data': {}, 'createTime': datetime.datetime.utcnow(), 'expiry': Deduplication.generate_max_expiry_time(1) }).inserted_id # Run for a bit self.assertTrue(cmd.server()) self.assertTrue(cmd.drain_jobs(ioc.getCollection('SourceData'))) result = ioc.getCollection('SourceData').find_one({'_id': ObjectId(jobid)}) self.assertTrue(result) self.assertTrue(result.get('grease_data').get('execution').get('executionSuccess')) self.assertTrue(result.get('grease_data').get('execution').get('commandSuccess')) ioc.getCollection('SourceData').drop() ioc.getCollection('Configuration').drop()
def test_real(self): ############################################# # SETUP UP TIME ############################################# ioc = GreaseContainer() pConf = PrototypeConfig(ioc) ioc.ensureRegistration() ioc.getCollection('JobServer').update_one( {'_id': ObjectId(ioc.getConfig().NodeIdentity)}, {'$set': { 'prototypes': ['scan', 'detect', 'schedule'] }}) ioc.getCollection('Configuration').insert_one({ 'active': True, 'type': 'prototype_config', "name": "full_stack_test", "job": "help", "exe_env": "general", "source": "url_source", "url": ['http://google.com'], "logic": { "Regex": [{ "field": "url", "pattern": ".*", 'variable': True, 'variable_name': 'url' }], 'Range': [{ 'field': 'status_code', 'min': 199, 'max': 201 }] }, 'constants': { 'test': 'ver' } }) pConf.load(reloadConf=True) ############################################# # EXECUTE SCANNING ############################################# Scanner = scan() Scanner.ioc.getLogger().getConfig().set('verbose', True, 'Logging') Scanner.ioc.getLogger().getConfig().set('trace', True, 'Logging') Scanner.ioc.getLogger().getConfig().set('config', 'full_stack_test', 'Sourcing') self.assertTrue(Scanner.execute({'loop': 1})) ############################################# # ASSERT SCANNING ############################################# self.assertTrue( ioc.getCollection('SourceData').find_one({ 'grease_data.sourcing.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.detection.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.detection.start': None, 'grease_data.detection.end': None })) ############################################# # EXECUTE DETECTION ############################################# Detect = detect() Detect.ioc.getLogger().getConfig().set('verbose', True, 'Logging') Detect.ioc.getLogger().getConfig().set('trace', True, 'Logging') Detect.ioc.getLogger().getConfig().set('config', 'full_stack_test', 'Sourcing') self.assertTrue(Detect.execute({'loop': 1})) ############################################# # ASSERT DETECTION ############################################# self.assertTrue( ioc.getCollection('SourceData').find_one({ 'grease_data.sourcing.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.detection.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.scheduling.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.scheduling.start': None, 'grease_data.scheduling.end': None })) ############################################# # EXECUTE SCHEDULING ############################################# Scheduling = schedule() Scheduling.ioc.getLogger().getConfig().set('verbose', True, 'Logging') Scheduling.ioc.getLogger().getConfig().set('trace', True, 'Logging') Scheduling.ioc.getLogger().getConfig().set('config', 'full_stack_test', 'Sourcing') self.assertTrue(Scheduling.execute({'loop': 1})) ############################################# # ASSERT SCHEDULING ############################################# self.assertTrue( ioc.getCollection('SourceData').find_one({ 'grease_data.sourcing.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.detection.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.scheduling.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.execution.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.execution.start': None, 'grease_data.execution.end': None })) ############################################# # EXECUTE JOBS ############################################# ioc.getCollection('JobServer').update_one( {'_id': ObjectId(ioc.getConfig().NodeIdentity)}, {'$set': { 'prototypes': [] }}) Daemon = DaemonProcess(ioc) Daemon.ioc.getLogger().getConfig().set('verbose', True, 'Logging') Daemon.ioc.getLogger().getConfig().set('trace', True, 'Logging') Daemon.ioc.getLogger().getConfig().set('config', 'full_stack_test', 'Sourcing') self.assertTrue(Daemon.server()) self.assertTrue(Daemon.drain_jobs(ioc.getCollection('SourceData'))) ############################################# # ASSERT JOB EXECUTION ############################################# # sleep a few for seconds to let help complete time.sleep(5) self.assertTrue( ioc.getCollection('SourceData').find_one({ 'grease_data.sourcing.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.detection.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.scheduling.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.execution.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.execution.commandSuccess': True, 'grease_data.execution.executionSuccess': True })) ############################################# # CLEAN UP TIME ############################################# ioc.getCollection('JobServer').update_one( {'_id': ObjectId(ioc.getConfig().NodeIdentity)}, {'$set': { 'prototypes': [] }}) ioc.getCollection('Configuration').drop() ioc.getCollection('SourceData').drop() ioc.getCollection('DeDup_Sourcing').drop() pConf.load(reloadConf=True)