def test_registration(self): ioc = GreaseContainer() cmd = DaemonProcess(ioc) self.assertTrue(cmd.register()) collection = ioc.getMongo().Client().get_database('grease').get_collection('JobServer') self.assertTrue(collection.find({}).count()) del collection del cmd del ioc
def test_prototype_execution(self): ioc = GreaseContainer() cmd = DaemonProcess(ioc) # add search path fil = open(ioc.getConfig().greaseConfigFile, 'r') data = json.loads(fil.read()) fil.close() fil = open(ioc.getConfig().greaseConfigFile, 'w') data['Import']['searchPath'].append('tgt_grease.router.Commands.tests') fil.write(json.dumps(data, sort_keys=True, indent=4)) fil.close() Configuration.ReloadConfig() # Update Node to run it ioc.getCollection('JobServer')\ .update_one( {'_id': ObjectId(ioc.getConfig().NodeIdentity)}, { '$set': { 'prototypes': ['TestProtoType'] } } ) # Sleeps are because mongo in Travis is slow sometimes to persist data time.sleep(1.5) self.assertTrue(cmd.server()) self.assertTrue(cmd.drain_jobs(ioc.getCollection('JobQueue'))) # ensure jobs drain out time.sleep(1.5) self.assertEqual( ioc.getCollection('TestProtoType').find({ 'runs': { '$exists': True } }).count(), 10) # clean up fil = open(ioc.getConfig().greaseConfigFile, 'r') data = json.loads(fil.read()) fil.close() # remove collection ioc.getCollection('TestProtoType').drop() # pop search path trash = data['Import']['searchPath'].pop() # close out fil = open(ioc.getConfig().greaseConfigFile, 'w') fil.write(json.dumps(data, sort_keys=True, indent=4)) fil.close() ioc.getCollection('JobServer') \ .update_one( {'_id': ObjectId(ioc.getConfig().NodeIdentity)}, { '$set': { 'prototypes': [] } } )
def test_job_execution(self): ioc = GreaseContainer() cmd = DaemonProcess(ioc) proto = PrototypeConfig(ioc) ioc.getCollection('Configuration').insert_one( { 'active': True, 'type': 'prototype_config', "name": "exe_test", "job": "help", "exe_env": "general", "source": "url_source", "logic": { "Regex": [ { "field": "url", "pattern": ".*", 'variable': True, 'variable_name': 'url' } ], 'Range': [ { 'field': 'status_code', 'min': 199, 'max': 201 } ] }, 'constants': { 'test': 'ver' } } ) proto.load(reloadConf=True) jobid = ioc.getCollection('SourceData').insert_one({ 'grease_data': { 'sourcing': { 'server': ObjectId(ioc.getConfig().NodeIdentity) }, 'detection': { 'server': ObjectId(ioc.getConfig().NodeIdentity), 'start': datetime.datetime.utcnow(), 'end': datetime.datetime.utcnow(), 'detection': {} }, 'scheduling': { 'server': ObjectId(ioc.getConfig().NodeIdentity), 'start': datetime.datetime.utcnow(), 'end': datetime.datetime.utcnow(), }, 'execution': { 'server': ObjectId(ioc.getConfig().NodeIdentity), 'assignmentTime': datetime.datetime.utcnow(), 'completeTime': None, 'returnData': {}, 'executionSuccess': False, 'commandSuccess': False, 'failures': 0 } }, 'source': 'dev', 'configuration': 'exe_test', 'data': {}, 'createTime': datetime.datetime.utcnow(), 'expiry': Deduplication.generate_max_expiry_time(1) }).inserted_id # Run for a bit self.assertTrue(cmd.server()) self.assertTrue(cmd.drain_jobs(ioc.getCollection('SourceData'))) result = ioc.getCollection('SourceData').find_one({'_id': ObjectId(jobid)}) self.assertTrue(result) self.assertTrue(result.get('grease_data').get('execution').get('executionSuccess')) self.assertTrue(result.get('grease_data').get('execution').get('commandSuccess')) ioc.getCollection('SourceData').drop() ioc.getCollection('Configuration').drop()
def test_real(self): ############################################# # SETUP UP TIME ############################################# ioc = GreaseContainer() pConf = PrototypeConfig(ioc) ioc.ensureRegistration() ioc.getCollection('JobServer').update_one( {'_id': ObjectId(ioc.getConfig().NodeIdentity)}, {'$set': { 'prototypes': ['scan', 'detect', 'schedule'] }}) ioc.getCollection('Configuration').insert_one({ 'active': True, 'type': 'prototype_config', "name": "full_stack_test", "job": "help", "exe_env": "general", "source": "url_source", "url": ['http://google.com'], "logic": { "Regex": [{ "field": "url", "pattern": ".*", 'variable': True, 'variable_name': 'url' }], 'Range': [{ 'field': 'status_code', 'min': 199, 'max': 201 }] }, 'constants': { 'test': 'ver' } }) pConf.load(reloadConf=True) ############################################# # EXECUTE SCANNING ############################################# Scanner = scan() Scanner.ioc.getLogger().getConfig().set('verbose', True, 'Logging') Scanner.ioc.getLogger().getConfig().set('trace', True, 'Logging') Scanner.ioc.getLogger().getConfig().set('config', 'full_stack_test', 'Sourcing') self.assertTrue(Scanner.execute({'loop': 1})) ############################################# # ASSERT SCANNING ############################################# self.assertTrue( ioc.getCollection('SourceData').find_one({ 'grease_data.sourcing.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.detection.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.detection.start': None, 'grease_data.detection.end': None })) ############################################# # EXECUTE DETECTION ############################################# Detect = detect() Detect.ioc.getLogger().getConfig().set('verbose', True, 'Logging') Detect.ioc.getLogger().getConfig().set('trace', True, 'Logging') Detect.ioc.getLogger().getConfig().set('config', 'full_stack_test', 'Sourcing') self.assertTrue(Detect.execute({'loop': 1})) ############################################# # ASSERT DETECTION ############################################# self.assertTrue( ioc.getCollection('SourceData').find_one({ 'grease_data.sourcing.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.detection.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.scheduling.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.scheduling.start': None, 'grease_data.scheduling.end': None })) ############################################# # EXECUTE SCHEDULING ############################################# Scheduling = schedule() Scheduling.ioc.getLogger().getConfig().set('verbose', True, 'Logging') Scheduling.ioc.getLogger().getConfig().set('trace', True, 'Logging') Scheduling.ioc.getLogger().getConfig().set('config', 'full_stack_test', 'Sourcing') self.assertTrue(Scheduling.execute({'loop': 1})) ############################################# # ASSERT SCHEDULING ############################################# self.assertTrue( ioc.getCollection('SourceData').find_one({ 'grease_data.sourcing.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.detection.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.scheduling.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.execution.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.execution.start': None, 'grease_data.execution.end': None })) ############################################# # EXECUTE JOBS ############################################# ioc.getCollection('JobServer').update_one( {'_id': ObjectId(ioc.getConfig().NodeIdentity)}, {'$set': { 'prototypes': [] }}) Daemon = DaemonProcess(ioc) Daemon.ioc.getLogger().getConfig().set('verbose', True, 'Logging') Daemon.ioc.getLogger().getConfig().set('trace', True, 'Logging') Daemon.ioc.getLogger().getConfig().set('config', 'full_stack_test', 'Sourcing') self.assertTrue(Daemon.server()) self.assertTrue(Daemon.drain_jobs(ioc.getCollection('SourceData'))) ############################################# # ASSERT JOB EXECUTION ############################################# # sleep a few for seconds to let help complete time.sleep(5) self.assertTrue( ioc.getCollection('SourceData').find_one({ 'grease_data.sourcing.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.detection.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.scheduling.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.execution.server': ObjectId(ioc.getConfig().NodeIdentity), 'grease_data.execution.commandSuccess': True, 'grease_data.execution.executionSuccess': True })) ############################################# # CLEAN UP TIME ############################################# ioc.getCollection('JobServer').update_one( {'_id': ObjectId(ioc.getConfig().NodeIdentity)}, {'$set': { 'prototypes': [] }}) ioc.getCollection('Configuration').drop() ioc.getCollection('SourceData').drop() ioc.getCollection('DeDup_Sourcing').drop() pConf.load(reloadConf=True)