def __init__(self, Logger=None):
if Logging and isinstance(Logger, Logging):
self.ioc = GreaseContainer(Logger)
else:
self.ioc = GreaseContainer()
self.variable_storage = self.ioc.getMongo()\
.Client()\
.get_database(self.ioc.getConfig().get('Connectivity', 'MongoDB').get('db', 'grease'))\
.get_collection(self.__class__.__name__)
self.start_time = datetime.utcnow()
self.exec_data = {'execVal': False, 'retVal': False, 'data': {}}
def __init__(self, ioc=None):
if ioc and isinstance(ioc, GreaseContainer):
self.ioc = ioc
else:
self.ioc = GreaseContainer()
self.conf = PrototypeConfig(self.ioc)
self.configs = []
def __init__(self, ioc=None):
if isinstance(ioc, GreaseContainer):
self.ioc = ioc
else:
self.ioc = GreaseContainer()
self.imp = ImportTool(self.ioc.getLogger())
self.monitor = NodeMonitoring(self.ioc)
def __init__(self, ioc=None):
if isinstance(ioc, GreaseContainer):
self.ioc = ioc
else:
self.ioc = GreaseContainer()
self.centralScheduler = Scheduling(self.ioc)
self.scheduler = Scheduler(self.ioc)
def test_object_score_low_duplication(self):
obj1 = {
'field1': 'value',
'field2': 'value1',
'field3': 'value2',
'field4': 'value3',
'field5': 'value4'
}
obj2 = {
'field1': str(uuid.uuid4()),
'field2': str(uuid.uuid4()),
'field3': str(uuid.uuid4()),
'field4': str(uuid.uuid4()),
'field5': str(uuid.uuid4())
}
ioc = GreaseContainer()
parent1 = ioc.getCollection('test_scoring').insert_one({
'expiry':
Deduplication.generate_expiry_time(1),
'max_expiry':
Deduplication.generate_max_expiry_time(1),
'type':
1,
'score':
1,
'source':
'test_source',
'hash':
Deduplication.generate_hash_from_obj(obj1)
}).inserted_id
score1 = Deduplication.object_field_score('test_scoring', ioc,
'test_source',
'test_configuration', obj1,
parent1, 1, 1)
parent2 = ioc.getCollection('test_scoring').insert_one({
'expiry':
Deduplication.generate_expiry_time(1),
'max_expiry':
Deduplication.generate_max_expiry_time(1),
'type':
1,
'score':
1,
'source':
'test_source',
'hash':
Deduplication.generate_hash_from_obj(obj2)
}).inserted_id
score2 = Deduplication.object_field_score('test_scoring', ioc,
'test_source',
'test_configuration', obj2,
parent2, 1, 1)
print("++++++++++++++++++++++++++++++++++")
print("score1: {0}".format(score1))
print("score2: {0}".format(score2))
print("++++++++++++++++++++++++++++++++++")
self.assertTrue(score1 == 0.0)
self.assertTrue(score2 <= 20.0)
ioc.getCollection('test_scoring').drop()
time.sleep(1.5)
def __init__(self, *args, **kwargs):
TestCase.__init__(self, *args, **kwargs)
self.configuration = None
self.enabled = False
self.mock_data = {}
self.expected_data = {}
self.ioc = GreaseContainer()
self.detect = Detect(self.ioc)
def __init__(self, ioc=None):
global GREASE_PROTOTYPE_CONFIGURATION
if ioc and isinstance(ioc, GreaseContainer):
self.ioc = ioc
else:
self.ioc = GreaseContainer()
if not GREASE_PROTOTYPE_CONFIGURATION:
GREASE_PROTOTYPE_CONFIGURATION = self.load()
def __init__(self, ioc=None):
if ioc and isinstance(ioc, GreaseContainer):
self.ioc = ioc
else:
self.ioc = GreaseContainer()
self.conf = PrototypeConfig(self.ioc)
self.impTool = ImportTool(self.ioc.getLogger())
self.dedup = Deduplication(self.ioc)
self.scheduler = Scheduling(self.ioc)
def test_registration(self):
ioc = GreaseContainer()
cmd = DaemonProcess(ioc)
self.assertTrue(cmd.register())
collection = ioc.getMongo().Client().get_database('grease').get_collection('JobServer')
self.assertTrue(collection.find({}).count())
del collection
del cmd
del ioc
def __init__(self, ioc):
if isinstance(ioc, GreaseContainer):
self.ioc = ioc
else:
self.ioc = GreaseContainer()
self.current_real_second = datetime.utcnow().second
if self.ioc.getConfig().NodeIdentity == "Unknown" and not self.register():
self.registered = False
self.impTool = ImportTool(self.ioc.getLogger())
self.conf = PrototypeConfig(self.ioc)
def test_prototype_execution(self):
ioc = GreaseContainer()
cmd = DaemonProcess(ioc)
# add search path
fil = open(ioc.getConfig().greaseConfigFile, 'r')
data = json.loads(fil.read())
fil.close()
fil = open(ioc.getConfig().greaseConfigFile, 'w')
data['Import']['searchPath'].append('tgt_grease.router.Commands.tests')
fil.write(json.dumps(data, sort_keys=True, indent=4))
fil.close()
Configuration.ReloadConfig()
# Update Node to run it
ioc.getCollection('JobServer')\
.update_one(
{'_id': ObjectId(ioc.getConfig().NodeIdentity)},
{
'$set': {
'prototypes': ['TestProtoType']
}
}
)
# Sleeps are because mongo in Travis is slow sometimes to persist data
time.sleep(1.5)
self.assertTrue(cmd.server())
self.assertTrue(cmd.drain_jobs(ioc.getCollection('JobQueue')))
# ensure jobs drain out
time.sleep(1.5)
self.assertEqual(
ioc.getCollection('TestProtoType').find({
'runs': {
'$exists': True
}
}).count(), 10)
# clean up
fil = open(ioc.getConfig().greaseConfigFile, 'r')
data = json.loads(fil.read())
fil.close()
# remove collection
ioc.getCollection('TestProtoType').drop()
# pop search path
trash = data['Import']['searchPath'].pop()
# close out
fil = open(ioc.getConfig().greaseConfigFile, 'w')
fil.write(json.dumps(data, sort_keys=True, indent=4))
fil.close()
ioc.getCollection('JobServer') \
.update_one(
{'_id': ObjectId(ioc.getConfig().NodeIdentity)},
{
'$set': {
'prototypes': []
}
}
)
def setUp(self):
self.ioc = GreaseContainer()
self.good_config = {
"source": "kafka",
"max_backlog": 20,
"min_backlog": 5,
"servers": ["server"],
"topics": ["topic"]
}
self.bad_config = {"source": "not kafka"}
self.mock_thread = MagicMock()
self.ks = KafkaSource()
def test_empty_conf(self):
ioc = GreaseContainer()
# clean up
for root, dirnames, filenames in os.walk(ioc.getConfig().get('Configuration', 'dir')):
for filename in fnmatch.filter(filenames, '*.config.json'):
self.assertIsNone(os.remove(os.path.join(root, filename)))
conf = PrototypeConfig()
conf.load(reloadConf=True)
self.assertTrue(conf.getConfiguration())
self.assertListEqual(conf.getConfiguration().get('configuration').get('pkg'), [])
self.assertListEqual(conf.getConfiguration().get('configuration').get('fs'), [])
self.assertListEqual(conf.getConfiguration().get('configuration').get('mongo'), [])
conf.load(reloadConf=True)
def test_empty_source_schedule(self):
ioc = GreaseContainer()
sch = Scheduling(ioc)
jServer = ioc.getCollection('JobServer')
jID = jServer.insert_one({
'jobs': 0,
'os': platform.system().lower(),
'roles': ["general"],
'prototypes': ["detect"],
'active': True,
'activationTime': datetime.datetime.utcnow()
}).inserted_id
time.sleep(1.5)
self.assertFalse(sch.scheduleDetection('test', 'test_conf', []))
jServer.delete_one({'_id': ObjectId(jID)})
ioc.getCollection('SourceData').drop()
def parse_source(self, configuration):
"""This will Query the SQL Server to find data
Args:
configuration (dict): Configuration of Source. See Class Documentation above for more info
Returns:
bool: If True data will be scheduled for ingestion after deduplication. If False the engine will bail out
"""
ioc = GreaseContainer()
if configuration.get('hour'):
if datetime.datetime.utcnow().hour != int(configuration.get('hour')):
# it is not the correct hour
return True
if configuration.get('minute'):
if datetime.datetime.utcnow().minute != int(configuration.get('minute')):
# it is not the correct hour
return True
if configuration.get('type') != 'postgresql':
ioc.getLogger().error("Unsupported SQL Server Type; Currently Only supporting PostgreSQL", notify=False)
return False
else:
# Attempt to get the DSN for the connection
if os.environ.get(configuration.get('dsn')) and configuration.get('query'):
# ensure the DSN is setup and the query is present
try:
DSN = os.environ.get(configuration.get('dsn'))
with psycopg2.connect(DSN) as conn:
with conn.cursor(cursor_factory=RealDictCursor) as cursor:
cursor.execute(configuration.get('query'))
data = cursor.fetchall()
for row in data:
self._data.append(row)
del ioc
return True
except Exception as e:
# Naked except to prevent issues around connections
ioc.getLogger().error("Error processing configuration; Error [{0}]".format(e.message), notify=False)
del ioc
return False
else:
# could not get the DSN
ioc.getLogger().error("Failed to locate the DSN variable", notify=False)
del ioc
return False
def test_deduplicate_object(self):
ioc = GreaseContainer()
ioc.getConfig().set('verbose', True, 'Logging')
obj = [{
'field': 'var',
'field1': 'var1',
'field2': 'var2',
'field3': 'var3',
'field4': 'var4',
'field5': 'var5',
}, {
'field': 'var',
'field1': 'var1',
'field2': 'var2',
'field3': 'var3',
'field4': 'var4',
'field5': 'var5',
}, {
'field': str(uuid.uuid4()),
'field1': 'var1',
'field2': str(uuid.uuid4()),
'field3': str(uuid.uuid4()),
'field4': 'var4',
'field5': str(uuid.uuid4()),
}]
finalObj = []
Deduplication.deduplicate_object(ioc, obj[0], 1, 1, 40.0,
'test_source', 'test_configuration',
finalObj, 'test_source')
self.assertEqual(len(finalObj), 1)
Deduplication.deduplicate_object(ioc, obj[1], 1, 1, 40.0,
'test_source', 'test_configuration',
finalObj, 'test_source')
self.assertEqual(len(finalObj), 1)
Deduplication.deduplicate_object(ioc, obj[2], 1, 1, 40.0,
'test_source', 'test_configuration',
finalObj, 'test_source')
self.assertGreaterEqual(len(finalObj), 1)
ioc.getConfig().set('verbose', False, 'Logging')
ioc.getCollection('test_source').drop()
time.sleep(1.5)
def test_logger(self):
ioc = GreaseContainer()
self.assertTrue(isinstance(ioc.getLogger(), Logging))
def test_registration(self):
ioc = GreaseContainer()
self.assertTrue(ioc.ensureRegistration())
def test_get_collection(self):
ioc = GreaseContainer()
self.assertTrue(isinstance(ioc.getMongo(), Mongo))
coll = ioc.getCollection('TestCollection')
self.assertTrue(isinstance(coll, Collection))
self.assertEqual(coll.name, "TestCollection")
def test_mongo(self):
ioc = GreaseContainer()
self.assertTrue(isinstance(ioc.getMongo(), Mongo))
def test_notifications(self):
ioc = GreaseContainer()
self.assertTrue(isinstance(ioc.getNotification(), Notifications))
def test_config(self):
ioc = GreaseContainer()
self.assertTrue(isinstance(ioc.getConfig(), Configuration))
def test_pkg_load_bad(self):
ioc = GreaseContainer()
# clean up
for root, dirnames, filenames in os.walk(pkg_resources.resource_filename('tgt_grease.enterprise.Model', 'config/')):
for filename in fnmatch.filter(filenames, '*.config.json'):
self.assertIsNone(os.remove(os.path.join(root, filename)))
configList = [
{
"name": "test1",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "badtest1",
"exe_env": "windows",
"source": "stackOverflow",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test3",
"job": "fakeJob",
"exe_env": "windows",
"source": "Google",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
],
"exists": [
{
"field": "var"
}
]
}
}
]
GoodConfigList = [
{
"name": "test1",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test3",
"job": "fakeJob",
"exe_env": "windows",
"source": "Google",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
],
"exists": [
{
"field": "var"
}
]
}
}
]
i = 0
for conf in configList:
with open(pkg_resources.resource_filename('tgt_grease.enterprise.Model', 'config/') + 'conf{0}.config.json'.format(i), 'w') as fil:
fil.write(json.dumps(conf, indent=4))
i += 1
conf = PrototypeConfig(ioc)
conf.load(reloadConf=True)
self.assertEqual(len(conf.getConfiguration().get('configuration').get('pkg')), len(GoodConfigList))
self.assertEqual(len(conf.getConfiguration().get('raw')), len(GoodConfigList))
self.assertEqual(len(conf.getConfiguration().get('source').get('swapi')), 1)
self.assertEqual(len(conf.getConfiguration().get('source').get('Google')), 1)
self.assertEqual(2, len(conf.get_sources()))
self.assertEqual(2, len(conf.get_names()))
self.assertEqual(len(conf.get_source('Google')), 1)
self.assertTrue(isinstance(conf.get_config('test1'), dict))
self.assertTrue(conf.get_config('test1'))
# clean up
for root, dirnames, filenames in os.walk(pkg_resources.resource_filename('tgt_grease.enterprise.Model', 'config/')):
for filename in fnmatch.filter(filenames, '*.config.json'):
self.assertIsNone(os.remove(os.path.join(root, filename)))
# clear the config
conf.load(reloadConf=True)
def test_scan(self):
# setup
configList = [
{
"name": "test1",
"job": "fakeJob",
"exe_env": "windows",
"source": "TestSource",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
}
]
ioc = GreaseContainer()
ioc.ensureRegistration()
ioc.getConfig().set('trace', True, 'Logging')
ioc.getConfig().set('verbose', True, 'Logging')
fil = open(ioc.getConfig().greaseConfigFile, 'r')
data = json.loads(fil.read())
fil.close()
fil = open(ioc.getConfig().greaseConfigFile, 'w')
data['Import']['searchPath'].append('tgt_grease.enterprise.Model.tests')
fil.write(json.dumps(data, sort_keys=True, indent=4))
fil.close()
Configuration.ReloadConfig()
jServer = ioc.getCollection('JobServer')
jID1 = jServer.insert_one({
'jobs': 0,
'os': platform.system().lower(),
'roles': ["general"],
'prototypes': ["detect"],
'active': True,
'activationTime': datetime.utcnow()
}).inserted_id
time.sleep(1)
jID2 = jServer.insert_one({
'jobs': 0,
'os': platform.system().lower(),
'roles': ["general"],
'prototypes': ["detect"],
'active': True,
'activationTime': datetime.utcnow()
}).inserted_id
# Begin Test
conf = PrototypeConfig(ioc)
conf.load(reloadConf=True, ConfigurationList=configList)
scanner = Scan(ioc)
# Scan Environment
self.assertTrue(scanner.Parse())
# Begin ensuring environment is how we expect
# we assert less or equal because sometimes uuid's are close :p
self.assertLessEqual(ioc.getCollection('SourceData').find({
'detectionServer': ObjectId(jID1)
}).count(), 3)
self.assertLessEqual(ioc.getCollection('SourceData').find({
'detectionServer': ObjectId(jID2)
}).count(), 3)
self.assertLessEqual(ioc.getCollection('JobServer').find_one({
'_id': ObjectId(jID1)
})['jobs'], 3)
self.assertLessEqual(ioc.getCollection('JobServer').find_one({
'_id': ObjectId(jID2)
})['jobs'], 3)
# clean up
fil = open(ioc.getConfig().greaseConfigFile, 'r')
data = json.loads(fil.read())
fil.close()
# remove collection
ioc.getCollection('TestProtoType').drop()
# remove prototypes
data['NodeInformation']['ProtoTypes'] = []
# pop search path
trash = data['Import']['searchPath'].pop()
# close out
fil = open(ioc.getConfig().greaseConfigFile, 'w')
fil.write(json.dumps(data, sort_keys=True, indent=4))
fil.close()
jServer.delete_one({'_id': ObjectId(jID1)})
jServer.delete_one({'_id': ObjectId(jID2)})
ioc.getCollection('SourceData').drop()
ioc.getCollection('Dedup_Sourcing').drop()
ioc.getConfig().set('trace', False, 'Logging')
ioc.getConfig().set('verbose', False, 'Logging')
Configuration.ReloadConfig()
def __init__(self, ioc=None):
if isinstance(ioc, GreaseContainer):
self.ioc = ioc
else:
self.ioc = GreaseContainer()
def test_all_load_bad(self):
ioc = GreaseContainer()
# clean up
for root, dirnames, filenames in os.walk(ioc.getConfig().get('Configuration', 'dir')):
for filename in fnmatch.filter(filenames, '*.config.json'):
self.assertIsNone(os.remove(os.path.join(root, filename)))
# clean up
for root, dirnames, filenames in os.walk(pkg_resources.resource_filename('tgt_grease.enterprise.Model', 'config/')):
for filename in fnmatch.filter(filenames, '*.config.json'):
self.assertIsNone(os.remove(os.path.join(root, filename)))
configList = [
{
"name": "test1",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "badtest1",
"exe_env": "windows",
"source": "stackOverflow",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test3",
"job": "fakeJob",
"exe_env": "windows",
"source": "Google",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
],
"exists": [
{
"field": "var"
}
]
}
}
]
GoodConfigList = [
{
"name": "test1",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test3",
"job": "fakeJob",
"exe_env": "windows",
"source": "Google",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
],
"exists": [
{
"field": "var"
}
]
}
}
]
i = 0
length = len(configList) - 1
while i <= length:
if i == 0:
with open(ioc.getConfig().get('Configuration', 'dir') + 'conf{0}.config.json'.format(i), 'w') as fil:
fil.write(json.dumps(configList[i], indent=4))
if i == 1:
with open(pkg_resources.resource_filename('tgt_grease.enterprise.Model',
'config/') + 'conf{0}.config.json'.format(i), 'w') as fil:
fil.write(json.dumps(configList[i], indent=4))
if i == 2:
ioc.getCollection('Configuration').insert_one(configList[i])
i += 1
ioc.getCollection('Configuration').update_many({}, {'$set': {'active': True, 'type': 'prototype_config'}})
# sleep because travis is slow
time.sleep(1.5)
conf = PrototypeConfig(ioc)
conf.load(reloadConf=True)
self.assertEqual(len(conf.getConfiguration().get('configuration').get('mongo')), 1)
self.assertEqual(len(conf.getConfiguration().get('configuration').get('pkg')), 0)
self.assertEqual(len(conf.getConfiguration().get('configuration').get('fs')), 1)
self.assertEqual(len(conf.getConfiguration().get('raw')), len(GoodConfigList))
self.assertEqual(len(conf.getConfiguration().get('source').get('swapi')), 1)
self.assertEqual(len(conf.getConfiguration().get('source').get('Google')), 1)
self.assertEqual(2, len(conf.get_names()))
self.assertEqual(len(conf.get_source('Google')), 1)
self.assertTrue(isinstance(conf.get_config('test1'), dict))
self.assertTrue(conf.get_config('test1'))
# clean up
ioc.getCollection('Configuration').drop()
for root, dirnames, filenames in os.walk(ioc.getConfig().get('Configuration', 'dir')):
for filename in fnmatch.filter(filenames, '*.config.json'):
self.assertIsNone(os.remove(os.path.join(root, filename)))
# clean up
for root, dirnames, filenames in os.walk(pkg_resources.resource_filename('tgt_grease.enterprise.Model', 'config/')):
for filename in fnmatch.filter(filenames, '*.config.json'):
self.assertIsNone(os.remove(os.path.join(root, filename)))
# clear the config
conf.load(reloadConf=True)
def __init__(self, ioc=None):
if isinstance(ioc, GreaseContainer):
self.ioc = ioc
else:
self.ioc = GreaseContainer()
self.ioc.ensureRegistration()
def test_detectionScheduling(self):
ioc = GreaseContainer()
ioc.ensureRegistration()
sch = Scheduling(ioc)
jServer = ioc.getCollection('JobServer')
jID1 = jServer.insert_one({
'jobs': 0,
'os': platform.system().lower(),
'roles': ["general"],
'prototypes': ["detect"],
'active': True,
'activationTime': datetime.datetime.utcnow()
}).inserted_id
time.sleep(1)
jID2 = jServer.insert_one({
'jobs': 0,
'os': platform.system().lower(),
'roles': ["general"],
'prototypes': ["detect"],
'active': True,
'activationTime': datetime.datetime.utcnow()
}).inserted_id
time.sleep(1)
self.assertTrue(
sch.scheduleDetection('test', 'test_conf', [
{
'test0': 'var0',
'test1': 'var1',
'test2': 'var2',
'test3': 'var3',
'test4': 'var4',
'test5': 'var5',
'test6': 'var6',
'test7': 'var7',
'test8': 'var8',
'test9': 'var9',
'test10': 'var10',
},
{
'test0': 'var0',
'test1': 'var1',
'test2': 'var2',
'test3': 'var3',
'test4': 'var4',
'test5': 'var5',
'test6': 'var6',
'test7': 'var7',
'test8': 'var8',
'test9': 'var9',
'test10': 'var10',
},
{
'test0': 'var0',
'test1': 'var1',
'test2': 'var2',
'test3': 'var3',
'test4': 'var4',
'test5': 'var5',
'test6': 'var6',
'test7': 'var7',
'test8': 'var8',
'test9': 'var9',
'test10': 'var10',
},
{
'test0': 'var0',
'test1': 'var1',
'test2': 'var2',
'test3': 'var3',
'test4': 'var4',
'test5': 'var5',
'test6': 'var6',
'test7': 'var7',
'test8': 'var8',
'test9': 'var9',
'test10': 'var10',
},
{
'test0': 'var0',
'test1': 'var1',
'test2': 'var2',
'test3': 'var3',
'test4': 'var4',
'test5': 'var5',
'test6': 'var6',
'test7': 'var7',
'test8': 'var8',
'test9': 'var9',
'test10': 'var10',
},
{
'test0': 'var0',
'test1': 'var1',
'test2': 'var2',
'test3': 'var3',
'test4': 'var4',
'test5': 'var5',
'test6': 'var6',
'test7': 'var7',
'test8': 'var8',
'test9': 'var9',
'test10': 'var10',
},
]))
time.sleep(1)
self.assertEqual(
ioc.getCollection('SourceData').find({
'grease_data.detection.server':
ObjectId(jID1)
}).count(), 3)
self.assertEqual(
ioc.getCollection('SourceData').find({
'grease_data.detection.server':
ObjectId(jID2)
}).count(), 3)
self.assertEqual(
ioc.getCollection('JobServer').find_one({'_id':
ObjectId(jID1)})['jobs'],
3)
self.assertEqual(
ioc.getCollection('JobServer').find_one({'_id':
ObjectId(jID2)})['jobs'],
3)
jServer.delete_one({'_id': ObjectId(jID1)})
jServer.delete_one({'_id': ObjectId(jID2)})
ioc.getCollection('SourceData').drop()
def test_deduplication(self):
ioc = GreaseContainer()
dedup = Deduplication(ioc)
ioc.getConfig().set('verbose', True, 'Logging')
obj = [{
'field': 'var',
'field1': 'var1',
'field2': 'var2',
'field3': 'var3',
'field4': 'var4',
'field5': 'var5',
}, {
'field': 'var',
'field1': 'var1',
'field2': 'var2',
'field3': 'var3',
'field4': 'var4',
'field5': 'var5',
}, {
'field': 'var',
'field1': 'var1',
'field2': 'var2',
'field3': 'var3',
'field4': 'var4',
'field5': 'var5',
}, {
'field': 'var',
'field1': 'var1',
'field2': 'var2',
'field3': 'var3',
'field4': 'var4',
'field5': 'var5',
}, {
'field': str(uuid.uuid4()),
'field1': str(uuid.uuid4()),
'field2': str(uuid.uuid4()),
'field3': str(uuid.uuid4()),
'field4': str(uuid.uuid4()),
'field5': str(uuid.uuid4())
}, {
'field': 'var',
'field1': 'var1',
'field2': 'var2',
'field3': 'var3',
'field4': 'var4',
'field5': 'var5',
}, {
'field': 'var',
'field1': 'var1',
'field2': 'var2',
'field3': 'var3',
'field4': 'var4',
'field5': 'var5',
}, {
'field': 'var',
'field1': 'var1',
'field2': 'var2',
'field3': 'var3',
'field4': 'var4',
'field5': 'var5',
}, {
'field': str(uuid.uuid4()),
'field1': str(uuid.uuid4()),
'field2': str(uuid.uuid4()),
'field3': str(uuid.uuid4()),
'field4': str(uuid.uuid4()),
'field5': str(uuid.uuid4())
}, {
'field': str(uuid.uuid4()),
'field1': str(uuid.uuid4()),
'field2': str(uuid.uuid4()),
'field3': str(uuid.uuid4()),
'field4': str(uuid.uuid4()),
'field5': str(uuid.uuid4())
}, {
'field': str(uuid.uuid4()),
'field1': str(uuid.uuid4()),
'field2': str(uuid.uuid4()),
'field3': str(uuid.uuid4()),
'field4': str(uuid.uuid4()),
'field5': str(uuid.uuid4())
}]
finalObj = dedup.Deduplicate(obj, 'test_source', 'test_configuration',
85.0, 1, 1, 'test_source')
self.assertGreaterEqual(len(finalObj), 4)
ioc.getConfig().set('verbose', False, 'Logging')
ioc.getCollection('test_source').drop()
time.sleep(1.5)
def test_mongo_load_bad(self):
ioc = GreaseContainer()
# clean up
for root, dirnames, filenames in os.walk(ioc.getConfig().get('Configuration', 'dir')):
for filename in fnmatch.filter(filenames, '*.config.json'):
self.assertIsNone(os.remove(os.path.join(root, filename)))
configList = [
{
"name": "test1",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "badtest1",
"exe_env": "windows",
"source": "stackOverflow",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test3",
"job": "fakeJob",
"exe_env": "windows",
"source": "Google",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
],
"exists": [
{
"field": "var"
}
]
}
}
]
GoodConfigList = [
{
"name": "test1",
"job": "fakeJob",
"exe_env": "windows",
"source": "swapi",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
]
}
},
{
"name": "test3",
"job": "fakeJob",
"exe_env": "windows",
"source": "Google",
"logic": {
"regex": [
{
"field": "character",
"pattern": ".*skywalker.*"
}
],
"exists": [
{
"field": "var"
}
]
}
}
]
for conf in configList:
ioc.getCollection('Configuration').insert_one(conf)
ioc.getCollection('Configuration').update_many({}, {'$set': {'active': True, 'type': 'prototype_config'}})
# sleep because travis is slow sometimes
time.sleep(1.5)
conf = PrototypeConfig(ioc)
conf.load(reloadConf=True)
self.assertEqual(len(conf.getConfiguration().get('configuration').get('mongo')), len(GoodConfigList))
self.assertEqual(len(conf.getConfiguration().get('raw')), len(GoodConfigList))
self.assertEqual(len(conf.getConfiguration().get('source').get('swapi')), 1)
self.assertEqual(len(conf.getConfiguration().get('source').get('Google')), 1)
self.assertEqual(2, len(conf.get_sources()))
self.assertEqual(2, len(conf.get_names()))
self.assertEqual(len(conf.get_source('Google')), 1)
self.assertTrue(isinstance(conf.get_config('test1'), dict))
self.assertTrue(conf.get_config('test1'))
# clean up
ioc.getCollection('Configuration').drop()
# clear the config
conf.load(reloadConf=True)