def test_writeContactHistory_currentFieldInc():
# Drop contactHistory before new UT
db.contactHistory.drop()
# Create ContactHistory with _currentField
dataOutOldRec = dwmAll(data=test_records.record_writeContactHistory_historyCurrent1, db=db, configName='test_writeContactHistory_writeConfig')
# Create new ContactHistory with _currentField and increment old record.
dataOutNextRec = dwmAll(data=test_records.record_writeContactHistory_historyCurrent2, db=db, configName='test_writeContactHistory_writeConfig')
# Get old rec with incremented _current
hist = db.contactHistory.find_one({"_id": dataOutOldRec[0]['historyId']})
assert hist['_current'] == 1
def test_writeContactHistory_writeConfig():
dataOut = dwmAll(data = test_records.record_writeContactHistory_writeConfig, db = db, configName='test_writeContactHistory_writeConfig')
hist = db.contactHistory.find_one({"_id": dataOut[0]['historyId']})
assert hist['configName'] == 'test_writeContactHistory_writeConfig'
def test_udf_beforeGenericValidation():
dataOut = dwmAll(data = test_records.record_udf_beforeGenericValidation, db = db, configName='test_udf_beforeGenericValidation', udfNamespace=__name__)
assert dataOut[0]['field1'] == 'goodvalue'
def test_lookupAll_genericLookup_notChecked():
dataOut = dwmAll(data = test_records.record_lookupAll_genericLookup_notChecked, db = db, configName='test_lookupAll_genericLookup')
assert dataOut[0]['field2'] != ''
def test_history_normRegex_caught():
dataOut = dwmAll(data = test_records.history_normRegex_caught, db = db, configName='test_regexAll_normRegex')
hist = db.contactHistory.find_one({"_id": dataOut[0]['historyId']})
assert hist['field1']['normRegex']['from'] == 'badvalue'
def test_verbose():
dataOut = dwmAll(data = test_records.record_lookupAll_genericLookup_caught, db = db, configName='test_lookupAll_genericLookup', verbose=True)
assert dataOut[0]['field1'] == ''
def test_dwmAll_requiresOneConfig():
dataOut = dwmAll(data = test_records.record_dwmAll_noConfig, db = db, config={"configName": "testConfig"}, configName="testConfig", udfNamespace=__name__)
def test_udf_beforeDeriveData():
dataOut = dwmAll(data = test_records.record_udf_beforeDeriveData, db = db, configName='test_udf_beforeDeriveData', udfNamespace=__name__)
assert dataOut[0]['field1'] == 'goodvalue'
def test_udf_sort():
dataOut = dwmAll(data = test_records.record_udf_sort, db = db, configName='test_udf_sort', udfNamespace=__name__)
assert dataOut[0]['field1'] == 'goodvalue'
def test_history_deriveIncludes_ends_caught():
dataOut = dwmAll(data = test_records.history_deriveIncludes_ends_caught, db = db, configName='test_deriveAll_deriveIncludes')
hist = db.contactHistory.find_one({"_id": dataOut[0]['historyId']})
assert hist['field1']['deriveIncludes']['from'] == ''
def test_history_deriveIncludes_notChecked():
dataOut = dwmAll(data = test_records.history_deriveIncludes_notChecked, db = db, configName='test_deriveAll_deriveIncludes')
hist = db.contactHistory.find_one({"_id": dataOut[0]['historyId']})
assert 'field3' not in hist.keys()
def test_history_deriveRegex_uncaught():
dataOut = dwmAll(data = test_records.history_deriveRegex_uncaught, db = db, configName='test_deriveAll_deriveRegex')
hist = db.contactHistory.find_one({"_id": dataOut[0]['historyId']})
assert 'field1' not in hist.keys()
def test_history_deriveRegex_blankIfNoMatch():
dataOut = dwmAll(data = test_records.history_deriveRegex_blankIfNoMatch, db = db, configName='test_deriveAll_deriveRegex_blankIfNoMatch')
hist = db.contactHistory.find_one({"_id": dataOut[0]['historyId']})
assert hist['field1']['deriveRegex']['pattern'] == 'no matching pattern'
def test_history_copyValue():
dataOut = dwmAll(data = test_records.history_copyValue, db = db, configName='test_deriveAll_copyValue')
hist = db.contactHistory.find_one({"_id": dataOut[0]['historyId']})
assert hist['field1']['copyValue']['from'] == ''
def test_history_normRegex_notChecked():
dataOut = dwmAll(data = test_records.history_normRegex_notChecked, db = db, configName='test_regexAll_normRegex')
hist = db.contactHistory.find_one({"_id": dataOut[0]['historyId']})
assert 'field2' not in hist.keys()
def test_history_normRegex_uncaught():
# do stuff for testing
dataOut = dwmAll(data = test_records.history_normRegex_uncaught, db = db, configName='test_regexAll_normRegex')
hist = db.contactHistory.find_one({"_id": dataOut[0]['historyId']})
assert 'field1' not in hist.keys()
def test_udf_beforeFieldSpecificRegex():
dataOut = dwmAll(data = test_records.record_udf_beforeFieldSpecificRegex, db = db, configName='test_udf_beforeFieldSpecificRegex', udfNamespace=__name__)
assert dataOut[0]['field1'] == 'goodvalue'
def test_history_deriveIncludes_overwriteFalse():
dataOut = dwmAll(data = test_records.history_deriveIncludes_overwriteFalse, db = db, configName='test_deriveAll_deriveIncludes_overwriteFalse')
hist = db.contactHistory.find_one({"_id": dataOut[0]['historyId']})
assert 'field1' not in hist.keys()
def test_udf_beforeNormalizationIncludes():
dataOut = dwmAll(data = test_records.record_udf_beforeNormalizationIncludes, db = db, configName='test_udf_beforeNormalizationIncludes', udfNamespace=__name__)
assert dataOut[0]['field1'] == 'goodvalue'
def test_history_deriveIncludes_blankIfNoMatch():
dataOut = dwmAll(data = test_records.history_deriveIncludes_blankIfNoMatch, db = db, configName='test_deriveAll_deriveIncludes_blankIfNoMatch')
hist = db.contactHistory.find_one({"_id": dataOut[0]['historyId']})
assert hist['field1']['deriveIncludes']['using']['blankIfNoMatch'] == 'no match found'
def test_udf_afterProcessing():
dataOut = dwmAll(data = test_records.record_udf_afterProcessing, db = db, configName='test_udf_afterProcessing', udfNamespace=__name__)
assert dataOut[0]['field1'] == 'goodvalue'
def test_returnHistoryId_False():
dataOut = dwmAll(data = test_records.record_returnHistoryId_False, db = db, configName='test_returnHistoryId_False')
assert 'historyId' not in dataOut[0].keys()
def test_udf_afterProcessing_invalidFcn():
dataOut = dwmAll(data = test_records.record_udf_afterProcessing_invalidFcn, db = db, configName='test_udf_afterProcessing_invalidFcn', udfNamespace=__name__)
def test_writeContactHistory_False():
dataOut = dwmAll(data = test_records.record_writeContactHistory_False, db = db, configName='test_writeContactHistory_False')
assert 'historyId' not in dataOut[0].keys()
def test_dwmAll_requiresConfigInput():
dataOut = dwmAll(data = test_records.record_dwmAll_noConfig, db = db, udfNamespace=__name__)
def test_history_normIncludes_caught():
# do stuff for testing
dataOut = dwmAll(data = test_records.history_normIncludes_included_caught, db = db, configName='test_normIncludes')
hist = db.contactHistory.find_one({"_id": dataOut[0]['historyId']})
assert hist['field1']['normIncludes']['from'] == 'findgoodinvaluejunk'
def test_lookupAll_genericLookup_uncaught():
dataOut = dwmAll(data = test_records.record_lookupAll_genericLookup_uncaught, db = db, configName='test_lookupAll_genericLookup')
assert dataOut[0]['field1'] != ''
def test_configDoesNotExist():
dataOut = dwmAll(data = test_records.record_configDoesNotExist, db = db, configName='test_configDoesNotExist')
clientConfig.close()
###############################################################################
## Run the DWM
###############################################################################
## connect to mongo
client = MongoClient(os.environ['MONGODB_URL'])
db = client['dwmdev']
logging.info("Connected to mongo")
## Run DWM
dwmStart = datetime.now()
dataOut = dwm.dwmAll(data=job,
db=db,
config=config,
udfNamespace=__name__,
verbose=True)
dwmEnd = datetime.now()
client.close()
###############################################################################
## Put them into the processedQueue; remove from exportQueue
###############################################################################
indicatorQueue.add(dataOut, transfer=True)
exportQueue.complete(job)
else:
def test_history_normIncludes_uncaught():
dataOut = dwmAll(data = test_records.history_normIncludes_included_uncaught, db = db, configName='test_normIncludes')
hist = db.contactHistory.find_one({"_id": dataOut[0]['historyId']})
assert 'field1' not in hist.keys()