def dbdata(connStr): utility.write_to_file( config.ConfigManager().LogFile, 'a', 'req_desc_file_extract running' + ' ' + str(datetime.datetime.now())) cursor = dbmanager.cursor_odbc_connection(connStr) query = custom.fetch_query(config.ConfigManager().STReqDocQueryId) configdocs = custom.retrieve_data_from_DB( int(config.ConfigManager().MongoDBPort), config.ConfigManager().DataCollectionDB, config.ConfigManager().ConfigCollection) query = query.replace('##jobDocumentID##', str(configdocs[0]['STreqDocpk'])) db_data_dict = dbmanager.cursor_execute(cursor, query) db_data = db_data_dict['dbdata'] db_data_cursorexec = db_data_dict['cursor_exec'] cursor_description = db_data_cursorexec.description column_headers = [column[0] for column in cursor_description] count = 0 data_dict = {} connection = dbmanager.mongoDB_connection( int(config.ConfigManager().MongoDBPort)) for row in db_data: try: data_dict = dict(utility.zip_list(column_headers, row)) reqdescfilepath = config.ConfigManager().ReqDocDirectory + '/' + '_' + \ str(data_dict['requirementID']) + '_' + \ str(data_dict['documentName']) filewrite = open(reqdescfilepath, 'wb') filewrite.write(data_dict['jobDocumentFile']) count += 1 except BaseException as ex: exception_message = '\n' + 'Exception:' + \ str(datetime.datetime.now()) + '\n' exception_message += 'File: ' + '\n' exception_message += '\n' + str(ex) + '\n' exception_message += '-' * 100 utility.write_to_file(config.ConfigManager().LogFile, 'a', exception_message) if 'jobDocumentID' in data_dict: primary_key = data_dict['jobDocumentID'] UpdateTemplateSet = utility.clean_dict() UpdateTemplateWhere = utility.clean_dict() UpdateTemplateSet['STreqDocpk'] = primary_key reqDocUpdateDelta = int(primary_key) - \ int(configdocs[0]['STreqDocpk']) utility.write_to_file( config.ConfigManager().LogFile, 'a', 'Req docs updated - ' + str(count) + ' ' + str(datetime.datetime.now())) UpdateTemplateWhere['_id'] = configdocs[0]['_id'] DBSet = utility.clean_dict() DBSet['$set'] = UpdateTemplateSet custom.update_data_to_Db_noupsert( int(config.ConfigManager().MongoDBPort), config.ConfigManager().DataCollectionDB, config.ConfigManager().ConfigCollection, UpdateTemplateWhere, DBSet, connection)
def updateconfigcollection(docid, dateTime, whereID): connection = dbmanager.mongoDB_connection( int(config.ConfigManager().MongoDBPort)) UpdateTemplateWhere = utility.clean_dict() UpdateTemplateSet = utility.clean_dict() UpdateTemplateWhere['_id'] = whereID UpdateTemplateSet['masterDocId'] = docid UpdateTemplateSet['stagingDateModified'] = dateTime DBSet = utility.clean_dict() DBSet['$set'] = UpdateTemplateSet custom.update_data_to_Db_noupsert( int(config.ConfigManager().MongoDBPort), config.ConfigManager().RatesDB, config.ConfigManager().RatesConfigCollection, UpdateTemplateWhere, DBSet, connection)
def generate_nounphrase_insert_into_db(data): global count try: status = "{:<8}".format(str(count)) + " :" status += str(datetime.datetime.now()) count += 1 mongoport = int(config.ConfigManager().MongoDBPort) col = config.ConfigManager().IntelligenceDataCollection desc = data['desc'] noun_phrases = dcrnlp.extract_nounphrases_sentences(desc) UpdateTemplateWhere = utility.clean_dict() UpdateTemplateSet = utility.clean_dict() DBSet = utility.clean_dict() UpdateTemplateWhere['_id'] = data['_id'] UpdateTemplateSet['nounPhrases'] = noun_phrases UpdateTemplateSet['description'] = desc DBSet['$set'] = UpdateTemplateSet status += " |" + str(datetime.datetime.now()) custom.update_data_to_Db_con(mongoport, config.ConfigManager().IntelligenceDb, col, UpdateTemplateWhere, DBSet, data['connection']) status += " |" + str(datetime.datetime.now()) status += " :" + "{:<9}".format(str(data['doc_id'])) print(status) except BaseException as ex: exception_message = '\n' + 'Exception:' + '\n' str(datetime.datetime.now()) + '\n' exception_message += 'File: ' + '\n' exception_message += '\n' + str(ex) + '\n' exception_message += '-' * 100 utility.write_to_file(dcrconfig.ConfigManager().SemanticGraphLogFile, 'a', exception_message)
def req_desc_file_append(): utility.write_to_file( config.ConfigManager().LogFile, 'a', 'req_desc_file_append running' + ' ' + str(datetime.datetime.now())) DBWhereConditon = utility.clean_dict() DBWhereConditon['documentType'] = 'job details' DBWhereConditon['dataSource'] = 'Smart Track' DBWhereConditon['isReqFileDescNew'] = 1 DBProjection = utility.clean_dict() DBProjection['descriptionOld'] = 1 DBProjection['reqFileDesc'] = 1 DBProjection['description'] = 1 description = '' connection = dbmanager.mongoDB_connection( int(config.ConfigManager().MongoDBPort)) descriptionUpdateCount = 0 docs = custom.retrieve_rowdata_from_DB_projection( int(config.ConfigManager().MongoDBPort), config.ConfigManager().DataCollectionDB, config.ConfigManager().STReqCollection, DBWhereConditon, DBProjection) UpdateTemplateWhere = utility.clean_dict() UpdateTemplateSet = utility.clean_dict() for doc in docs: try: if not 'descriptionAppend' in doc: if not doc['descriptionOld'] is None: print('Inside if') if 'reqFileDesc' in doc: description = doc['descriptionOld'] + \ '. ' + doc['reqFileDesc'] else: description = doc['descriptionOld'] UpdateTemplateSet = utility.clean_dict() UpdateTemplateSet['description'] = description UpdateTemplateSet['descriptionAppend'] = 1 descriptionUpdateCount += 1 else: print('Inside else') description = doc['reqFileDesc'] UpdateTemplateSet = utility.clean_dict() UpdateTemplateSet['description'] = description UpdateTemplateSet['descriptionAppend'] = 1 descriptionUpdateCount += 1 UpdateTemplateSet['matchIndexProcess'] = 0 UpdateTemplateWhere = utility.clean_dict() UpdateTemplateWhere['_id'] = doc['_id'] DBSet = utility.clean_dict() DBSet['$set'] = UpdateTemplateSet custom.update_data_to_Db_con( int(config.ConfigManager().MongoDBPort), config.ConfigManager().DataCollectionDB, config.ConfigManager().STReqCollection, UpdateTemplateWhere, DBSet, connection) except BaseException as ex: utility.log_exception(ex) utility.write_to_file( config.ConfigManager().LogFile, 'a', 'Number of descriptions updated - ' + str(descriptionUpdateCount) + ' ' + str(datetime.datetime.now()))
from collections import OrderedDict from pyexcel_ods3 import save_data import os orderedDict = collections.OrderedDict() data = OrderedDict() # global variable declaration totalrecords = 0 invalidrecords = 0 emptydesc = 0 incompletedesc = 0 smalldesc = 0 nonedesc = 0 nodesc = 0 totaljobsdict = utility.clean_dict() jobsitedict = utility.clean_dict() listdata_uniqueids = [] current_date = (str(datetime.datetime.now()).split(" "))[0] analysis_file = config.ConfigManager().PCDataAnalysisResultsFile.replace( '.ods', current_date + '.ods').replace('/mnt/nlpdata/', '') def analyze_data(filepaths): global totalrecords global invalidrecords global emptydesc global incompletedesc global smalldesc global nonedesc global nodesc
'industry data read running' + ' ' + str(datetime.datetime.now())) connection = dbmanager.mongoDB_connection( int(config.ConfigManager().MongoDBPort)) configdocs = custom.retrieve_data_from_DB( int(config.ConfigManager().MongoDBPort), config.ConfigManager().RatesDB, config.ConfigManager().RatesConfigCollection) industryID = int(configdocs[0]['industryID']) print(industryID) industryID = custom.master_data_transfer_from_sql( config.ConfigManager().STConnStr, config.ConfigManager().IndustryQueryID, config.ConfigManager().industryDetails, config.ConfigManager().ST, industryID, config.ConfigManager().ExternalHost) UpdateTemplateWhere = utility.clean_dict() UpdateTemplateSet = utility.clean_dict() UpdateTemplateWhere['_id'] = configdocs[0]['_id'] if industryID != 0: UpdateTemplateSet['industryID'] = industryID else: UpdateTemplateSet['industryID'] = int(configdocs[0]['industryID']) DBSet = utility.clean_dict() DBSet['$set'] = UpdateTemplateSet custom.update_data_to_Db_noupsert( int(config.ConfigManager().MongoDBPort), config.ConfigManager().RatesDB, config.ConfigManager().RatesConfigCollection, UpdateTemplateWhere, DBSet, connection)
def job_info_analysis(page, filepath, dbrecordcount): global totalrecords global invalidrecords global emptydesc global incompletedesc global smalldesc global nonedesc global nodesc global totaljobsdict global jobsitedict # Fetching current config paramters configdocs = custom.retrieve_data_from_DB(int(config.ConfigManager(). MongoDBPort), config.ConfigManager(). IntelligenceDb, config.ConfigManager(). ConfigCollection) docid_count = int(configdocs[0]['docid_count']) dict_object_record_list = [] for jobinfo in page.findall('record'): try: # creating dictionary from xml tag contents dict_object = utility.xml_to_dict(ET.tostring(jobinfo)) totaljobsdict = fill_job_by_site(filepath) totalrecords += 1 # outer if check is jobdescription tag is in the xml if 'jobdescription' in (dict_object['record']): # checking if job description is none if ((dict_object['record'])['jobdescription'] is not None): # variable to determine if record needs to be # updated in DB incorrectjobdescription = 0 # checking if job description is empty if (((dict_object['record'])['jobdescription']) .strip()) == '': write_fileinfo(filepath, dict_object) invalidrecords += 1 emptydesc += 1 incorrectjobdescription = 1 jobsitedict = fill_job_site_data(filepath) # checking if job desc has less than 20 chars if (len(((dict_object['record'])['jobdescription']) ) < 20): incorrectjobdescription = 1 # eliminating the incomplete desc case if (((dict_object['record'])['jobdescription']) .strip()[-3:]) == '...': print('Do nothing') else: write_fileinfo(filepath, dict_object) invalidrecords += 1 smalldesc += 1 jobsitedict = fill_job_site_data(filepath) # checking the incomplete desc case if (((dict_object['record'])['jobdescription']) .strip()[-3:]) == '...': incorrectjobdescription = 1 write_fileinfo(filepath, dict_object) invalidrecords += 1 incompletedesc += 1 jobsitedict = fill_job_site_data(filepath) if (incorrectjobdescription == 0): docid_count += 1 (dict_object['record'])['doc_id'] = docid_count (dict_object['record'])['description'] = ((dict_object['record'])['jobdescription']) (dict_object['record'])['nounPhrases'] = "" dict_object_record_list.append(dict_object['record']) dbrecordcount += 1 # checking if job description is none if (dict_object['record'])['jobdescription'] is None: write_fileinfo(filepath, dict_object) invalidrecords += 1 nonedesc += 1 jobsitedict = fill_job_site_data(filepath) else: write_fileinfo(filepath, dict_object) invalidrecords += 1 nodesc += 1 jobsitedict = fill_job_site_data(filepath) except BaseException as ex: utility.log_exception_file(ex, dcrconfig.ConfigManager().SemanticGraphLogFile) if dict_object_record_list: insert_to_db(dict_object_record_list) # updating doc_id in config table UpdateTemplateWhere = utility.clean_dict() UpdateTemplateSet = utility.clean_dict() UpdateTemplateWhere['_id'] = configdocs[0]['_id'] UpdateTemplateSet['docid_count'] = docid_count DBSet = utility.clean_dict() DBSet['$set'] = UpdateTemplateSet custom.update_data_to_Db_noupsert(int(config.ConfigManager().MongoDBPort), config.ConfigManager().IntelligenceDb, config.ConfigManager().ConfigCollection, UpdateTemplateWhere, DBSet, connection) return dbrecordcount
def stcandidate_update(): utility.write_to_file( config.ConfigManager().LogFile, 'a', 'in stcandidates update currency Code running.!' + ' ' + str(datetime.datetime.now())) recordnumber = 0 query = custom.fetch_query( config.ConfigManager().STCandidateCurrencyQueryId) print(query) query = custom.query_variable_replace( query, config.ConfigManager().STCandidateCurrencyDetails, config.ConfigManager().ST) print(query) cursor = dbmanager.cursor_odbc_connection(config.ConfigManager().STConnStr) db_data_dict = dbmanager.cursor_execute(cursor, query) db_data = db_data_dict['dbdata'] db_data_cursorexec = db_data_dict['cursor_exec'] cursor_description = db_data_cursorexec.description column_headers = [column[0] for column in cursor_description] connection = dbmanager.mongoDB_connection( int(config.ConfigManager().MongoDBPort)) data_dict1 = {} req_list = [] candidateDatesList = [] for row1 in db_data: try: print(data_dict1) strtimestamp = str(datetime.datetime.now()) recordnumber += 1 print(recordnumber) data_dict1 = dict(utility.zip_list(column_headers, row1)) STCandidateCollection.update( { "$and": [{ "candidateid": data_dict1['candidateid'] }, { "requirementRateStatusList": { "$elemMatch": { "requirementId": data_dict1['requirementid'] } } }] }, { "$set": { "requirementRateStatusList.$.currencyCode": data_dict1['currencycode'], "requirementRateStatusList.$.SupplierCurrencyCode": data_dict1['SupplierCurrencyCode'], "requirementRateStatusList.$.supplierRegBillRateEX": str(data_dict1['supplierRegBillRateEX']) } }) candidateDatesList.append(data_dict1['dateCreated']) except BaseException as ex: print(ex) utility.log_exception_file(ex, config.ConfigManager().LogFile) if 'dateCreated' in data_dict1: maxCandDate = max(candidateDatesList) UpdateTemplateWhere = utility.clean_dict() UpdateTemplateSet = utility.clean_dict() UpdateTemplateWhere['_id'] = configdocs[0]['_id'] print(maxCandDate) print(str(maxCandDate)) UpdateTemplateSet['STCandidateCurrencyCodeLastDate'] = str(maxCandDate) DBSet = utility.clean_dict() DBSet['$set'] = UpdateTemplateSet custom.update_data_to_Db_noupsert( int(config.ConfigManager().MongoDBPort), config.ConfigManager().DataCollectionDB, config.ConfigManager().ConfigCollection, UpdateTemplateWhere, DBSet, connection)
def route_dataread(filepaths): file_count = 0 docreadcount = 0 antiwordemptycount = 0 connection = dbmanager.mongoDB_connection( int(config.ConfigManager().MongoDBPort)) utility.write_to_file( config.ConfigManager().LogFile, 'a', 'resumeread running' + ' ' + str(datetime.datetime.now())) for filepath in filepaths: strtimestamp = str(datetime.datetime.now()) data_text = '' try: file_count += 1 if filepath[-4:].lower() == ".txt": data_text = datareadfiletypes.read_text_text( filepath, data_text) elif filepath[-4:].lower() == ".pdf": data_text = datareadfiletypes.read_pdf_text( filepath, data_text) elif filepath[-5:].lower() == ".docx": data_text = datareadfiletypes.read_docx_text( filepath, data_text) elif filepath[-4:].lower() == ".doc": data_text = datareadfiletypes.read_doc_text( filepath, data_text) if data_text == '': antiwordemptycount += 1 data_text = datareadfiletypes.read_doc_text_catdoc( filepath, data_text) if data_text == '': utility.write_to_file(config.ConfigManager().LogFile, 'a', 'Filepath is: ' + filepath) elif filepath[-4:].lower() == ".xls": data_text = datareadfiletypes.read_excel_text( filepath, data_text) elif filepath[-5:].lower() == ".xlsx": data_text = datareadfiletypes.read_excel_text( filepath, data_text) elif filepath[-4:].lower() == ".csv": data_text = datareadfiletypes.read_csv_text( filepath, data_text) elif filepath[-4:].lower() == ".odt": data_text = datareadfiletypes.read_odt_text( filepath, data_text) elif filepath[-4:].lower() == ".xml": docid_count = custom.process_xml_data(filepath, docid_count) strtimestamp += ' ' + str(datetime.datetime.now()) if not data_text == '': filepath = filepath.replace('\\', '') file_name = utility.filename_from_filepath(filepath) candidateId = (file_name[1:]).split('_')[0] print(candidateId) UpdateTemplateWhere = utility.clean_dict() UpdateTemplateSet = utility.clean_dict() UpdateTemplateWhere['candidateid'] = int(candidateId) # UpdateTemplateWhere['documentType'] = 'candidate details' # UpdateTemplateWhere['dataSource'] = 'Smart Track' UpdateTemplateSet['resumeText'] = data_text UpdateTemplateSet['isResumeTextNew'] = 1 DBSet = utility.clean_dict() DBSet['$set'] = UpdateTemplateSet custom.update_data_to_Db_noupsert( int(config.ConfigManager().MongoDBPort), config.ConfigManager().DataCollectionDB, config.ConfigManager().STCandidateCollection, UpdateTemplateWhere, DBSet, connection) strtimestamp += ' ' + str(datetime.datetime.now()) if filepath[-4:].lower() == ".doc": docreadcount += 1 print('Total doc read count:' + str(docreadcount)) print('File : ' + str(file_count) + ' ' + strtimestamp) print('Antiword empty count:' + str(antiwordemptycount)) except BaseException as ex: exception_message = '\n' + 'Exception:' + \ str(datetime.datetime.now()) + '\n' exception_message += 'File: ' + '\n' exception_message += '\n' + str(ex) + '\n' exception_message += '-' * 100 # .encode('utf8')) utility.write_to_file(config.ConfigManager().LogFile, 'a', exception_message) utility.write_to_file( config.ConfigManager().LogFile, 'a', 'Number of resumes read - ' + str(file_count) + ' ' + str(datetime.datetime.now()))
def dbdata(connStr): utility.write_to_file( config.ConfigManager().LogFile, 'a', 'resume_extract running' + ' ' + str(datetime.datetime.now())) st_db_name_list = utility.find_string_inbetween( config.ConfigManager().STConnStr, "DATABASE=", ";UID") cursor = dbmanager.cursor_odbc_connection(connStr) query = custom.fetch_query( config.ConfigManager().STCandidateResumesQueryId) configdocs = custom.retrieve_data_from_DB( int(config.ConfigManager().MongoDBPort), config.ConfigManager().DataCollectionDB, config.ConfigManager().ConfigCollection) query = query.replace('##candidateResumeID##', str(configdocs[0]['STcandidateResumepk'])).replace( '##STDB##', st_db_name_list[0]) db_data_dict = dbmanager.cursor_execute(cursor, query) db_data = db_data_dict['dbdata'] db_data_cursorexec = db_data_dict['cursor_exec'] cursor_description = db_data_cursorexec.description column_headers = [column[0] for column in cursor_description] count = 0 data_dict = {} connection = dbmanager.mongoDB_connection( int(config.ConfigManager().MongoDBPort)) for row in db_data: try: data_dict = dict(utility.zip_list(column_headers, row)) resumefilepath = config.ConfigManager().ResumeDirectory + '/' + '_' + \ str(data_dict['candidateID']) + '_' + \ str(data_dict['documentName']) filewrite = open(resumefilepath, 'wb') f = io.BytesIO(data_dict['ResumeFile']) filewrite.write(data_dict['ResumeFile']) filefilepath = config.ConfigManager().fileDirectory + '/' + '_' + \ str(data_dict['candidateResumeID'])+'-'+str(data_dict['supplierID']) + '_' + \ str(data_dict['documentName']) file_write = open(filefilepath, 'wb') file_write.write(data_dict['ResumeFile']) except BaseException as ex: exception_message = '\n' + 'Exception:' + \ str(datetime.datetime.now()) + '\n' exception_message += 'File: ' + '\n' exception_message += '\n' + str(ex) + '\n' exception_message += '-' * 100 utility.write_to_file(config.ConfigManager().LogFile, 'a', exception_message) if 'candidateResumeID' in data_dict: primary_key = data_dict['candidateResumeID'] UpdateTemplateWhere = utility.clean_dict() UpdateTemplateSet = utility.clean_dict() UpdateTemplateSet['STcandidateResumepk'] = primary_key resumeUpdateDelta = int(primary_key) - \ int(configdocs[0]['STcandidateResumepk']) utility.write_to_file( config.ConfigManager().LogFile, 'a', 'Resumes updated - ' + str(resumeUpdateDelta) + ' ' + str(datetime.datetime.now())) UpdateTemplateWhere['_id'] = configdocs[0]['_id'] DBSet = utility.clean_dict() DBSet['$set'] = UpdateTemplateSet custom.update_data_to_Db_noupsert( int(config.ConfigManager().MongoDBPort), config.ConfigManager().DataCollectionDB, config.ConfigManager().ConfigCollection, UpdateTemplateWhere, DBSet, connection)
def requirement_update(): utility.write_to_file( config.ConfigManager().LogFile, 'a', 'requpdatefastest running' + ' ' + str(datetime.datetime.now())) recordnumber = 0 configdocs = custom.retrieve_data_from_DB( int(config.ConfigManager().MongoDBPort), config.ConfigManager().DataCollectionDB, config.ConfigManager().ConfigCollection) query = custom.fetch_query( config.ConfigManager().STCandidateSubmissionsQueryId) if '.' in str(configdocs[0]['submissionsdateCreated']): query = query.replace( '##dateCreated##', str(configdocs[0]['submissionsdateCreated']).split('.')[0] + ('.') + str(configdocs[0]['submissionsdateCreated']).split('.')[1][:3]) else: query = query.replace('##dateCreated##', str(configdocs[0]['submissionsdateCreated'])) # query = query.replace('##dateCreated##', str((datetime.datetime.now() - datetime.timedelta(minutes=480))).split('.')[0] + ('.') + str((datetime.datetime.now() - datetime.timedelta(minutes=480))).split('.')[1][:3]) print(query) cursor = dbmanager.cursor_odbc_connection(config.ConfigManager().STConnStr) db_data_dict = dbmanager.cursor_execute(cursor, query) db_data = db_data_dict['dbdata'] db_data_cursorexec = db_data_dict['cursor_exec'] cursor_description = db_data_cursorexec.description column_headers = [column[0] for column in cursor_description] connection = dbmanager.mongoDB_connection( int(config.ConfigManager().MongoDBPort)) data_dict1 = {} req_list = [] for row1 in db_data: try: strtimestamp = str(datetime.datetime.now()) recordnumber += 1 print(recordnumber) data_dict1 = dict(utility.zip_list(column_headers, row1)) if not (data_dict1['requirementID']).strip(): requirementIDListperCandidate = [] reqratelist = [] reqratestatdictlist = [] else: reqratelist = [] reqratestatdictlist = [] requirementIDListperCandidate = [ reqID for reqID in (data_dict1['requirementID']).split(',') ] for reqratecombID in requirementIDListperCandidate: reqratelistinit = [ reqrateID for reqrateID in (reqratecombID.strip().replace( '~!@- ', '~!@-')).split('~!@-') ] print(reqratelistinit[0]) reqratestatdict = {} req_list.append(int(reqratelistinit[1])) if reqratelistinit[0] == "0.00": reqratelistinit[0] = "" reqratestatdict['rate'] = reqratelistinit[0] reqratestatdict['requirementId'] = int(reqratelistinit[1]) reqratestatdict['candidateStatus'] = reqratelistinit[2] reqratelist.append(reqratelistinit) reqratestatdictlist.append(reqratestatdict) strtimestamp += ' ' + str(datetime.datetime.now()) UpdateTemplateSet = utility.clean_dict() UpdateTemplateWhere = utility.clean_dict() UpdateTemplateSet[ 'requirementIDList'] = reqratelist # requirementIDListperCandidate UpdateTemplateSet[ 'requirementRateStatusList'] = reqratestatdictlist UpdateTemplateWhere['candidateid'] = data_dict1['CandidateID'] UpdateTemplateWhere['documentType'] = 'candidate details' UpdateTemplateWhere['dataSource'] = 'Smart Track' DBSet = utility.clean_dict() DBSet['$set'] = UpdateTemplateSet print(UpdateTemplateSet['requirementRateStatusList']) custom.update_data_to_Db_noupsert( int(config.ConfigManager().MongoDBPort), config.ConfigManager().DataCollectionDB, config.ConfigManager().STCandidateCollection, UpdateTemplateWhere, DBSet, connection) strtimestamp += ' ' + str(datetime.datetime.now()) except BaseException as ex: utility.log_exception(ex) if 'dateCreated' in data_dict1: if not data_dict1['dateCreated'] is None: UpdateTemplateSet = utility.clean_dict() UpdateTemplateWhere = utility.clean_dict() UpdateTemplateSet['submissionsdateCreated'] = data_dict1[ 'dateCreated'] UpdateTemplateWhere['_id'] = configdocs[0]['_id'] DBSet = utility.clean_dict() DBSet['$set'] = UpdateTemplateSet custom.update_data_to_Db_noupsert( int(config.ConfigManager().MongoDBPort), config.ConfigManager().DataCollectionDB, config.ConfigManager().ConfigCollection, UpdateTemplateWhere, DBSet, connection) if req_list: print(list(set(req_list))) utility.write_to_file( config.ConfigManager().LogFile, 'a', 'requirement id list - ' + str(json.dumps(list(set(req_list)))) + ' ' + str(datetime.datetime.now())) db_requirements_candidate.generate_req_candidate_file_selected_req( list(set(req_list))) utility.write_to_file( config.ConfigManager().LogFile, 'a', 'Number of records for which requirement list was updated - ' + str(recordnumber) + ' ' + str(datetime.datetime.now()))