def dbdata(connStr):
utility.write_to_file(
config.ConfigManager().LogFile, 'a',
'req_desc_file_extract running' + ' ' + str(datetime.datetime.now()))
cursor = dbmanager.cursor_odbc_connection(connStr)
query = custom.fetch_query(config.ConfigManager().STReqDocQueryId)
configdocs = custom.retrieve_data_from_DB(
int(config.ConfigManager().MongoDBPort),
config.ConfigManager().DataCollectionDB,
config.ConfigManager().ConfigCollection)
query = query.replace('##jobDocumentID##',
str(configdocs[0]['STreqDocpk']))
db_data_dict = dbmanager.cursor_execute(cursor, query)
db_data = db_data_dict['dbdata']
db_data_cursorexec = db_data_dict['cursor_exec']
cursor_description = db_data_cursorexec.description
column_headers = [column[0] for column in cursor_description]
count = 0
data_dict = {}
connection = dbmanager.mongoDB_connection(
int(config.ConfigManager().MongoDBPort))
for row in db_data:
try:
data_dict = dict(utility.zip_list(column_headers, row))
reqdescfilepath = config.ConfigManager().ReqDocDirectory + '/' + '_' + \
str(data_dict['requirementID']) + '_' + \
str(data_dict['documentName'])
filewrite = open(reqdescfilepath, 'wb')
filewrite.write(data_dict['jobDocumentFile'])
count += 1
except BaseException as ex:
exception_message = '\n' + 'Exception:' + \
str(datetime.datetime.now()) + '\n'
exception_message += 'File: ' + '\n'
exception_message += '\n' + str(ex) + '\n'
exception_message += '-' * 100
utility.write_to_file(config.ConfigManager().LogFile, 'a',
exception_message)
if 'jobDocumentID' in data_dict:
primary_key = data_dict['jobDocumentID']
UpdateTemplateSet = utility.clean_dict()
UpdateTemplateWhere = utility.clean_dict()
UpdateTemplateSet['STreqDocpk'] = primary_key
reqDocUpdateDelta = int(primary_key) - \
int(configdocs[0]['STreqDocpk'])
utility.write_to_file(
config.ConfigManager().LogFile, 'a', 'Req docs updated - ' +
str(count) + ' ' + str(datetime.datetime.now()))
UpdateTemplateWhere['_id'] = configdocs[0]['_id']
DBSet = utility.clean_dict()
DBSet['$set'] = UpdateTemplateSet
custom.update_data_to_Db_noupsert(
int(config.ConfigManager().MongoDBPort),
config.ConfigManager().DataCollectionDB,
config.ConfigManager().ConfigCollection, UpdateTemplateWhere,
DBSet, connection)
def update_DB(configdocs, latestdate):
connection = dbmanager.mongoDB_connection(
int(config.ConfigManager().MongoDBPort))
dictionaries.UpdateTemplateSet = {}
dictionaries.UpdateTemplateWhere = {}
dictionaries.UpdateTemplateSet['PClastDate'] = latestdate
dictionaries.UpdateTemplateWhere['_id'] = configdocs[0]['_id']
dictionaries.DBSet['$set'] = dictionaries.UpdateTemplateSet
custom.update_data_to_Db_noupsert(int(config.ConfigManager().MongoDBPort),
config.ConfigManager().DataCollectionDB,
config.ConfigManager().ConfigCollection,
dictionaries.UpdateTemplateWhere,
dictionaries.DBSet, connection)
def updateconfigcollection(docid, dateTime, whereID):
connection = dbmanager.mongoDB_connection(
int(config.ConfigManager().MongoDBPort))
UpdateTemplateWhere = utility.clean_dict()
UpdateTemplateSet = utility.clean_dict()
UpdateTemplateWhere['_id'] = whereID
UpdateTemplateSet['masterDocId'] = docid
UpdateTemplateSet['stagingDateModified'] = dateTime
DBSet = utility.clean_dict()
DBSet['$set'] = UpdateTemplateSet
custom.update_data_to_Db_noupsert(
int(config.ConfigManager().MongoDBPort),
config.ConfigManager().RatesDB,
config.ConfigManager().RatesConfigCollection, UpdateTemplateWhere,
DBSet, connection)
def requirement_update():
(dictionaries.DBWhereCondition1)['documentType'] = 'candidate details'
(dictionaries.DBWhereCondition1)['dataSource'] = 'Smart Track'
docs = custom.retrieve_rowdata_from_DB_notimeout(
int(config.ConfigManager().MongoDBPort),
config.ConfigManager().DataCollectionDB,
config.ConfigManager().DataCollectionDBCollection,
dictionaries.DBWhereCondition1)
recordnumber = 0
for doc in docs:
recordnumber += 1
requirementIDList = []
query = custom.fetch_query(
config.ConfigManager().STCandidateSubmissionsQueryId)
cursor = dbmanager.cursor_odbc_connection(
config.ConfigManager().STConnStr)
db_data_dict = dbmanager.cursor_execute(cursor, query)
db_data = db_data_dict['dbdata']
db_data_cursorexec = db_data_dict['cursor_exec']
cursor_description = db_data_cursorexec.description
column_headers = [column[0] for column in cursor_description]
for row in db_data:
try:
data_dict = dict(utility.zip_list(column_headers, row))
if (data_dict['CandidateID'] == doc['candidateid']):
requirementIDList.append(data_dict['requirementID'])
except BaseException as ex:
exception_message = '\n' + 'Exception:' + \
str(datetime.datetime.now()) + '\n'
exception_message += 'File: ' + '\n'
exception_message += '\n' + str(ex) + '\n'
exception_message += '-' * 100
utility.write_to_file(config.ConfigManager().LogFile, 'a',
exception_message)
dictionaries.UpdateTemplateSet['requirementIDList'] = requirementIDList
dictionaries.UpdateTemplateWhere['candidateid'] = doc['candidateid']
dictionaries.DBSet['$set'] = dictionaries.UpdateTemplateSet
print(recordnumber, doc['candidateid'], requirementIDList)
custom.update_data_to_Db_noupsert(
int(config.ConfigManager().MongoDBPort),
config.ConfigManager().DataCollectionDB,
config.ConfigManager().DataCollectionDBCollection,
dictionaries.UpdateTemplateWhere, dictionaries.DBSet)
def route_dataread(filepaths):
data_read_count = int(utility.read_from_file(
config.ConfigManager().ExecutioncountFile, 'r'))
file_read_count = 0
file_path_count = 0
configdocs = custom.retrieve_data_from_DB(int(config.ConfigManager().MongoDBPort), config.ConfigManager(
).DataCollectionDB, config.ConfigManager().ConfigCollection)
docid_count = int(configdocs[0]['docid_count'])
connection = dbmanager.mongoDB_connection(
int(config.ConfigManager().MongoDBPort))
utility.write_to_file(config.ConfigManager().LogFile,
'a', 'dataread running')
for filepath in filepaths:
data_text = ''
try:
file_path_count += 1
print('File number: ' + str(file_path_count))
print('Processing file..' + filepath)
if filepath[-4:].lower() == ".txt":
data_text = datareadfiletypes.read_text_text(
filepath, data_text)
elif filepath[-4:].lower() == ".pdf":
data_text = datareadfiletypes.read_pdf_text(
filepath, data_text)
elif filepath[-5:].lower() == ".docx":
data_text = datareadfiletypes.read_docx_text(
filepath, data_text)
elif filepath[-4:].lower() == ".doc":
data_text = datareadfiletypes.read_doc_text(
filepath, data_text)
elif filepath[-4:].lower() == ".xls":
# data_text = datareadfiletypes.read_excel_text(
# filepath, data_text)
docid_count = custom.process_excel_rowdata(
filepath, docid_count)
elif filepath[-5:].lower() == ".xlsx":
# data_text = datareadfiletypes.read_excel_text(
# filepath, data_text)
docid_count = custom.process_excel_rowdata(
filepath, docid_count)
elif filepath[-4:].lower() == ".csv":
data_text = datareadfiletypes.read_csv_text(
filepath, data_text)
elif filepath[-4:].lower() == ".odt":
data_text = datareadfiletypes.read_odt_text(
filepath, data_text)
elif filepath[-4:].lower() == ".xml":
docid_count = custom.process_xml_data(filepath, docid_count)
if not data_text == '':
docid_count += 1
file_read_count += 1
# dcrnlp.extract_nounphrases_sentences(data_text)
noun_phrases = ''
dictionaries.DataProperties['description'] = data_text
dictionaries.DataProperties['nounPhrases'] = noun_phrases
dictionaries.DataProperties[
'documentType'] = utility.filefolder_from_filepath(filepath)
dictionaries.DataProperties[
'dataSource'] = config.ConfigManager().Misc # config.ConfigManager().JobPortal
dictionaries.DataProperties['doc_id'] = docid_count
dictionaries.DataProperties[
'documentTitle'] = utility.filename_from_filepath(filepath)
dictionaries.DataProperties['documentDesc'] = (
dictionaries.DataProperties['description'])[0:200]
jsonfordatastore = custom.prepare_json_for_datastore(
dictionaries.DataProperties)
jsonfordatastore_deserialized = utility.jsonstring_deserialize(
jsonfordatastore)
custom.insert_data_to_DB(
jsonfordatastore_deserialized, connection)
phrases_file_data = custom.prepare_phrases_file_data(
noun_phrases, data_read_count, file_read_count)
utility.write_to_file(
config.ConfigManager().PhraseFile, 'a', phrases_file_data)
except BaseException as ex:
exception_message = '\n' + 'Exception:' + \
str(datetime.datetime.now()) + '\n'
exception_message += 'File: ' + filepath + '\n'
exception_message += '\n' + str(ex) + '\n'
exception_message += '-' * 100
utility.write_to_file(
config.ConfigManager().LogFile, 'a', exception_message)
data_read_count += 1
utility.write_to_file(config.ConfigManager(
).ExecutioncountFile, 'w', str(data_read_count))
dictionaries.UpdateTemplateWhere['_id'] = configdocs[0]['_id']
dictionaries.UpdateTemplateSet['docid_count'] = docid_count
dictionaries.DBSet['$set'] = dictionaries.UpdateTemplateSet
custom.update_data_to_Db_noupsert(int(config.ConfigManager().MongoDBPort), config.ConfigManager().DataCollectionDB, config.ConfigManager(
).ConfigCollection, dictionaries.UpdateTemplateWhere, dictionaries.DBSet, connection)
'industry data read running' + ' ' + str(datetime.datetime.now()))
connection = dbmanager.mongoDB_connection(
int(config.ConfigManager().MongoDBPort))
configdocs = custom.retrieve_data_from_DB(
int(config.ConfigManager().MongoDBPort),
config.ConfigManager().RatesDB,
config.ConfigManager().RatesConfigCollection)
industryID = int(configdocs[0]['industryID'])
print(industryID)
industryID = custom.master_data_transfer_from_sql(
config.ConfigManager().STConnStr,
config.ConfigManager().IndustryQueryID,
config.ConfigManager().industryDetails,
config.ConfigManager().ST, industryID,
config.ConfigManager().ExternalHost)
UpdateTemplateWhere = utility.clean_dict()
UpdateTemplateSet = utility.clean_dict()
UpdateTemplateWhere['_id'] = configdocs[0]['_id']
if industryID != 0:
UpdateTemplateSet['industryID'] = industryID
else:
UpdateTemplateSet['industryID'] = int(configdocs[0]['industryID'])
DBSet = utility.clean_dict()
DBSet['$set'] = UpdateTemplateSet
custom.update_data_to_Db_noupsert(
int(config.ConfigManager().MongoDBPort),
config.ConfigManager().RatesDB,
config.ConfigManager().RatesConfigCollection, UpdateTemplateWhere,
DBSet, connection)
def job_info_analysis(page, filepath, dbrecordcount):
global totalrecords
global invalidrecords
global emptydesc
global incompletedesc
global smalldesc
global nonedesc
global nodesc
global totaljobsdict
global jobsitedict
# Fetching current config paramters
configdocs = custom.retrieve_data_from_DB(int(config.ConfigManager().
MongoDBPort),
config.ConfigManager().
IntelligenceDb,
config.ConfigManager().
ConfigCollection)
docid_count = int(configdocs[0]['docid_count'])
dict_object_record_list = []
for jobinfo in page.findall('record'):
try:
# creating dictionary from xml tag contents
dict_object = utility.xml_to_dict(ET.tostring(jobinfo))
totaljobsdict = fill_job_by_site(filepath)
totalrecords += 1
# outer if check is jobdescription tag is in the xml
if 'jobdescription' in (dict_object['record']):
# checking if job description is none
if ((dict_object['record'])['jobdescription'] is not
None):
# variable to determine if record needs to be
# updated in DB
incorrectjobdescription = 0
# checking if job description is empty
if (((dict_object['record'])['jobdescription'])
.strip()) == '':
write_fileinfo(filepath, dict_object)
invalidrecords += 1
emptydesc += 1
incorrectjobdescription = 1
jobsitedict = fill_job_site_data(filepath)
# checking if job desc has less than 20 chars
if (len(((dict_object['record'])['jobdescription'])
) < 20):
incorrectjobdescription = 1
# eliminating the incomplete desc case
if (((dict_object['record'])['jobdescription'])
.strip()[-3:]) == '...':
print('Do nothing')
else:
write_fileinfo(filepath, dict_object)
invalidrecords += 1
smalldesc += 1
jobsitedict = fill_job_site_data(filepath)
# checking the incomplete desc case
if (((dict_object['record'])['jobdescription'])
.strip()[-3:]) == '...':
incorrectjobdescription = 1
write_fileinfo(filepath, dict_object)
invalidrecords += 1
incompletedesc += 1
jobsitedict = fill_job_site_data(filepath)
if (incorrectjobdescription == 0):
docid_count += 1
(dict_object['record'])['doc_id'] = docid_count
(dict_object['record'])['description'] = ((dict_object['record'])['jobdescription'])
(dict_object['record'])['nounPhrases'] = ""
dict_object_record_list.append(dict_object['record'])
dbrecordcount += 1
# checking if job description is none
if (dict_object['record'])['jobdescription'] is None:
write_fileinfo(filepath, dict_object)
invalidrecords += 1
nonedesc += 1
jobsitedict = fill_job_site_data(filepath)
else:
write_fileinfo(filepath, dict_object)
invalidrecords += 1
nodesc += 1
jobsitedict = fill_job_site_data(filepath)
except BaseException as ex:
utility.log_exception_file(ex, dcrconfig.ConfigManager().SemanticGraphLogFile)
if dict_object_record_list:
insert_to_db(dict_object_record_list)
# updating doc_id in config table
UpdateTemplateWhere = utility.clean_dict()
UpdateTemplateSet = utility.clean_dict()
UpdateTemplateWhere['_id'] = configdocs[0]['_id']
UpdateTemplateSet['docid_count'] = docid_count
DBSet = utility.clean_dict()
DBSet['$set'] = UpdateTemplateSet
custom.update_data_to_Db_noupsert(int(config.ConfigManager().MongoDBPort),
config.ConfigManager().IntelligenceDb,
config.ConfigManager().ConfigCollection,
UpdateTemplateWhere, DBSet, connection)
return dbrecordcount
def stcandidate_update():
utility.write_to_file(
config.ConfigManager().LogFile, 'a',
'in stcandidates update currency Code running.!' + ' ' +
str(datetime.datetime.now()))
recordnumber = 0
query = custom.fetch_query(
config.ConfigManager().STCandidateCurrencyQueryId)
print(query)
query = custom.query_variable_replace(
query,
config.ConfigManager().STCandidateCurrencyDetails,
config.ConfigManager().ST)
print(query)
cursor = dbmanager.cursor_odbc_connection(config.ConfigManager().STConnStr)
db_data_dict = dbmanager.cursor_execute(cursor, query)
db_data = db_data_dict['dbdata']
db_data_cursorexec = db_data_dict['cursor_exec']
cursor_description = db_data_cursorexec.description
column_headers = [column[0] for column in cursor_description]
connection = dbmanager.mongoDB_connection(
int(config.ConfigManager().MongoDBPort))
data_dict1 = {}
req_list = []
candidateDatesList = []
for row1 in db_data:
try:
print(data_dict1)
strtimestamp = str(datetime.datetime.now())
recordnumber += 1
print(recordnumber)
data_dict1 = dict(utility.zip_list(column_headers, row1))
STCandidateCollection.update(
{
"$and": [{
"candidateid": data_dict1['candidateid']
}, {
"requirementRateStatusList": {
"$elemMatch": {
"requirementId": data_dict1['requirementid']
}
}
}]
}, {
"$set": {
"requirementRateStatusList.$.currencyCode":
data_dict1['currencycode'],
"requirementRateStatusList.$.SupplierCurrencyCode":
data_dict1['SupplierCurrencyCode'],
"requirementRateStatusList.$.supplierRegBillRateEX":
str(data_dict1['supplierRegBillRateEX'])
}
})
candidateDatesList.append(data_dict1['dateCreated'])
except BaseException as ex:
print(ex)
utility.log_exception_file(ex, config.ConfigManager().LogFile)
if 'dateCreated' in data_dict1:
maxCandDate = max(candidateDatesList)
UpdateTemplateWhere = utility.clean_dict()
UpdateTemplateSet = utility.clean_dict()
UpdateTemplateWhere['_id'] = configdocs[0]['_id']
print(maxCandDate)
print(str(maxCandDate))
UpdateTemplateSet['STCandidateCurrencyCodeLastDate'] = str(maxCandDate)
DBSet = utility.clean_dict()
DBSet['$set'] = UpdateTemplateSet
custom.update_data_to_Db_noupsert(
int(config.ConfigManager().MongoDBPort),
config.ConfigManager().DataCollectionDB,
config.ConfigManager().ConfigCollection, UpdateTemplateWhere,
DBSet, connection)
def route_dataread(filepaths):
file_count = 0
docreadcount = 0
antiwordemptycount = 0
connection = dbmanager.mongoDB_connection(
int(config.ConfigManager().MongoDBPort))
utility.write_to_file(
config.ConfigManager().LogFile, 'a',
'resumeread running' + ' ' + str(datetime.datetime.now()))
for filepath in filepaths:
strtimestamp = str(datetime.datetime.now())
data_text = ''
try:
file_count += 1
if filepath[-4:].lower() == ".txt":
data_text = datareadfiletypes.read_text_text(
filepath, data_text)
elif filepath[-4:].lower() == ".pdf":
data_text = datareadfiletypes.read_pdf_text(
filepath, data_text)
elif filepath[-5:].lower() == ".docx":
data_text = datareadfiletypes.read_docx_text(
filepath, data_text)
elif filepath[-4:].lower() == ".doc":
data_text = datareadfiletypes.read_doc_text(
filepath, data_text)
if data_text == '':
antiwordemptycount += 1
data_text = datareadfiletypes.read_doc_text_catdoc(
filepath, data_text)
if data_text == '':
utility.write_to_file(config.ConfigManager().LogFile, 'a',
'Filepath is: ' + filepath)
elif filepath[-4:].lower() == ".xls":
data_text = datareadfiletypes.read_excel_text(
filepath, data_text)
elif filepath[-5:].lower() == ".xlsx":
data_text = datareadfiletypes.read_excel_text(
filepath, data_text)
elif filepath[-4:].lower() == ".csv":
data_text = datareadfiletypes.read_csv_text(
filepath, data_text)
elif filepath[-4:].lower() == ".odt":
data_text = datareadfiletypes.read_odt_text(
filepath, data_text)
elif filepath[-4:].lower() == ".xml":
docid_count = custom.process_xml_data(filepath, docid_count)
strtimestamp += ' ' + str(datetime.datetime.now())
if not data_text == '':
filepath = filepath.replace('\\', '')
file_name = utility.filename_from_filepath(filepath)
candidateId = (file_name[1:]).split('_')[0]
print(candidateId)
UpdateTemplateWhere = utility.clean_dict()
UpdateTemplateSet = utility.clean_dict()
UpdateTemplateWhere['candidateid'] = int(candidateId)
# UpdateTemplateWhere['documentType'] = 'candidate details'
# UpdateTemplateWhere['dataSource'] = 'Smart Track'
UpdateTemplateSet['resumeText'] = data_text
UpdateTemplateSet['isResumeTextNew'] = 1
DBSet = utility.clean_dict()
DBSet['$set'] = UpdateTemplateSet
custom.update_data_to_Db_noupsert(
int(config.ConfigManager().MongoDBPort),
config.ConfigManager().DataCollectionDB,
config.ConfigManager().STCandidateCollection,
UpdateTemplateWhere, DBSet, connection)
strtimestamp += ' ' + str(datetime.datetime.now())
if filepath[-4:].lower() == ".doc":
docreadcount += 1
print('Total doc read count:' + str(docreadcount))
print('File : ' + str(file_count) + ' ' + strtimestamp)
print('Antiword empty count:' + str(antiwordemptycount))
except BaseException as ex:
exception_message = '\n' + 'Exception:' + \
str(datetime.datetime.now()) + '\n'
exception_message += 'File: ' + '\n'
exception_message += '\n' + str(ex) + '\n'
exception_message += '-' * 100
# .encode('utf8'))
utility.write_to_file(config.ConfigManager().LogFile, 'a',
exception_message)
utility.write_to_file(
config.ConfigManager().LogFile, 'a', 'Number of resumes read - ' +
str(file_count) + ' ' + str(datetime.datetime.now()))
def dbdata(connStr):
utility.write_to_file(
config.ConfigManager().LogFile, 'a',
'resume_extract running' + ' ' + str(datetime.datetime.now()))
st_db_name_list = utility.find_string_inbetween(
config.ConfigManager().STConnStr, "DATABASE=", ";UID")
cursor = dbmanager.cursor_odbc_connection(connStr)
query = custom.fetch_query(
config.ConfigManager().STCandidateResumesQueryId)
configdocs = custom.retrieve_data_from_DB(
int(config.ConfigManager().MongoDBPort),
config.ConfigManager().DataCollectionDB,
config.ConfigManager().ConfigCollection)
query = query.replace('##candidateResumeID##',
str(configdocs[0]['STcandidateResumepk'])).replace(
'##STDB##', st_db_name_list[0])
db_data_dict = dbmanager.cursor_execute(cursor, query)
db_data = db_data_dict['dbdata']
db_data_cursorexec = db_data_dict['cursor_exec']
cursor_description = db_data_cursorexec.description
column_headers = [column[0] for column in cursor_description]
count = 0
data_dict = {}
connection = dbmanager.mongoDB_connection(
int(config.ConfigManager().MongoDBPort))
for row in db_data:
try:
data_dict = dict(utility.zip_list(column_headers, row))
resumefilepath = config.ConfigManager().ResumeDirectory + '/' + '_' + \
str(data_dict['candidateID']) + '_' + \
str(data_dict['documentName'])
filewrite = open(resumefilepath, 'wb')
f = io.BytesIO(data_dict['ResumeFile'])
filewrite.write(data_dict['ResumeFile'])
filefilepath = config.ConfigManager().fileDirectory + '/' + '_' + \
str(data_dict['candidateResumeID'])+'-'+str(data_dict['supplierID']) + '_' + \
str(data_dict['documentName'])
file_write = open(filefilepath, 'wb')
file_write.write(data_dict['ResumeFile'])
except BaseException as ex:
exception_message = '\n' + 'Exception:' + \
str(datetime.datetime.now()) + '\n'
exception_message += 'File: ' + '\n'
exception_message += '\n' + str(ex) + '\n'
exception_message += '-' * 100
utility.write_to_file(config.ConfigManager().LogFile, 'a',
exception_message)
if 'candidateResumeID' in data_dict:
primary_key = data_dict['candidateResumeID']
UpdateTemplateWhere = utility.clean_dict()
UpdateTemplateSet = utility.clean_dict()
UpdateTemplateSet['STcandidateResumepk'] = primary_key
resumeUpdateDelta = int(primary_key) - \
int(configdocs[0]['STcandidateResumepk'])
utility.write_to_file(
config.ConfigManager().LogFile, 'a', 'Resumes updated - ' +
str(resumeUpdateDelta) + ' ' + str(datetime.datetime.now()))
UpdateTemplateWhere['_id'] = configdocs[0]['_id']
DBSet = utility.clean_dict()
DBSet['$set'] = UpdateTemplateSet
custom.update_data_to_Db_noupsert(
int(config.ConfigManager().MongoDBPort),
config.ConfigManager().DataCollectionDB,
config.ConfigManager().ConfigCollection, UpdateTemplateWhere,
DBSet, connection)
def requirement_update():
utility.write_to_file(
config.ConfigManager().LogFile, 'a',
'requpdatefastest running' + ' ' + str(datetime.datetime.now()))
recordnumber = 0
configdocs = custom.retrieve_data_from_DB(
int(config.ConfigManager().MongoDBPort),
config.ConfigManager().DataCollectionDB,
config.ConfigManager().ConfigCollection)
query = custom.fetch_query(
config.ConfigManager().STCandidateSubmissionsQueryId)
if '.' in str(configdocs[0]['submissionsdateCreated']):
query = query.replace(
'##dateCreated##',
str(configdocs[0]['submissionsdateCreated']).split('.')[0] +
('.') +
str(configdocs[0]['submissionsdateCreated']).split('.')[1][:3])
else:
query = query.replace('##dateCreated##',
str(configdocs[0]['submissionsdateCreated']))
# query = query.replace('##dateCreated##', str((datetime.datetime.now() - datetime.timedelta(minutes=480))).split('.')[0] + ('.') + str((datetime.datetime.now() - datetime.timedelta(minutes=480))).split('.')[1][:3])
print(query)
cursor = dbmanager.cursor_odbc_connection(config.ConfigManager().STConnStr)
db_data_dict = dbmanager.cursor_execute(cursor, query)
db_data = db_data_dict['dbdata']
db_data_cursorexec = db_data_dict['cursor_exec']
cursor_description = db_data_cursorexec.description
column_headers = [column[0] for column in cursor_description]
connection = dbmanager.mongoDB_connection(
int(config.ConfigManager().MongoDBPort))
data_dict1 = {}
req_list = []
for row1 in db_data:
try:
strtimestamp = str(datetime.datetime.now())
recordnumber += 1
print(recordnumber)
data_dict1 = dict(utility.zip_list(column_headers, row1))
if not (data_dict1['requirementID']).strip():
requirementIDListperCandidate = []
reqratelist = []
reqratestatdictlist = []
else:
reqratelist = []
reqratestatdictlist = []
requirementIDListperCandidate = [
reqID for reqID in (data_dict1['requirementID']).split(',')
]
for reqratecombID in requirementIDListperCandidate:
reqratelistinit = [
reqrateID
for reqrateID in (reqratecombID.strip().replace(
'~!@- ', '~!@-')).split('~!@-')
]
print(reqratelistinit[0])
reqratestatdict = {}
req_list.append(int(reqratelistinit[1]))
if reqratelistinit[0] == "0.00":
reqratelistinit[0] = ""
reqratestatdict['rate'] = reqratelistinit[0]
reqratestatdict['requirementId'] = int(reqratelistinit[1])
reqratestatdict['candidateStatus'] = reqratelistinit[2]
reqratelist.append(reqratelistinit)
reqratestatdictlist.append(reqratestatdict)
strtimestamp += ' ' + str(datetime.datetime.now())
UpdateTemplateSet = utility.clean_dict()
UpdateTemplateWhere = utility.clean_dict()
UpdateTemplateSet[
'requirementIDList'] = reqratelist # requirementIDListperCandidate
UpdateTemplateSet[
'requirementRateStatusList'] = reqratestatdictlist
UpdateTemplateWhere['candidateid'] = data_dict1['CandidateID']
UpdateTemplateWhere['documentType'] = 'candidate details'
UpdateTemplateWhere['dataSource'] = 'Smart Track'
DBSet = utility.clean_dict()
DBSet['$set'] = UpdateTemplateSet
print(UpdateTemplateSet['requirementRateStatusList'])
custom.update_data_to_Db_noupsert(
int(config.ConfigManager().MongoDBPort),
config.ConfigManager().DataCollectionDB,
config.ConfigManager().STCandidateCollection,
UpdateTemplateWhere, DBSet, connection)
strtimestamp += ' ' + str(datetime.datetime.now())
except BaseException as ex:
utility.log_exception(ex)
if 'dateCreated' in data_dict1:
if not data_dict1['dateCreated'] is None:
UpdateTemplateSet = utility.clean_dict()
UpdateTemplateWhere = utility.clean_dict()
UpdateTemplateSet['submissionsdateCreated'] = data_dict1[
'dateCreated']
UpdateTemplateWhere['_id'] = configdocs[0]['_id']
DBSet = utility.clean_dict()
DBSet['$set'] = UpdateTemplateSet
custom.update_data_to_Db_noupsert(
int(config.ConfigManager().MongoDBPort),
config.ConfigManager().DataCollectionDB,
config.ConfigManager().ConfigCollection, UpdateTemplateWhere,
DBSet, connection)
if req_list:
print(list(set(req_list)))
utility.write_to_file(
config.ConfigManager().LogFile, 'a',
'requirement id list - ' + str(json.dumps(list(set(req_list)))) +
' ' + str(datetime.datetime.now()))
db_requirements_candidate.generate_req_candidate_file_selected_req(
list(set(req_list)))
utility.write_to_file(
config.ConfigManager().LogFile, 'a',
'Number of records for which requirement list was updated - ' +
str(recordnumber) + ' ' + str(datetime.datetime.now()))