time.mktime(
time.strptime(
str(
str(lastDayOfMonth) + '/' + str(month) + '/' + str(year) +
" 23:59:59"), "%d/%m/%Y %H:%M:%S")))
holidayOfMonth = mongodb.getOne(MONGO_COLLECTION=common.getSubUser(
subUserType, 'Report_off_sys'),
WHERE={'off_date': todayTimeStamp})
if holidayOfMonth != None:
sys.exit()
mongodb.remove_document(MONGO_COLLECTION=collection,
WHERE={
'createdAt': {
'$gte': todayTimeStamp,
'$lte': endTodayTimeStamp
}
})
aggregate_blockcard = [{
"$group": {
"_id": 'null',
"acc_blook_card": {
'$push': '$account_number'
},
}
}]
blook_acc = mongodb.aggregate_pipeline(
MONGO_COLLECTION=collection, aggregate_pipeline=aggregate_blockcard)
temp['error_mesg'] = 'Sai kiểu dữ liệu nhập'
temp['result'] = 'error'
result = False
temp['created_by'] = 'system'
temp['created_at'] = time.time()
temp['import_id'] = str(importLogId)
if (result == False):
errorData.append(temp)
else:
insertData.append(temp)
result = True
complete += 1
if (len(errorData) > 0):
mongodbresult.remove_document(
MONGO_COLLECTION=common.getSubUser(subUserType, (
'Lawsuit_' + str(year) + str(month) + str(day))))
mongodbresult.batch_insert(
common.getSubUser(
subUserType, ('Lawsuit_' + str(year) + str(month) + str(day))),
errorData)
mongodb.update(MONGO_COLLECTION=common.getSubUser(
subUserType, 'Import'),
WHERE={'_id': importLogId},
VALUE={
'status': 0,
'complete_import': time.time(),
'total': total,
'complete': complete
})
else:
subtype = json.loads(model['sub_type'])
if 'format' in subtype.keys():
modelFormat[model['field']] = subtype['format']
modelFormat1.append(subtype['format'])
else:
modelFormat[model['field']] = ''
modelFormat1.append('')
if 'column' in subtype.keys():
modelPosition[model['field']] = subtype['column']
modelPosition1.append(subtype['column'])
else:
modelPosition[model['field']] = ''
modelPosition1.append('')
mongodb.remove_document(MONGO_COLLECTION=collection)
with open(importLogInfo['file_path'],
'r',
newline='\n',
encoding='ISO-8859-1') as fin:
csv_reader = csv.reader(fin, delimiter=' ', quotechar='"')
result = True
for rowRaw in csv_reader:
if len(rowRaw) > 0:
row = list(filter(None, rowRaw))
if len(row) > 1:
if isinstance(
row[1],
str) and len(row[1]) > 12 and row[1].isdigit():
total += 1
modelFormat[model['field']] = ''
modelFormat1.append('')
if 'column' in subtype.keys():
modelPosition[model['field']] = subtype['column']
modelPosition1.append(subtype['column'])
else:
modelPosition[model['field']] = ''
modelPosition1.append('')
filenameExtension = fileName.split('.')
if len(filenameExtension) < 2:
filenameExtension.append('txt')
mongodb.remove_document(MONGO_COLLECTION=collection)
if filenameExtension[1] in ['csv', 'xlsx']:
if(filenameExtension[1] == 'csv'):
inputDataRaw = excel.getDataCSV(file_path=importLogInfo['file_path'], dtype=object, sep=sep, header=None, names=modelColumns, na_values='')
else:
inputDataRaw = excel.getDataExcel(file_path=importLogInfo['file_path'], header=None, names=modelColumns, na_values='')
inputData = inputDataRaw.to_dict('records')
for idx, row in enumerate(inputData):
total += 1
temp = {}
result = True
if row['account_number'] not in ['', None]:
for cell in row:
try:
temp['error_mesg'] = 'Sai kiểu dữ liệu nhập'
temp['result'] = 'error'
result = False
temp['created_by'] = 'system'
temp['created_at'] = time.time()
temp['import_id'] = str(importLogId)
if (result == False):
errorData.append(temp)
else:
insertData.append(temp)
result = True
complete += 1
if (len(errorData) > 0):
mongodbresult.remove_document(
MONGO_COLLECTION=common.getSubUser(subUserType, (
'Investigation_file_' + str(year) + str(month) + str(day))))
mongodbresult.batch_insert(
common.getSubUser(
subUserType,
('Investigation_file_' + str(year) + str(month) + str(day))),
errorData)
mongodb.update(MONGO_COLLECTION=common.getSubUser(
subUserType, 'Import'),
WHERE={'_id': importLogId},
VALUE={
'status': 0,
'complete_import': time.time(),
'total': total,
'complete': complete
})
try:
temp[cell] = common.convertDataType(data=row[cell], datatype=modelConverters[cell], formatType=modelFormat[cell])
result = True
except Exception as errorConvertType:
temp['error_cell'] = modelPosition[cell] + str(idx + 1)
temp['type'] = modelConverters[cell]
temp['error_mesg'] = 'Sai kiểu dữ liệu nhập'
temp['result'] = 'error'
result = False
temp['created_by'] = 'system'
temp['created_at'] = time.time()
temp['import_id'] = str(importLogId)
if(result == False):
errorData.append(temp)
else:
temp['result'] = 'success'
insertData.append(temp)
result = True
if(len(errorData) > 0):
mongodb.batch_insert(common.getSubUser(subUserType, 'Site_result_result'), errorData)
mongodb.update(MONGO_COLLECTION=common.getSubUser(subUserType, 'Import'), WHERE={'_id': importLogId}, VALUE={'status': 0, 'complete_import': time.time(), 'total': total, 'complete': complete})
else:
if len(insertData) > 0:
mongodb.remove_document(MONGO_COLLECTION=collection)
mongodb.batch_insert(MONGO_COLLECTION=collection, insert_data=insertData)
mongodb.batch_insert(common.getSubUser(subUserType, 'Site_result_result'), insert_data=insertData)
mongodb.update(MONGO_COLLECTION=common.getSubUser(subUserType, 'Import'), WHERE={'_id': importLogId}, VALUE={'status': 1, 'complete_import': time.time(), 'total': total, 'complete': complete})
except Exception as e:
log.write(now.strftime("%d/%m/%Y, %H:%M:%S") + ': ' + str(e) + '\n')
temp['error_mesg'] = 'Sai kiểu dữ liệu nhập'
temp['result'] = 'error'
result = False
temp['created_by'] = 'system'
temp['created_at'] = time.time()
temp['import_id'] = str(importLogId)
if (result == False):
errorData.append(temp)
else:
insertData.append(temp)
result = True
complete += 1
if (len(errorData) > 0):
mongodbresult.remove_document(
MONGO_COLLECTION=common.getSubUser(subUserType, (
'Collateral_info_' + str(year) + str(month) + str(day))))
mongodbresult.batch_insert(
common.getSubUser(
subUserType,
('Collateral_info_' + str(year) + str(month) + str(day))),
errorData)
mongodb.update(MONGO_COLLECTION=common.getSubUser(
subUserType, 'Import'),
WHERE={'_id': importLogId},
VALUE={
'status': 0,
'complete_import': time.time(),
'total': total,
'complete': complete
})
subtype = json.loads(model['sub_type'])
if 'format' in subtype.keys():
modelFormat[model['field']] = subtype['format']
modelFormat1.append(subtype['format'])
else:
modelFormat[model['field']] = ''
modelFormat1.append('')
if 'column' in subtype.keys():
modelPosition[model['field']] = subtype['column']
modelPosition1.append(subtype['column'])
else:
modelPosition[model['field']] = ''
modelPosition1.append('')
mongodb.remove_document(MONGO_COLLECTION=collection)
with open(importLogInfo['file_path'],
'r',
newline='\n',
encoding='ISO-8859-1') as fin:
csv_reader = csv.reader(fin, delimiter=' ', quotechar='"')
for idx, rowRaw in enumerate(csv_reader):
if len(rowRaw) > 0:
row = list(filter(None, rowRaw))
if len(row) > 1:
if isinstance(
row[1],
str) and len(row[1]) > 12 and row[1].isdigit():
result = True
temp = {}
time.mktime(
time.strptime(str(todayString + " 23:59:59"),
"%d/%m/%Y %H:%M:%S")))
collectionName = 'LO_input_report_' + str(year) + str(month)
if day == 1:
mongodb.create_db(collectionName)
mongodbReport = Mongodb(collectionName)
lnjc05InfoFull = mongodb.get(
MONGO_COLLECTION=common.getSubUser(subUserType, 'LNJC05'))
lnjc05ColName = 'LNJC05_' + str(year) + str(month) + str(day)
mongodbReport.create_col(COL_NAME=lnjc05ColName)
mongodbReport.remove_document(MONGO_COLLECTION=lnjc05ColName)
mongodbReport.batch_insert(MONGO_COLLECTION=lnjc05ColName,
insert_data=lnjc05InfoFull)
listOfAccFull = mongodb.get(MONGO_COLLECTION=common.getSubUser(
subUserType, 'List_of_account_in_collection'))
listOfAccColName = 'List_of_account_in_collection_' + str(year) + str(
month) + str(day)
mongodbReport.create_col(COL_NAME=listOfAccColName)
mongodbReport.remove_document(MONGO_COLLECTION=listOfAccColName)
mongodbReport.batch_insert(MONGO_COLLECTION=listOfAccColName,
insert_data=listOfAccFull)
mainProduct = {}
mainProductRaw = mongodb.get(
MONGO_COLLECTION=common.getSubUser(subUserType, 'Product'))
temp['error_mesg'] = 'Sai kiểu dữ liệu nhập'
temp['result'] = 'error'
result = False
temp['created_by'] = 'system'
temp['created_at'] = time.time()
temp['import_id'] = str(importLogId)
if (result == False):
errorData.append(temp)
else:
insertData.append(temp)
result = True
complete += 1
if (len(errorData) > 0):
mongodbresult.remove_document(
MONGO_COLLECTION=common.getSubUser(subUserType, (
'Report_release_sale_' + str(year) + str(month) + str(day))))
mongodbresult.batch_insert(
common.getSubUser(
subUserType,
('Report_release_sale_' + str(year) + str(month) + str(day))),
errorData)
mongodb.update(MONGO_COLLECTION=common.getSubUser(
subUserType, 'Import'),
WHERE={'_id': importLogId},
VALUE={
'status': 0,
'complete_import': time.time(),
'total': total,
'complete': complete
})
MONGO_COLLECTION=collection,
WHERE={'CONTRACTNR': temp['CONTRACTNR']})
if checkInfo is not None:
temp['updated_by'] = 'system'
temp['updated_at'] = time.time()
updateDate.append(temp)
else:
temp['created_by'] = 'system'
temp['created_at'] = time.time()
insertData.append(temp)
result = True
complete += 1
if (len(errorData) > 0):
mongodbresult.remove_document(
MONGO_COLLECTION=common.getSubUser(subUserType, (
'Cus_assigned_partner_' + str(year) + str(month) + str(day))))
mongodbresult.batch_insert(
common.getSubUser(
subUserType,
('Cus_assigned_partner_' + str(year) + str(month) + str(day))),
errorData)
mongodb.update(MONGO_COLLECTION=common.getSubUser(
subUserType, 'Import'),
WHERE={'_id': importLogId},
VALUE={
'status': 0,
'complete_import': time.time(),
'total': total,
'complete': complete
})
