def splitBatches(splitFactor_batchid):
from fup.helpers.batchdirs import createSplitDirs
from fup.helpers.batch import batchInfo
from fup.utils.commun import current_date, listifyString
from fup.utils.dbwrap import sql_insertDict, sql_deleteRow
splitBatchidli = splitFactor_batchid.split('_')
splitFactor, batchid = int(splitBatchidli[0]), splitBatchidli[1]
oldBatchinfo = batchInfo(batchid)
infodirs = createSplitDirs(splitFactor, batchid)
if infodirs != False:
prevBatchID = infodirs['oldid']
newBatchID = infodirs['newids']
else:
return False
prevloginfo = [str(l) for l in oldBatchinfo['ChangesLog']]
loginfo = ''.join(prevloginfo) + ', Batch "{}" was splited in batches: "{}", on {}'.format(prevBatchID, newBatchID, current_date())
oldBatchinfo['ChangesLog'] = loginfo
newBIDli = listifyString(newBatchID)
for bid in newBIDli:
oldBatchinfo['BatchID'] = bid
oldBatchinfo.pop('EstimatedTaskNbr', None)
oldBatchinfo.pop('EstimatedFdgNbr', None)
if sql_insertDict('followup', oldBatchinfo) == False:
return False
if sql_deleteRow('followup', 'BatchID', batchid) == False:
return False
return newBatchID
def appendNewFilesToBatch(batchID, orgfilesname, newfilespath, filesId):
from fup.utils.dbwrap import sql_updateDict
from fup.utils.commun import listifyString, uniquelist
from fup.helpers.batch import batchInfo
from fup.utils.commun import current_date, cleanPath
colstoChange = ['OriginalFilesName', 'OriginalFilesPath', 'FilesID', 'ChangesLog', 'BatchID']
infodict_previous = batchInfo(batchID)
changeInfodict = {}
for kcol, val in infodict_previous.items():
if kcol in colstoChange:
if kcol == 'ChangesLog':
changeInfodict[kcol] = val[0] + ', New files added on {}'.format(current_date())
elif kcol == 'OriginalFilesName':
changeInfodict[kcol] = val[0] + ',\n' + orgfilesname
elif kcol == 'OriginalFilesPath':
#print("newfilespath", val[0], ' yuhuu ',newfilespath)
if newfilespath in uniquelist(listifyString(val[0])):
changeInfodict[kcol] = cleanPath(newfilespath)
elif kcol == 'FilesID':
changeInfodict[kcol] = val[0] + ',\n' + filesId
elif kcol == 'BatchID':
changeInfodict[kcol] = batchID
if sql_updateDict('followup', changeInfodict, 'BatchID') == False:
return False
else:
return True
def verifyStatus(infoBatchdict):
#print('verifyStatus: ',infoBatchdict)
try:
try: #if user
#'TO BE CHECKED'
if infoBatchdict['ResponsibleStatus'] == 'TO BE CHECKED':
infoBatchdict['OverallStatus'] = 'TO BE CHECKED'
return infoBatchdict
elif infoBatchdict['ResponsibleStatus'] != 'UNASSIGNED':
infoBatchdict['OverallStatus'] = 'ONGOING'
return infoBatchdict
except: #if proofreader
#"TO BE IMPORTED, FINISHED, REWORK, STANDBY, UNRECORDABLE"
if infoBatchdict['ProofreaderStatus'] == 'REWORK':
infoBatchdict['OverallStatus'] = 'REWORK'
infoBatchdict['ResponsibleStatus'] = 'REWORK'
return infoBatchdict
elif infoBatchdict['ProofreaderStatus'] == 'STANDBY':
from fup.helpers.batch import batchInfo
from fup.utils.commun import current_date
from fup.utils.dbwrap import sql_updateDict
prevInfoDictAll = batchInfo(infoBatchdict['BatchID'])
prevlog = [str(l) for l in prevInfoDictAll['ChangesLog']]
infolog = str("SET to STANDBY on {}".format(current_date()))
log = ', '.join(list(set(prevlog))) + ',\n' + infolog
upd = {
"ChangesLog": log,
"StartDate": "-",
"BatchID": infoBatchdict['BatchID']
}
sql_updateDict('followup', upd, "BatchID")
infoBatchdict['OverallStatus'] = 'STANDBY'
infoBatchdict['ResponsibleStatus'] = 'STANDBY'
return infoBatchdict
elif infoBatchdict['ProofreaderStatus'] == 'UNRECORDABLE':
infoBatchdict['OverallStatus'] = 'UNRECORDABLE'
infoBatchdict['ResponsibleStatus'] = 'UNRECORDABLE'
return infoBatchdict
elif infoBatchdict['ProofreaderStatus'] == 'FINISHED':
infoBatchdict['OverallStatus'] = 'FINISHED'
infoBatchdict['ResponsibleStatus'] = 'FINISHED'
return infoBatchdict
elif infoBatchdict['ProofreaderStatus'] == 'TO BE IMPORTED':
infoBatchdict['OverallStatus'] = 'TO BE IMPORTED'
return infoBatchdict
except:
return infoBatchdict
def resetStartDate(infodict):
from fup.helpers.batch import batchInfo
from fup.utils.commun import current_date
from fup.utils.dbwrap import sql_updateDict
bid = infodict["BatchID"]#"eyi0hT"
newStatus = infodict["OverallStatus"]
prevInfoDictAll = batchInfo(bid)
standby = prevInfoDictAll['OverallStatus'][0] == 'STANDBY'
startdate = prevInfoDictAll['StartDate'][0] == '-'
if standby and startdate and newStatus != "STANDBY":
date = current_date()
sql_updateDict('followup', {"StartDate": date, "BatchID": bid}, "BatchID")
def mergeBatches(batchidstrli):
from fup.helpers.batchdirs import mergeDirBatches
from fup.utils.commun import current_date, listifyString
from fup.helpers.batch import batchInfo
from fup.utils.dbwrap import sql_insertDict, sql_updateDict, sql_deleteRow
import pandas as pd
mergedInfodict = mergeDirBatches(batchidstrli)
bidli = listifyString(batchidstrli)
if isinstance(mergedInfodict, dict):
prevInfodict = {}
for batch in mergedInfodict['batchidli']:
previnfo = batchInfo(batch)
previnfo.pop('EstimatedTaskNbr', None)
previnfo.pop('EstimatedFdgNbr', None)
prevInfodict[batch] = previnfo
else:
print('Cannot merge dirs!')
return False
#gather prev info in a df then a dict
dfli = []
for bid in prevInfodict.keys():
df = pd.DataFrame.from_dict(prevInfodict[bid]) #make df from dict
dfli.append(df)
dfall = pd.concat(dfli, axis=0)
prevInfoDictAll = dfall.to_dict('list')
prevInfoDictAll['BatchID'] = mergedInfodict['mergedID']
infolog = 'Batch merged from "{}" on {}'.format(batchidstrli,
current_date())
prevlog = [str(l) for l in prevInfoDictAll['ChangesLog']]
prevInfoDictAll['ChangesLog'] = ', '.join(list(
set(prevlog))) + ', ' + infolog
prevInfoDictAll['AddedDate'] = current_date()
if sql_insertDict('followup', prevInfoDictAll) == False:
return False
for bid in bidli:
if sql_deleteRow('followup', 'BatchID', bid) == False:
return False
return mergedInfodict['mergedID']
def cleanBatchPath(batchstr):
from fup.utils.commun import cleanPath, listifyString
from fup.helpers.batch import batchInfo
from fup.utils.dbwrap import sql_updateDict
batches = listifyString(batchstr)
for batch in batches:
infoBatch = batchInfo(batch)
#Prepare dict for sql_updateDict func (in future maybe move this to sql_updateDict func)
prepinfoBatch = {}
for k, v in infoBatch.items():
if isinstance(v, list):
val = v[0]
if val == 'None' or val == None:
val = ''
prepinfoBatch[k] = val
else:
prepinfoBatch[k] = v
prepinfoBatch['OriginalFilesPath'] = cleanPath(prepinfoBatch['OriginalFilesPath'])
sql_updateDict('followup', prepinfoBatch, 'BatchID')
def updateDBforNewFiles():
#Verify if new files were added to a existing batch if so, update db
import os, re
import pandas as pd
from fup.utils.dbwrap import sql_insertDict, sql_updateDict, get_dftable, sql_deleteRow
from fup.helpers.batch import batchInfo
from fup.helpers.files import getfileSizeMtime
from fup.utils.commun import list_duplicates
#Update followup with the new file added to the batch
followupdf = get_dftable('followup')
orgpaths = followupdf['OriginalFilesPath'].tolist()
orgpaths_nodups = list(set(orgpaths))
newtempbid = {}
for opath in orgpaths_nodups:
bid = opath.split("\\")[-1].split('BID_')[-1].strip()
followupdf_bid = followupdf[
followupdf['OriginalFilesPath'].str.contains('|'.join([bid]),
na=False)]
bids = followupdf_bid["BatchID"].tolist()
bidtodelli = [b for b in bids if b != bid]
tempd = {}
for biddel in bidtodelli:
infobatch_previous = batchInfo(biddel)
if infobatch_previous != False:
for k in list(infobatch_previous.keys()):
if k not in [
'OriginalFilesName', 'FilesID', 'ChangesLog',
'BatchID'
]:
infobatch_previous.pop(k, None)
tempd["prevInfo"] = infobatch_previous
# else:
# response_notfound = "BatchID {} is not in database! Please delete from unassigned folder {}!".format(existingBatchID, existingBatchID)
# tempd["prevInfo"] = response_notfound
# #return response_notfound, response_notfound, response_notfound
newtempbid[bid] = tempd
orgpaths_dups = list_duplicates(orgpaths)
existingbid = {}
for opath in orgpaths_dups:
tempd = {}
bid = opath.split("\\")[-1].split('BID_')[-1].strip()
infobatch_previous = batchInfo(bid)
if infobatch_previous != False:
for k in list(infobatch_previous.keys()):
if k not in [
'OriginalFilesName', 'FilesID', 'ChangesLog', 'BatchID'
]:
infobatch_previous.pop(k, None)
#print('OK ',infobatch_previous)
tempd["prevInfo"] = infobatch_previous
# else:
# response_notfound = "BatchID {} is not in database! Please delete from unassigned folder {}!".format(existingBatchID, existingBatchID)
# #print('NOK ',response_notfound)
# tempd["prevInfo"] = response_notfound
# #return response_notfound, response_notfound, response_notfound
existingbid[bid] = tempd
tempbidtodel = []
for bidorg, dorg in existingbid.items():
for bidtemp, dtemp in newtempbid.items():
if bidorg == bidtemp:
#make df from dict
dforg = pd.DataFrame.from_dict(dorg['prevInfo'])
dftemp = pd.DataFrame.from_dict(dtemp['prevInfo'])
todelli = dftemp['BatchID'].tolist()
for b in todelli:
tempbidtodel.append(b)
bidtodelli = list(set(tempbidtodel))
dfconcat = pd.concat([dforg, dftemp], axis=0)
dfdict = dfconcat.to_dict('list')
#Create dict to update followup
joineddict = {}
for kcol, vrow in dfdict.items():
if kcol == "BatchID":
vrow = list(set(vrow).difference(set(bidtodelli)))
try:
li = list(set(filter(None, vrow)))
vrow = ', '.join(li)
except:
pass
joineddict[kcol] = vrow
if sql_updateDict('followup', joineddict, 'BatchID') == False:
updatefup_failed = "Update in followup failed for BID_{} file {}..".format(
joineddict['BatchID'], joineddict['OriginalFilesName'])
#print(updatefup_failed)
return updatefup_failed
#Delete new temp bid from db
for bid in bidtodelli:
if sql_deleteRow('followup', 'BatchID', bid):
pass
else:
#print("NOK")
return "Please delete from database {}".format(
str(bidtodelli))
#Update fileshistory table in db
fileshistorydf = get_dftable('fileshistory')
fileInfoli = []
for fpath in orgpaths_nodups:
fileInfo = {}
bid = fpath.split("\\")[-1].split('BID_')[-1].strip()
fhistdf_filtered = fileshistorydf[fileshistorydf["AddedInBatch"] ==
bid]
fids = fhistdf_filtered["FileID"].tolist()
files = os.listdir(fpath)
fidorgli = []
for file in files:
fidorg = file.split(' ')[0].split('_')[-1]
fidorgli.append(fidorg)
newfid = list(set(fids).symmetric_difference(set(
fidorgli))) # difference of/from 2 lists [1,2] and [1,2,3] => [3]
#print(newfid)
newfilepathli = []
for fid in newfid:
for file in files:
if fid == file.split(' ')[0].split('_')[-1]:
#print(fid, file)
newfilepath = os.path.join(fpath, file)
newfilepathli.append(newfilepath)
for newfilepath in newfilepathli:
fileSpec = getfileSizeMtime(newfilepath)
fileName = ' '.join(newfilepath.split('\\')[-1].split(' ')[1:])
fileInfo = {
'FileID': newfid,
'AddedInBatch': [bid],
'ModificationDate': [fileSpec['ModificationDate']],
'FileName': [fileName],
'FileSizeBytes': [fileSpec['FileSizeBytes']]
}
fileInfoli.append(fileInfo)
for finfodict in fileInfoli:
if sql_insertDict('fileshistory', finfodict) == False:
return "Please update manually in fileshistory {}".format(
str(finfodict))
#print("update manually")
#print("return True")
return True
def applyUpdateBatchChanges():
batchChangesdict = {}
config = configInfo()
batchChangesdict['BatchID'] = request.form['batchid']
batchid = batchChangesdict['BatchID']
#print('applyUpdateBatchChanges: ', batchid)
batchChangesdict['ResponsibleStatus'] = str(
request.form['responsibleStatus']).replace("**", "")
batchChangesdict['ProofreaderStatus'] = str(
request.form['proofreaderStatus']).replace("**", "")
batchChangesdict['OverallStatus'] = request.form['overallStatus']
batchChangesdict['Aircraft'] = request.form['aircraft']
batchChangesdict['Responsible'] = request.form['reAssignBatch']
try:
batchChangesdict['splitBatchFactor'] = request.form['splitBatch']
splitFactor = batchChangesdict['splitBatchFactor']
except:
pass
try:
fileobli = request.files.getlist("files2upload")
batchChangesdict['filestoupload'] = len(fileobli)
except:
pass
try:
batchChangesdict['EstimatedTaskNbr'] = request.form['aproxtasknr']
except:
batchChangesdict['EstimatedTaskNbr'] = ''
pass
try:
batchChangesdict['EstimatedFdgNbr'] = request.form['aproxfdgnr']
except:
batchChangesdict['EstimatedFdgNbr'] = ''
pass
try:
batchChangesdict['comments'] = request.form['comments']
except:
batchChangesdict['comments'] = ''
pass
updateResponse = checkupdate(batchChangesdict)
print('updateResponse: ', updateResponse)
if updateResponse != False:
if updateResponse == 'merge':
#Deal with Merge Batches
batches = str(batchid)
return redirect(
url_for('updatebatch.applyMergeBatches', batches=batches))
elif updateResponse == 'add':
batchStatus = batchInfo(batchid)
batchStatus = batchStatus['OverallStatus'][0]
#Deal with the adding more files to one batch
if batchStatus != False:
if batchStatus != "UNASSIGNED":
bidDirAssigned = os.path.abspath(
config['path_to_batches_assigned'])
assginedDirsli = os.listdir(bidDirAssigned)
assignedDir = [
folderName for folderName in assginedDirsli
if re.search(batchid, folderName)
][0]
path = os.path.join(bidDirAssigned, assignedDir)
filesnameli = []
pathsli = []
fileIDli = []
for fileob in fileobli:
filename = secure_filename(fileob.filename)
fileid = generateID()
newFileName = 'FID_' + fileid + ' ' + filename
save_path = os.path.join(path, newFileName)
fileob.save(save_path)
#Check if file was added before
fileinfo = getfileSizeMtime(save_path)
fileinfo['FileID'], fileinfo[
'FileName'] = fileid, filename
fileinfo['AddedInBatch'] = batchid
responseFileInfo = checkFileInfo(fileinfo)
if responseFileInfo != True:
os.remove(save_path)
errormessage = responseFileInfo
return redirect(
url_for('comm.showFailedPage',
errormessage=errormessage))
else:
sql_insertDict('fileshistory', fileinfo)
filesnameli.append(filename)
pathsli.append(path)
fileIDli.append(fileid)
orgfilesname = ', '.join(filesnameli)
newfilespath = ', '.join(pathsli)
filesId = ', '.join(fileIDli)
if appendNewFilesToBatch(batchid, orgfilesname,
newfilespath, filesId) == True:
return redirect(url_for('comm.showSuccessPage'))
else:
errormessage = "Changes not saved into the database!"
return redirect(
url_for('comm.showFailedPage',
errormessage=errormessage))
elif batchStatus == "UNASSIGNED":
errormessage = "Barch is UNASSIGNED! You can add new files using this method only if this batch is ASSIGNED!"
return redirect(
url_for('comm.showFailedPage',
errormessage=errormessage))
elif updateResponse == 'split':
#Deal with the splitBatch
splitFactor_Batch = str(splitFactor) + '_' + str(batchid)
return redirect(
url_for('updatebatch.applySplitBatches',
splitFactor_Batch=splitFactor_Batch))
elif updateResponse == 'update':
#Just update the batch in the database
if updateBatchinFollowup(batchChangesdict):
return redirect(url_for('comm.showSuccessPage'))
else:
errormessage = str(
"Moving BID_{} folders failed or DCS info not found! Check docs for more info.."
.format(batchid))
return redirect(
url_for('comm.showFailedPage', errormessage=errormessage))
else:
print(updateResponse)
errormessage = "Only one change can be applyed for options with '*' sign! Reset to defaults by clicking '| Update Batches' title"
return redirect(
url_for('comm.showFailedPage', errormessage=errormessage))
def updateBatchOptions(batchlink=''):
#get batch update options depending on the user type (user/responsible, admin or proofreader)
import re
from fup.utils.commun import listifyString
from fup.utils.jsoninfo import configInfo, sessionInfo
from fup.helpers.user import get_usersdict
from fup.helpers.batch import viewBatches
from fup.helpers.batch import batchInfo
#print('updateBatchOptions: ', batchlink)
config = configInfo()
update_options_responsible = listifyString(
config['batch_status_options_responsible'])
update_options_proofreader = listifyString(
config['batch_status_options_proofreader'])
update_options_overall = listifyString(
config['batch_status_options_overall'])
aircraft = listifyString(config['aircraft'])
split_batch_factor = listifyString(config['split_batch_factor'])
allusers = get_usersdict(True)
allusers = [user for user in allusers if re.search('\.', user)]
print("users: ", allusers)
session = sessionInfo()
current_user_rights = session['current_user_rights']
infoBatch = batchInfo(batchlink)
try:
infoBatch["Operator"]
except:
infoBatch = {
'Operator': '',
'Aircraft': '',
'OverallStatus': '',
'AddedDate': '',
'StartDate': '',
'ImportedDateISAIM': ''
}
update_batch_dict = {
"responsibleStatus": update_options_responsible,
"proofreaderStatus": update_options_proofreader,
"overallStatus": update_options_overall,
"aircraft": aircraft,
"splitBatch": split_batch_factor,
"allusers": allusers,
"batchlink": batchlink,
"userBatches": viewBatches(user_batches=True),
"disableCommentResponsible": '',
"disableCommentProofreader": '',
"disableCheckbox": '',
"infoBatch": infoBatch
}
if current_user_rights == 'user':
update_batch_dict["proofreaderStatus"] = ['You cannot change this']
update_batch_dict["allusers"] = ['You cannot change this']
update_batch_dict["overallStatus"] = ['You cannot change this']
update_batch_dict["disableCommentProofreader"] = "disabled"
update_batch_dict["disableCheckbox"] = "disabled"
return update_batch_dict
elif current_user_rights == 'proofreader':
update_batch_dict["responsibleStatus"] = ['You cannot change this']
update_batch_dict["disableCommentResponsible"] = "disabled"
return update_batch_dict
elif current_user_rights == 'admin':
return update_batch_dict