def verifyStatus(infoBatchdict): #print('verifyStatus: ',infoBatchdict) try: try: #if user #'TO BE CHECKED' if infoBatchdict['ResponsibleStatus'] == 'TO BE CHECKED': infoBatchdict['OverallStatus'] = 'TO BE CHECKED' return infoBatchdict elif infoBatchdict['ResponsibleStatus'] != 'UNASSIGNED': infoBatchdict['OverallStatus'] = 'ONGOING' return infoBatchdict except: #if proofreader #"TO BE IMPORTED, FINISHED, REWORK, STANDBY, UNRECORDABLE" if infoBatchdict['ProofreaderStatus'] == 'REWORK': infoBatchdict['OverallStatus'] = 'REWORK' infoBatchdict['ResponsibleStatus'] = 'REWORK' return infoBatchdict elif infoBatchdict['ProofreaderStatus'] == 'STANDBY': from fup.helpers.batch import batchInfo from fup.utils.commun import current_date from fup.utils.dbwrap import sql_updateDict prevInfoDictAll = batchInfo(infoBatchdict['BatchID']) prevlog = [str(l) for l in prevInfoDictAll['ChangesLog']] infolog = str("SET to STANDBY on {}".format(current_date())) log = ', '.join(list(set(prevlog))) + ',\n' + infolog upd = { "ChangesLog": log, "StartDate": "-", "BatchID": infoBatchdict['BatchID'] } sql_updateDict('followup', upd, "BatchID") infoBatchdict['OverallStatus'] = 'STANDBY' infoBatchdict['ResponsibleStatus'] = 'STANDBY' return infoBatchdict elif infoBatchdict['ProofreaderStatus'] == 'UNRECORDABLE': infoBatchdict['OverallStatus'] = 'UNRECORDABLE' infoBatchdict['ResponsibleStatus'] = 'UNRECORDABLE' return infoBatchdict elif infoBatchdict['ProofreaderStatus'] == 'FINISHED': infoBatchdict['OverallStatus'] = 'FINISHED' infoBatchdict['ResponsibleStatus'] = 'FINISHED' return infoBatchdict elif infoBatchdict['ProofreaderStatus'] == 'TO BE IMPORTED': infoBatchdict['OverallStatus'] = 'TO BE IMPORTED' return infoBatchdict except: return infoBatchdict
def resetStartDate(infodict): from fup.helpers.batch import batchInfo from fup.utils.commun import current_date from fup.utils.dbwrap import sql_updateDict bid = infodict["BatchID"]#"eyi0hT" newStatus = infodict["OverallStatus"] prevInfoDictAll = batchInfo(bid) standby = prevInfoDictAll['OverallStatus'][0] == 'STANDBY' startdate = prevInfoDictAll['StartDate'][0] == '-' if standby and startdate and newStatus != "STANDBY": date = current_date() sql_updateDict('followup', {"StartDate": date, "BatchID": bid}, "BatchID")
def appendNewFilesToBatch(batchID, orgfilesname, newfilespath, filesId): from fup.utils.dbwrap import sql_updateDict from fup.utils.commun import listifyString, uniquelist from fup.helpers.batch import batchInfo from fup.utils.commun import current_date, cleanPath colstoChange = ['OriginalFilesName', 'OriginalFilesPath', 'FilesID', 'ChangesLog', 'BatchID'] infodict_previous = batchInfo(batchID) changeInfodict = {} for kcol, val in infodict_previous.items(): if kcol in colstoChange: if kcol == 'ChangesLog': changeInfodict[kcol] = val[0] + ', New files added on {}'.format(current_date()) elif kcol == 'OriginalFilesName': changeInfodict[kcol] = val[0] + ',\n' + orgfilesname elif kcol == 'OriginalFilesPath': #print("newfilespath", val[0], ' yuhuu ',newfilespath) if newfilespath in uniquelist(listifyString(val[0])): changeInfodict[kcol] = cleanPath(newfilespath) elif kcol == 'FilesID': changeInfodict[kcol] = val[0] + ',\n' + filesId elif kcol == 'BatchID': changeInfodict[kcol] = batchID if sql_updateDict('followup', changeInfodict, 'BatchID') == False: return False else: return True
def cleanBatchPath(batchstr): from fup.utils.commun import cleanPath, listifyString from fup.helpers.batch import batchInfo from fup.utils.dbwrap import sql_updateDict batches = listifyString(batchstr) for batch in batches: infoBatch = batchInfo(batch) #Prepare dict for sql_updateDict func (in future maybe move this to sql_updateDict func) prepinfoBatch = {} for k, v in infoBatch.items(): if isinstance(v, list): val = v[0] if val == 'None' or val == None: val = '' prepinfoBatch[k] = val else: prepinfoBatch[k] = v prepinfoBatch['OriginalFilesPath'] = cleanPath(prepinfoBatch['OriginalFilesPath']) sql_updateDict('followup', prepinfoBatch, 'BatchID')
def updateBatchinFollowup(batchdict): from fup.utils.dbwrap import sql_updateDict, tb_cols_placeholder from fup.utils.commun import cleanDict from fup.utils.jsoninfo import sessionInfo from fup.helpers.batchdirs import moveDirsforUpdate, renameAssgnDir from fup.models.batch import verifyStatus, resetStartDate #print('updateBatchinFollowup: ', batchdict) session = sessionInfo() try: cleanedBatch = cleanDict(batchdict) comment = cleanedBatch['comments'] cleanedBatch.pop('comments', None) if session["current_user_rights"] == 'user': cleanedBatch['ResponsibleComment'] = comment elif session["current_user_rights"] == 'proofreader' or session[ "current_user_rights"] == 'admin': cleanedBatch['ProofreaderComment'] = comment except: #no comments pass followupCols = list(tb_cols_placeholder('followup')['columns']) infoBatchdict = {} for k, v in cleanedBatch.items(): if k in followupCols: infoBatchdict[k] = v #print('updateBatchinFollowup-infoBatchdict: ', infoBatchdict) movedir_response = moveDirsforUpdate(infoBatchdict) #print(movedir_response) if movedir_response == False: return False infoBatchdict = verifyStatus(infoBatchdict) resetStartDate(infoBatchdict) #print('infoBatchdict ', infoBatchdict) try: if isinstance(infoBatchdict['Responsible'], str): renameAssgnDir(infoBatchdict) except: pass if sql_updateDict('followup', infoBatchdict, 'BatchID') == False: return False else: return True
def updateDBforNewFiles(): #Verify if new files were added to a existing batch if so, update db import os, re import pandas as pd from fup.utils.dbwrap import sql_insertDict, sql_updateDict, get_dftable, sql_deleteRow from fup.helpers.batch import batchInfo from fup.helpers.files import getfileSizeMtime from fup.utils.commun import list_duplicates #Update followup with the new file added to the batch followupdf = get_dftable('followup') orgpaths = followupdf['OriginalFilesPath'].tolist() orgpaths_nodups = list(set(orgpaths)) newtempbid = {} for opath in orgpaths_nodups: bid = opath.split("\\")[-1].split('BID_')[-1].strip() followupdf_bid = followupdf[ followupdf['OriginalFilesPath'].str.contains('|'.join([bid]), na=False)] bids = followupdf_bid["BatchID"].tolist() bidtodelli = [b for b in bids if b != bid] tempd = {} for biddel in bidtodelli: infobatch_previous = batchInfo(biddel) if infobatch_previous != False: for k in list(infobatch_previous.keys()): if k not in [ 'OriginalFilesName', 'FilesID', 'ChangesLog', 'BatchID' ]: infobatch_previous.pop(k, None) tempd["prevInfo"] = infobatch_previous # else: # response_notfound = "BatchID {} is not in database! Please delete from unassigned folder {}!".format(existingBatchID, existingBatchID) # tempd["prevInfo"] = response_notfound # #return response_notfound, response_notfound, response_notfound newtempbid[bid] = tempd orgpaths_dups = list_duplicates(orgpaths) existingbid = {} for opath in orgpaths_dups: tempd = {} bid = opath.split("\\")[-1].split('BID_')[-1].strip() infobatch_previous = batchInfo(bid) if infobatch_previous != False: for k in list(infobatch_previous.keys()): if k not in [ 'OriginalFilesName', 'FilesID', 'ChangesLog', 'BatchID' ]: infobatch_previous.pop(k, None) #print('OK ',infobatch_previous) tempd["prevInfo"] = infobatch_previous # else: # response_notfound = "BatchID {} is not in database! Please delete from unassigned folder {}!".format(existingBatchID, existingBatchID) # #print('NOK ',response_notfound) # tempd["prevInfo"] = response_notfound # #return response_notfound, response_notfound, response_notfound existingbid[bid] = tempd tempbidtodel = [] for bidorg, dorg in existingbid.items(): for bidtemp, dtemp in newtempbid.items(): if bidorg == bidtemp: #make df from dict dforg = pd.DataFrame.from_dict(dorg['prevInfo']) dftemp = pd.DataFrame.from_dict(dtemp['prevInfo']) todelli = dftemp['BatchID'].tolist() for b in todelli: tempbidtodel.append(b) bidtodelli = list(set(tempbidtodel)) dfconcat = pd.concat([dforg, dftemp], axis=0) dfdict = dfconcat.to_dict('list') #Create dict to update followup joineddict = {} for kcol, vrow in dfdict.items(): if kcol == "BatchID": vrow = list(set(vrow).difference(set(bidtodelli))) try: li = list(set(filter(None, vrow))) vrow = ', '.join(li) except: pass joineddict[kcol] = vrow if sql_updateDict('followup', joineddict, 'BatchID') == False: updatefup_failed = "Update in followup failed for BID_{} file {}..".format( joineddict['BatchID'], joineddict['OriginalFilesName']) #print(updatefup_failed) return updatefup_failed #Delete new temp bid from db for bid in bidtodelli: if sql_deleteRow('followup', 'BatchID', bid): pass else: #print("NOK") return "Please delete from database {}".format( str(bidtodelli)) #Update fileshistory table in db fileshistorydf = get_dftable('fileshistory') fileInfoli = [] for fpath in orgpaths_nodups: fileInfo = {} bid = fpath.split("\\")[-1].split('BID_')[-1].strip() fhistdf_filtered = fileshistorydf[fileshistorydf["AddedInBatch"] == bid] fids = fhistdf_filtered["FileID"].tolist() files = os.listdir(fpath) fidorgli = [] for file in files: fidorg = file.split(' ')[0].split('_')[-1] fidorgli.append(fidorg) newfid = list(set(fids).symmetric_difference(set( fidorgli))) # difference of/from 2 lists [1,2] and [1,2,3] => [3] #print(newfid) newfilepathli = [] for fid in newfid: for file in files: if fid == file.split(' ')[0].split('_')[-1]: #print(fid, file) newfilepath = os.path.join(fpath, file) newfilepathli.append(newfilepath) for newfilepath in newfilepathli: fileSpec = getfileSizeMtime(newfilepath) fileName = ' '.join(newfilepath.split('\\')[-1].split(' ')[1:]) fileInfo = { 'FileID': newfid, 'AddedInBatch': [bid], 'ModificationDate': [fileSpec['ModificationDate']], 'FileName': [fileName], 'FileSizeBytes': [fileSpec['FileSizeBytes']] } fileInfoli.append(fileInfo) for finfodict in fileInfoli: if sql_insertDict('fileshistory', finfodict) == False: return "Please update manually in fileshistory {}".format( str(finfodict)) #print("update manually") #print("return True") return True
def assignBatchtoUser(batchID, assignedtoProofreader): from fup.utils.jsoninfo import sessionInfo from fup.utils.commun import current_date from fup.helpers.batch import getUnassignedBatch from fup.helpers.batchdirs import createAssignedDirFiles from fup.models.batch import checkOverallStatus from fup.utils.dbwrap import sql_updateDict from fup.helpers.batch import batchExists from fup.helpers.user import getuserProofreader if checkOverallStatus() == True: date = current_date() userinfo = sessionInfo() responsible_user = userinfo["current_user_working"] defaultProofreader = getuserProofreader(responsible_user) tableName = 'followup' updatedict = { "BatchID": batchID, "Responsible": responsible_user, "ResponsibleStatus": "ASSIGNED", "Proofreader": defaultProofreader, "ProofreaderStatus": "ASSIGNED", "ChangesLog": '' } if defaultProofreader == "UNASSIGNED": updatedict["ProofreaderStatus"] = "UNASSIGNED" updatedict_fallback = { "BatchID": batchID, "Responsible": "UNASSIGNED", "ResponsibleStatus": "UNASSIGNED", "Proofreader": "UNASSIGNED", "ProofreaderStatus": "UNASSIGNED", "ChangesLog": '' } colIDName = "BatchID" #print(updatedict) if (batchID == '' or batchExists(batchID)) and (assignedtoProofreader == True): unassignedBatch = getUnassignedBatch(batchID, 'ProofreaderStatus') updatedict["BatchID"] = unassignedBatch updatedict_fallback["BatchID"] = unassignedBatch elif (batchID == '' or batchExists(batchID)) and (assignedtoProofreader == False): unassignedBatch = getUnassignedBatch(batchID, 'ResponsibleStatus') updatedict["BatchID"] = unassignedBatch updatedict_fallback["BatchID"] = unassignedBatch if assignedtoProofreader == True: updatedict.pop("Responsible", None) updatedict.pop("ResponsibleStatus", None) updatedict.pop("ChangesLog", None) if sql_updateDict(tableName, updatedict, colIDName) == True: checkOverallStatus() return True else: print('1fallback', tableName, updatedict, colIDName) sql_updateDict(tableName, updatedict_fallback, colIDName) return False elif assignedtoProofreader == False: loginfo = "ASSIGNED to {} on {}".format(responsible_user, date) updatedict["ChangesLog"] = loginfo updatedict["StartDate"] = date if (sql_updateDict(tableName, updatedict, colIDName)) == True and (createAssignedDirFiles( updatedict["BatchID"]) == True): checkOverallStatus() return True else: print('2fallback', tableName, updatedict, colIDName) sql_updateDict(tableName, updatedict_fallback, colIDName) return False else: return False
def moveDirsforUpdate(infoBatchdict): import os, re from fup.helpers.batchdirs import moveDirName from fup.utils.jsoninfo import configInfo from fup.helpers.files import dcsinfo from fup.utils.dbwrap import sql_updateDict from fup.utils.commun import current_date #infoBatchdict = {'ResponsibleStatus': 'TO BE CHECKED', 'BatchID': 'VqUSKc'} batchid = infoBatchdict['BatchID'] config = configInfo() for kcol, val in infoBatchdict.items(): #print(kcol, val) if kcol == 'ResponsibleStatus' and val == 'TO BE CHECKED': ok = moveDirName(batchid, "path_to_batches_assigned", "path_to_batches_tobechecked") if ok: return True else: print("[info]Batch {} not found in ASSIGNED folder".format( batchid)) return False elif kcol == 'ProofreaderStatus' and val == 'REWORK': ok = moveDirName(batchid, "path_to_batches_tobechecked", "path_to_batches_assigned") if ok: return True else: print( "[info]Batch {} not found in TO BE CHECKED folder".format( batchid)) return False elif kcol == 'ProofreaderStatus' and val == 'STANDBY': ok = moveDirName(batchid, "path_to_batches_tobechecked", "path_to_batches_instandby") if ok: return True else: print( "[info]Batch {} not found in TO BE CHECKED folder".format( batchid)) return False elif kcol == 'ProofreaderStatus' and val == 'UNRECORDABLE': ok = moveDirName(batchid, "path_to_batches_tobechecked", "path_to_batches_unrecordable") if ok: return True else: print( "[info]Batch {} not found in TO BE CHECKED folder".format( batchid)) return False elif kcol == 'ProofreaderStatus' and val == 'TO BE IMPORTED': try: dcspath = config['path_to_dcs_info'] dcsli = os.listdir(dcspath) dcsli = [f for f in dcsli if re.search(batchid, f)] dcsli = [f for f in dcsli if re.search('DCS', f)] dcsli = [f for f in dcsli if re.search('.xml', f)] dcsfilepath = os.path.join(dcspath, dcsli[0]) dcsdictinfo = dcsinfo(dcsfilepath) dcsdictinfo['BatchID'] = batchid if sql_updateDict('followup', dcsdictinfo, 'BatchID') == False: print("Cannot update the DCS information to database!") return False ok = moveDirName(batchid, "path_to_batches_tobechecked", "path_to_batches_tbimported") if ok: return True else: print("Batch {} not found in TO BE CHECKED folder".format( batchid)) return False except: print('Cannot open/read the DCS xml file!') return False elif kcol == 'ProofreaderStatus' and val == 'FINISHED': ok = moveDirName(batchid, "path_to_batches_tbimported", "path_to_batches_finished") if ok: cdate = current_date() importedDatedict = { 'BatchID': batchid, 'ImportedDateISAIM': cdate, 'ResponsibleStatus': 'FINISHED' } sql_updateDict('followup', importedDatedict, 'BatchID') else: print("Batch {} not found in TO BE IMPORTED folder".format( batchid)) return False