def mergeDirBatches(batchidstrli): import os, re, shutil, time from fup.utils.commun import copytree, generateID, listifyString from fup.utils.jsoninfo import configInfo, sessionInfo config = configInfo() session = sessionInfo() try: user = session['current_user_working'] if re.search('@', user): user = user.split('@')[0] batchidli = listifyString(batchidstrli) dir_assigned = config['path_to_batches_assigned'] dirs_assigned = os.listdir(dir_assigned) #Make a dir for the merged batches mergedID = generateID() mergeddirpath = os.path.join( dir_assigned, '{}-___ A___ BID_{}'.format(user, mergedID)) os.mkdir(mergeddirpath) #Get names of the folders from Assigned folder by checking the BID dirstomergeli = [] for batchid in batchidli: dir_bidAssigned = [ d for d in dirs_assigned if re.search(batchid, d) ] dirstomergeli.append(dir_bidAssigned[0]) #Copy contents of the old batches into the new created merged folder for folderName in dirstomergeli: assignedPathOld = os.path.join(dir_assigned, folderName) src, dst = assignedPathOld, mergeddirpath copytree(src, dst, symlinks=False, ignore=None) files_not_deleted = True while files_not_deleted: try: shutil.rmtree(assignedPathOld) #delete folders files_not_deleted = False except: print("Please close file(s) open in folder {}".format( assignedPathOld)) time.sleep(2) mergedInfodict = { 'mergedID': mergedID, 'mergeddirpath': mergeddirpath, 'batchidli': batchidli } return mergedInfodict except Exception as e: print('mergeDirBatches/helpers Got :', e) return False
def createSplitDirs(splitFactor, batchid): import os, re, shutil from fup.utils.commun import copytree, deletetree, generateID from fup.utils.jsoninfo import configInfo from fup.helpers.batch import batchExists config = configInfo() if batchExists(batchid) == False: return False #Get path to ASSIGNED and filer by BID dir_assigned = config['path_to_batches_assigned'] dirs_assigned = os.listdir(dir_assigned) dir_bidAssigned = [d for d in dirs_assigned if re.search(batchid, d)] if len(dir_bidAssigned) == 0: return False #Copy the main directory in the range of the splitFactor assignedPathOld = os.path.join(dir_assigned, dir_bidAssigned[0]) oldDirName = dir_bidAssigned[0] respNameAC = ' '.join(oldDirName.split(' ')[0:-1]) newSplitpathsd = {} idli = [] for splitedID in range(splitFactor): newid = generateID() newdirName = respNameAC + ' BID_' + newid assignedPathNew = os.path.join(dir_assigned, newdirName) src, dst = assignedPathOld, assignedPathNew copytree(src, dst, symlinks=False, ignore=None) newSplitpathsd[newdirName] = assignedPathNew idli.append(newid) newSplitpathsd['newids'] = ', '.join(idli) newSplitpathsd['oldid'] = batchid deletetree(assignedPathOld) return newSplitpathsd
def autoNewDirs(): #do a check before generating batches resultCheckNew = checkNew() if isinstance(resultCheckNew, str): return resultCheckNew, '', '' import os, shutil from fup.helpers.files import originalFilesPaths, getfileSizeMtime from fup.utils.commun import generateID, current_date from fup.utils.jsoninfo import configInfo config = configInfo() bindir = os.path.abspath(config["path_to_bin"]) filesinfodict = originalFilesPaths(infoDict={}, auto=True) #print('filesinfodict', str(filesinfodict).encode("utf-8")) newdirsNames = list(filesinfodict.keys()) unassignedpath = os.path.abspath(config['path_to_batches_unassigned']) unassigneddirli = os.listdir(unassignedpath) unsdict = {} for d in unassigneddirli: commName = d.split('BID_')[0].strip() unsdict[commName] = d unassigneddirNames = list(unsdict.keys()) communliBatch = list(set(newdirsNames).intersection(unassigneddirNames)) auto = False infoDictli = [] tobinli = [] for opac, vdict in filesinfodict.items(): #similar to uploadFilesCreateBatch, but without flask file object print("\nProcessing {}..".format(opac)) batchID = generateID() operator = opac.split(' ')[0] aircraft = opac.split(' ')[1] bindir_batch = os.path.join(bindir, batchID) if opac not in communliBatch: batchNameFolder = operator + ' ' + aircraft + ' BID_' + batchID path = os.path.join(unassignedpath, batchNameFolder) os.mkdir(path) else: auto = True communOpAc = list(set([opac]).intersection(communliBatch)) batchNameFolder = unsdict[communOpAc[0]] path = os.path.join(unassignedpath, batchNameFolder) existingBatchID = batchNameFolder.split('BID_')[-1].replace( '_', '') bindir_batch = os.path.join(bindir, existingBatchID) tobinli.append({ 'source': vdict['rootpath'], 'destination': bindir_batch }) filesnameli = [] fileIDli = [] for filepath in vdict['files']: if auto: fileinfo = getfileSizeMtime(filepath) #print("autoNewDirs: getfileSizeMtime, fileinfo> ", fileinfo) fileinfo["FileName"] = filepath.split("\\")[-1] responseFileInfo = checkFileInfo(fileinfo) if responseFileInfo != True: return responseFileInfo, auto, auto filename = filepath.split('\\')[-1].replace(',', ' ') #print("autoNewDirs: filename> ", filename, file) fileid = generateID() newFileName = 'FID_' + fileid + ' ' + filename save_path = os.path.join(path, newFileName) try: print("Copying this file:\n{}\nto this path:\n{}".format( filepath, save_path)) shutil.copy2(filepath, save_path) filesnameli.append(filename) fileIDli.append(fileid) except Exception as e: errmsg = "Make sure that all folders from NEW contains ONLY FILES! Got: {}".format( str(e)) return str(errmsg), str(e), str(e) orgfilesname = ', '.join(filesnameli) orgfilespath = path filesId = ', '.join(fileIDli) addedDate = current_date() infoaddDict = { 'BatchID': batchID, 'Aircraft': aircraft, 'Operator': operator, 'OriginalFilesName': orgfilesname, 'OriginalFilesPath': orgfilespath, 'FilesID': filesId, 'AddedDate': addedDate } infoDictli.append(infoaddDict) print("Info about:\ninfoDictli:\n{}\n, auto:\n{}\n, tobinli:\n{}\n".format( infoDictli, auto, tobinli)) return infoDictli, auto, tobinli
def applyUpdateBatchChanges(): batchChangesdict = {} config = configInfo() batchChangesdict['BatchID'] = request.form['batchid'] batchid = batchChangesdict['BatchID'] #print('applyUpdateBatchChanges: ', batchid) batchChangesdict['ResponsibleStatus'] = str( request.form['responsibleStatus']).replace("**", "") batchChangesdict['ProofreaderStatus'] = str( request.form['proofreaderStatus']).replace("**", "") batchChangesdict['OverallStatus'] = request.form['overallStatus'] batchChangesdict['Aircraft'] = request.form['aircraft'] batchChangesdict['Responsible'] = request.form['reAssignBatch'] try: batchChangesdict['splitBatchFactor'] = request.form['splitBatch'] splitFactor = batchChangesdict['splitBatchFactor'] except: pass try: fileobli = request.files.getlist("files2upload") batchChangesdict['filestoupload'] = len(fileobli) except: pass try: batchChangesdict['EstimatedTaskNbr'] = request.form['aproxtasknr'] except: batchChangesdict['EstimatedTaskNbr'] = '' pass try: batchChangesdict['EstimatedFdgNbr'] = request.form['aproxfdgnr'] except: batchChangesdict['EstimatedFdgNbr'] = '' pass try: batchChangesdict['comments'] = request.form['comments'] except: batchChangesdict['comments'] = '' pass updateResponse = checkupdate(batchChangesdict) print('updateResponse: ', updateResponse) if updateResponse != False: if updateResponse == 'merge': #Deal with Merge Batches batches = str(batchid) return redirect( url_for('updatebatch.applyMergeBatches', batches=batches)) elif updateResponse == 'add': batchStatus = batchInfo(batchid) batchStatus = batchStatus['OverallStatus'][0] #Deal with the adding more files to one batch if batchStatus != False: if batchStatus != "UNASSIGNED": bidDirAssigned = os.path.abspath( config['path_to_batches_assigned']) assginedDirsli = os.listdir(bidDirAssigned) assignedDir = [ folderName for folderName in assginedDirsli if re.search(batchid, folderName) ][0] path = os.path.join(bidDirAssigned, assignedDir) filesnameli = [] pathsli = [] fileIDli = [] for fileob in fileobli: filename = secure_filename(fileob.filename) fileid = generateID() newFileName = 'FID_' + fileid + ' ' + filename save_path = os.path.join(path, newFileName) fileob.save(save_path) #Check if file was added before fileinfo = getfileSizeMtime(save_path) fileinfo['FileID'], fileinfo[ 'FileName'] = fileid, filename fileinfo['AddedInBatch'] = batchid responseFileInfo = checkFileInfo(fileinfo) if responseFileInfo != True: os.remove(save_path) errormessage = responseFileInfo return redirect( url_for('comm.showFailedPage', errormessage=errormessage)) else: sql_insertDict('fileshistory', fileinfo) filesnameli.append(filename) pathsli.append(path) fileIDli.append(fileid) orgfilesname = ', '.join(filesnameli) newfilespath = ', '.join(pathsli) filesId = ', '.join(fileIDli) if appendNewFilesToBatch(batchid, orgfilesname, newfilespath, filesId) == True: return redirect(url_for('comm.showSuccessPage')) else: errormessage = "Changes not saved into the database!" return redirect( url_for('comm.showFailedPage', errormessage=errormessage)) elif batchStatus == "UNASSIGNED": errormessage = "Barch is UNASSIGNED! You can add new files using this method only if this batch is ASSIGNED!" return redirect( url_for('comm.showFailedPage', errormessage=errormessage)) elif updateResponse == 'split': #Deal with the splitBatch splitFactor_Batch = str(splitFactor) + '_' + str(batchid) return redirect( url_for('updatebatch.applySplitBatches', splitFactor_Batch=splitFactor_Batch)) elif updateResponse == 'update': #Just update the batch in the database if updateBatchinFollowup(batchChangesdict): return redirect(url_for('comm.showSuccessPage')) else: errormessage = str( "Moving BID_{} folders failed or DCS info not found! Check docs for more info.." .format(batchid)) return redirect( url_for('comm.showFailedPage', errormessage=errormessage)) else: print(updateResponse) errormessage = "Only one change can be applyed for options with '*' sign! Reset to defaults by clicking '| Update Batches' title" return redirect( url_for('comm.showFailedPage', errormessage=errormessage))
def autoNewDirs(): import os, shutil from fup.helpers.files import originalFilesPaths, getfileSizeMtime from fup.utils.commun import generateID, current_date from fup.utils.jsoninfo import configInfo config = configInfo() bindir = os.path.abspath(config["path_to_bin"]) filesinfodict = originalFilesPaths(infoDict={}, auto=True) newdirsNames = list(filesinfodict.keys()) unassignedpath = os.path.abspath(config['path_to_batches_unassigned']) unassigneddirli = os.listdir(unassignedpath) unsdict = {} for d in unassigneddirli: commName = d.split('BID_')[0].strip() unsdict[commName] = d unassigneddirNames = list(unsdict.keys()) communliBatch = list(set(newdirsNames).intersection(unassigneddirNames)) auto = False infoDictli = [] tobinli = [] for opac, vdict in filesinfodict.items(): #similar to uploadFilesCreateBatch, but without flask file object batchID = generateID() operator = opac.split(' ')[0] aircraft = opac.split(' ')[1] bindir_batch = os.path.join(bindir, batchID) if opac not in communliBatch: batchNameFolder = operator + ' ' + aircraft + ' BID_' + batchID path = os.path.join(unassignedpath, batchNameFolder) os.mkdir(path) else: auto = True communOpAc = list(set([opac]).intersection(communliBatch)) batchNameFolder = unsdict[communOpAc[0]] path = os.path.join(unassignedpath, batchNameFolder) existingBatchID = batchNameFolder.split('BID_')[-1].replace( '_', '') bindir_batch = os.path.join(bindir, existingBatchID) tobinli.append({ 'source': vdict['rootpath'], 'destination': bindir_batch }) filesnameli = [] fileIDli = [] for file in vdict['files']: if auto: #print("yuhuu file",file) filepath = file fileinfo = getfileSizeMtime(filepath) fileinfo["FileName"] = file.split("\\")[-1] responseFileInfo = checkFileInfo(fileinfo) if responseFileInfo != True: return responseFileInfo, auto, auto filename = file.split('\\')[-1] fileid = generateID() newFileName = 'FID_' + fileid + ' ' + filename save_path = os.path.join(path, newFileName) filesnameli.append(filename) fileIDli.append(fileid) try: shutil.copy2(file, save_path) except Exception as e: return str(e), str(e), str(e) orgfilesname = ', '.join(filesnameli) orgfilespath = path filesId = ', '.join(fileIDli) addedDate = current_date() infoaddDict = { 'BatchID': batchID, 'Aircraft': aircraft, 'Operator': operator, 'OriginalFilesName': orgfilesname, 'OriginalFilesPath': orgfilespath, 'FilesID': filesId, 'AddedDate': addedDate } infoDictli.append(infoaddDict) #print(infoaddDict) return infoDictli, auto, tobinli
def autoNewDirs(): #do a check before generating batches resultCheckNew = checkNew() if isinstance(resultCheckNew, str): return resultCheckNew, '', '' import os, shutil from fup.helpers.files import originalFilesPaths, getfileSizeMtime from fup.utils.commun import generateID, current_date from fup.utils.jsoninfo import configInfo config = configInfo() bindir = os.path.abspath(config["path_to_bin"]) filesinfodict = originalFilesPaths(infoDict={}, auto=True) newdirsNames = list(filesinfodict.keys()) unassignedpath = os.path.abspath(config['path_to_batches_unassigned']) unassigneddirli = os.listdir(unassignedpath) unsdict = {} for d in unassigneddirli: commName = d.split('BID_')[0].strip() unsdict[commName] = d unassigneddirNames = list(unsdict.keys()) communliBatch = list(set(newdirsNames).intersection(unassigneddirNames)) auto = False infoDictli = [] tobinli = [] for opac, vdict in filesinfodict.items(): #similar to uploadFilesCreateBatch, but without flask file object batchID = generateID() operator = opac.split(' ')[0] aircraft = opac.split(' ')[1] bindir_batch = os.path.join(bindir, batchID) if opac not in communliBatch: batchNameFolder = operator + ' ' + aircraft + ' BID_' + batchID path = os.path.join(unassignedpath, batchNameFolder) os.mkdir(path) else: auto = True communOpAc = list(set([opac]).intersection(communliBatch)) batchNameFolder = unsdict[communOpAc[0]] path = os.path.join(unassignedpath, batchNameFolder) existingBatchID = batchNameFolder.split('BID_')[-1].replace( '_', '') bindir_batch = os.path.join(bindir, existingBatchID) tobinli.append({ 'source': vdict['rootpath'], 'destination': bindir_batch }) filesnameli = [] fileIDli = [] # errtxt_path = os.path.join(unassignedpath, "ERRORS_ifEmptyOK.txt") # try: # os.remove(errtxt_path) # except: # pass # errtxt = open(errtxt_path, "a") for file in vdict['files']: if auto: #print("yuhuu file",file) filepath = file fileinfo = getfileSizeMtime(filepath) fileinfo["FileName"] = file.split("\\")[-1] responseFileInfo = checkFileInfo(fileinfo) if responseFileInfo != True: return responseFileInfo, auto, auto filename = file.split('\\')[-1] fileid = generateID() newFileName = 'FID_' + fileid + ' ' + filename save_path = os.path.join(path, newFileName) try: shutil.copy2(file, save_path) filesnameli.append(filename) fileIDli.append(fileid) except Exception as e: errmsg = "Make sure that all folders from NEW contains ONLY FILES! Please delete from UNASSIGNED all new created folders" #errmsg = "\n\nERROR:\n{} \n\nPlease check if all files are in\n{}\nfor\n{}\nPlease move the files needed manually!\n\n".format(str(e), save_path, path) print(errmsg) #errtxt.write(errmsg) return str(errmsg), str(e), str(e) #errtxt.close() orgfilesname = ', '.join(filesnameli) orgfilespath = path filesId = ', '.join(fileIDli) addedDate = current_date() infoaddDict = { 'BatchID': batchID, 'Aircraft': aircraft, 'Operator': operator, 'OriginalFilesName': orgfilesname, 'OriginalFilesPath': orgfilespath, 'FilesID': filesId, 'AddedDate': addedDate } infoDictli.append(infoaddDict) #print(infoaddDict) print("yuhuuu infoDictli, auto, tobinli", infoDictli, auto, tobinli) return infoDictli, auto, tobinli