def geracsv(book, records, arq, signal):
try:
basename = os.path.basename(book).split('.')[0]
col = Columns()
js = col.columns(book, signal=signal)
fj = [json.loads(line.decode('utf-8')) for line in js]
ff = Fixed_files(fj, obj=True)
rec_in = []
for record in records:
rec_in.append(ff.parse(record))
flds = [str(att['field']) for att in fj]
csvs = ';'.join(
str(flds)[1:-1].upper().replace(basename + '_', '').replace(
"'", "").split(',')) + '\n'
csvs = re.sub('FILLER_..', 'FILLER', csvs)
fmt = '''"{}"'''.format(str(['{' + f + '}' for f in flds
])[1:-1]).replace("'",
"").replace(',', ';')
fmt += ".format(**r)"
for r in rec_in:
for n, f in enumerate(r):
csvs += r[n] + ';'
csvs += '\n'
outcsv = arq.split('.')[0] + '.csv'
out = open(outcsv, 'w')
out.write(csvs)
out.close()
return (True, outcsv)
except:
return (False, traceback.format_exc(sys.exc_info))
def gerpgm(self):
try:
bookin = file(self.book).readlines()
lengthin = str(calc_length(bookin)['lrecl'])
col = Columns()
bkout = col.columns(self.book, fmt='cbl', signal=self.signal)
bookout = ''
for b in bkout:
bookout += b
lengthout = str(calc_length(bookout)['lrecl'])
regout = 'WRK-ARQOUTZD-REGISTRO'
bookin = Homogenize(bookin)
if int(word(bookin[0], 1)) == 1:
regin = word(bookin[0], 2).replace('.', '')
bookout = change({regin: regout}, bookout)
lvl01bookin = ''
lvl01bookout = ''
else:
regin = 'WRK-ARQINPPD-REGISTRO'
lvl01bookin = '{:>9} {}.\n'.format('01', regin)
lvl01bookout = '{:>9} {}.\n'.format('01', regout)
formatout = ''
for line in bookin:
if 'PIC' in line:
if word(line, 2) == 'FILLER':
continue
fld = word(line, 2)
formatout += '{:>15} {:31} OF {}\n'.format(
'MOVE', fld, regin)
formatout += '{:>15} {:31} OF {}\n'.format(
'TO', fld, regout)
dicProg = {
'@PGMID': self.programId,
'@DATE': date.today().strftime('%d %b %Y').upper(),
'@BOOKIN': os.path.basename(self.book).split('.')[0].upper(),
'@SINAL': 'COM' if self.signal else 'SEM',
'@BOOKOUT\n': bookout,
'@REGIN': regin,
'@REGOUT': regout,
'@LVL01BKIN\n': lvl01bookin,
'@LVL01BKOUT\n': lvl01bookout,
'@LENGTHIN': lengthin,
'@LENGTHOUT': lengthout,
'@FORMATOUT\n': formatout
}
prog = insAster72(change(dicProg, file('legrpdzd.cbl').read()))
progName = os.path.join(self.path, '{}.cbl'.format(self.programId))
progWrite = open(progName, 'w')
progWrite.write(prog)
progWrite.close()
return True, None
except:
return (False, traceback.format_exc(sys.exc_info))
def gerjob(self):
try:
basename = os.path.basename(self.book).split('.')[0].upper()
col = Columns()
bookout = col.columns(self.book, fmt='cbl', signal=False)
book_zonado = os.path.join(self.path, '{}_ZD.cpy'.format(basename))
with open(book_zonado, 'w') as bkzd:
bkzd.writelines(bookout)
lengthout = str(calc_length(bookout)['lrecl'])
formatout = ''
start = 1
bookin = file(self.book).readlines()
bookin = Homogenize(bookin, cbl=True)
for line in bookin:
if 'PIC' not in line:
continue
splt_pic = line.split('PIC')[1]
repl_pic = splt_pic.replace(' USAGE ', '').replace(
'COMP-3', '').replace('COMP', '').rstrip()
ap = ' OUTREC FIELDS=(' if start == 1 else ' '
length = int(
str(
calc_length(line.replace(splt_pic,
repl_pic))['lrecl']))
lenpd = calc_length(line)['lrecl']
pd2zd = ('PD,TO=ZD,LENGTH={:03},'.format(length)
if 'COMP-3' in splt_pic else
'BI,TO=ZD,LENGTH={:03},'.format(length)
if 'COMP' in splt_pic else '')
formatout += '{}{:03},{:03},{}\n'.format(
ap, start, lenpd, pd2zd)
start += lenpd
formatout = formatout[:-2] + ')\n'
dicjob = {
'@JOBNAME': '{:8}'.format(self.jobname),
'@BOOK': basename,
'@SORTIN': self.sortin,
'@SORTOUT': self.sortout,
'@LRECL': lengthout,
'@OUTREC\n': formatout
}
job = change(dicjob, file('jobpk2zd.template').read())
jobName = os.path.join(self.path, '{}.jcl'.format(self.jobname))
with open(jobName, 'w') as jobWrite:
jobWrite.write(job)
return True, None
except:
return (False, traceback.format_exc(sys.exc_info))
def geratxt(book, lines, csv):
try:
col = Columns()
js = col.columns(book)
ff = Fixed_files(js, obj=True)
lenbook = calc_length(file(book).readlines())['lrecl']
record = ff.parse('0' * lenbook)
fj = [json.loads(line.decode('utf-8')) for line in js]
fldtype = [(str(att['field']), str(att['type'])) for att in fj]
txt = ''
for line in lines:
r = line.split(';')
recordreplace = ''
for n, f in enumerate(fldtype):
recordreplace += "{} = {},".format(fldtype[n][0]
,r[n] if fldtype[n][1] == 'int'
else "'"+r[n]+"'")
record = eval('record._replace({})'.format(recordreplace))
txt += ff.unparse(record)
outtxt = csv.split('.')[0]
nt = 0
while True:
filetxt = '{}{}.txt'.format(outtxt, '' if not nt else '({})'.format(nt))
if os.path.exists(filetxt):
nt += 1
else:
break
out = open(filetxt, 'w')
out.write(txt)
out.close()
return (True, filetxt)
except:
return (False, traceback.format_exc(sys.exc_info))
from columns import Columns col = Columns() book = r'E:\GFCT\Evidencias\cpy\GFCTB092.CPY' bookwrite = open(r'E:\GFCT\Evidencias\cpy\gfctb092.json', 'w') bookwrite.writelines(col.columns(book, signal=False)) bookwrite.close()
def __init__(self, properties = 'pathProp.txt'):
self.properties = properties
self.col = Columns()
def __init__(self, config='pathProp.txt'):
self.config = config
self.col = Columns()
def branchAndPrice(data,
outputTreePickle=None,
printStatus=True,
gapLimit = 1e-9,
timeLimit = None, # Time limit in seconds
reducedTreeSize = True, # Reduce size of tree by deleting unnecessary data
resourceVec = ['TWMin', 'TWMin_g', 'TV'], # Resources used in the sub problem
removeIllegalColumns = False, # Remove all illegal columns before solving a problem
coverConstraint = '=', # Cover constraint. = or >=.
partialCG = False, # Only solve SPs until one with neg red cost is found
orderStrategy = 'random', # Strategy for choosing order of solving SP's
partialCG_constructionHeuristic = False, # Only solve SPs until one with neg red cost is found in construction heuristic
constructionHeuristic = 'CGartificialVariables', # Specify what construction heuristic to use
labelExtensionLimits = [], # Increments in SP label extension limit
SPSolutionsCount = 1, # Maximum number of SP solutions returned from one SPPRC iteration
sizeLimit = None, # Limit size of tree
CGOptimalityGapLimit=1e-9, # CG stop criterion optimality gap
CGImprovementStepSize=1, # CG improvement criterion step size
CGImprovementThreshold=1e-9, # CG improvement criterion threshold
branchOnUpperBound=False, # Branch once CG finds objective value below tree upper bound
branchingStrategy = 'xVars'
):
'''Branch-and-Price algorithm for solving the roster problem. If a tree
pickle filename is given, branch-and-price continues on the given tree.
'''
'''Initializations'''
# Start time
refTime = time.time()
# Start time for initializations
start = time.time()
# Set margin for numerical error
epsilon = 1e-9
terminate = False
# Retrieve inputs to branch and price function
# Array to store configuration
configuration = {}
# Retrieve function arguments
frame = inspect.currentframe()
arguments, _, _, values = inspect.getargvalues(frame)
# Save all arguments relevant
for argument in arguments:
if argument not in ['data', 'outputTreePickle']:
configuration[argument] = values[argument]
# Problem k to be solved
k = 0
# Initialize branch-and-bound tree
tree = Tree(configuration, refTime)
# Initialize columns and graphs
graphs = dict.fromkeys(employee for employee in data['Employees'])
for employee in data['Employees']:
graphs[employee] = Graph(data=data, employee=employee)
stop = time.time()
tree.times['Branch and price']['Initializations'] += stop - start
start = time.time()
# Generate initial columns
columns = Columns(data=data, graphs = graphs,
constructionHeuristic = constructionHeuristic,
printStatus = printStatus,
orderStrategy = orderStrategy,
partialCG_constructionHeuristic = partialCG_constructionHeuristic,
coverConstraint = coverConstraint,
removeIllegalColumns = removeIllegalColumns,
resourceVec = resourceVec)
stop = time.time()
tree.times['Branch and price']['Construction heuristic'] += stop - start
start = time.time()
# Define root problem
rootProblem = Problem(ID=k, columns=columns, graphs=graphs, data=data,
coverConstraint=coverConstraint)
# Add root problem to tree
tree.addProblem(problem=rootProblem)
stop = time.time()
tree.times['Branch and price']['Initializations'] += stop - start
tree.times['Total'] += time.time() - tree.times['refTime']
tree.times['refTime'] = time.time()
'''Main Loop'''
while not terminate:
# Select problem k from unprocessedProblems
Pk = tree.unprocessedProblems[k]
# If required and not root problem, remove illegal columns from Pk
if k > 0 and removeIllegalColumns:
# Remove from columns object
removedColumnNumbers = Pk.columns.removeIllegalColumns(data = data)
# Delete all removed columns from master problem object
Pk.masterProblem.delVariable(['lambda({},{})'.format(e, k) for e in removedColumnNumbers for k in removedColumnNumbers[e]])
# Prepare timing for solving problem k
tree.times['Total'] += time.time() - tree.times['refTime']
tree.times['refTime'] = time.time()
# Check if time limit is reached while not already terminated
if timeLimit != None and tree.times['Total'] > timeLimit:
if printStatus:
print('Time limit {:.0f} s reached. Process terminated.'.format(timeLimit))
savePickle(tree, outputTreePickle)
# Terminate function
return
# Solve problem k
Pk.solve(data = data, epsilon = epsilon, partialCG = partialCG,
orderStrategy = orderStrategy,
partialCG_constructionHeuristic = partialCG_constructionHeuristic,
labelExtensionLimits=labelExtensionLimits,
SPSolutionsCount=SPSolutionsCount,
removeIllegalColumns = removeIllegalColumns,
CGOptimalityGapLimit=CGOptimalityGapLimit,
CGImprovementStepSize=CGImprovementStepSize,
CGImprovementThreshold=CGImprovementThreshold,
branchOnUpperBound=branchOnUpperBound,
treeUpperBound=tree.upperBoundProblem.upperBound,
times = tree.times,
resourceVec = resourceVec,
timeLimit=timeLimit)
stop = time.time()
tree.times['Total'] += time.time() - tree.times['refTime']
tree.times['refTime'] = time.time()
# Check if time limit is reached while not already terminated
if timeLimit != None and tree.times['Total'] > timeLimit:
if Pk.processed:
# Remove master problem and reduce size
Pk.masterProblem = None
Pk.reduceSize(data = data)
# Move problem k from unprocessedProblems to processedProblems
tree.processedProblems[k] = tree.unprocessedProblems.pop(k)
if printStatus:
print('Time limit {:.0f} s reached. Process terminated.'.format(timeLimit))
savePickle(tree, outputTreePickle)
# Terminate function
return
# Move problem k from unprocessedProblems to processedProblems
tree.processedProblems[k] = tree.unprocessedProblems.pop(k)
# If Pk infeasible, or without potential of improvement
if not Pk.feasible or Pk.lowerBound > tree.upperBoundProblem.upperBound:
# Prune node
Pk.pruned = True
# Delete master problem in node
Pk.masterProblem = None
# Else if the final solution is integer
elif Pk.integer:
# Update upper bound if better solution found
if Pk.upperBound < tree.upperBoundProblem.upperBound:
tree.upperBoundProblem = Pk
# Prune all unprocessed problem with lower bound above new upper bound
tree.pruneOnUpperBound()
# Prune node (integer)
Pk.pruned = True
# Delete master problem in node
Pk.masterProblem = None
# Otherwise, update upper bound if Pk had a better integer solution than
# the current upper bound problem and branch problem
else:
if Pk.upperBound and Pk.upperBound < tree.upperBoundProblem.upperBound:
tree.upperBoundProblem = Pk
# Prune all unprocessed problem with lower bound above new upper bound
tree.pruneOnUpperBound()
start = time.time()
# Branch the problem
tree.branch(problem = Pk, branchingStrategy = branchingStrategy, data = data)
stop = time.time()
tree.times['Branch and price']['Branch'] += stop - start
tree.times['Total'] += time.time() - tree.times['refTime']
tree.times['refTime'] = time.time()
# Reduce size of problem by removing unnecessary attributes
Pk.reduceSize(data)
# Update lower bound and optimality gap
tree.calculateLowerBound()
tree.calculateOptimalityGap()
tree.times['Total'] += time.time() - tree.times['refTime']
tree.times['refTime'] = time.time()
# Check termination criterion
if tree.terminationCriterion(sizeLimit=sizeLimit, gapLimit = gapLimit):
terminate = True
tree.complete = True
savePickle(tree, outputTreePickle)
# Terminate function
return
# Else: update k according to search strategy
else:
# If no integer solution found (no UBD): use depth first search
if tree.upperBoundProblem.upperBound == float('inf'):
searchStrategy = 'DepthFirst_Up'
# Else: use best first search
else:
searchStrategy = 'BestFirst'
# Search and store time
start = time.time()
k = tree.search(searchStrategy)
stop = time.time()
tree.times['Branch and price']['Search'] += stop - start
tree.times['Total'] += time.time() - tree.times['refTime']
tree.times['refTime'] = time.time()
if printStatus:
nProcessedProblems = len(tree.processedProblems)
if tree.upperBoundProblem == Pk:
print('New upper bound found')
if tree.upperBoundProblem == Pk or (nProcessedProblems<=10 or
(nProcessedProblems<=50 and nProcessedProblems%2 == 0) or
nProcessedProblems%10 == 0):
printMessage(nProcessedProblems, tree.upperBoundProblem.upperBound,
tree.lowerBound, tree.gap, tree.times['Total'])
