def geracsv(book, records, arq, signal):
try:
basename = os.path.basename(book).split('.')[0]
col = Columns()
js = col.columns(book, signal=signal)
fj = [json.loads(line.decode('utf-8')) for line in js]
ff = Fixed_files(fj, obj=True)
rec_in = []
for record in records:
rec_in.append(ff.parse(record))
flds = [str(att['field']) for att in fj]
csvs = ';'.join(
str(flds)[1:-1].upper().replace(basename + '_', '').replace(
"'", "").split(',')) + '\n'
csvs = re.sub('FILLER_..', 'FILLER', csvs)
fmt = '''"{}"'''.format(str(['{' + f + '}' for f in flds
])[1:-1]).replace("'",
"").replace(',', ';')
fmt += ".format(**r)"
for r in rec_in:
for n, f in enumerate(r):
csvs += r[n] + ';'
csvs += '\n'
outcsv = arq.split('.')[0] + '.csv'
out = open(outcsv, 'w')
out.write(csvs)
out.close()
return (True, outcsv)
except:
return (False, traceback.format_exc(sys.exc_info))
def gerpgm(self):
try:
bookin = file(self.book).readlines()
lengthin = str(calc_length(bookin)['lrecl'])
col = Columns()
bkout = col.columns(self.book, fmt='cbl', signal=self.signal)
bookout = ''
for b in bkout:
bookout += b
lengthout = str(calc_length(bookout)['lrecl'])
regout = 'WRK-ARQOUTZD-REGISTRO'
bookin = Homogenize(bookin)
if int(word(bookin[0], 1)) == 1:
regin = word(bookin[0], 2).replace('.', '')
bookout = change({regin: regout}, bookout)
lvl01bookin = ''
lvl01bookout = ''
else:
regin = 'WRK-ARQINPPD-REGISTRO'
lvl01bookin = '{:>9} {}.\n'.format('01', regin)
lvl01bookout = '{:>9} {}.\n'.format('01', regout)
formatout = ''
for line in bookin:
if 'PIC' in line:
if word(line, 2) == 'FILLER':
continue
fld = word(line, 2)
formatout += '{:>15} {:31} OF {}\n'.format(
'MOVE', fld, regin)
formatout += '{:>15} {:31} OF {}\n'.format(
'TO', fld, regout)
dicProg = {
'@PGMID': self.programId,
'@DATE': date.today().strftime('%d %b %Y').upper(),
'@BOOKIN': os.path.basename(self.book).split('.')[0].upper(),
'@SINAL': 'COM' if self.signal else 'SEM',
'@BOOKOUT\n': bookout,
'@REGIN': regin,
'@REGOUT': regout,
'@LVL01BKIN\n': lvl01bookin,
'@LVL01BKOUT\n': lvl01bookout,
'@LENGTHIN': lengthin,
'@LENGTHOUT': lengthout,
'@FORMATOUT\n': formatout
}
prog = insAster72(change(dicProg, file('legrpdzd.cbl').read()))
progName = os.path.join(self.path, '{}.cbl'.format(self.programId))
progWrite = open(progName, 'w')
progWrite.write(prog)
progWrite.close()
return True, None
except:
return (False, traceback.format_exc(sys.exc_info))
def write_columns(self):
d = files.input_coldir(self.name, self.version)
if os.path.exists(d):
raise ValueError("coldir exists, start fresh: '%s'" % d)
outcols = Columns(d)
outcols.create()
if self.logic is None:
self.select()
w=where1(self.mag_and_mask_logic)
print("\nkeeping those that pass mag and mask logic: %s/%s" % (w.size, self.logic.size))
colnames = ['photoid',
'ra',
'dec',
'objc_flags',
'objc_flags2',
'ingood',
'instar',
'inbadfield']
for col in colnames:
print('Creating:',col)
data = self.cols[col][:]
data = data[w]
outcols.write_column(col, data)
del data
combcols = ['flags','flags2',
'modelflux','modelflux_ivar',
'cmodelflux','cmodelflux_ivar',
'extinction']
for ccol in combcols:
print('Creating:',ccol)
colnames = [ccol+'_'+f for f in ['u','g','r','i','z'] ]
data = self.cols.read_columns(colnames, rows=w, verbose=True)
dts = data[ccol+'_'+f].dtype.descr[0][1]
dt = [(ccol, dts, 5)]
data = data.view(dt)
# don't want it to show up as a .rec
rawdata = data[ccol]
outcols.write_column(ccol, rawdata)
del data
print('Adding more restrictive flags to "keep"')
keep = zeros(w.size, dtype='u1')
wrest = where1( self.logic[w] )
keep[wrest] = 1
outcols.write_column('keep', keep)
def gerjob(self):
try:
basename = os.path.basename(self.book).split('.')[0].upper()
col = Columns()
bookout = col.columns(self.book, fmt='cbl', signal=False)
book_zonado = os.path.join(self.path, '{}_ZD.cpy'.format(basename))
with open(book_zonado, 'w') as bkzd:
bkzd.writelines(bookout)
lengthout = str(calc_length(bookout)['lrecl'])
formatout = ''
start = 1
bookin = file(self.book).readlines()
bookin = Homogenize(bookin, cbl=True)
for line in bookin:
if 'PIC' not in line:
continue
splt_pic = line.split('PIC')[1]
repl_pic = splt_pic.replace(' USAGE ', '').replace(
'COMP-3', '').replace('COMP', '').rstrip()
ap = ' OUTREC FIELDS=(' if start == 1 else ' '
length = int(
str(
calc_length(line.replace(splt_pic,
repl_pic))['lrecl']))
lenpd = calc_length(line)['lrecl']
pd2zd = ('PD,TO=ZD,LENGTH={:03},'.format(length)
if 'COMP-3' in splt_pic else
'BI,TO=ZD,LENGTH={:03},'.format(length)
if 'COMP' in splt_pic else '')
formatout += '{}{:03},{:03},{}\n'.format(
ap, start, lenpd, pd2zd)
start += lenpd
formatout = formatout[:-2] + ')\n'
dicjob = {
'@JOBNAME': '{:8}'.format(self.jobname),
'@BOOK': basename,
'@SORTIN': self.sortin,
'@SORTOUT': self.sortout,
'@LRECL': lengthout,
'@OUTREC\n': formatout
}
job = change(dicjob, file('jobpk2zd.template').read())
jobName = os.path.join(self.path, '{}.jcl'.format(self.jobname))
with open(jobName, 'w') as jobWrite:
jobWrite.write(job)
return True, None
except:
return (False, traceback.format_exc(sys.exc_info))
def add_colc_nmgypercount(self):
"""
self.cols is the sweeps
"""
import es_sdsspy
d = files.input_coldir(self.name, self.version)
ccols = Columns(d)
print("cluster input cols:",ccols.dir)
scols = self.cols
print("sweep columns:",scols.dir)
print("reading sweep photoid")
spid = scols['photoid'][:]
print("reading cluster input photoid")
cpid = ccols['photoid'][:]
print("matching")
ms,mc = esutil.numpy_util.match(spid, cpid)
n=cpid.size
if mc.size != n:
raise ValueError("not all matched: %d/%d" % (mt.size/n))
# problem is the columns code always sorts the requested
# indices so we will have lost order information. Need
# to pre-sort both sets of indices according to the sweep
# row number
print("ordering matches with sweep cols")
s=ms.argsort()
ms = ms[s]
mc = mc[s]
for colname in ['colc','nmgypercount']:
data=numpy.zeros((n,5),dtype='f4')
for i,band in enumerate(['u','g','r','i','z']):
scolname='%s_%s' % (colname,band)
print(" %s" % scolname)
sdata=scols[scolname][ms]
data[mc,i] = sdata
del sdata
print("writing: %s" % colname)
ccols.write_column(colname, data)
del data
def add_inbadfield(self):
import es_sdsspy
m=es_sdsspy.mangle_masks.load('boss','badfield')
d = files.input_coldir(self.name, self.version)
c = Columns(d)
print("reading ra")
ra=c['ra'][:]
print("reading dec")
dec=c['dec'][:]
print("checking mask")
cont = m.contains(ra,dec).astype('i1')
c.write_column('inbadfield', cont)
def geratxt(book, lines, csv):
try:
col = Columns()
js = col.columns(book)
ff = Fixed_files(js, obj=True)
lenbook = calc_length(file(book).readlines())['lrecl']
record = ff.parse('0' * lenbook)
fj = [json.loads(line.decode('utf-8')) for line in js]
fldtype = [(str(att['field']), str(att['type'])) for att in fj]
txt = ''
for line in lines:
r = line.split(';')
recordreplace = ''
for n, f in enumerate(fldtype):
recordreplace += "{} = {},".format(fldtype[n][0]
,r[n] if fldtype[n][1] == 'int'
else "'"+r[n]+"'")
record = eval('record._replace({})'.format(recordreplace))
txt += ff.unparse(record)
outtxt = csv.split('.')[0]
nt = 0
while True:
filetxt = '{}{}.txt'.format(outtxt, '' if not nt else '({})'.format(nt))
if os.path.exists(filetxt):
nt += 1
else:
break
out = open(filetxt, 'w')
out.write(txt)
out.close()
return (True, filetxt)
except:
return (False, traceback.format_exc(sys.exc_info))
class Montajson(object):
def __init__(self, properties = 'pathProp.txt'):
self.properties = properties
self.col = Columns()
def montajson(self):
path = ''.join(open(self.properties).readline().replace("'", "").split())
config = open(os.path.join(path,'config.properties')).readlines()
diccnfg = {line.split()[0]:line.split()[1] for line in config}
isbook = lambda book: book[-3:].upper() == diccnfg['EXTCPY']
dirfilelist = DirFileList()
dirfilelist.setDirFileList(diccnfg['DIRSOULIB'])
booklist = dirfilelist.getDirFileList()
for book in filter(isbook, booklist):
basename = os.path.basename(book).split('.')[0]
print book
bookwrite = open('{}{}'.format(diccnfg['DIRCNVJSON'], basename + '.json'), 'w')
bookwrite.writelines(self.col.columns(book))
bookwrite.close()
from columns import Columns col = Columns() book = r'E:\GFCT\Evidencias\cpy\GFCTB092.CPY' bookwrite = open(r'E:\GFCT\Evidencias\cpy\gfctb092.json', 'w') bookwrite.writelines(col.columns(book, signal=False)) bookwrite.close()
def __init__(self, properties = 'pathProp.txt'):
self.properties = properties
self.col = Columns()
def __init__(self, config='pathProp.txt'):
self.config = config
self.col = Columns()
def branchAndPrice(data,
outputTreePickle=None,
printStatus=True,
gapLimit = 1e-9,
timeLimit = None, # Time limit in seconds
reducedTreeSize = True, # Reduce size of tree by deleting unnecessary data
resourceVec = ['TWMin', 'TWMin_g', 'TV'], # Resources used in the sub problem
removeIllegalColumns = False, # Remove all illegal columns before solving a problem
coverConstraint = '=', # Cover constraint. = or >=.
partialCG = False, # Only solve SPs until one with neg red cost is found
orderStrategy = 'random', # Strategy for choosing order of solving SP's
partialCG_constructionHeuristic = False, # Only solve SPs until one with neg red cost is found in construction heuristic
constructionHeuristic = 'CGartificialVariables', # Specify what construction heuristic to use
labelExtensionLimits = [], # Increments in SP label extension limit
SPSolutionsCount = 1, # Maximum number of SP solutions returned from one SPPRC iteration
sizeLimit = None, # Limit size of tree
CGOptimalityGapLimit=1e-9, # CG stop criterion optimality gap
CGImprovementStepSize=1, # CG improvement criterion step size
CGImprovementThreshold=1e-9, # CG improvement criterion threshold
branchOnUpperBound=False, # Branch once CG finds objective value below tree upper bound
branchingStrategy = 'xVars'
):
'''Branch-and-Price algorithm for solving the roster problem. If a tree
pickle filename is given, branch-and-price continues on the given tree.
'''
'''Initializations'''
# Start time
refTime = time.time()
# Start time for initializations
start = time.time()
# Set margin for numerical error
epsilon = 1e-9
terminate = False
# Retrieve inputs to branch and price function
# Array to store configuration
configuration = {}
# Retrieve function arguments
frame = inspect.currentframe()
arguments, _, _, values = inspect.getargvalues(frame)
# Save all arguments relevant
for argument in arguments:
if argument not in ['data', 'outputTreePickle']:
configuration[argument] = values[argument]
# Problem k to be solved
k = 0
# Initialize branch-and-bound tree
tree = Tree(configuration, refTime)
# Initialize columns and graphs
graphs = dict.fromkeys(employee for employee in data['Employees'])
for employee in data['Employees']:
graphs[employee] = Graph(data=data, employee=employee)
stop = time.time()
tree.times['Branch and price']['Initializations'] += stop - start
start = time.time()
# Generate initial columns
columns = Columns(data=data, graphs = graphs,
constructionHeuristic = constructionHeuristic,
printStatus = printStatus,
orderStrategy = orderStrategy,
partialCG_constructionHeuristic = partialCG_constructionHeuristic,
coverConstraint = coverConstraint,
removeIllegalColumns = removeIllegalColumns,
resourceVec = resourceVec)
stop = time.time()
tree.times['Branch and price']['Construction heuristic'] += stop - start
start = time.time()
# Define root problem
rootProblem = Problem(ID=k, columns=columns, graphs=graphs, data=data,
coverConstraint=coverConstraint)
# Add root problem to tree
tree.addProblem(problem=rootProblem)
stop = time.time()
tree.times['Branch and price']['Initializations'] += stop - start
tree.times['Total'] += time.time() - tree.times['refTime']
tree.times['refTime'] = time.time()
'''Main Loop'''
while not terminate:
# Select problem k from unprocessedProblems
Pk = tree.unprocessedProblems[k]
# If required and not root problem, remove illegal columns from Pk
if k > 0 and removeIllegalColumns:
# Remove from columns object
removedColumnNumbers = Pk.columns.removeIllegalColumns(data = data)
# Delete all removed columns from master problem object
Pk.masterProblem.delVariable(['lambda({},{})'.format(e, k) for e in removedColumnNumbers for k in removedColumnNumbers[e]])
# Prepare timing for solving problem k
tree.times['Total'] += time.time() - tree.times['refTime']
tree.times['refTime'] = time.time()
# Check if time limit is reached while not already terminated
if timeLimit != None and tree.times['Total'] > timeLimit:
if printStatus:
print('Time limit {:.0f} s reached. Process terminated.'.format(timeLimit))
savePickle(tree, outputTreePickle)
# Terminate function
return
# Solve problem k
Pk.solve(data = data, epsilon = epsilon, partialCG = partialCG,
orderStrategy = orderStrategy,
partialCG_constructionHeuristic = partialCG_constructionHeuristic,
labelExtensionLimits=labelExtensionLimits,
SPSolutionsCount=SPSolutionsCount,
removeIllegalColumns = removeIllegalColumns,
CGOptimalityGapLimit=CGOptimalityGapLimit,
CGImprovementStepSize=CGImprovementStepSize,
CGImprovementThreshold=CGImprovementThreshold,
branchOnUpperBound=branchOnUpperBound,
treeUpperBound=tree.upperBoundProblem.upperBound,
times = tree.times,
resourceVec = resourceVec,
timeLimit=timeLimit)
stop = time.time()
tree.times['Total'] += time.time() - tree.times['refTime']
tree.times['refTime'] = time.time()
# Check if time limit is reached while not already terminated
if timeLimit != None and tree.times['Total'] > timeLimit:
if Pk.processed:
# Remove master problem and reduce size
Pk.masterProblem = None
Pk.reduceSize(data = data)
# Move problem k from unprocessedProblems to processedProblems
tree.processedProblems[k] = tree.unprocessedProblems.pop(k)
if printStatus:
print('Time limit {:.0f} s reached. Process terminated.'.format(timeLimit))
savePickle(tree, outputTreePickle)
# Terminate function
return
# Move problem k from unprocessedProblems to processedProblems
tree.processedProblems[k] = tree.unprocessedProblems.pop(k)
# If Pk infeasible, or without potential of improvement
if not Pk.feasible or Pk.lowerBound > tree.upperBoundProblem.upperBound:
# Prune node
Pk.pruned = True
# Delete master problem in node
Pk.masterProblem = None
# Else if the final solution is integer
elif Pk.integer:
# Update upper bound if better solution found
if Pk.upperBound < tree.upperBoundProblem.upperBound:
tree.upperBoundProblem = Pk
# Prune all unprocessed problem with lower bound above new upper bound
tree.pruneOnUpperBound()
# Prune node (integer)
Pk.pruned = True
# Delete master problem in node
Pk.masterProblem = None
# Otherwise, update upper bound if Pk had a better integer solution than
# the current upper bound problem and branch problem
else:
if Pk.upperBound and Pk.upperBound < tree.upperBoundProblem.upperBound:
tree.upperBoundProblem = Pk
# Prune all unprocessed problem with lower bound above new upper bound
tree.pruneOnUpperBound()
start = time.time()
# Branch the problem
tree.branch(problem = Pk, branchingStrategy = branchingStrategy, data = data)
stop = time.time()
tree.times['Branch and price']['Branch'] += stop - start
tree.times['Total'] += time.time() - tree.times['refTime']
tree.times['refTime'] = time.time()
# Reduce size of problem by removing unnecessary attributes
Pk.reduceSize(data)
# Update lower bound and optimality gap
tree.calculateLowerBound()
tree.calculateOptimalityGap()
tree.times['Total'] += time.time() - tree.times['refTime']
tree.times['refTime'] = time.time()
# Check termination criterion
if tree.terminationCriterion(sizeLimit=sizeLimit, gapLimit = gapLimit):
terminate = True
tree.complete = True
savePickle(tree, outputTreePickle)
# Terminate function
return
# Else: update k according to search strategy
else:
# If no integer solution found (no UBD): use depth first search
if tree.upperBoundProblem.upperBound == float('inf'):
searchStrategy = 'DepthFirst_Up'
# Else: use best first search
else:
searchStrategy = 'BestFirst'
# Search and store time
start = time.time()
k = tree.search(searchStrategy)
stop = time.time()
tree.times['Branch and price']['Search'] += stop - start
tree.times['Total'] += time.time() - tree.times['refTime']
tree.times['refTime'] = time.time()
if printStatus:
nProcessedProblems = len(tree.processedProblems)
if tree.upperBoundProblem == Pk:
print('New upper bound found')
if tree.upperBoundProblem == Pk or (nProcessedProblems<=10 or
(nProcessedProblems<=50 and nProcessedProblems%2 == 0) or
nProcessedProblems%10 == 0):
printMessage(nProcessedProblems, tree.upperBoundProblem.upperBound,
tree.lowerBound, tree.gap, tree.times['Total'])
from columns import Columns
col = Columns()
cols = col.columns(r'C:\BNB\S303\ORIGINAIS\BOOK\B303031.txt', signal=False)
for c in cols:
print c
cols = col.columns(r'C:\BNB\S303\ORIGINAIS\BOOK\B303001.txt', fmt='cob',signal=False)
for c in cols:
print c
class CiccWm:
def __init__(self, keyOp, winOp):
self.keyOp = keyOp
self.winOp = winOp
self.windows = {}
self.columns = Columns()
def start(self):
self.keyOp.setUserKeyListener(self.keyListener_)
for (cb, key, mods) in [
(self.onCycleLeft_, "H", ["alt"]),
(self.onCycleDown_, "J", ["alt"]),
(self.onCycleUp_, "K", ["alt"]),
(self.onCycleRight_, "L", ["alt"]),
(self.onShiftLeft_, "H", ["alt", "shift"]),
(self.onShiftDown_, "J", ["alt", "shift"]),
(self.onShiftUp_, "K", ["alt", "shift"]),
(self.onShiftRight_, "L", ["alt", "shift"]),
]:
self.keyOp.addHotkeyListener(
cb,
ord(key),
withWin="win" in mods,
withAlt="alt" in mods,
withShift="shift" in mods,
withCtrl="ctrl" in mods,
)
self.keyOp.startEvents()
def quit(self):
self.keyOp.stopEvents()
def windowListener_(self, win):
if win not in self.windows:
self.newWindow_(win)
def newWindow_(self, win):
wi = WinInfo(win)
self.windows[win] = wi
def getWi(self, win):
return self.windows[win]
def startManaging_(self, win):
wi = self.getWi(win)
if wi.isManaged():
return
wi.manage()
self.columns.addAsRow(wi)
def onCycleUp_(self):
self.columns.cycleUpActCol()
self.columns.doLayout()
def onCycleDown_(self):
self.columns.cycleDownActCol()
self.columns.doLayout()
def onCycleRight_(self):
self.columns.cycleRight()
self.columns.doLayout()
def onCycleLeft_(self):
self.columns.cycleLeft()
self.columns.doLayout()
def onShiftUp_(self):
self.columns.shiftUpActWin()
self.columns.doLayout()
def onShiftDown_(self):
self.columns.shiftDownActWin()
self.columns.doLayout()
def onShiftRight_(self):
self.columns.shiftRightActWin()
self.columns.doLayout()
def onShiftLeft_(self):
self.columns.shiftLeftActWin()
self.columns.doLayout()
def keyListener_(self, ev):
if ev.keyId in KeyCode.MODIFIERS or not ev.isPress:
return True
"""
print("key press: {0}, id: {1}, alt: {alt}, ctrl: {ctrl}, shift: {shift}, window: {wincap})".format(
ev.isPress, ev.keyId, alt=ev.withAlt, ctrl=ev.withCtrl,
shift=ev.withShift, wincap=ev.forWindow.getCaption()) )
"""
self.windowListener_(ev.forWindow)
if ev.withAlt and ev.withShift and ev.keyId == ord("Q"):
self.quit()
return False
if ev.withAlt and ev.withShift and ev.keyId == ord("A"):
self.startManaging_(ev.forWindow)
self.columns.doLayout()
return False
if ev.withAlt and ev.withShift and ev.keyId == ord("M"):
self.columns.setActColLayout(layout.MaxingLayout())
self.columns.doLayout()
return False
if ev.withAlt and ev.withShift and ev.keyId == ord("D"):
self.columns.setActColLayout(layout.DistributingLayout())
self.columns.doLayout()
return False
if ev.withAlt and ev.withShift and ev.keyId == ord("S"):
self.columns.setActColLayout(layout.StackLayout())
self.columns.doLayout()
return False
# if ev.withAlt and ev.keyId == ord('K'):
# self.columns.cycleUpActCol()
# return False
if ev.withAlt and ev.withShift and ev.keyId == ord("R"):
print("Layout")
self.columns.doLayout()
return False
return True
def __init__(self, keyOp, winOp):
self.keyOp = keyOp
self.winOp = winOp
self.windows = {}
self.columns = Columns()
from util.HOFs import * from columns import Columns book = r'c:\Publico\Flavio\CMTF\CPY\CMTFW22A.cpy' col = Columns() bkout = col.columns(book, fmt='cbl') bkout from calc_length import * calc_length(bkout) bkin = file(book).readlines() calc_length(bkin) bkin = map(l672, filter(all3(isNotRem, isNotBlank, isNotEjectOrSkip), bkin)) word(bkin[0], 1) #teste do commit via pycharm #teste do commit via pycharm 5.0.4 #teste do commit via pycharm 5.0.4
