def geracsv(book, records, arq, signal): try: basename = os.path.basename(book).split('.')[0] col = Columns() js = col.columns(book, signal=signal) fj = [json.loads(line.decode('utf-8')) for line in js] ff = Fixed_files(fj, obj=True) rec_in = [] for record in records: rec_in.append(ff.parse(record)) flds = [str(att['field']) for att in fj] csvs = ';'.join( str(flds)[1:-1].upper().replace(basename + '_', '').replace( "'", "").split(',')) + '\n' csvs = re.sub('FILLER_..', 'FILLER', csvs) fmt = '''"{}"'''.format(str(['{' + f + '}' for f in flds ])[1:-1]).replace("'", "").replace(',', ';') fmt += ".format(**r)" for r in rec_in: for n, f in enumerate(r): csvs += r[n] + ';' csvs += '\n' outcsv = arq.split('.')[0] + '.csv' out = open(outcsv, 'w') out.write(csvs) out.close() return (True, outcsv) except: return (False, traceback.format_exc(sys.exc_info))
def gerpgm(self): try: bookin = file(self.book).readlines() lengthin = str(calc_length(bookin)['lrecl']) col = Columns() bkout = col.columns(self.book, fmt='cbl', signal=self.signal) bookout = '' for b in bkout: bookout += b lengthout = str(calc_length(bookout)['lrecl']) regout = 'WRK-ARQOUTZD-REGISTRO' bookin = Homogenize(bookin) if int(word(bookin[0], 1)) == 1: regin = word(bookin[0], 2).replace('.', '') bookout = change({regin: regout}, bookout) lvl01bookin = '' lvl01bookout = '' else: regin = 'WRK-ARQINPPD-REGISTRO' lvl01bookin = '{:>9} {}.\n'.format('01', regin) lvl01bookout = '{:>9} {}.\n'.format('01', regout) formatout = '' for line in bookin: if 'PIC' in line: if word(line, 2) == 'FILLER': continue fld = word(line, 2) formatout += '{:>15} {:31} OF {}\n'.format( 'MOVE', fld, regin) formatout += '{:>15} {:31} OF {}\n'.format( 'TO', fld, regout) dicProg = { '@PGMID': self.programId, '@DATE': date.today().strftime('%d %b %Y').upper(), '@BOOKIN': os.path.basename(self.book).split('.')[0].upper(), '@SINAL': 'COM' if self.signal else 'SEM', '@BOOKOUT\n': bookout, '@REGIN': regin, '@REGOUT': regout, '@LVL01BKIN\n': lvl01bookin, '@LVL01BKOUT\n': lvl01bookout, '@LENGTHIN': lengthin, '@LENGTHOUT': lengthout, '@FORMATOUT\n': formatout } prog = insAster72(change(dicProg, file('legrpdzd.cbl').read())) progName = os.path.join(self.path, '{}.cbl'.format(self.programId)) progWrite = open(progName, 'w') progWrite.write(prog) progWrite.close() return True, None except: return (False, traceback.format_exc(sys.exc_info))
def write_columns(self): d = files.input_coldir(self.name, self.version) if os.path.exists(d): raise ValueError("coldir exists, start fresh: '%s'" % d) outcols = Columns(d) outcols.create() if self.logic is None: self.select() w=where1(self.mag_and_mask_logic) print("\nkeeping those that pass mag and mask logic: %s/%s" % (w.size, self.logic.size)) colnames = ['photoid', 'ra', 'dec', 'objc_flags', 'objc_flags2', 'ingood', 'instar', 'inbadfield'] for col in colnames: print('Creating:',col) data = self.cols[col][:] data = data[w] outcols.write_column(col, data) del data combcols = ['flags','flags2', 'modelflux','modelflux_ivar', 'cmodelflux','cmodelflux_ivar', 'extinction'] for ccol in combcols: print('Creating:',ccol) colnames = [ccol+'_'+f for f in ['u','g','r','i','z'] ] data = self.cols.read_columns(colnames, rows=w, verbose=True) dts = data[ccol+'_'+f].dtype.descr[0][1] dt = [(ccol, dts, 5)] data = data.view(dt) # don't want it to show up as a .rec rawdata = data[ccol] outcols.write_column(ccol, rawdata) del data print('Adding more restrictive flags to "keep"') keep = zeros(w.size, dtype='u1') wrest = where1( self.logic[w] ) keep[wrest] = 1 outcols.write_column('keep', keep)
def gerjob(self): try: basename = os.path.basename(self.book).split('.')[0].upper() col = Columns() bookout = col.columns(self.book, fmt='cbl', signal=False) book_zonado = os.path.join(self.path, '{}_ZD.cpy'.format(basename)) with open(book_zonado, 'w') as bkzd: bkzd.writelines(bookout) lengthout = str(calc_length(bookout)['lrecl']) formatout = '' start = 1 bookin = file(self.book).readlines() bookin = Homogenize(bookin, cbl=True) for line in bookin: if 'PIC' not in line: continue splt_pic = line.split('PIC')[1] repl_pic = splt_pic.replace(' USAGE ', '').replace( 'COMP-3', '').replace('COMP', '').rstrip() ap = ' OUTREC FIELDS=(' if start == 1 else ' ' length = int( str( calc_length(line.replace(splt_pic, repl_pic))['lrecl'])) lenpd = calc_length(line)['lrecl'] pd2zd = ('PD,TO=ZD,LENGTH={:03},'.format(length) if 'COMP-3' in splt_pic else 'BI,TO=ZD,LENGTH={:03},'.format(length) if 'COMP' in splt_pic else '') formatout += '{}{:03},{:03},{}\n'.format( ap, start, lenpd, pd2zd) start += lenpd formatout = formatout[:-2] + ')\n' dicjob = { '@JOBNAME': '{:8}'.format(self.jobname), '@BOOK': basename, '@SORTIN': self.sortin, '@SORTOUT': self.sortout, '@LRECL': lengthout, '@OUTREC\n': formatout } job = change(dicjob, file('jobpk2zd.template').read()) jobName = os.path.join(self.path, '{}.jcl'.format(self.jobname)) with open(jobName, 'w') as jobWrite: jobWrite.write(job) return True, None except: return (False, traceback.format_exc(sys.exc_info))
def add_colc_nmgypercount(self): """ self.cols is the sweeps """ import es_sdsspy d = files.input_coldir(self.name, self.version) ccols = Columns(d) print("cluster input cols:",ccols.dir) scols = self.cols print("sweep columns:",scols.dir) print("reading sweep photoid") spid = scols['photoid'][:] print("reading cluster input photoid") cpid = ccols['photoid'][:] print("matching") ms,mc = esutil.numpy_util.match(spid, cpid) n=cpid.size if mc.size != n: raise ValueError("not all matched: %d/%d" % (mt.size/n)) # problem is the columns code always sorts the requested # indices so we will have lost order information. Need # to pre-sort both sets of indices according to the sweep # row number print("ordering matches with sweep cols") s=ms.argsort() ms = ms[s] mc = mc[s] for colname in ['colc','nmgypercount']: data=numpy.zeros((n,5),dtype='f4') for i,band in enumerate(['u','g','r','i','z']): scolname='%s_%s' % (colname,band) print(" %s" % scolname) sdata=scols[scolname][ms] data[mc,i] = sdata del sdata print("writing: %s" % colname) ccols.write_column(colname, data) del data
def add_inbadfield(self): import es_sdsspy m=es_sdsspy.mangle_masks.load('boss','badfield') d = files.input_coldir(self.name, self.version) c = Columns(d) print("reading ra") ra=c['ra'][:] print("reading dec") dec=c['dec'][:] print("checking mask") cont = m.contains(ra,dec).astype('i1') c.write_column('inbadfield', cont)
def geratxt(book, lines, csv): try: col = Columns() js = col.columns(book) ff = Fixed_files(js, obj=True) lenbook = calc_length(file(book).readlines())['lrecl'] record = ff.parse('0' * lenbook) fj = [json.loads(line.decode('utf-8')) for line in js] fldtype = [(str(att['field']), str(att['type'])) for att in fj] txt = '' for line in lines: r = line.split(';') recordreplace = '' for n, f in enumerate(fldtype): recordreplace += "{} = {},".format(fldtype[n][0] ,r[n] if fldtype[n][1] == 'int' else "'"+r[n]+"'") record = eval('record._replace({})'.format(recordreplace)) txt += ff.unparse(record) outtxt = csv.split('.')[0] nt = 0 while True: filetxt = '{}{}.txt'.format(outtxt, '' if not nt else '({})'.format(nt)) if os.path.exists(filetxt): nt += 1 else: break out = open(filetxt, 'w') out.write(txt) out.close() return (True, filetxt) except: return (False, traceback.format_exc(sys.exc_info))
class Montajson(object): def __init__(self, properties = 'pathProp.txt'): self.properties = properties self.col = Columns() def montajson(self): path = ''.join(open(self.properties).readline().replace("'", "").split()) config = open(os.path.join(path,'config.properties')).readlines() diccnfg = {line.split()[0]:line.split()[1] for line in config} isbook = lambda book: book[-3:].upper() == diccnfg['EXTCPY'] dirfilelist = DirFileList() dirfilelist.setDirFileList(diccnfg['DIRSOULIB']) booklist = dirfilelist.getDirFileList() for book in filter(isbook, booklist): basename = os.path.basename(book).split('.')[0] print book bookwrite = open('{}{}'.format(diccnfg['DIRCNVJSON'], basename + '.json'), 'w') bookwrite.writelines(self.col.columns(book)) bookwrite.close()
from columns import Columns col = Columns() book = r'E:\GFCT\Evidencias\cpy\GFCTB092.CPY' bookwrite = open(r'E:\GFCT\Evidencias\cpy\gfctb092.json', 'w') bookwrite.writelines(col.columns(book, signal=False)) bookwrite.close()
def __init__(self, properties = 'pathProp.txt'): self.properties = properties self.col = Columns()
def __init__(self, config='pathProp.txt'): self.config = config self.col = Columns()
def branchAndPrice(data, outputTreePickle=None, printStatus=True, gapLimit = 1e-9, timeLimit = None, # Time limit in seconds reducedTreeSize = True, # Reduce size of tree by deleting unnecessary data resourceVec = ['TWMin', 'TWMin_g', 'TV'], # Resources used in the sub problem removeIllegalColumns = False, # Remove all illegal columns before solving a problem coverConstraint = '=', # Cover constraint. = or >=. partialCG = False, # Only solve SPs until one with neg red cost is found orderStrategy = 'random', # Strategy for choosing order of solving SP's partialCG_constructionHeuristic = False, # Only solve SPs until one with neg red cost is found in construction heuristic constructionHeuristic = 'CGartificialVariables', # Specify what construction heuristic to use labelExtensionLimits = [], # Increments in SP label extension limit SPSolutionsCount = 1, # Maximum number of SP solutions returned from one SPPRC iteration sizeLimit = None, # Limit size of tree CGOptimalityGapLimit=1e-9, # CG stop criterion optimality gap CGImprovementStepSize=1, # CG improvement criterion step size CGImprovementThreshold=1e-9, # CG improvement criterion threshold branchOnUpperBound=False, # Branch once CG finds objective value below tree upper bound branchingStrategy = 'xVars' ): '''Branch-and-Price algorithm for solving the roster problem. If a tree pickle filename is given, branch-and-price continues on the given tree. ''' '''Initializations''' # Start time refTime = time.time() # Start time for initializations start = time.time() # Set margin for numerical error epsilon = 1e-9 terminate = False # Retrieve inputs to branch and price function # Array to store configuration configuration = {} # Retrieve function arguments frame = inspect.currentframe() arguments, _, _, values = inspect.getargvalues(frame) # Save all arguments relevant for argument in arguments: if argument not in ['data', 'outputTreePickle']: configuration[argument] = values[argument] # Problem k to be solved k = 0 # Initialize branch-and-bound tree tree = Tree(configuration, refTime) # Initialize columns and graphs graphs = dict.fromkeys(employee for employee in data['Employees']) for employee in data['Employees']: graphs[employee] = Graph(data=data, employee=employee) stop = time.time() tree.times['Branch and price']['Initializations'] += stop - start start = time.time() # Generate initial columns columns = Columns(data=data, graphs = graphs, constructionHeuristic = constructionHeuristic, printStatus = printStatus, orderStrategy = orderStrategy, partialCG_constructionHeuristic = partialCG_constructionHeuristic, coverConstraint = coverConstraint, removeIllegalColumns = removeIllegalColumns, resourceVec = resourceVec) stop = time.time() tree.times['Branch and price']['Construction heuristic'] += stop - start start = time.time() # Define root problem rootProblem = Problem(ID=k, columns=columns, graphs=graphs, data=data, coverConstraint=coverConstraint) # Add root problem to tree tree.addProblem(problem=rootProblem) stop = time.time() tree.times['Branch and price']['Initializations'] += stop - start tree.times['Total'] += time.time() - tree.times['refTime'] tree.times['refTime'] = time.time() '''Main Loop''' while not terminate: # Select problem k from unprocessedProblems Pk = tree.unprocessedProblems[k] # If required and not root problem, remove illegal columns from Pk if k > 0 and removeIllegalColumns: # Remove from columns object removedColumnNumbers = Pk.columns.removeIllegalColumns(data = data) # Delete all removed columns from master problem object Pk.masterProblem.delVariable(['lambda({},{})'.format(e, k) for e in removedColumnNumbers for k in removedColumnNumbers[e]]) # Prepare timing for solving problem k tree.times['Total'] += time.time() - tree.times['refTime'] tree.times['refTime'] = time.time() # Check if time limit is reached while not already terminated if timeLimit != None and tree.times['Total'] > timeLimit: if printStatus: print('Time limit {:.0f} s reached. Process terminated.'.format(timeLimit)) savePickle(tree, outputTreePickle) # Terminate function return # Solve problem k Pk.solve(data = data, epsilon = epsilon, partialCG = partialCG, orderStrategy = orderStrategy, partialCG_constructionHeuristic = partialCG_constructionHeuristic, labelExtensionLimits=labelExtensionLimits, SPSolutionsCount=SPSolutionsCount, removeIllegalColumns = removeIllegalColumns, CGOptimalityGapLimit=CGOptimalityGapLimit, CGImprovementStepSize=CGImprovementStepSize, CGImprovementThreshold=CGImprovementThreshold, branchOnUpperBound=branchOnUpperBound, treeUpperBound=tree.upperBoundProblem.upperBound, times = tree.times, resourceVec = resourceVec, timeLimit=timeLimit) stop = time.time() tree.times['Total'] += time.time() - tree.times['refTime'] tree.times['refTime'] = time.time() # Check if time limit is reached while not already terminated if timeLimit != None and tree.times['Total'] > timeLimit: if Pk.processed: # Remove master problem and reduce size Pk.masterProblem = None Pk.reduceSize(data = data) # Move problem k from unprocessedProblems to processedProblems tree.processedProblems[k] = tree.unprocessedProblems.pop(k) if printStatus: print('Time limit {:.0f} s reached. Process terminated.'.format(timeLimit)) savePickle(tree, outputTreePickle) # Terminate function return # Move problem k from unprocessedProblems to processedProblems tree.processedProblems[k] = tree.unprocessedProblems.pop(k) # If Pk infeasible, or without potential of improvement if not Pk.feasible or Pk.lowerBound > tree.upperBoundProblem.upperBound: # Prune node Pk.pruned = True # Delete master problem in node Pk.masterProblem = None # Else if the final solution is integer elif Pk.integer: # Update upper bound if better solution found if Pk.upperBound < tree.upperBoundProblem.upperBound: tree.upperBoundProblem = Pk # Prune all unprocessed problem with lower bound above new upper bound tree.pruneOnUpperBound() # Prune node (integer) Pk.pruned = True # Delete master problem in node Pk.masterProblem = None # Otherwise, update upper bound if Pk had a better integer solution than # the current upper bound problem and branch problem else: if Pk.upperBound and Pk.upperBound < tree.upperBoundProblem.upperBound: tree.upperBoundProblem = Pk # Prune all unprocessed problem with lower bound above new upper bound tree.pruneOnUpperBound() start = time.time() # Branch the problem tree.branch(problem = Pk, branchingStrategy = branchingStrategy, data = data) stop = time.time() tree.times['Branch and price']['Branch'] += stop - start tree.times['Total'] += time.time() - tree.times['refTime'] tree.times['refTime'] = time.time() # Reduce size of problem by removing unnecessary attributes Pk.reduceSize(data) # Update lower bound and optimality gap tree.calculateLowerBound() tree.calculateOptimalityGap() tree.times['Total'] += time.time() - tree.times['refTime'] tree.times['refTime'] = time.time() # Check termination criterion if tree.terminationCriterion(sizeLimit=sizeLimit, gapLimit = gapLimit): terminate = True tree.complete = True savePickle(tree, outputTreePickle) # Terminate function return # Else: update k according to search strategy else: # If no integer solution found (no UBD): use depth first search if tree.upperBoundProblem.upperBound == float('inf'): searchStrategy = 'DepthFirst_Up' # Else: use best first search else: searchStrategy = 'BestFirst' # Search and store time start = time.time() k = tree.search(searchStrategy) stop = time.time() tree.times['Branch and price']['Search'] += stop - start tree.times['Total'] += time.time() - tree.times['refTime'] tree.times['refTime'] = time.time() if printStatus: nProcessedProblems = len(tree.processedProblems) if tree.upperBoundProblem == Pk: print('New upper bound found') if tree.upperBoundProblem == Pk or (nProcessedProblems<=10 or (nProcessedProblems<=50 and nProcessedProblems%2 == 0) or nProcessedProblems%10 == 0): printMessage(nProcessedProblems, tree.upperBoundProblem.upperBound, tree.lowerBound, tree.gap, tree.times['Total'])
from columns import Columns col = Columns() cols = col.columns(r'C:\BNB\S303\ORIGINAIS\BOOK\B303031.txt', signal=False) for c in cols: print c cols = col.columns(r'C:\BNB\S303\ORIGINAIS\BOOK\B303001.txt', fmt='cob',signal=False) for c in cols: print c
class CiccWm: def __init__(self, keyOp, winOp): self.keyOp = keyOp self.winOp = winOp self.windows = {} self.columns = Columns() def start(self): self.keyOp.setUserKeyListener(self.keyListener_) for (cb, key, mods) in [ (self.onCycleLeft_, "H", ["alt"]), (self.onCycleDown_, "J", ["alt"]), (self.onCycleUp_, "K", ["alt"]), (self.onCycleRight_, "L", ["alt"]), (self.onShiftLeft_, "H", ["alt", "shift"]), (self.onShiftDown_, "J", ["alt", "shift"]), (self.onShiftUp_, "K", ["alt", "shift"]), (self.onShiftRight_, "L", ["alt", "shift"]), ]: self.keyOp.addHotkeyListener( cb, ord(key), withWin="win" in mods, withAlt="alt" in mods, withShift="shift" in mods, withCtrl="ctrl" in mods, ) self.keyOp.startEvents() def quit(self): self.keyOp.stopEvents() def windowListener_(self, win): if win not in self.windows: self.newWindow_(win) def newWindow_(self, win): wi = WinInfo(win) self.windows[win] = wi def getWi(self, win): return self.windows[win] def startManaging_(self, win): wi = self.getWi(win) if wi.isManaged(): return wi.manage() self.columns.addAsRow(wi) def onCycleUp_(self): self.columns.cycleUpActCol() self.columns.doLayout() def onCycleDown_(self): self.columns.cycleDownActCol() self.columns.doLayout() def onCycleRight_(self): self.columns.cycleRight() self.columns.doLayout() def onCycleLeft_(self): self.columns.cycleLeft() self.columns.doLayout() def onShiftUp_(self): self.columns.shiftUpActWin() self.columns.doLayout() def onShiftDown_(self): self.columns.shiftDownActWin() self.columns.doLayout() def onShiftRight_(self): self.columns.shiftRightActWin() self.columns.doLayout() def onShiftLeft_(self): self.columns.shiftLeftActWin() self.columns.doLayout() def keyListener_(self, ev): if ev.keyId in KeyCode.MODIFIERS or not ev.isPress: return True """ print("key press: {0}, id: {1}, alt: {alt}, ctrl: {ctrl}, shift: {shift}, window: {wincap})".format( ev.isPress, ev.keyId, alt=ev.withAlt, ctrl=ev.withCtrl, shift=ev.withShift, wincap=ev.forWindow.getCaption()) ) """ self.windowListener_(ev.forWindow) if ev.withAlt and ev.withShift and ev.keyId == ord("Q"): self.quit() return False if ev.withAlt and ev.withShift and ev.keyId == ord("A"): self.startManaging_(ev.forWindow) self.columns.doLayout() return False if ev.withAlt and ev.withShift and ev.keyId == ord("M"): self.columns.setActColLayout(layout.MaxingLayout()) self.columns.doLayout() return False if ev.withAlt and ev.withShift and ev.keyId == ord("D"): self.columns.setActColLayout(layout.DistributingLayout()) self.columns.doLayout() return False if ev.withAlt and ev.withShift and ev.keyId == ord("S"): self.columns.setActColLayout(layout.StackLayout()) self.columns.doLayout() return False # if ev.withAlt and ev.keyId == ord('K'): # self.columns.cycleUpActCol() # return False if ev.withAlt and ev.withShift and ev.keyId == ord("R"): print("Layout") self.columns.doLayout() return False return True
def __init__(self, keyOp, winOp): self.keyOp = keyOp self.winOp = winOp self.windows = {} self.columns = Columns()
from util.HOFs import * from columns import Columns book = r'c:\Publico\Flavio\CMTF\CPY\CMTFW22A.cpy' col = Columns() bkout = col.columns(book, fmt='cbl') bkout from calc_length import * calc_length(bkout) bkin = file(book).readlines() calc_length(bkin) bkin = map(l672, filter(all3(isNotRem, isNotBlank, isNotEjectOrSkip), bkin)) word(bkin[0], 1) #teste do commit via pycharm #teste do commit via pycharm 5.0.4 #teste do commit via pycharm 5.0.4