def __init__(self, mlu): syscnv = easytorq.ICUConverter() syscnv.setencoding("char") self.__syscnv = syscnv self.__mlu = mlu d, f = os_path_split(sys.argv[0], self.__syscnv) usage = """ Usage 1: %(argv0)s PREPROCESS_SCRIPT OPTIONS -i filelist Performs preprocessing. Options -c encodingName: character encoding in source file. -n dir: directory where preprocessed files are created. -r value: preprocess options. -v: verbose. --removeobsolute: removes obsolete preprocessed files. --threads=maxWorkerThreads: uses up to maxWorkerThreads processes. --errorfiles=output: don't stop preprocessing when parse error occured, and writes such file names to output (redirected to stderr if '-' is given). Usage 2: %(argv0)s PREPROCESS_SCRIPT --listoptions OPTIONS Prints out available options of the preprocess script. Options -s: prints raw (unformatted) text. Usage 3: %(argv0)s PREPROCESS_SCRIPT --preprocessedextension OPTIONS Prints out extension of preprocessed file. Options -o output: output file. -r value: preprocess options. Usage 4: %(argv0)s PREPROCESS_SCRIPT --getdefaultparameterizing OPTIONS Prints out default matching option for each parameter names. Options -o output: output file. """ % { "argv0": f } usage = usage[1:-1] self.__usage = usage
def __preprocess_files(self, filesToBePreprocessed, extensionStr, preprocessModule, tempFileSeed=None, options=list(), verbose=False, parseErrorFiles=None): filesToBePreprocessed = list(filesToBePreprocessed) encodingName = None preprocessorOptions = None prepDirs = list() for k, v in options: if k == '-c': encodingName = v elif k == '-r': preprocessorOptions = v elif k == '-n': prepDirs.append(v) cnv = easytorq.ICUConverter() if encodingName: cnv.setencoding(encodingName) prep = preprocessModule.getpreprocessor() if preprocessorOptions: prep.setoptions(preprocessorOptions) if verbose: progressBar = utility.ProgressReporter(len(filesToBePreprocessed)) else: progressBar = utility.ProgressReporter(0) if len(filesToBePreprocessed) > 0: prefetch = FileReader(filesToBePreprocessed[0]) prefetch.start() else: prefetch = None for i in xrange(len(filesToBePreprocessed)): if i + 1 < len(filesToBePreprocessed): nextPrefetch = FileReader(filesToBePreprocessed[i + 1]) nextPrefetch.start() else: nextPrefetch = None prefetch.join() preprocessedFname = prefetch.filename + extensionStr #print "prepDirs=", prepDirs #debug #print "preprocessedFname=", preprocessedFname #debug preprocessedFname = to_filename_in_prepdir(preprocessedFname, prepDirs) #print "preprocessedFname=", preprocessedFname #debug if prefetch.error is not None: print >> sys.stderr, "warning: not found file '%s'" % prefetch.filename else: try: strUtf8 = cnv.decode(prefetch.content) except TypeError: print >> sys.stderr, "error: invalid string (wrong character encoding?) in file '%s'" % prefetch.filename raise parseResult = None try: parseResult = prep.parse(strUtf8) except ValueError, e: if parseErrorFiles is not None: parseErrorFiles.append(prefetch.filename) else: print >> sys.stderr, "error: failure to parse file '%s'" % prefetch.filename raise e if parseResult: preorocessedFnameTemp = preprocessedFname + "-temp" try: f = fopen(preorocessedFnameTemp, "wb") except IOError: try: d = os_path_split(preprocessedFname, self.__syscnv)[0] os.makedirs(d) except: # Rarely, another process makes the directory while this process is trying to make the directory. # In such case, the above mkdirs() fails, since the directory already exists. if not os.path.exists(d): raise # the directory does not exist and this process fails to make the directory f = fopen(preorocessedFnameTemp, "wb") try: f.write(parseResult) f.close() rename_file( preorocessedFnameTemp, preprocessedFname ) # rarely causes an error, the reason is unknown except EnvironmentError, e: print >> sys.stderr, "debug info: preorocessedFnameTemp=%s preprocessedFname=%s" % ( preorocessedFnameTemp, preprocessedFname) remove_file_neglecting_error(preorocessedFnameTemp) raise e progressBar.proceed(i + 1)
CCFX_PREPDIR = ".ccfxprepdir" # walkaround to avoid \x5c character problem in os.path.split def os_path_split(pathstr, cnv): s = cnv.decode(pathstr) d, f = os.path.split(s) return cnv.encode(d), cnv.encode(f) # walkaround to avoid limit of length of file path in Windows if platform.architecture() == ('32bit', 'WindowsPE'): import win32getshortname __converter_file_funcs = easytorq.ICUConverter() __converter_file_funcs.setencoding("char") def __shorten(filepath): dirName, fileName = os_path_split(filepath, __converter_file_funcs) shortDirName = win32getshortname.getShortPathName( dirName) # may return None if shortDirName: fp = os.path.join(shortDirName, fileName) if fp: return fp return filepath def fopen(filepath, modestr): return file(__shorten(filepath), modestr)
return {"id": pp.Param.P_MATCH} def parse(self, sourceCodeStrInUtf8): if self.pat == None: self.setoptions(None) t = easytorq.Tree(sourceCodeStrInUtf8) self.pat.apply(t) s = self.fmt.format(t) return s def getpreprocessor(): return JavaPreprocessor() if __name__ == '__main__': cnv = easytorq.ICUConverter() cnv.setencoding("char") f = file(sys.argv[1], "rb") str = f.read() f.close() strUtf8 = cnv.decode(str) prep = getpreprocessor() prep.setoptions('default') s = prep.parse(strUtf8) sys.stdout.write(s)