def __init__(self, mlu):
syscnv = easytorq.ICUConverter()
syscnv.setencoding("char")
self.__syscnv = syscnv
self.__mlu = mlu
d, f = os_path_split(sys.argv[0], self.__syscnv)
usage = """
Usage 1: %(argv0)s PREPROCESS_SCRIPT OPTIONS -i filelist
Performs preprocessing.
Options
-c encodingName: character encoding in source file.
-n dir: directory where preprocessed files are created.
-r value: preprocess options.
-v: verbose.
--removeobsolute: removes obsolete preprocessed files.
--threads=maxWorkerThreads: uses up to maxWorkerThreads processes.
--errorfiles=output: don't stop preprocessing when parse error occured, and
writes such file names to output (redirected to stderr if '-' is given).
Usage 2: %(argv0)s PREPROCESS_SCRIPT --listoptions OPTIONS
Prints out available options of the preprocess script.
Options
-s: prints raw (unformatted) text.
Usage 3: %(argv0)s PREPROCESS_SCRIPT --preprocessedextension OPTIONS
Prints out extension of preprocessed file.
Options
-o output: output file.
-r value: preprocess options.
Usage 4: %(argv0)s PREPROCESS_SCRIPT --getdefaultparameterizing OPTIONS
Prints out default matching option for each parameter names.
Options
-o output: output file.
""" % {
"argv0": f
}
usage = usage[1:-1]
self.__usage = usage
def __preprocess_files(self,
filesToBePreprocessed,
extensionStr,
preprocessModule,
tempFileSeed=None,
options=list(),
verbose=False,
parseErrorFiles=None):
filesToBePreprocessed = list(filesToBePreprocessed)
encodingName = None
preprocessorOptions = None
prepDirs = list()
for k, v in options:
if k == '-c':
encodingName = v
elif k == '-r':
preprocessorOptions = v
elif k == '-n':
prepDirs.append(v)
cnv = easytorq.ICUConverter()
if encodingName:
cnv.setencoding(encodingName)
prep = preprocessModule.getpreprocessor()
if preprocessorOptions:
prep.setoptions(preprocessorOptions)
if verbose:
progressBar = utility.ProgressReporter(len(filesToBePreprocessed))
else:
progressBar = utility.ProgressReporter(0)
if len(filesToBePreprocessed) > 0:
prefetch = FileReader(filesToBePreprocessed[0])
prefetch.start()
else:
prefetch = None
for i in xrange(len(filesToBePreprocessed)):
if i + 1 < len(filesToBePreprocessed):
nextPrefetch = FileReader(filesToBePreprocessed[i + 1])
nextPrefetch.start()
else:
nextPrefetch = None
prefetch.join()
preprocessedFname = prefetch.filename + extensionStr
#print "prepDirs=", prepDirs #debug
#print "preprocessedFname=", preprocessedFname #debug
preprocessedFname = to_filename_in_prepdir(preprocessedFname,
prepDirs)
#print "preprocessedFname=", preprocessedFname #debug
if prefetch.error is not None:
print >> sys.stderr, "warning: not found file '%s'" % prefetch.filename
else:
try:
strUtf8 = cnv.decode(prefetch.content)
except TypeError:
print >> sys.stderr, "error: invalid string (wrong character encoding?) in file '%s'" % prefetch.filename
raise
parseResult = None
try:
parseResult = prep.parse(strUtf8)
except ValueError, e:
if parseErrorFiles is not None:
parseErrorFiles.append(prefetch.filename)
else:
print >> sys.stderr, "error: failure to parse file '%s'" % prefetch.filename
raise e
if parseResult:
preorocessedFnameTemp = preprocessedFname + "-temp"
try:
f = fopen(preorocessedFnameTemp, "wb")
except IOError:
try:
d = os_path_split(preprocessedFname,
self.__syscnv)[0]
os.makedirs(d)
except:
# Rarely, another process makes the directory while this process is trying to make the directory.
# In such case, the above mkdirs() fails, since the directory already exists.
if not os.path.exists(d):
raise # the directory does not exist and this process fails to make the directory
f = fopen(preorocessedFnameTemp, "wb")
try:
f.write(parseResult)
f.close()
rename_file(
preorocessedFnameTemp, preprocessedFname
) # rarely causes an error, the reason is unknown
except EnvironmentError, e:
print >> sys.stderr, "debug info: preorocessedFnameTemp=%s preprocessedFname=%s" % (
preorocessedFnameTemp, preprocessedFname)
remove_file_neglecting_error(preorocessedFnameTemp)
raise e
progressBar.proceed(i + 1)
CCFX_PREPDIR = ".ccfxprepdir"
# walkaround to avoid \x5c character problem in os.path.split
def os_path_split(pathstr, cnv):
s = cnv.decode(pathstr)
d, f = os.path.split(s)
return cnv.encode(d), cnv.encode(f)
# walkaround to avoid limit of length of file path in Windows
if platform.architecture() == ('32bit', 'WindowsPE'):
import win32getshortname
__converter_file_funcs = easytorq.ICUConverter()
__converter_file_funcs.setencoding("char")
def __shorten(filepath):
dirName, fileName = os_path_split(filepath, __converter_file_funcs)
shortDirName = win32getshortname.getShortPathName(
dirName) # may return None
if shortDirName:
fp = os.path.join(shortDirName, fileName)
if fp:
return fp
return filepath
def fopen(filepath, modestr):
return file(__shorten(filepath), modestr)
return {"id": pp.Param.P_MATCH}
def parse(self, sourceCodeStrInUtf8):
if self.pat == None:
self.setoptions(None)
t = easytorq.Tree(sourceCodeStrInUtf8)
self.pat.apply(t)
s = self.fmt.format(t)
return s
def getpreprocessor():
return JavaPreprocessor()
if __name__ == '__main__':
cnv = easytorq.ICUConverter()
cnv.setencoding("char")
f = file(sys.argv[1], "rb")
str = f.read()
f.close()
strUtf8 = cnv.decode(str)
prep = getpreprocessor()
prep.setoptions('default')
s = prep.parse(strUtf8)
sys.stdout.write(s)
