def sweep_dir(path, srcs, results):
dname = os.path.basename(path)
if dname not in ignore_dirs:
filelist = os.listdir(path)
filelist.sort(key=lambda x: x[0:20])
for f in filelist:
pf = os.path.join(path, f)
if util.dir_exists(pf):
sweep_dir(pf, srcs, results)
elif util.file_exists(pf):
_, ext = os.path.splitext(f)
passed_filters = False
if f not in ignore_files and ext not in ignore_exts:
passed_filters = True
if len(only_exts) > 0:
if ext not in only_exts:
passed_filters = False
if passed_filters:
fd = None
try:
fd = open(pf, 'r')
lines = fd.readlines()
lineno = 0
for line in lines:
lineno += 1
for src in srcs:
if line.find(src) != -1:
elog.info("found '%s': [%s:%d]", src,
os.path.relpath(pf, APPPATH),
lineno)
elog.force_clean("%s", line)
if pf not in results:
results.append(pf)
except:
elog.error("%r: %s", sys.exc_info(), pf)
finally:
util.close_file_nothrow(fd)
else:
#elog.warn("ignore file: %s", pf)
pass
else:
elog.warn("ignore path: %s", path)
pass
def main(parser, appConfig, loggerConfig):
import utils.logger
(options, args) = parser.parse_args(args=None, values=None)
loggerDictConfig = utils.logger.set_logger(loggerConfig, options.log_path,
options.log_level)
elog.force("%s-%s starting", APPNAME, APPVER)
# 当前脚本绝对路径
abspath = util.script_abspath(inspect.currentframe())
util.print_options_attrs(options, ['source_dbicfg', 'dest_sqlfile'])
absYamlFile = os.path.abspath(options.source_dbicfg)
absSqlFile = os.path.abspath(options.dest_sqlfile)
module, _ = os.path.splitext(os.path.basename(absYamlFile))
util.info2(absYamlFile)
util.info2(absSqlFile)
# 打开配置文件
fd = open(absYamlFile)
data = fd.read()
fd.close()
# 载入配置
dict = yaml.load(data)
# 创建 sqlfile
create_phoenix_cresql(dict, module.lower(), absSqlFile, options.force,
dict.get('constants'))
util.warn(
"NOW YOU CAN USE BELOW COMMAND TO CREATE TABLES IN HBASE-PHOENIX !")
elog.force_clean(" $ sqlline.py zkhost:zkport /path/to/file.cresql")
util.info("Sample:\n $ ./sqlline.py localhost:2182 %s" % absSqlFile)
pass
def printReport(self, procNameList, title):
totalSeconds = self.elapsedSeconds()
elog.force_clean("[%s Report by Process (%d seconds)]", title,
totalSeconds)
elog.force_clean("---------------------------------------")
elog.force_clean("process key value")
elog.force_clean("---------------------------------------")
keyStatsTotal = {}
for pname in procNameList:
for statKey, statValue in self.statDict.items():
keys = statKey.split(':')
if keys[0] == pname and keys[1] == 'stats':
key = keys[2]
elog.force_clean("%s %s %d", pname, key,
statValue)
totalValue = keyStatsTotal.get(key, 0) + statValue
keyStatsTotal[key] = totalValue
elog.force_clean("=======================================")
if len(procNameList) > 0:
for (key, total) in keyStatsTotal.items():
elog.force_clean("%s=%d (avg %d per second)", key, total,
int(total / totalSeconds))
pass