def sweep_path(path, file_exts, recursive, author, curtime):
filelist = os.listdir(path)
filelist.sort(key=lambda x:x[0:20])
for f in filelist:
try:
pf = os.path.join(path, f)
fs = os.stat(pf)
mod = fs.st_mode
if stat.S_ISDIR(mod):
# is dir
if util.dir_exists(pf):
if recursive:
sweep_path(pf, file_exts, recursive, author, curtime)
pass
pass
elif stat.S_ISREG(mod):
# is file
ignored = False
if not util.file_exists(pf) or pf == APPFILE or f == "__init__.py":
ignored = True
if not ignored:
_, ext = os.path.splitext(f)
if ext in file_exts:
update_file(pf, f, fs, author, curtime)
pass
except OSError as e:
elog.warn("%r: %s" % (e, pf))
pass
def create_sb2web_project(appConfig, options):
artifactDir = os.path.join(appConfig.projectsDir, options.artifact)
elog.info("starting create project: %s", artifactDir)
if util.dir_exists(artifactDir):
util.warn("artifact already exists: %s" % artifactDir)
if not options.force:
util.warn("using '--force' to ovewrite it")
sys.exit(0)
pass
try:
shutil.rmtree(artifactDir)
except:
pass
pairConfig = (util.DotDict(), util.DotDict())
# 载入模板工程的配置文件
templateDict = {}
templateYaml = os.path.join(appConfig.sb2webRoot, "j2template.yaml")
if util.file_exists(templateYaml):
fd = open(templateYaml)
templateDict = yaml.load(fd.read())
fd.close()
renderConfig = util.DotDict(
LICENSE_HEADER=read_file_content(
os.path.join(APPHOME, 'license_header.txt')),
j2envRoot=appConfig.sb2webRoot,
j2env=Environment(loader=FileSystemLoader(appConfig.sb2webRoot)),
templateDict=templateDict,
springbootVer=options.springboot,
groupId=options.group,
artifactId=options.artifact,
artifactName=options.name,
artifactVer=options.ver,
description=options.description,
packageName=options.packagename,
packaging=options.packaging,
javaVersion=options.java,
serverPort=options.port,
contextPath=options.context_path)
# 复制目录树, 同时处理模板文件
util.copydirtree(appConfig.sb2webRoot, artifactDir, pairConfig, True,
copy_template, renderConfig)
elog.info("success create project: %s", artifactDir)
if options.run:
elog.info("starting run: %s", artifactDir)
start_run_project(artifactDir)
pass
def sweep_dir(path, srcs, results):
dname = os.path.basename(path)
if dname not in ignore_dirs:
filelist = os.listdir(path)
filelist.sort(key=lambda x: x[0:20])
for f in filelist:
pf = os.path.join(path, f)
if util.dir_exists(pf):
sweep_dir(pf, srcs, results)
elif util.file_exists(pf):
_, ext = os.path.splitext(f)
passed_filters = False
if f not in ignore_files and ext not in ignore_exts:
passed_filters = True
if len(only_exts) > 0:
if ext not in only_exts:
passed_filters = False
if passed_filters:
fd = None
try:
fd = open(pf, 'r')
lines = fd.readlines()
lineno = 0
for line in lines:
lineno += 1
for src in srcs:
if line.find(src) != -1:
elog.info("found '%s': [%s:%d]", src,
os.path.relpath(pf, APPPATH),
lineno)
elog.force_clean("%s", line)
if pf not in results:
results.append(pf)
except:
elog.error("%r: %s", sys.exc_info(), pf)
finally:
util.close_file_nothrow(fd)
else:
#elog.warn("ignore file: %s", pf)
pass
else:
elog.warn("ignore path: %s", path)
pass
def main(parser, appConfig, loggerConfig):
import utils.logger
(options, args) = parser.parse_args(args=None, values=None)
loggerDictConfig = utils.logger.set_logger(loggerConfig, options.log_path,
options.log_level)
elog.force("%s-%s start", APPNAME, APPVER)
# 当前脚本绝对路径
abspath = util.script_abspath(inspect.currentframe())
options.artifactName = options.artifactName \
.replace('${artifactId}', options.artifactId) \
.replace('-', '_') \
.replace('.', '_')
options.artifactRootdir = options.artifactRootdir \
.replace('${projectRootdir}', appConfig['projectRootdir']) \
.replace('${artifactId}', options.artifactId) \
.replace('${artifactName}', options.artifactName)
if options.create_project:
util.info2("projectRootdir = '%s'" % appConfig['projectRootdir'])
util.info2("sb2template = '%s'" % appConfig['sb2template'])
util.print_options_attrs(options, [
'groupId', 'artifactId', 'artifactName', 'artifactVersion',
'artifactDescription', 'artifactRootdir'
])
if util.dir_exists(options.artifactRootdir) and not options.force:
elog.warn(
"artifactRootdir has already existed. (using '--force' to overwrite it.)"
)
sys.exit(-1)
pass
create_sb2_project(appConfig, options)
elog.force("%s-%s exit.", APPNAME, APPVER)
pass
def generate(parser, dictcfg, templateRoot, j2env, options):
# 不可更改 !
module = 'xstp'
module_ver = "%s-%s" % (module, dictcfg['Version'])
module_prefix = os.path.join(os.path.dirname(APPHOME), options.output,
module_ver)
util.info2("template root: %s" % templateRoot)
util.info2("create module: %s" % module_prefix)
if util.dir_exists(module_prefix):
util.warn("module already exists: %s" % module_prefix)
if not options.force:
util.warn("using '--force' to ovewrite it")
sys.exit(0)
pass
try:
shutil.rmtree(module_prefix)
except:
pass
license_header = read_content(os.path.join(APPHOME, dictcfg['License']))
dictcfg['Update'] = util.nowtime()
dictcfg['Datatypes'] = Datatypes
modcfg = {
'module': module,
'module_ver': module_ver,
'dictcfg': dictcfg,
'license_header': license_header,
'j2env': j2env
}
# 复制目录树, 同时处理模板文件
util.copydirtree(templateRoot, module_prefix, None, True, copy_module,
modcfg)
pass
def init_data(self, logfile):
self.restore_position()
if not util.dir_exists(self.log_prefix):
elog.warn("create dir for stash log: %s", self.log_prefix)
os.makedirs(self.log_prefix)
elog.debug('log config: %r', self.dictcfg)
elog.info('stash prefix: %s', self.log_prefix)
elog.info('start tstamp: %d', self.start_tstamp)
elog.info('start rowid: %d', self.start_rowid)
elog.info('batch rows: %d', self.batch_rows)
file_dests = os.path.join(self.plugins_dir, 'config' , 'dests.csv')
file_proxys = os.path.join(self.plugins_dir, 'config' , 'proxys.csv')
file_keywds = os.path.join(self.plugins_dir, 'config' , 'keywds.csv')
elog.info("dests file: %s", file_dests)
elog.info("proxys file: %s", file_proxys)
elog.info("keywds file: %s", file_keywds)
with open(file_dests, 'r') as fd:
dests = fd.readlines()
with open(file_proxys, 'r') as fd:
proxys = fd.readlines()
with open(file_keywds, 'r') as fd:
keywds = fd.readlines()
self.dests = []
for n in range(0, len(dests)):
# id, ip, port, host
# 100005,67.64.46.91,80,www.zhibo8.cc
self.dests.append(tuple(dests[n].strip('\n').split(',')))
del dests
self.proxys = []
for n in range(0, len(proxys)):
# ip, port, type
# 121.232.144.158,9000,HTTP
self.proxys.append(tuple(proxys[n].strip('\n').split(',')))
del proxys
self.keywds = []
for n in range(0, len(keywds)):
# id, word
self.keywds.append(tuple(keywds[n].strip('\n').split(',')))
del keywds
self.max_dests = len(self.dests) - 1
self.max_proxys = len(self.proxys) - 1
self.max_keywds = len(self.keywds) - 1
# update dictcfg with logfile
elog.update_log_config(self.dictcfg, self.logger_name, logfile, 'INFO')
# reload config
logging.config.dictConfig(self.dictcfg)
# update logger
self.logger = logging.getLogger(self.logger_name)
self.logfile = logfile
(self.a, self.b, self.c, self.d, self.p) = ((1, 220), (10, 230), (20, 240), (30, 250), (10000, 60000))
self.fields = (
'rowid',
'timestr',
'timeint',
'destid',
'sourip',
'sourport',
'destip',
'destport',
'desturl',
'proxyip',
'proxyport',
'proxytype',
'keywdid')
pass
def main(parser):
(options, args) = parser.parse_args(args=None, values=None)
# 当前脚本绝对路径
abspath = util.script_abspath(inspect.currentframe())
# 产品文件夹全路径
artifactRoot = options.artifactRoot
# 产品文件夹名称
artifactRootName = os.path.basename(artifactRoot)
artifactId = options.artifactId
if not artifactId:
artifactId = artifactRootName
artifactName = options.artifactName
if not artifactName:
artifactName = artifactId
if not options.groupId:
exit(-1)
if artifactRootName != artifactId:
print "WARN:", "产品ID与文件夹名称不一致, 可能导致歧义:"
print "产品文件夹:", artifactRoot
print "产品ID:", artifactId
print "期望的产品ID:", artifactRootName
pass
# 解压模板文件
demoZipfile = os.path.join(abspath, "demo.zip")
if not util.file_exists(demoZipfile):
print "ERROR 模板文件不存在", demoZipfile
exit(-1)
demoRootDir = os.path.join(abspath, "build", "tmp", "demo")
if util.dir_exists(demoRootDir):
shutil.rmtree(demoRootDir)
pass
util.unzip_file(demoZipfile, demoRootDir)
if not util.dir_exists(demoRootDir):
print "ERROR 模板路径不存在", demoRootDir
exit(-1)
if not options.forceUpdate:
create_artifact(artifactRoot=artifactRoot,
artifactId=artifactId,
artifactName=artifactName,
groupId=options.groupId,
artifactVersion=options.artifactVersion,
springbootVersion=options.springbootVersion,
javaVersion=options.javaVersion,
description=options.description,
author=options.author)
else:
print "当前不支持!"
pass
# 使用完毕删除模板
shutil.rmtree(demoRootDir)
pass