def remove_logger(config, found_workers):
import yaml
NN = len(found_workers) - 1
if NN == 0:
elog.warn("no logger can be removed")
return
loggerPrefix = found_workers[0].split('-')[0]
delLoggerName = "%s-%d" % (loggerPrefix, NN)
while delLoggerName not in found_workers and NN < LOGGER_WORKERS_MAX:
NN = NN + 1
delLoggerName = "%s-%d" % (loggerPrefix, NN)
if delLoggerName not in found_workers:
elog.warn("no logger can be removed")
return
# remove file loggerNN:
loggerNN = os.path.join(config['loggers_abspath'], "%s.py" % delLoggerName)
loggerNNPosition = os.path.join(config['loggers_abspath'], ".%s.position" % delLoggerName)
elog.info("%s: %s", delLoggerName, loggerNN)
(fr, fd) = (None, None)
try:
loggingConfigYaml = config['logger']['logging_config']
loggingConfigYamlDefault = "%s.0" % loggingConfigYaml
shutil.copy(loggingConfigYaml, loggingConfigYamlDefault)
fr = open(loggingConfigYaml)
cfg = yaml.load(fr)
fr.close()
fr = None
del cfg['loggers'][delLoggerName]
fd = util.open_file(loggingConfigYaml)
yaml.dump(cfg, fd, default_flow_style=False)
fd.close()
fd = None
os.remove(loggerNN)
os.remove(loggerNNPosition)
shutil.copy(loggingConfigYaml, loggingConfigYamlDefault)
elog.info("success: %s", delLoggerName)
except:
shutil.copy(loggingConfigYamlDefault, loggingConfigYaml)
elog.error("failed: %s", delLoggerName)
pass
finally:
if fr:
fr.close()
if fd:
fd.close()
pass
def sweep_path(path, file_exts, recursive, author, curtime):
filelist = os.listdir(path)
filelist.sort(key=lambda x:x[0:20])
for f in filelist:
try:
pf = os.path.join(path, f)
fs = os.stat(pf)
mod = fs.st_mode
if stat.S_ISDIR(mod):
# is dir
if util.dir_exists(pf):
if recursive:
sweep_path(pf, file_exts, recursive, author, curtime)
pass
pass
elif stat.S_ISREG(mod):
# is file
ignored = False
if not util.file_exists(pf) or pf == APPFILE or f == "__init__.py":
ignored = True
if not ignored:
_, ext = os.path.splitext(f)
if ext in file_exts:
update_file(pf, f, fs, author, curtime)
pass
except OSError as e:
elog.warn("%r: %s" % (e, pf))
pass
def add_logger(config, found_workers):
import yaml
from copy import deepcopy
NN = len(found_workers)
if NN > LOGGER_WORKERS_MAX:
elog.warn("too many loggers(>%d) to add", LOGGER_WORKERS_MAX)
return
loggerPrefix = found_workers[0].split('-')[0]
newLoggerName = "%s-%d" % (loggerPrefix, NN)
while newLoggerName in found_workers:
NN = NN + 1
newLoggerName = "%s-%d" % (loggerPrefix, NN)
# add loggerNN:
logger0 = os.path.join(config['loggers_abspath'], "%s.py" % loggerPrefix)
loggerNN = os.path.join(config['loggers_abspath'], "%s.py" % newLoggerName)
elog.info("%s: %s", newLoggerName, loggerNN)
(fr, fd) = (None, None)
try:
loggingConfigYaml = config['logger']['logging_config']
loggingConfigYamlDefault = "%s.0" % loggingConfigYaml
shutil.copy(loggingConfigYaml, loggingConfigYamlDefault)
fr = open(loggingConfigYaml)
cfg = yaml.load(fr)
fr.close()
fr = None
fd = util.open_file(loggingConfigYaml)
cfg['loggers'][newLoggerName] = deepcopy(cfg['loggers'][loggerPrefix])
yaml.dump(cfg, fd, default_flow_style=False)
fd.close()
fd = None
shutil.copy(logger0, loggerNN)
shutil.copy(loggingConfigYaml, loggingConfigYamlDefault)
elog.info("success: %s", newLoggerName)
except:
shutil.copy(loggingConfigYamlDefault, loggingConfigYaml)
elog.error("failed: %s", newLoggerName)
pass
finally:
if fr:
fr.close()
if fd:
fd.close()
pass
def sweep_dir(path, srcs, results):
dname = os.path.basename(path)
if dname not in ignore_dirs:
filelist = os.listdir(path)
filelist.sort(key=lambda x: x[0:20])
for f in filelist:
pf = os.path.join(path, f)
if util.dir_exists(pf):
sweep_dir(pf, srcs, results)
elif util.file_exists(pf):
_, ext = os.path.splitext(f)
passed_filters = False
if f not in ignore_files and ext not in ignore_exts:
passed_filters = True
if len(only_exts) > 0:
if ext not in only_exts:
passed_filters = False
if passed_filters:
fd = None
try:
fd = open(pf, 'r')
lines = fd.readlines()
lineno = 0
for line in lines:
lineno += 1
for src in srcs:
if line.find(src) != -1:
elog.info("found '%s': [%s:%d]", src,
os.path.relpath(pf, APPPATH),
lineno)
elog.force_clean("%s", line)
if pf not in results:
results.append(pf)
except:
elog.error("%r: %s", sys.exc_info(), pf)
finally:
util.close_file_nothrow(fd)
else:
#elog.warn("ignore file: %s", pf)
pass
else:
elog.warn("ignore path: %s", path)
pass
def main(parser, appConfig, loggerConfig):
import utils.logger
(options, args) = parser.parse_args(args=None, values=None)
loggerDictConfig = utils.logger.set_logger(loggerConfig, options.log_path,
options.log_level)
elog.force("%s-%s start", APPNAME, APPVER)
# 当前脚本绝对路径
abspath = util.script_abspath(inspect.currentframe())
options.artifactName = options.artifactName \
.replace('${artifactId}', options.artifactId) \
.replace('-', '_') \
.replace('.', '_')
options.artifactRootdir = options.artifactRootdir \
.replace('${projectRootdir}', appConfig['projectRootdir']) \
.replace('${artifactId}', options.artifactId) \
.replace('${artifactName}', options.artifactName)
if options.create_project:
util.info2("projectRootdir = '%s'" % appConfig['projectRootdir'])
util.info2("sb2template = '%s'" % appConfig['sb2template'])
util.print_options_attrs(options, [
'groupId', 'artifactId', 'artifactName', 'artifactVersion',
'artifactDescription', 'artifactRootdir'
])
if util.dir_exists(options.artifactRootdir) and not options.force:
elog.warn(
"artifactRootdir has already existed. (using '--force' to overwrite it.)"
)
sys.exit(-1)
pass
create_sb2_project(appConfig, options)
elog.force("%s-%s exit.", APPNAME, APPVER)
pass
def main(parser):
(options, args) = parser.parse_args(args=None, values=None)
# 当前脚本绝对路径
abspath = util.script_abspath(inspect.currentframe())
if not options.path:
elog.warn("No path specified. using: -P, --path=PATH")
exit(-1)
# 取得配置项 options.path 的绝对路径
root_path = util.source_abspath(APPFILE, options.path, abspath)
# 取得文件扩展名数组
file_exts = []
filters = parse_strarr(options.filter)
for filter in filters:
if filter.startswith('.'):
if filter not in file_exts:
file_exts.append(filter)
if filter in source_filters_dict.keys():
for ext in source_filters_dict[filter]:
if ext not in file_exts:
file_exts.append(ext)
curtime = time.time()
elog.force("path: %r", root_path)
elog.force("exts: %r", file_exts)
elog.force("recursive: %r", options.recursive)
elog.force("timestamp: %r", curtime)
if options.author:
elog.force("author: %r", options.author)
sweep_path(root_path, file_exts, options.recursive, options.author,
curtime)
pass
def main(parser):
(options, args) = parser.parse_args(args=None, values=None)
# 子进程退出后向父进程发送的信号
## signal.signal(signal.SIGCHLD, util.sig_chld)
# 进程退出信号
signal.signal(signal.SIGINT, util.sig_int)
signal.signal(signal.SIGTERM, util.sig_term)
# 当前脚本绝对路径
abspath = util.script_abspath(inspect.currentframe())
if not options.path:
options.path = os.getcwd()
elog.warn(
"No path specified. using current working dir or using: --path='PATH'"
)
if not options.srcs:
elog.error("No source strings specified. using: --srcs='STRINGS'")
sys.exit(1)
if not options.dsts:
elog.warn("No destigation strings specified. using: --dsts='STRINGS'")
# 取得配置项options.path的绝对路径
root_path = util.source_abspath(APPFILE, options.path, abspath)
srcs = parse_strarr(options.srcs)
dsts = parse_strarr(options.dsts)
elog.force("path: %s", root_path)
elog.force("sour = %r", srcs)
elog.force("dest = %r", dsts)
founds = []
sweep_dir(root_path, srcs, founds)
elog.force("Total %d files found", len(founds))
if len(founds) > 0:
if options.replace:
if len(srcs) == len(dsts):
for pf in founds:
ctime, mtime = None, None
fts = file_times(pf)
if fts:
ctime, mtime = fts
else:
elog.warn("missing file: %s", pf)
continue
for i in range(0, len(srcs)):
srcstr = srcs[i]
dststr = None
if i < len(dsts):
dststr = dsts[i]
if dststr:
ds = dststr.replace('$(mtime)', mtime).replace(
'$(ctime)', ctime)
if options.whole_line:
cmd = "sed -i 's/%s.*/%s/g' '%s'" % (srcstr,
ds, pf)
else:
cmd = "sed -i 's/%s/%s/g' '%s'" % (srcstr, ds,
pf)
elog.debug(cmd)
(status, output) = commands.getstatusoutput(cmd)
if status != 0:
elog.error(
"failed to command: \"%s\", output: %r",
sed, output)
elog.force("Total %d files replaced", len(founds))
else:
elog.error(
"Failed to replace for srcs(%r) mismatched with dsts(%r)",
srcs, dsts)
pass
else:
elog.warn("No files to be replaced. Using: --replace")
pass
pass
group.add_option("--startup",
action="store_true", dest="startup", default=False,
help="startup given worker logger")
group.add_option("--batch-rows",
action="store", dest="batch_rows", type=int, default=5000,
help="specify batch rows for logger. 5000 default",
metavar="ROWS")
group.add_option("--force",
action="store_true", dest="force", default=False,
help="force apply on all workers")
if len(sys.argv) == 1:
elog.warn("WORKERs not specified")
print "--------------------------------"
parser.print_help()
print "--------------------------------"
exit(1)
workers = None
firstarg = sys.argv[1]
if not firstarg.startswith('-'):
workers = []
names = firstarg.split(',')
for name in names:
workers.append(name.strip(' '))
pass
config = {
def init_data(self, logfile):
self.restore_position()
if not util.dir_exists(self.log_prefix):
elog.warn("create dir for stash log: %s", self.log_prefix)
os.makedirs(self.log_prefix)
elog.debug('log config: %r', self.dictcfg)
elog.info('stash prefix: %s', self.log_prefix)
elog.info('start tstamp: %d', self.start_tstamp)
elog.info('start rowid: %d', self.start_rowid)
elog.info('batch rows: %d', self.batch_rows)
file_dests = os.path.join(self.plugins_dir, 'config' , 'dests.csv')
file_proxys = os.path.join(self.plugins_dir, 'config' , 'proxys.csv')
file_keywds = os.path.join(self.plugins_dir, 'config' , 'keywds.csv')
elog.info("dests file: %s", file_dests)
elog.info("proxys file: %s", file_proxys)
elog.info("keywds file: %s", file_keywds)
with open(file_dests, 'r') as fd:
dests = fd.readlines()
with open(file_proxys, 'r') as fd:
proxys = fd.readlines()
with open(file_keywds, 'r') as fd:
keywds = fd.readlines()
self.dests = []
for n in range(0, len(dests)):
# id, ip, port, host
# 100005,67.64.46.91,80,www.zhibo8.cc
self.dests.append(tuple(dests[n].strip('\n').split(',')))
del dests
self.proxys = []
for n in range(0, len(proxys)):
# ip, port, type
# 121.232.144.158,9000,HTTP
self.proxys.append(tuple(proxys[n].strip('\n').split(',')))
del proxys
self.keywds = []
for n in range(0, len(keywds)):
# id, word
self.keywds.append(tuple(keywds[n].strip('\n').split(',')))
del keywds
self.max_dests = len(self.dests) - 1
self.max_proxys = len(self.proxys) - 1
self.max_keywds = len(self.keywds) - 1
# update dictcfg with logfile
elog.update_log_config(self.dictcfg, self.logger_name, logfile, 'INFO')
# reload config
logging.config.dictConfig(self.dictcfg)
# update logger
self.logger = logging.getLogger(self.logger_name)
self.logfile = logfile
(self.a, self.b, self.c, self.d, self.p) = ((1, 220), (10, 230), (20, 240), (30, 250), (10000, 60000))
self.fields = (
'rowid',
'timestr',
'timeint',
'destid',
'sourip',
'sourport',
'destip',
'destport',
'desturl',
'proxyip',
'proxyport',
'proxytype',
'keywdid')
pass