def start_spider(host, source_dir, subdir, spider_cmd):
cmds = [
"cd %s" % os.path.join(source_dir, subdir),
"nohup {0} > /dev/null 2>&1 &".format(spider_cmd)
]
cmd = "; ".join(cmds)
rcmd(host, cmd)
def kill_spider(host, spider_cmd):
print("### ssh to %s ..." % host)
cmd = "ps -ef|grep -vE 'grep|monitor.py|kill.py'|grep '%s'" % spider_cmd
pss = rcmd(host, cmd)
for ps in pss.readlines():
pid = re.split(r'\s+', ps)[1]
print(ps)
print("kill %s ..." % pid)
rcmd(host, "kill %s" % pid)
def build_spider(source_dir, subdir, spider_cmd):
cmd_file = os.path.join(subdir, spider_cmd)
py_file = "%s.py" % cmd_file
cmds = [
"cd %s" % source_dir,
"rm -rf %s" % cmd_file,
"pyinstaller -w -F %s --distpath %s" % (py_file, subdir),
"rm -rf ./build"
]
rcmd(local_ip, "; ".join(cmds))
def kill_spider(host, spider_cmd):
print("### ssh to %s ..." % host)
cmd = "ps -ef|grep -vE 'grep|monitor.py|kill.py'|grep '%s'" % spider_cmd
pss = rcmd(host, cmd)
pids = []
for ps in pss.readlines():
pids.append(re.split(r'\s+', ps)[1])
print(ps.strip())
for pid in pids:
print("kill %s ..." % pid)
rcmd(host, "kill %s" % pid)
print("")
def check_spider(host, spider_cmd):
cmd = "ps -ef|grep -vE 'grep|monitor.py|check.py'|grep '%s'" % spider_cmd
return rcmd(host, cmd).read()
def rm_remote_src(host, dir_name):
cmd = "rm -rf %s" % dir_name
print("Remove remote host %s source dir: %s" % (host, dir_name))
rcmd(host, cmd)