def running_pid(proc_required=True, context=None):
"""
Return a list of pids if the crawler is running (per ps(1)) or [] otherwise
"""
cfg = CrawlConfig.add_config()
rval = []
if proc_required:
result = pidcmd()
for line in result.split("\n"):
if 'crawl start' in line:
pid = int(line.split()[0])
pfpath = "%s/%d" % (CrawlConfig.pid_dir(), pid)
if os.path.exists(pfpath):
(ctx, xpath) = util.contents(pfpath).strip().split()
rval.append((pid, ctx, xpath))
elif not os.path.exists(pfpath + '.DEFUNCT'):
# crawler is running but the pid file has been lost
ctx = context or cfg.get('crawler', 'context')
xpath = cfg.get_d('crawler', 'exitpath', '%s.exit' % ctx)
make_pidfile(pid, ctx, xpath)
rval.append((pid, ctx, xpath))
# if pfpath + '.DEFUNCT' exists, the crawler is shutting down
# so we don't want to recreate the pid file.
else:
pid_l = glob.glob("%s/*" % CrawlConfig.pid_dir())
for pid_n in pid_l:
pid = int(os.path.basename(pid_n))
(ctx, xpath) = util.contents(pid_n).strip().split()
rval.append((pid, ctx, xpath))
return rval
def running_pid(proc_required=True, context=None):
"""
Return a list of pids if the crawler is running (per ps(1)) or [] otherwise
"""
cfg = CrawlConfig.add_config()
rval = []
if proc_required:
result = pidcmd()
for line in result.split("\n"):
if 'crawl start' in line:
pid = int(line.split()[0])
pfpath = "%s/%d" % (CrawlConfig.pid_dir(), pid)
if os.path.exists(pfpath):
(ctx, xpath) = util.contents(pfpath).strip().split()
rval.append((pid, ctx, xpath))
elif not os.path.exists(pfpath + '.DEFUNCT'):
# crawler is running but the pid file has been lost
ctx = context or cfg.get('crawler', 'context')
xpath = cfg.get_d('crawler', 'exitpath', '%s.exit' % ctx)
make_pidfile(pid, ctx, xpath)
rval.append((pid, ctx, xpath))
# if pfpath + '.DEFUNCT' exists, the crawler is shutting down
# so we don't want to recreate the pid file.
else:
pid_l = glob.glob("%s/*" % CrawlConfig.pid_dir())
for pid_n in pid_l:
pid = int(os.path.basename(pid_n))
(ctx, xpath) = util.contents(pid_n).strip().split()
rval.append((pid, ctx, xpath))
return rval
def __init__(self, *args, **kwargs):
"""
Set piddir for the object from the configuration, then call the
parent's constructor.
"""
self.piddir = CrawlConfig.pid_dir()
super(CrawlDaemon, self).__init__(*args, **kwargs)
def make_pidfile(pid, context, exitpath, just_check=False):
"""
Generate a pid file in the pid directory (defined in CrawlDaemon), creating
the directory if necessary.
"""
ok = False
piddir = CrawlConfig.pid_dir()
if not os.path.exists(piddir):
os.mkdir(piddir)
ok = True
if not ok:
pf_l = [
x for x in glob.glob("%s/*" % piddir) if not x.endswith('.DEFUNCT')
]
for pf_n in pf_l:
data = util.contents(pf_n)
if 0 == len(data):
continue
(ctx, xp) = data.strip().split()
if ctx == context:
raise StandardError("The pidfile for context %s exists" %
context)
pfname = "%s/%d" % (piddir, pid)
if just_check:
return pfname
with open(pfname, 'w') as f:
f.write("%s %s\n" % (context, exitpath))
return pfname
def __init__(self, *args, **kwargs):
"""
Set piddir for the object from the configuration, then call the
parent's constructor.
"""
self.piddir = CrawlConfig.pid_dir()
super(CrawlDaemon, self).__init__(*args, **kwargs)
def make_pidfile(pid, context, exitpath, just_check=False):
"""
Generate a pid file in the pid directory (defined in CrawlDaemon), creating
the directory if necessary.
"""
ok = False
piddir = CrawlConfig.pid_dir()
if not os.path.exists(piddir):
os.mkdir(piddir)
ok = True
if not ok:
pf_l = [x for x in glob.glob("%s/*" % piddir)
if not x.endswith('.DEFUNCT')]
for pf_n in pf_l:
data = util.contents(pf_n)
if 0 == len(data):
continue
(ctx, xp) = data.strip().split()
if ctx == context:
raise StandardError("The pidfile for context %s exists" %
context)
pfname = "%s/%d" % (piddir, pid)
if just_check:
return pfname
with open(pfname, 'w') as f:
f.write("%s %s\n" % (context, exitpath))
return pfname
def clean_defunct_pidfiles(context):
"""
Remove .DEFUNCT pid files for *context*
"""
cfg = CrawlConfig.add_config()
pdir = CrawlConfig.pid_dir()
for path in glob.glob(os.path.join(pdir, '*.DEFUNCT')):
c = util.contents(path)
if context in c:
os.unlink(path)
def clean_defunct_pidfiles(context):
"""
Remove .DEFUNCT pid files for *context*
"""
cfg = CrawlConfig.add_config()
pdir = CrawlConfig.pid_dir()
for path in glob.glob(os.path.join(pdir, '*.DEFUNCT')):
c = util.contents(path)
if context in c:
os.unlink(path)