def run_cmd(url_tuple, out_dir, timeout=cm.HARD_TIME_OUT,
flash_support=cm.FLASH_ENABLE, cookie_support=cm.COOKIE_ALLOW_ALL):
if isfile(cm.STOP_CRAWL_FILE):
print "Stop crawl file exists, won't run"
return
cpu_util = psutil.cpu_percent()
if cpu_util > cm.MIN_CPU_UTIL_FOR_SLEEP:
sleep_dur = (cm.MAX_PRE_CRAWL_SLEEP * cpu_util * random()) / 100
print "CPU util is %s, will sleep: %s on before visiting %s" %\
(cpu_util, sleep_dur, url_tuple)
sleep(sleep_dur)
debug_log = join(out_dir, "debug.log")
cmd = "timeout -k %d %d python crawler/get.py --url %s"\
" --rank %d --out_dir %s --flash %d --cookie %s 2>&1 >> %s" %\
(cm.KILL_TIME_OUT, timeout, pipes.quote(url_tuple[1]),
int(url_tuple[0]), pipes.quote(out_dir), int(flash_support),
int(cookie_support), pipes.quote(debug_log))
print "Will run:", cmd
# print "status, output:", ut.run_cmd(cmd)
ut.run_cmd(cmd)
def run_cmd(url_tuple,
out_dir,
timeout=cm.HARD_TIME_OUT,
flash_support=cm.FLASH_ENABLE,
cookie_support=cm.COOKIE_ALLOW_ALL):
if isfile(cm.STOP_CRAWL_FILE):
print "Stop crawl file exists, won't run"
return
cpu_util = psutil.cpu_percent()
if cpu_util > cm.MIN_CPU_UTIL_FOR_SLEEP:
sleep_dur = (cm.MAX_PRE_CRAWL_SLEEP * cpu_util * random()) / 100
print "CPU util is %s, will sleep: %s on before visiting %s" %\
(cpu_util, sleep_dur, url_tuple)
sleep(sleep_dur)
debug_log = join(out_dir, "debug.log")
cmd = "timeout -k %d %d python crawler/get.py --url %s"\
" --rank %d --out_dir %s --flash %d --cookie %s 2>&1 >> %s" %\
(cm.KILL_TIME_OUT, timeout, pipes.quote(url_tuple[1]),
int(url_tuple[0]), pipes.quote(out_dir), int(flash_support),
int(cookie_support), pipes.quote(debug_log))
print "Will run:", cmd
# print "status, output:", ut.run_cmd(cmd)
ut.run_cmd(cmd)
def is_installed(self, pkg_name):
"""Check if a Linux package is installed."""
cmd = 'which %s' % pkg_name
status, _ = ut.run_cmd(cmd)
return False if status else True