Пример #1
0
def run_cmd(url_tuple, out_dir, timeout=cm.HARD_TIME_OUT,
            flash_support=cm.FLASH_ENABLE, cookie_support=cm.COOKIE_ALLOW_ALL):
    if isfile(cm.STOP_CRAWL_FILE):
        print "Stop crawl file exists, won't run"
        return
    cpu_util = psutil.cpu_percent()
    if cpu_util > cm.MIN_CPU_UTIL_FOR_SLEEP:
        sleep_dur = (cm.MAX_PRE_CRAWL_SLEEP * cpu_util * random()) / 100
        print "CPU util is %s, will sleep: %s on before visiting %s" %\
            (cpu_util, sleep_dur, url_tuple)
        sleep(sleep_dur)
    debug_log = join(out_dir, "debug.log")
    cmd = "timeout -k %d %d python crawler/get.py --url %s"\
        " --rank %d  --out_dir %s --flash %d  --cookie %s 2>&1 >> %s" %\
        (cm.KILL_TIME_OUT, timeout, pipes.quote(url_tuple[1]),
         int(url_tuple[0]), pipes.quote(out_dir), int(flash_support),
         int(cookie_support), pipes.quote(debug_log))

    print "Will run:", cmd
    # print "status, output:", ut.run_cmd(cmd)
    ut.run_cmd(cmd)
Пример #2
0
def run_cmd(url_tuple,
            out_dir,
            timeout=cm.HARD_TIME_OUT,
            flash_support=cm.FLASH_ENABLE,
            cookie_support=cm.COOKIE_ALLOW_ALL):
    if isfile(cm.STOP_CRAWL_FILE):
        print "Stop crawl file exists, won't run"
        return
    cpu_util = psutil.cpu_percent()
    if cpu_util > cm.MIN_CPU_UTIL_FOR_SLEEP:
        sleep_dur = (cm.MAX_PRE_CRAWL_SLEEP * cpu_util * random()) / 100
        print "CPU util is %s, will sleep: %s on before visiting %s" %\
            (cpu_util, sleep_dur, url_tuple)
        sleep(sleep_dur)
    debug_log = join(out_dir, "debug.log")
    cmd = "timeout -k %d %d python crawler/get.py --url %s"\
        " --rank %d  --out_dir %s --flash %d  --cookie %s 2>&1 >> %s" %\
        (cm.KILL_TIME_OUT, timeout, pipes.quote(url_tuple[1]),
         int(url_tuple[0]), pipes.quote(out_dir), int(flash_support),
         int(cookie_support), pipes.quote(debug_log))

    print "Will run:", cmd
    # print "status, output:", ut.run_cmd(cmd)
    ut.run_cmd(cmd)
Пример #3
0
 def is_installed(self, pkg_name):
     """Check if a Linux package is installed."""
     cmd = 'which %s' % pkg_name
     status, _ = ut.run_cmd(cmd)
     return False if status else True