示例#1
0
def main(cfg, log):
    if cfg.playlist:
        for url in cfg.urls:
            outpath, cfg.urls = parsers.getPlayListParser(url).info(url)
            cfg.outpath = pjoin(cfg.outpath, outpath)
            util.assure_path(cfg.outpath)
            with open(pjoin(cfg.outpath, 'url.txt'), 'w') as fp:
                fp.writelines([url + "\n\n"])
                for i, clip in enumerate(cfg.urls):
                    fp.writelines(["[%03d] %s\n"%(i, clip)])

    bar = ProgressBar()
    ws = WorkShop(tmin=cfg.tmin, tmax=cfg.tmax, log=log)
    dlvs = []
    for i, url in enumerate(cfg.urls):
        dlvideo = VUrlTask(url, vidfmt=cfg.vidfmt, npf=cfg.npf,
                           outpath=cfg.outpath, bar=bar, log=log)
        dlvs.append(dlvideo)
    try:
        ws.serve()
        ws.addTasks(dlvs)
        while len(dlvs) > 0:
            for i, dlv in enumerate(dlvs):
                if dlv.isArchived() or dlv.isError():
                    del dlvs[i]
            _sleep(1)
    except Exception as e:
        log.exception(e)
    finally:
        ws.setToStop()
        ws.join()
示例#2
0
def recode(url, duration=None, vfmt=2, outpath='./',
           npf=3, freq=10, tmin=5, tmax=20, proxy=None, log=None):
    assert duration is None or duration > 0
    name = '%s.%s.ts' % (_util.get_time_string(), hash(url))
    outfile = pjoin(outpath, name)
    log.info("|=> begin: %s", url)
    if duration:
        log.info("|=>duration: %d", duration)
    log.info("|=> output: %s", outfile)

    _util.assure_path(outpath)
    axel = WorkShop(tmin=tmin, tmax=tmax, log=log)
    m3u8 = M3u8Stream(axel=axel, proxy=proxy,log=log)
    fetcher = HttpFetcher()
    start_at = time.time()
    try:
        with open(outfile, 'wb') as fp:
            if url.find('m3u8') > 0 or __is_url_file(url):
                axel.serve()
                m3u8.recode(url=url, duration=duration, vfmt=vfmt, fp=fp, npf=npf, freq=cfg.freq)
            else:
                fetcher.fetch(url=url, fp=fp)
        log.info("|=> end: total=%.2fs, out=%s", time.time() - start_at, outfile)
    finally:
        if axel.isAlive():
            axel.setToStop()
            axel.join()
示例#3
0
def main(cfg, log):
    if cfg.playlist:
        for url in cfg.urls:
            outpath, cfg.urls = parsers.getPlayListParser(url).info(url)
            cfg.outpath = pjoin(cfg.outpath, outpath)
            util.assure_path(cfg.outpath)
            with open(pjoin(cfg.outpath, 'url.txt'), 'w') as fp:
                fp.writelines([url + "\n\n"])
                for i, clip in enumerate(cfg.urls):
                    fp.writelines(["[%03d] %s\n" % (i, clip)])

    bar = ProgressBar()
    ws = WorkShop(tmin=cfg.tmin, tmax=cfg.tmax, log=log)
    dlvs = []
    for i, url in enumerate(cfg.urls):
        dlvideo = VUrlTask(url,
                           vidfmt=cfg.vidfmt,
                           npf=cfg.npf,
                           outpath=cfg.outpath,
                           bar=bar,
                           log=log)
        dlvs.append(dlvideo)
    try:
        ws.serve()
        ws.addTasks(dlvs)
        while len(dlvs) > 0:
            for i, dlv in enumerate(dlvs):
                if dlv.isArchived() or dlv.isError():
                    del dlvs[i]
            _sleep(1)
    except Exception as e:
        log.exception(e)
    finally:
        ws.setToStop()
        ws.join()
示例#4
0
def main():
    urls = [
        # 'http://v.youku.com/v_show/id_XNzUyNDE4MTQw.html'
        # 'http://i.youku.com/u/UNTc4NzI3MjY0',
        # 'http://v.youku.com/v_show/id_XNzQ5NDAwMDIw.html?from=y1.1-2.10001-0.1-1',
        # 'http://v.youku.com/v_show/id_XNzUwMTE2MDQw.html?f=22611771',
        # 'http://v.youku.com/v_show/id_XNzQ3MjMxMTYw.html',
        'http://video.sina.com.cn/p/ent/v/m/2014-08-14/102164094039.html'
    ]
    log = util.get_logger()
    bar = ProgressBar()
    ws = WorkShop(tmin=1, tmax=2, log=log)
    dlvs = []
    for i, url in enumerate(urls):
        dlvideo = VUrlTask(url, 0, 3, './tmp', bar=bar, log=log)
        dlvs.append(dlvideo)
    try:
        ws.serve()
        ws.addTasks(dlvs)
        while len(dlvs) > 0:
            for i, dlv in enumerate(dlvs):
                if dlv.isArchived() or dlv.isError():
                    del dlvs[i]
            _sleep(1)
    except KeyboardInterrupt:
        pass
    except Exception as e:
        log.exception(e)
    finally:
        ws.setToStop()
        ws.join()
示例#5
0
文件: vaxel.py 项目: pkhopper/scripts
def main():
    urls = [
        # 'http://v.youku.com/v_show/id_XNzUyNDE4MTQw.html'
        # 'http://i.youku.com/u/UNTc4NzI3MjY0',
        # 'http://v.youku.com/v_show/id_XNzQ5NDAwMDIw.html?from=y1.1-2.10001-0.1-1',
        # 'http://v.youku.com/v_show/id_XNzUwMTE2MDQw.html?f=22611771',
        # 'http://v.youku.com/v_show/id_XNzQ3MjMxMTYw.html',
        'http://video.sina.com.cn/p/ent/v/m/2014-08-14/102164094039.html'
    ]
    log = util.get_logger()
    bar = ProgressBar()
    ws = WorkShop(tmin=1, tmax=2, log=log)
    dlvs = []
    for i, url in enumerate(urls):
        dlvideo = VUrlTask(url, 0, 3, './tmp', bar=bar, log=log)
        dlvs.append(dlvideo)
    try:
        ws.serve()
        ws.addTasks(dlvs)
        while len(dlvs) > 0:
            for i, dlv in enumerate(dlvs):
                if dlv.isArchived() or dlv.isError():
                    del dlvs[i]
            _sleep(1)
    except KeyboardInterrupt:
        pass
    except Exception as e:
        log.exception(e)
    finally:
        ws.setToStop()
        ws.join()
示例#6
0
def main(argv):
    cfg = config.MiniAxelConfig()
    cfg.read_cmdline_config('miniaxel.ini', script=__file__, argv=sys.argv)
    log = cfg.log
    bar = ProgressBar()
    axel = WorkShop(tmin=cfg.tmin, tmax=cfg.tmax, log=log)
    try:
        if not axel.serve(timeout=3):
            raise ValueError('server not started')
        if hasattr(cfg, 'urls'):
            for url in cfg.urls:
                log.info('add %s', url)
                name = pjoin(cfg.outpath, find_name(url))
                urltask = UrlTask(url,
                                  out=name,
                                  npf=cfg.npf,
                                  bar=bar,
                                  retrans=True,
                                  log=cfg.log)
                axel.addTask(urltask)
        while True:
            cmd = raw_input('>>')
            if cmd in ('q'):
                break
            elif cmd in ('h'):
                print usage()
            elif cmd in ('test'):
                mainTest(axel, bar, log)
            else:
                name, url = process_cmd(cmd)
                if name:
                    name = pjoin(cfg.outpath, name)
                else:
                    name = pjoin(cfg.outpath, find_name(url))
                urltask = UrlTask(url,
                                  out=name,
                                  npf=cfg.npf,
                                  bar=bar,
                                  retrans=True,
                                  log=cfg.log)
                axel.addTask(urltask)
    except KeyboardInterrupt as e:
        pass
    except Exception as e:
        log.exception(e)
        raise
    finally:
        axel.setToStop()
        axel.join()
示例#7
0
def recode(url,
           duration=None,
           vfmt=2,
           outpath='./',
           npf=3,
           freq=10,
           tmin=5,
           tmax=20,
           proxy=None,
           log=None):
    assert duration is None or duration > 0
    name = '%s.%s.ts' % (_util.get_time_string(), hash(url))
    outfile = pjoin(outpath, name)
    log.info("|=> begin: %s", url)
    if duration:
        log.info("|=>duration: %d", duration)
    log.info("|=> output: %s", outfile)

    _util.assure_path(outpath)
    axel = WorkShop(tmin=tmin, tmax=tmax, log=log)
    m3u8 = M3u8Stream(axel=axel, proxy=proxy, log=log)
    fetcher = HttpFetcher()
    start_at = time.time()
    try:
        with open(outfile, 'wb') as fp:
            if url.find('m3u8') > 0 or __is_url_file(url):
                axel.serve()
                m3u8.recode(url=url,
                            duration=duration,
                            vfmt=vfmt,
                            fp=fp,
                            npf=npf,
                            freq=cfg.freq)
            else:
                fetcher.fetch(url=url, fp=fp)
        log.info("|=> end: total=%.2fs, out=%s",
                 time.time() - start_at, outfile)
    finally:
        if axel.isAlive():
            axel.setToStop()
            axel.join()
示例#8
0
def main(argv):
    cfg = config.MiniAxelConfig()
    cfg.read_cmdline_config('miniaxel.ini', script=__file__, argv=sys.argv)
    log = cfg.log
    bar = ProgressBar()
    axel = WorkShop(tmin=cfg.tmin, tmax=cfg.tmax, log=log)
    try:
        if not axel.serve(timeout=3):
            raise ValueError('server not started')
        if hasattr(cfg, 'urls'):
            for url in cfg.urls:
                log.info('add %s', url)
                name = pjoin(cfg.outpath, find_name(url))
                urltask = UrlTask(url, out=name, npf=cfg.npf, bar=bar, retrans=True, log=cfg.log)
                axel.addTask(urltask)
        while True:
            cmd = raw_input('>>')
            if cmd in ('q'):
                break
            elif cmd in ('h'):
                print usage()
            elif cmd in('test'):
                mainTest(axel, bar, log)
            else:
                name, url = process_cmd(cmd)
                if name:
                    name = pjoin(cfg.outpath, name)
                else:
                    name = pjoin(cfg.outpath, find_name(url))
                urltask = UrlTask(url, out=name, npf=cfg.npf, bar=bar, retrans=True, log=cfg.log)
                axel.addTask(urltask)
    except KeyboardInterrupt as e:
        pass
    except Exception as e:
        log.exception(e)
        raise
    finally:
        axel.setToStop()
        axel.join()
示例#9
0
            fTestFunc(axel, bar, url, md5, n, log)
            # mTestFunc(axel, url, md5, n, log)
            log.debug('add a test work: %s,%s,%d', url, md5, n)


from vavava.threadutil import WorkShop
from vavava.util import get_logger
if __name__ == '__main__':
    log = get_logger()
    bar = ProgressBar()
    axel = WorkShop(tmin=2, tmax=5, log=log)
    try:
        if not axel.serve(timeout=3):
            raise ValueError('server not started')
        mainTest(axel, bar, log)
        while True:
            _sleep(1)
            if axel.allTasksDone():
                if raw_input('again ??') in ('y'):
                    mainTest(axel, bar, log)
                else:
                    break
    except KeyboardInterrupt as e:
        pass
    except Exception as e:
        log.exception(e)
        raise
    finally:
        axel.setToStop()
        axel.join()
示例#10
0
        for md5, url in test_urls.items():
            fTestFunc(axel, bar, url, md5, n, log)
            # mTestFunc(axel, url, md5, n, log)
            log.debug('add a test work: %s,%s,%d', url, md5, n)

from vavava.threadutil import WorkShop
from vavava.util import get_logger
if __name__ == '__main__':
    log = get_logger()
    bar = ProgressBar()
    axel = WorkShop(tmin=2, tmax=5, log=log)
    try:
        if not axel.serve(timeout=3):
            raise ValueError('server not started')
        mainTest(axel, bar, log)
        while True:
            _sleep(1)
            if axel.allTasksDone():
                if raw_input('again ??') in ('y'):
                    mainTest(axel, bar, log)
                else:
                    break
    except KeyboardInterrupt as e:
        pass
    except Exception as e:
        log.exception(e)
        raise
    finally:
        axel.setToStop()
        axel.join()