def main(cfg, log):
if cfg.playlist:
for url in cfg.urls:
outpath, cfg.urls = parsers.getPlayListParser(url).info(url)
cfg.outpath = pjoin(cfg.outpath, outpath)
util.assure_path(cfg.outpath)
with open(pjoin(cfg.outpath, 'url.txt'), 'w') as fp:
fp.writelines([url + "\n\n"])
for i, clip in enumerate(cfg.urls):
fp.writelines(["[%03d] %s\n" % (i, clip)])
bar = ProgressBar()
ws = WorkShop(tmin=cfg.tmin, tmax=cfg.tmax, log=log)
dlvs = []
for i, url in enumerate(cfg.urls):
dlvideo = VUrlTask(url,
vidfmt=cfg.vidfmt,
npf=cfg.npf,
outpath=cfg.outpath,
bar=bar,
log=log)
dlvs.append(dlvideo)
try:
ws.serve()
ws.addTasks(dlvs)
while len(dlvs) > 0:
for i, dlv in enumerate(dlvs):
if dlv.isArchived() or dlv.isError():
del dlvs[i]
_sleep(1)
except Exception as e:
log.exception(e)
finally:
ws.setToStop()
ws.join()
def main():
urls = [
# 'http://v.youku.com/v_show/id_XNzUyNDE4MTQw.html'
# 'http://i.youku.com/u/UNTc4NzI3MjY0',
# 'http://v.youku.com/v_show/id_XNzQ5NDAwMDIw.html?from=y1.1-2.10001-0.1-1',
# 'http://v.youku.com/v_show/id_XNzUwMTE2MDQw.html?f=22611771',
# 'http://v.youku.com/v_show/id_XNzQ3MjMxMTYw.html',
'http://video.sina.com.cn/p/ent/v/m/2014-08-14/102164094039.html'
]
log = util.get_logger()
bar = ProgressBar()
ws = WorkShop(tmin=1, tmax=2, log=log)
dlvs = []
for i, url in enumerate(urls):
dlvideo = VUrlTask(url, 0, 3, './tmp', bar=bar, log=log)
dlvs.append(dlvideo)
try:
ws.serve()
ws.addTasks(dlvs)
while len(dlvs) > 0:
for i, dlv in enumerate(dlvs):
if dlv.isArchived() or dlv.isError():
del dlvs[i]
_sleep(1)
except KeyboardInterrupt:
pass
except Exception as e:
log.exception(e)
finally:
ws.setToStop()
ws.join()
def main(cfg, log):
if cfg.playlist:
for url in cfg.urls:
outpath, cfg.urls = parsers.getPlayListParser(url).info(url)
cfg.outpath = pjoin(cfg.outpath, outpath)
util.assure_path(cfg.outpath)
with open(pjoin(cfg.outpath, 'url.txt'), 'w') as fp:
fp.writelines([url + "\n\n"])
for i, clip in enumerate(cfg.urls):
fp.writelines(["[%03d] %s\n"%(i, clip)])
bar = ProgressBar()
ws = WorkShop(tmin=cfg.tmin, tmax=cfg.tmax, log=log)
dlvs = []
for i, url in enumerate(cfg.urls):
dlvideo = VUrlTask(url, vidfmt=cfg.vidfmt, npf=cfg.npf,
outpath=cfg.outpath, bar=bar, log=log)
dlvs.append(dlvideo)
try:
ws.serve()
ws.addTasks(dlvs)
while len(dlvs) > 0:
for i, dlv in enumerate(dlvs):
if dlv.isArchived() or dlv.isError():
del dlvs[i]
_sleep(1)
except Exception as e:
log.exception(e)
finally:
ws.setToStop()
ws.join()
def main():
urls = [
# 'http://v.youku.com/v_show/id_XNzUyNDE4MTQw.html'
# 'http://i.youku.com/u/UNTc4NzI3MjY0',
# 'http://v.youku.com/v_show/id_XNzQ5NDAwMDIw.html?from=y1.1-2.10001-0.1-1',
# 'http://v.youku.com/v_show/id_XNzUwMTE2MDQw.html?f=22611771',
# 'http://v.youku.com/v_show/id_XNzQ3MjMxMTYw.html',
'http://video.sina.com.cn/p/ent/v/m/2014-08-14/102164094039.html'
]
log = util.get_logger()
bar = ProgressBar()
ws = WorkShop(tmin=1, tmax=2, log=log)
dlvs = []
for i, url in enumerate(urls):
dlvideo = VUrlTask(url, 0, 3, './tmp', bar=bar, log=log)
dlvs.append(dlvideo)
try:
ws.serve()
ws.addTasks(dlvs)
while len(dlvs) > 0:
for i, dlv in enumerate(dlvs):
if dlv.isArchived() or dlv.isError():
del dlvs[i]
_sleep(1)
except KeyboardInterrupt:
pass
except Exception as e:
log.exception(e)
finally:
ws.setToStop()
ws.join()
def recode(url, duration=None, vfmt=2, outpath='./',
npf=3, freq=10, tmin=5, tmax=20, proxy=None, log=None):
assert duration is None or duration > 0
name = '%s.%s.ts' % (_util.get_time_string(), hash(url))
outfile = pjoin(outpath, name)
log.info("|=> begin: %s", url)
if duration:
log.info("|=>duration: %d", duration)
log.info("|=> output: %s", outfile)
_util.assure_path(outpath)
axel = WorkShop(tmin=tmin, tmax=tmax, log=log)
m3u8 = M3u8Stream(axel=axel, proxy=proxy,log=log)
fetcher = HttpFetcher()
start_at = time.time()
try:
with open(outfile, 'wb') as fp:
if url.find('m3u8') > 0 or __is_url_file(url):
axel.serve()
m3u8.recode(url=url, duration=duration, vfmt=vfmt, fp=fp, npf=npf, freq=cfg.freq)
else:
fetcher.fetch(url=url, fp=fp)
log.info("|=> end: total=%.2fs, out=%s", time.time() - start_at, outfile)
finally:
if axel.isAlive():
axel.setToStop()
axel.join()
def main(argv):
cfg = config.MiniAxelConfig()
cfg.read_cmdline_config('miniaxel.ini', script=__file__, argv=sys.argv)
log = cfg.log
bar = ProgressBar()
axel = WorkShop(tmin=cfg.tmin, tmax=cfg.tmax, log=log)
try:
if not axel.serve(timeout=3):
raise ValueError('server not started')
if hasattr(cfg, 'urls'):
for url in cfg.urls:
log.info('add %s', url)
name = pjoin(cfg.outpath, find_name(url))
urltask = UrlTask(url, out=name, npf=cfg.npf, bar=bar, retrans=True, log=cfg.log)
axel.addTask(urltask)
while True:
cmd = raw_input('>>')
if cmd in ('q'):
break
elif cmd in ('h'):
print usage()
elif cmd in('test'):
mainTest(axel, bar, log)
else:
name, url = process_cmd(cmd)
if name:
name = pjoin(cfg.outpath, name)
else:
name = pjoin(cfg.outpath, find_name(url))
urltask = UrlTask(url, out=name, npf=cfg.npf, bar=bar, retrans=True, log=cfg.log)
axel.addTask(urltask)
except KeyboardInterrupt as e:
pass
except Exception as e:
log.exception(e)
raise
finally:
axel.setToStop()
axel.join()
def recode(url,
duration=None,
vfmt=2,
outpath='./',
npf=3,
freq=10,
tmin=5,
tmax=20,
proxy=None,
log=None):
assert duration is None or duration > 0
name = '%s.%s.ts' % (_util.get_time_string(), hash(url))
outfile = pjoin(outpath, name)
log.info("|=> begin: %s", url)
if duration:
log.info("|=>duration: %d", duration)
log.info("|=> output: %s", outfile)
_util.assure_path(outpath)
axel = WorkShop(tmin=tmin, tmax=tmax, log=log)
m3u8 = M3u8Stream(axel=axel, proxy=proxy, log=log)
fetcher = HttpFetcher()
start_at = time.time()
try:
with open(outfile, 'wb') as fp:
if url.find('m3u8') > 0 or __is_url_file(url):
axel.serve()
m3u8.recode(url=url,
duration=duration,
vfmt=vfmt,
fp=fp,
npf=npf,
freq=cfg.freq)
else:
fetcher.fetch(url=url, fp=fp)
log.info("|=> end: total=%.2fs, out=%s",
time.time() - start_at, outfile)
finally:
if axel.isAlive():
axel.setToStop()
axel.join()
def mainTest(axel, bar, log):
cmd = "1" #'1,2,3,4,5,6'# raw_input('n=')
for n in cmd.split(','):
n = int(n)
for md5, url in test_urls.items():
fTestFunc(axel, bar, url, md5, n, log)
# mTestFunc(axel, url, md5, n, log)
log.debug('add a test work: %s,%s,%d', url, md5, n)
from vavava.threadutil import WorkShop
from vavava.util import get_logger
if __name__ == '__main__':
log = get_logger()
bar = ProgressBar()
axel = WorkShop(tmin=2, tmax=5, log=log)
try:
if not axel.serve(timeout=3):
raise ValueError('server not started')
mainTest(axel, bar, log)
while True:
_sleep(1)
if axel.allTasksDone():
if raw_input('again ??') in ('y'):
mainTest(axel, bar, log)
else:
break
except KeyboardInterrupt as e:
pass
except Exception as e:
log.exception(e)
def main(argv):
cfg = config.MiniAxelConfig()
cfg.read_cmdline_config('miniaxel.ini', script=__file__, argv=sys.argv)
log = cfg.log
bar = ProgressBar()
axel = WorkShop(tmin=cfg.tmin, tmax=cfg.tmax, log=log)
try:
if not axel.serve(timeout=3):
raise ValueError('server not started')
if hasattr(cfg, 'urls'):
for url in cfg.urls:
log.info('add %s', url)
name = pjoin(cfg.outpath, find_name(url))
urltask = UrlTask(url,
out=name,
npf=cfg.npf,
bar=bar,
retrans=True,
log=cfg.log)
axel.addTask(urltask)
while True:
cmd = raw_input('>>')
if cmd in ('q'):
break
elif cmd in ('h'):
print usage()
elif cmd in ('test'):
mainTest(axel, bar, log)
else:
name, url = process_cmd(cmd)
if name:
name = pjoin(cfg.outpath, name)
else:
name = pjoin(cfg.outpath, find_name(url))
urltask = UrlTask(url,
out=name,
npf=cfg.npf,
bar=bar,
retrans=True,
log=cfg.log)
axel.addTask(urltask)
except KeyboardInterrupt as e:
pass
except Exception as e:
log.exception(e)
raise
finally:
axel.setToStop()
axel.join()
def mainTest(axel, bar, log):
cmd = "1" #'1,2,3,4,5,6'# raw_input('n=')
for n in cmd.split(','):
n = int(n)
for md5, url in test_urls.items():
fTestFunc(axel, bar, url, md5, n, log)
# mTestFunc(axel, url, md5, n, log)
log.debug('add a test work: %s,%s,%d', url, md5, n)
from vavava.threadutil import WorkShop
from vavava.util import get_logger
if __name__ == '__main__':
log = get_logger()
bar = ProgressBar()
axel = WorkShop(tmin=2, tmax=5, log=log)
try:
if not axel.serve(timeout=3):
raise ValueError('server not started')
mainTest(axel, bar, log)
while True:
_sleep(1)
if axel.allTasksDone():
if raw_input('again ??') in ('y'):
mainTest(axel, bar, log)
else:
break
except KeyboardInterrupt as e:
pass
except Exception as e:
log.exception(e)
