def __init__(self, urls, dest=None, progress_bar=True, fix_urls=True, logger=None, connect_default_logger=False):
self.mirrors = [urls] if isinstance(urls, basestring) else urls
if fix_urls:
self.mirrors = [utils.url_fix(x) for x in self.mirrors]
self.url = self.mirrors.pop(0)
fn = os.path.basename(urlparse(self.url).path)
self.dest = dest or os.path.join(tempfile.gettempdir(), 'pySmartDL', fn)
if self.dest[-1] == os.sep:
if os.path.exists(self.dest[:-1]) and os.path.isfile(self.dest[:-1]):
os.unlink(self.dest[:-1])
self.dest += fn
if os.path.isdir(self.dest):
self.dest = os.path.join(self.dest, fn)
self.progress_bar = progress_bar
if logger:
self.logger = logger
elif connect_default_logger:
self.logger = utils.create_debugging_logger()
else:
self.logger = utils.DummyLogger()
self.headers = {'User-Agent': utils.get_random_useragent()}
self.threads_count = 3
self.timeout = 4
self.current_attemp = 1
self.attemps_limit = 4
self.minChunkFile = 1024**2*2 # 2MB
self.filesize = 0
self.shared_var = multiprocessing.Value(c_int, 0) # a ctypes var that counts the bytes already downloaded
self.thread_shared_cmds = {}
self.status = "ready"
self.verify_hash = False
self._killed = False
self._failed = False
self._start_func_blocking = True
self.errors = []
self.post_threadpool_thread = None
self.control_thread = None
if not os.path.exists(os.path.dirname(self.dest)):
self.logger.debug('Folder "%s" does not exist. Creating...' % os.path.dirname(self.dest))
os.makedirs(os.path.dirname(self.dest))
if not utils.is_HTTPRange_supported(self.url):
self.logger.warning("Server does not support HTTPRange. threads_count is set to 1.")
self.threads_count = 1
if os.path.exists(self.dest):
self.logger.warning('Destination "%s" already exists. Existing file will be removed.' % self.dest)
if not os.path.exists(os.path.dirname(self.dest)):
self.logger.warning('Directory "%s" does not exist. Creating it...' % os.path.dirname(self.dest))
os.makedirs(os.path.dirname(self.dest))
self.pool = utils.ManagedThreadPoolExecutor(self.threads_count)
def __init__(self, urls, dest=None, progress_bar=True, fix_urls=True, logger=None, connect_default_logger=False):
self.mirrors = [urls] if isinstance(urls, basestring) else urls
if fix_urls:
self.mirrors = [utils.url_fix(x) for x in self.mirrors]
self.url = self.mirrors.pop(0)
fn = urllib2.unquote(os.path.basename(urlparse(self.url).path)).decode('utf-8')
self.dest = dest or os.path.join(tempfile.gettempdir(), 'pySmartDL', fn)
if self.dest[-1] == os.sep:
if os.path.exists(self.dest[:-1]) and os.path.isfile(self.dest[:-1]):
os.unlink(self.dest[:-1])
self.dest += fn
if os.path.isdir(self.dest):
self.dest = os.path.join(self.dest, fn)
self.progress_bar = progress_bar
if logger:
self.logger = logger
elif connect_default_logger:
self.logger = utils.create_debugging_logger()
else:
self.logger = utils.DummyLogger()
self.headers = {'User-Agent': utils.get_random_useragent()}
self.threads_count = 5
self.timeout = 4
self.current_attemp = 1
self.attemps_limit = 4
self.minChunkFile = 1024**2*2 # 2MB
self.filesize = 0
self.shared_var = multiprocessing.Value(c_int, 0) # a ctypes var that counts the bytes already downloaded
self.thread_shared_cmds = {}
self.status = "ready"
self.verify_hash = False
self._killed = False
self._failed = False
self._start_func_blocking = True
self.errors = []
self.post_threadpool_thread = None
self.control_thread = None
if not os.path.exists(os.path.dirname(self.dest)):
self.logger.debug('Folder "%s" does not exist. Creating...' % os.path.dirname(self.dest))
os.makedirs(os.path.dirname(self.dest))
if not utils.is_HTTPRange_supported(self.url):
self.logger.warning("Server does not support HTTPRange. threads_count is set to 1.")
self.threads_count = 1
if os.path.exists(self.dest):
self.logger.warning('Destination "%s" already exists. Existing file will be removed.' % self.dest)
if not os.path.exists(os.path.dirname(self.dest)):
self.logger.warning('Directory "%s" does not exist. Creating it...' % os.path.dirname(self.dest))
os.makedirs(os.path.dirname(self.dest))
self.pool = utils.ManagedThreadPoolExecutor(self.threads_count)
def __init__(self,
urls,
dest=None,
progress_bar=True,
fix_urls=True,
threads=5,
logger=None,
connect_default_logger=False,
proxy=None):
global DEFAULT_LOGGER_CREATED
self.mirrors = [urls] if isinstance(urls, basestring) else urls
if fix_urls:
self.mirrors = [utils.url_fix(x) for x in self.mirrors]
self.url = self.mirrors.pop(0)
if proxy is not None:
proxy = urllib2.ProxyHandler({'http': proxy, 'https': proxy})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
fn = urllib2.unquote(os.path.basename(urlparse(self.url).path))
if sys.version_info < (3, 0):
fn = fn.decode('utf-8') # required only on python 2
self.dest = dest or os.path.join(tempfile.gettempdir(), 'pySmartDL',
fn)
if self.dest[-1] == os.sep:
if os.path.exists(self.dest[:-1]) and os.path.isfile(
self.dest[:-1]):
os.unlink(self.dest[:-1])
self.dest += fn
if os.path.isdir(self.dest):
self.dest = os.path.join(self.dest, fn)
self.progress_bar = progress_bar
if logger:
self.logger = logger
elif connect_default_logger:
if not DEFAULT_LOGGER_CREATED:
self.logger = utils.create_debugging_logger()
DEFAULT_LOGGER_CREATED = True
else:
self.logger = logging.getLogger('pySmartDL')
else:
self.logger = utils.DummyLogger()
self.headers = {'User-Agent': utils.get_random_useragent()}
self.threads_count = threads
self.timeout = 4
self.current_attemp = 1
self.attemps_limit = 4
self.minChunkFile = 1024**2 * 2 # 2MB
self.filesize = 0
self.shared_var = multiprocessing.Value(
c_int, 0) # a ctypes var that counts the bytes already downloaded
self.thread_shared_cmds = {}
self.status = "ready"
self.verify_hash = False
self._killed = False
self._failed = False
self._start_func_blocking = True
self.errors = []
self.post_threadpool_thread = None
self.control_thread = None
if not os.path.exists(os.path.dirname(self.dest)):
self.logger.info('Folder "%s" does not exist. Creating...' %
os.path.dirname(self.dest))
os.makedirs(os.path.dirname(self.dest))
if not utils.is_HTTPRange_supported(self.url):
self.logger.warning(
"Server does not support HTTPRange. threads_count is set to 1."
)
self.threads_count = 1
if os.path.exists(self.dest):
self.logger.warning(
'Destination "%s" already exists. Existing file will be removed.'
% self.dest)
if not os.path.exists(os.path.dirname(self.dest)):
self.logger.warning(
'Directory "%s" does not exist. Creating it...' %
os.path.dirname(self.dest))
os.makedirs(os.path.dirname(self.dest))
self.logger.info("Creating a ThreadPool of %d thread(s).",
self.threads_count)
self.pool = utils.ManagedThreadPoolExecutor(self.threads_count)