def process(link, dest):
info = FSMODS_CHECK.match(link)
dl_link = 'http://www.freespacemods.net/request.php?' + info.group(2)
info_link = 'http://www.freespacemods.net/download.php?view.' + info.group(2)
with open(dest, 'wb') as stream:
return util.download(dl_link, stream, headers={'Referer': info_link})
def download(self, dest, sel_files=None):
count = 0
num = 0
for u, files in self.urls:
if sel_files is not None:
files = set(files) & sel_files
count += len(files)
for urls, files in self.urls:
if not isinstance(urls, list):
urls = [urls]
if sel_files is not None:
files = set(files) & sel_files
for filename in files:
done = False
for link in urls:
with open(os.path.join(dest, filename), 'wb') as dl:
progress.start_task(float(num) / count, 1.0 / count, '%d/%d: %%s' % (num + 1, count))
if download(pjoin(link, filename), dl):
num += 1
done = True
progress.finish_task()
break
progress.finish_task()
if not done:
logging.error('Failed to download "%s"!', filename)
def download(self, dest, sel_files=None):
count = 0
num = 0
for u, files in self.urls:
if sel_files is not None:
files = set(files) & sel_files
count += len(files)
for urls, files in self.urls:
if not isinstance(urls, list):
urls = [urls]
if sel_files is not None:
files = set(files) & sel_files
for filename in files:
done = False
for link in urls:
with open(os.path.join(dest, filename), 'wb') as dl:
progress.start_task(
float(num) / count, 1.0 / count,
'%d/%d: %%s' % (num + 1, count))
if download(pjoin(link, filename), dl):
num += 1
done = True
progress.finish_task()
break
progress.finish_task()
if not done:
logging.error('Failed to download "%s"!', filename)
def _download(self, links, path, tstamp):
from . import download
all_links = links[:]
retries = 5
# Remove all indirect links.
for i, link in reversed(list(enumerate(links))):
if not download.is_direct(link):
del links[i]
while retries > 0:
retries -= 1
for link in all_links:
if self.dl_mirror is not None and link.startswith(self.dl_mirror):
link_path = os.path.join(self.dl_path, link[len(self.dl_mirror):].lstrip('/'))
if os.path.isfile(link_path):
shutil.copyfile(link_path, path)
return True
with open(path, 'wb') as stream:
res = util.download(link, stream, headers={'If-Modified': str(tstamp)})
if res == 304 or res:
return res
return False
def process(link, dest):
info = FSMODS_CHECK.match(link)
dl_link = 'http://www.freespacemods.net/request.php?' + info.group(2)
info_link = 'http://www.freespacemods.net/download.php?view.' + info.group(
2)
with open(dest, 'wb') as stream:
return util.download(dl_link,
stream,
headers={'Referer': info_link})
def process(link, dest):
data = util.get(link, random_ua=True)
link_info = BOX_CHECK.match(link)
info = BOX_EXTRACT.search(data)
if not info:
logging.error('Failed to process the download page for "%s"!',
link)
return False
dl_link = (
'https://app.box.com/index.php?rm=box_download_shared_file&shared_name=%s&file_id=%s'
% (link_info.group(1), info.group(1)))
with open(dest, 'wb') as stream:
return util.download(dl_link, stream, random_ua=True)
def process(link, dest):
data = util.get(link, random_ua=True)
link_info = BOX_CHECK.match(link)
info = BOX_EXTRACT.search(data)
if not info:
logging.error('Failed to process the download page for "%s"!', link)
return False
dl_link = (
'https://app.box.com/index.php?rm=box_download_shared_file&shared_name=%s&file_id=%s'
% (link_info.group(1), info.group(1))
)
with open(dest, 'wb') as stream:
return util.download(dl_link, stream, random_ua=True)
def _download(self, links, path, tstamp):
from . import download
all_links = links[:]
# Remove all indirect links.
for i, link in reversed(list(enumerate(links))):
if not download.is_direct(link):
del links[i]
for link in all_links:
res = download.download(link, path)
if res is None:
with open(path, 'wb') as stream:
res = util.download(link,
stream,
headers={'If-Modified': tstamp})
if res == 304 or res:
return res
return False
def process(link, dest):
tries = 3
while tries > 0:
if tries < 3:
time.sleep(0.3)
tries -= 1
with open(dest, 'wb') as stream:
res = util.download(link, stream, random_ua=True)
if not res:
return False
with open(dest, 'r') as stream:
if '<html' not in stream.read(1024):
# Let's assume we downloaded the file.
return True
stream.seek(0)
data = stream.read()
break
tries += 1
while tries > 0:
if tries < 3:
time.sleep(0.3)
tries -= 1
if 'solvemedia.com/papi/' in data:
logging.info('Detected SolveMedia captcha in "%s".', link)
try:
sv = SolveMedia(data)
code = sv.get_code()
challenge = sv.get_challenge(code)
except:
logging.exception('Captcha solving failed!')
else:
data = {
'adcopy_challenge': challenge,
'adcopy_response': code.replace(' ', '+')
}
data = util.post(link, data, random_ua=True)
if not data:
logging.error('Failed to submit captcha response!')
elif re.search(r'(api\.recaptcha\.net|google\.com/recaptcha/api/)',
data):
logging.info('Detected ReCaptcha in "%s".', link)
try:
rc = ReCaptcha(data)
info = re.search(r'challenge\?k=(.+?)"', data)
if info:
rc.id_ = info.group(1)
rc.ask_for_code(link)
data = rc.data
else:
logging.error('Can\'t find the ReCaptcha challenge!')
except:
logging.exception('Captcha solving failed!')
else:
info = MEDIAFIRE_EXTRACT.search(data)
if not info:
logging.error(
'Failed to parse MediaFire\'s response! Probably a CAPTCHA...'
)
continue
else:
with open(dest, 'wb') as stream:
return util.download(info.group(1),
stream,
random_ua=True)
def process(link, dest):
tries = 3
while tries > 0:
if tries < 3:
time.sleep(0.3)
tries -= 1
with open(dest, 'wb') as stream:
res = util.download(link, stream, random_ua=True)
if not res:
return False
with open(dest, 'r') as stream:
if '<html' not in stream.read(1024):
# Let's assume we downloaded the file.
return True
stream.seek(0)
data = stream.read()
break
tries += 1
while tries > 0:
if tries < 3:
time.sleep(0.3)
tries -= 1
if 'solvemedia.com/papi/' in data:
logging.info('Detected SolveMedia captcha in "%s".', link)
try:
sv = SolveMedia(data)
code = sv.get_code()
challenge = sv.get_challenge(code)
except:
logging.exception('Captcha solving failed!')
else:
data = {
'adcopy_challenge': challenge,
'adcopy_response': code.replace(' ', '+')
}
data = util.post(link, data, random_ua=True)
if not data:
logging.error('Failed to submit captcha response!')
elif re.search(r'(api\.recaptcha\.net|google\.com/recaptcha/api/)', data):
logging.info('Detected ReCaptcha in "%s".', link)
try:
rc = ReCaptcha(data)
info = re.search(r'challenge\?k=(.+?)"', data)
if info:
rc.id_ = info.group(1)
rc.ask_for_code(link)
data = rc.data
else:
logging.error('Can\'t find the ReCaptcha challenge!')
except:
logging.exception('Captcha solving failed!')
else:
info = MEDIAFIRE_EXTRACT.search(data)
if not info:
logging.error('Failed to parse MediaFire\'s response! Probably a CAPTCHA...')
continue
else:
with open(dest, 'wb') as stream:
return util.download(info.group(1), stream, random_ua=True)