def add(self): # wait_func: wait method from thread_managed
"""
TODO: Refactory.
"""
link_file = None
err_msg = None
source = None
wait = None
found = False
try:
# Remove the filename from the url
tmp = self.link.split("/file/")[1].split("/")[0]
self.link = "%s/file/%s" % (BASE_URL, tmp)
file_id = self.link.split("/")[-1].strip("/")
cookie = cookielib.CookieJar()
opener = URLOpen(cookie) # cookielib
# form = urllib.urlencode([("checkTimeLimit", "check")]) #deprecated by fileserve
form = urllib.urlencode([("checkDownload", "check")])
post_result = opener.open(self.link, form).read()
if "success" not in post_result:
if "timeLimit" in post_result:
raise LimitExceededException("Limit Exceeded")
else:
raise LinkErrorException("Link Error")
with URLClose(opener.open(self.link)) as s:
for line in s:
if "reCAPTCHA_publickey=" in line:
tmp = line.split("'")[1].split("'")[0]
recaptcha_link = "http://www.google.com/recaptcha/api/challenge?k=%s" % tmp
if self.wait_func(): # wait... if true: download was stopped
return self.link, None, err_msg
c = Recaptcha(BASE_URL, recaptcha_link, self.wait_func)
for retry in range(3):
if self.wait_func(): # wait... if true: download was stopped
return self.link, None, err_msg
challenge, response = c.solve_captcha()
if response is not None:
# Submit the input to the recaptcha system
form = urllib.urlencode(
[
("recaptcha_challenge_field", challenge),
("recaptcha_response_field", response),
("recaptcha_shortencode_field", file_id),
]
)
recaptcha_url = "%s/checkReCaptcha.php" % BASE_URL
# Captcha is good
# on error: {"success":0,"error":"incorrect-captcha-sol"}
# on succes: {"success":1}
with URLClose(opener.open(recaptcha_url, form)) as sa:
if "error" not in sa.read():
form = urllib.urlencode([("downloadLink", "wait")])
with URLClose(opener.open(self.link, form)) as sb:
wait = int(sb.read()[-2:]) # somethimes gives fail404
if self.wait_func(wait): # wait... if true: download was stopped
return self.link, None, err_msg
form = urllib.urlencode([("downloadLink", "show")])
with URLClose(opener.open(self.link, form)) as sc:
if self.wait_func(): # wait... if true: download was stopped
return self.link, None, err_msg
sc.read()
form = urllib.urlencode([("download", "normal")])
with URLClose(
opener.open(self.link, form, range=(self.content_range, None)),
always_close=False,
) as sd:
if sd.url == self.link: # link not found or weird countdown issue
# logger.debug(sd.read())
raise LinkErrorException("Link Error, redirected")
else:
source = sd # ,content_range)
break
else:
err_msg = "Wrong captcha"
else:
raise CaptchaException("No response from the user")
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
err_msg = err
except (LimitExceededException, LinkErrorException, CaptchaException) as err:
if isinstance(err, LimitExceededException):
self.set_limit_exceeded(True)
err_msg = err
logger.info(err)
except Exception as err:
logger.exception(err)
err_msg = err
return self.link, source, err_msg # puede ser el objeto archivo o None.
def add(self): #wait_func: wait method from thread_managed
"""
TODO: Refactory.
"""
link_file = None
err_msg = None
source = None
wait = None
found = False
try:
#Remove the filename from the url
tmp = self.link.split("/file/")[1].split("/")[0]
self.link = "%s/file/%s" % (BASE_URL, tmp)
file_id = self.link.split("/")[-1].strip("/")
cookie = cookielib.CookieJar()
opener = URLOpen(cookie) #cookielib
#form = urllib.urlencode([("checkTimeLimit", "check")]) #deprecated by fileserve
form = urllib.urlencode([("checkDownload", "check")])
post_result = opener.open(self.link, form).read()
if "success" not in post_result:
if "timeLimit" in post_result:
raise LimitExceededException("Limit Exceeded")
else:
raise LinkErrorException("Link Error")
with URLClose(opener.open(self.link)) as s:
for line in s:
if 'reCAPTCHA_publickey=' in line:
tmp = line.split("'")[1].split("'")[0]
recaptcha_link = "http://www.google.com/recaptcha/api/challenge?k=%s" % tmp
if self.wait_func(
): #wait... if true: download was stopped
return self.link, None, err_msg
c = Recaptcha(BASE_URL, recaptcha_link, self.wait_func)
for retry in range(3):
if self.wait_func(
): #wait... if true: download was stopped
return self.link, None, err_msg
challenge, response = c.solve_captcha()
if response is not None:
#Submit the input to the recaptcha system
form = urllib.urlencode([
("recaptcha_challenge_field", challenge),
("recaptcha_response_field", response),
("recaptcha_shortencode_field", file_id)
])
recaptcha_url = "%s/checkReCaptcha.php" % BASE_URL
#Captcha is good
#on error: {"success":0,"error":"incorrect-captcha-sol"}
#on succes: {"success":1}
with URLClose(opener.open(recaptcha_url,
form)) as sa:
if "error" not in sa.read():
form = urllib.urlencode([
("downloadLink", "wait")
])
with URLClose(
opener.open(self.link,
form)) as sb:
wait = int(sb.read(
)[-2:]) #somethimes gives fail404
if self.wait_func(
wait
): #wait... if true: download was stopped
return self.link, None, err_msg
form = urllib.urlencode([
("downloadLink", "show")
])
with URLClose(
opener.open(
self.link,
form)) as sc:
if self.wait_func(
): #wait... if true: download was stopped
return self.link, None, err_msg
sc.read()
form = urllib.urlencode([
("download", "normal")
])
with URLClose(
opener.open(
self.link,
form,
range=(
self.
content_range,
None)),
always_close=False
) as sd:
if sd.url == self.link: #link not found or weird countdown issue
#logger.debug(sd.read())
raise LinkErrorException(
"Link Error, redirected"
)
else:
source = sd #,content_range)
break
else:
err_msg = "Wrong captcha"
else:
raise CaptchaException(
"No response from the user")
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
err_msg = err
except (LimitExceededException, LinkErrorException,
CaptchaException) as err:
if isinstance(err, LimitExceededException):
self.set_limit_exceeded(True)
err_msg = err
logger.info(err)
except Exception as err:
logger.exception(err)
err_msg = err
return self.link, source, err_msg #puede ser el objeto archivo o None.
def add(self): #wait_func: wait method from thread_managed
"""
TODO: Refactory.
"""
link_file = None
err_msg = None
source = None
wait = WAITING
try:
file_id = self.link.split("turbobit.net/")[-1].split(
"/")[0].rstrip(".html")
self.link = BASE_URL + "/download/free/" + file_id
cookie = cookielib.CookieJar()
opener = URLOpen(cookie) #cookielib
with URLClose(opener.open(self.link)) as s1:
key = None
for line in s1:
if "challenge?k=" in line:
key = line.split('challenge?k=')[-1].split('"')[0]
recaptcha_link = "http://www.google.com/recaptcha/api/challenge?k=%s" % key
c = Recaptcha(BASE_URL, recaptcha_link, self.wait_func)
break
if key is None:
raise LimitExceededException("Limit Exceeded")
for retry in range(3):
challenge, response = c.solve_captcha()
if response is not None:
form = urllib.urlencode([
("recaptcha_challenge_field", challenge),
("recaptcha_response_field", response),
("captcha_type", "recaptcha"),
("captcha_subtype", "")
])
with URLClose(opener.open(self.link, form)) as s2:
found = False
for line in s2:
if "limit :" in line:
found = True
try:
wait = int(
line.split(":")[-1].split(",")
[0]) / 100 #ms
except Exception as err:
logger.exception(err)
wait = WAITING
elif "captcha-error" in line:
err_msg = "Wrong captcha"
if found:
if self.wait_func(wait + 1):
return self.link, None, err_msg
url = BASE_URL + "/download/getLinkAfterTimeout/" + file_id
print url
with URLClose(opener.open(url)) as s3:
for line in s3:
print line
if "href='/download/redirect" in line:
tmp = line.split(
"href='")[-1].split("'")[0]
redir_url = BASE_URL + tmp
print redir_url
with URLClose(
opener.open(
redir_url)) as s4:
for line in s4:
if 'href="' in line:
link_file = line.split(
'href="'
)[-1].split('"')[0]
#print link_file
with URLClose(
opener.open(
link_file,
range=
(self.
content_range,
None)),
always_close=
False) as s5:
source = s5
raise FileLinkFoundException(
)
else:
raise CaptchaException("No response from the user")
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
err_msg = err
except (FileLinkFoundException, LimitExceededException,
LinkErrorException, CaptchaException) as err:
if isinstance(err, LimitExceededException):
self.set_limit_exceeded(True)
err_msg = err
logger.info(err)
except Exception as err:
logger.exception(err)
err_msg = err
return link_file, source, err_msg
def add(self): #wait_func: wait method from thread_managed
"""
TODO: Refactory.
"""
link_file = None
err_msg = None
source = None
wait = WAITING
try:
if "/ul.to/" in self.link:
file_id = self.link.split("/ul.to/")[-1].split("/")[0]
else:
file_id = self.link.split("/file/")[-1].split("/")[0]
self.link = BASE_URL + "/file/" + file_id
cookie = cookielib.CookieJar()
opener = URLOpen(cookie) #cookielib
with URLClose(opener.open(self.link)) as s1:
if self.wait_func():
return self.link, None, err_msg
for line in s1:
if 'class="free' in line:
try:
s1.next()
wait = int(s1.next().split("<span>")[-1].split(
"</span>")[0])
except Exception as err:
logger.exception(err)
wait = WAITING
break
form = urllib.urlencode({})
form_url = BASE_URL + "/io/ticket/slot/" + file_id
with URLClose(opener.open(form_url, form)) as s2:
s = s2.read()
if "succ:true" in s:
if self.wait_func(wait):
return self.link, None, err_msg
js_url = BASE_URL + "/js/download.js"
with URLClose(opener.open(js_url)) as s3:
if self.wait_func():
return self.link, None, err_msg
for line in s3:
if 'Recaptcha.create("' in line:
key = line.split('Recaptcha.create("')[
-1].split('"')[0].strip()
break
print key
recaptcha_link = "http://www.google.com/recaptcha/api/challenge?k=%s" % key
c = Recaptcha(BASE_URL, recaptcha_link, self.wait_func)
for retry in range(3):
challenge, response = c.solve_captcha()
if response is not None:
form_url = BASE_URL + "/io/ticket/captcha/" + file_id
form = urllib.urlencode([
("recaptcha_challenge_field", challenge),
("recaptcha_response_field", response)
])
with URLClose(opener.open(form_url,
form)) as s4:
if self.wait_func():
return self.link, None, err_msg
s = s4.read()
if "download" in s:
link_file = s.split("url:'")[-1].split(
"'")[0]
print link_file
with URLClose(
opener.open(
link_file,
range=(self.content_range,
None)),
always_close=False) as s5:
source = s5
raise FileLinkFoundException()
elif "limit-dl" in s:
raise LimitExceededException(
"Limit Exceeded")
else: #{err:"captcha"}
print s
err_msg = "Wrong captcha"
else:
raise CaptchaException(
"No response from the user")
else:
LinkErrorException("Link not found")
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
err_msg = err
except (FileLinkFoundException, LimitExceededException,
LinkErrorException, CaptchaException) as err:
if isinstance(err, LimitExceededException):
self.set_limit_exceeded(True)
err_msg = err
logger.info(err)
except Exception as err:
logger.exception(err)
err_msg = err
return link_file, source, err_msg
def add(self):
"""
TODO: Refactory.
"""
link_file = None
err_msg = None
source = None
wait = WAITING
try:
cookie = cookielib.CookieJar()
opener = URLOpen(cookie) #cookielib
#url parse
if "file_id" in self.link: #most likely not.
file_id = self.link.split("file_id=")[-1].split("&")[0]
else:
file_id = self.link.split("netload.in/datei")[-1].split(
"/")[0].split(".")[0]
self.link = BASE_URL + "/" + "index.php?id=10&file_id=" + file_id
with URLClose(opener.open(self.link)) as s1:
if self.wait_func():
return self.link, None, err_msg
for line in s1:
if 'class="Free_dl' in line:
id = line.split("?id=")[-1].split("&")[0]
url = BASE_URL + "/" + line.split('href="')[-1].split(
'"')[0].replace("&", "&")
break
with URLClose(opener.open(url)) as s2:
for line in s2:
if "captcha.php" in line:
captcha_url = BASE_URL + "/" + line.split(
'src="')[-1].split('"')[0]
elif ">countdown(" in line:
try:
wait = int(
line.split(">countdown(")[-1].split(",")
[0]) / 100 #ms
except Exception as err:
logger.exception(err)
wait = WAITING
if self.wait_func(wait + 1):
return self.link, None, err_msg
captcha_result = tesseract.get_solved_captcha(
captcha_url, cookie, self.filter)
form = urllib.urlencode([("file_id", file_id),
("captcha_check", captcha_result),
("start", "")])
captcha_form_url = BASE_URL + "/" + "index.php?id=" + id
with URLClose(opener.open(captcha_form_url, form)) as s3:
for line in s3:
if ">countdown(" in line:
try:
wait = int(
line.split(">countdown(")[-1].split(
",")[0]) / 100 #ms
except Exception as err:
logger.exception(err)
wait = WAITING
elif 'class="Orange_Link' in line:
link_file = line.split('href="')[-1].split(
'"')[0]
if wait > 600: # 10 minutes
raise LimitExceededException("Limit exceeded")
if self.wait_func(wait + 1):
return self.link, None, err_msg
with URLClose(opener.open(link_file,
range=(self.content_range,
None)),
always_close=False) as s4:
source = s4
raise FileLinkFoundException()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
err_msg = err
except (FileLinkFoundException, LimitExceededException,
LinkErrorException, CaptchaException) as err:
if isinstance(err, LimitExceededException):
self.set_limit_exceeded(True)
err_msg = err
logger.info(err)
except Exception as err:
logger.exception(err)
err_msg = err
return link_file, source, err_msg
def add(self): #wait_func: wait method from thread_managed
"""
http://api.wupload.com/
"""
link_file = None
err_msg = None
source = None
wait = WAITING
found = False
try:
#Remove the filename from the url
tmp = self.link.split("/file/")[1].split("/")[0]
self.link = "%s/file/%s" % (BASE_URL, tmp)
file_id = self.link.split("/")[-1].strip("/")
cookie = cookielib.CookieJar()
opener = URLOpen(cookie) #cookielib
#form = urllib.urlencode([("checkTimeLimit", "check")]) #deprecated by fileserve
if self.wait_func(): #wait... if true: download was stopped
return self.link, None, err_msg
"""
form_action = "{0}?start=1".format(link)
it = opener.open(form_action)
form_action = "{0}?start=1".format(it.geturl()) #get redirect url
#end = form_action.split(".")[2].split("/")[0] #get .com replacement
form_action2 = "{0}/{1}?start=1".format(link, file_id)
#form_action2 = form_action2.replace(".com", end)
form = urllib.urlencode([("foo", "foo")]) #force urllib2 to do a post
headers = {"X-Requested-With": "XMLHttpRequest", }
"""
it = opener.open(self.link)
form_action = "{0}?start=1".format(it.geturl())
form = urllib.urlencode({})
headers = {
"X-Requested-With": "XMLHttpRequest",
}
with URLClose(opener.open(form_action, form,
headers=headers)) as s:
if self.wait_func(): #wait... if true: download was stopped
return self.link, None, err_msg
#when there is a countdown, the page need to be reloaded and search for the captcha again.
for countdown in range(3):
for line in s:
if 'Recaptcha.create("' in line:
tmp = line.split('"')[1].split('"')[0]
recaptcha_link = "http://www.google.com/recaptcha/api/challenge?k=%s" % tmp
if self.wait_func(
): #wait... if true: download was stopped
return self.link, None, err_msg
c = Recaptcha(BASE_URL, recaptcha_link,
self.wait_func)
for retry in range(3):
challenge, response = c.solve_captcha()
if self.wait_func(
): #wait... if true: download was stopped
return self.link, None, err_msg
if response is not None:
#Submit the input to the recaptcha system
form = urllib.urlencode([
("recaptcha_challenge_field",
challenge),
("recaptcha_response_field", response)
])
#recaptcha_url = "%s/checkReCaptcha.php" % BASE_URL
with URLClose(
opener.open(form_action,
form)) as sa:
for line in sa:
if 'downloadLink' in line:
sa.next()
link_file = sa.next().split(
'href="')[-1].split('"')[0]
with URLClose(
opener.open(
link_file,
range=(
self.
content_range,
None)),
always_close=False
) as sd:
source = sd #,content_range)
raise FileLinkFoundException()
else:
raise CaptchaException(
"No response from the user")
if retry == 2:
raise CaptchaException(
"Captcha, max retries reached")
#Link already there O.o
elif 'downloadLink' in line:
s.next()
link_file = s.next().split('href="')[-1].split(
'"')[0]
with URLClose(opener.open(
link_file,
range=(self.content_range, None)),
always_close=False) as sd:
source = sd #,content_range)
raise FileLinkFoundException()
#waiting... ?
elif "name='tm'" in line:
tm = line.split("value='")[-1].split("'")[0]
tm_hash = s.next().split("value='")[-1].split(
"'")[0]
form = urllib.urlencode([("tm", tm),
("tm_hash", tm_hash)])
#waiting...
elif "var countDownDelay" in line:
wait = int(line.split("=")[1].split(";")[0])
if wait < 60:
if self.wait_func(wait):
return self.link, None, err_msg
s = opener.open(
form_action, form
) #fetch the page again. but posting the tm, tm_hash
break
else:
raise LimitExceededException("Limit Exceeded")
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
err_msg = err
except (FileLinkFoundException, CaptchaException,
LimitExceededException, LinkNotFoundException) as err:
if isinstance(err, LimitExceededException):
self.set_limit_exceeded(True)
elif not isinstance(err, FileLinkFoundException):
logger.info(err)
err_msg = err
except Exception as err:
logger.exception(err)
err_msg = err
return self.link, source, err_msg #puede ser el objeto archivo o None.
def add(self): #wait_func: wait method from thread_managed
""""""
link_file = None
err_msg = None
source = None
wait = WAITING
try:
file_id = self.link.split("/files/")[1].split("/")[0]
self.link = BASE_URL + "/files/" + file_id
cookie = cookielib.CookieJar()
opener = URLOpen(cookie) #cookielib
form = urllib.urlencode([
("gateway_result", "1"),
])
with URLClose(opener.open(self.link)) as s1:
if self.wait_func():
return self.link, None, err_msg
s1.read()
with URLClose(opener.open(self.link, form)) as s2:
for line in s2:
if 'download_waiter_remain">' in line:
wait = int(
line.split('download_waiter_remain">')
[-1].split('<')[0])
elif "Recaptcha.create('" in line:
key = line.split("Recaptcha.create('")[-1].split(
"'")[0]
elif "var fid" in line:
fid = line.split("'")[1]
elif 'limit_interval">' in line:
wait = int(
line.split('limit_interval">')[-1].split("<")
[0])
if wait > 320:
raise LimitExceededException("Limit Exceeded")
if self.wait_func(wait):
return self.link, None, err_msg
if key is not None:
recaptcha_link = "http://www.google.com/recaptcha/api/challenge?k=%s" % key
c = Recaptcha(BASE_URL, recaptcha_link, self.wait_func)
for retry in range(3):
if self.wait_func(
): #wait... if true: download was stopped
return self.link, None, err_msg
challenge, response = c.solve_captcha()
if response is not None:
recaptcha_url = BASE_URL + "/get_file.php?fid=" + fid + "&challenge=" + challenge + "&response=" + response
with URLClose(
opener.open(recaptcha_url)) as s3:
if self.wait_func():
return self.link, None, err_msg
for line in s3:
if 'form action="' in line and not "recaptcha" in line:
link_file = line.split(
'form action="')[-1].split(
'"')[0]
#print link_file
with URLClose(
opener.open(
link_file,
range=(
self.content_range,
None)),
always_close=False) as s4:
source = s4
raise FileLinkFoundException()
err_msg = "Wrong captcha"
else:
raise CaptchaException(
"No response from the user")
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
err_msg = err
except (FileLinkFoundException, LimitExceededException,
LinkErrorException, CaptchaException) as err:
if isinstance(err, LimitExceededException):
self.set_limit_exceeded(True)
err_msg = err
logger.info(err)
except Exception as err:
logger.exception(err)
err_msg = err
return link_file, source, err_msg
def add(self): # wait_func: wait method from thread_managed
""""""
link_file = None
err_msg = None
source = None
wait = WAITING
try:
file_id = self.link.split("/files/")[1].split("/")[0]
self.link = BASE_URL + "/files/" + file_id
cookie = cookielib.CookieJar()
opener = URLOpen(cookie) # cookielib
form = urllib.urlencode([("gateway_result", "1")])
with URLClose(opener.open(self.link)) as s1:
if self.wait_func():
return self.link, None, err_msg
s1.read()
with URLClose(opener.open(self.link, form)) as s2:
for line in s2:
if 'download_waiter_remain">' in line:
wait = int(line.split('download_waiter_remain">')[-1].split("<")[0])
elif "Recaptcha.create('" in line:
key = line.split("Recaptcha.create('")[-1].split("'")[0]
elif "var fid" in line:
fid = line.split("'")[1]
elif 'limit_interval">' in line:
wait = int(line.split('limit_interval">')[-1].split("<")[0])
if wait > 320:
raise LimitExceededException("Limit Exceeded")
if self.wait_func(wait):
return self.link, None, err_msg
if key is not None:
recaptcha_link = "http://www.google.com/recaptcha/api/challenge?k=%s" % key
c = Recaptcha(BASE_URL, recaptcha_link, self.wait_func)
for retry in range(3):
if self.wait_func(): # wait... if true: download was stopped
return self.link, None, err_msg
challenge, response = c.solve_captcha()
if response is not None:
recaptcha_url = (
BASE_URL
+ "/get_file.php?fid="
+ fid
+ "&challenge="
+ challenge
+ "&response="
+ response
)
with URLClose(opener.open(recaptcha_url)) as s3:
if self.wait_func():
return self.link, None, err_msg
for line in s3:
if 'form action="' in line and not "recaptcha" in line:
link_file = line.split('form action="')[-1].split('"')[0]
# print link_file
with URLClose(
opener.open(link_file, range=(self.content_range, None)),
always_close=False,
) as s4:
source = s4
raise FileLinkFoundException()
err_msg = "Wrong captcha"
else:
raise CaptchaException("No response from the user")
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
err_msg = err
except (FileLinkFoundException, LimitExceededException, LinkErrorException, CaptchaException) as err:
if isinstance(err, LimitExceededException):
self.set_limit_exceeded(True)
err_msg = err
logger.info(err)
except Exception as err:
logger.exception(err)
err_msg = err
return link_file, source, err_msg
def add(self): #wait_func: wait method from thread_managed
"""
TODO: Refactory.
"""
link_file = None
err_msg = None
source = None
wait = WAITING
try:
file_id = self.link.split("turbobit.net/")[-1].split("/")[0].rstrip(".html")
self.link = BASE_URL + "/download/free/" + file_id
cookie = cookielib.CookieJar()
opener = URLOpen(cookie) #cookielib
with URLClose(opener.open(self.link)) as s1:
key = None
for line in s1:
if "challenge?k=" in line:
key = line.split('challenge?k=')[-1].split('"')[0]
recaptcha_link = "http://www.google.com/recaptcha/api/challenge?k=%s" % key
c = Recaptcha(BASE_URL, recaptcha_link, self.wait_func)
break
if key is None:
raise LimitExceededException("Limit Exceeded")
for retry in range(3):
challenge, response = c.solve_captcha()
if response is not None:
form = urllib.urlencode([("recaptcha_challenge_field", challenge),
("recaptcha_response_field", response),
("captcha_type", "recaptcha"),
("captcha_subtype", "")])
with URLClose(opener.open(self.link, form)) as s2:
found = False
for line in s2:
if "limit :" in line:
found = True
try:
wait = int(line.split(":")[-1].split(",")[0]) / 100 #ms
except Exception as err:
logger.exception(err)
wait = WAITING
elif "captcha-error" in line:
err_msg = "Wrong captcha"
if found:
if self.wait_func(wait+1):
return self.link, None, err_msg
url = BASE_URL + "/download/getLinkAfterTimeout/" + file_id
print url
with URLClose(opener.open(url)) as s3:
for line in s3:
print line
if "href='/download/redirect" in line:
tmp = line.split("href='")[-1].split("'")[0]
redir_url = BASE_URL + tmp
print redir_url
with URLClose(opener.open(redir_url)) as s4:
for line in s4:
if 'href="' in line:
link_file = line.split('href="')[-1].split('"')[0]
#print link_file
with URLClose(opener.open(link_file, range=(self.content_range, None)), always_close=False) as s5:
source = s5
raise FileLinkFoundException()
else:
raise CaptchaException("No response from the user")
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
err_msg = err
except (FileLinkFoundException, LimitExceededException, LinkErrorException, CaptchaException) as err:
if isinstance(err, LimitExceededException):
self.set_limit_exceeded(True)
err_msg = err
logger.info(err)
except Exception as err:
logger.exception(err)
err_msg = err
return link_file, source, err_msg
def add(self):
""""""
link_file = None
err_msg = None
source = None
wait = WAITING
max_retries = 3
try:
#Remove the filename from the url
tmp = self.link.split("/file/")[1].split("/")[0]
self.link = "%s/file/%s" % (BASE_URL, tmp)
cookie = cookielib.CookieJar()
opener = URLOpen(cookie) #cookielib
with URLClose(opener.open(self.link)) as s:
if self.wait_func():
return self.link, None, err_msg
for line in s:
if 'check:' in line:
check = line.split("check:'")[1].replace("'",
"").strip()
elif "Recaptcha.create" in line:
tmp = line.split('("')[1].split('"')[0]
recaptcha_link = "http://www.google.com/recaptcha/api/challenge?k=%s" % tmp
c = Recaptcha(BASE_URL, recaptcha_link, self.wait_func)
for retry in range(3):
if self.wait_func():
return self.link, None, err_msg
if retry < (max_retries + 1):
challenge, response = c.solve_captcha()
if response is not None:
#Filefactory perfoms as check on its server by doing an
#Ajax request sending the following data
form = urllib.urlencode([
("recaptcha_challenge_field",
challenge),
("recaptcha_response_field", response),
("recaptcha_shortencode_field",
"undefined"), ("check", check)
])
recaptcha_link = "%s/file/checkCaptcha.php" % BASE_URL
#Getting the result back, status:{"ok"|"fail"}
with URLClose(
opener.open(recaptcha_link,
form)) as sa:
if self.wait_func():
return self.link, None, err_msg
for line in sa:
if 'status:"ok"' in line:
tmp = line.split(
'path:"')[-1].split('"')[0]
tmp_link = "%s%s" % (BASE_URL,
tmp)
with URLClose(
opener.open(
tmp_link)) as sb:
if self.wait_func():
return self.link, None, err_msg
for line in sb:
if 'countdown">' in line:
#Try to get WAIT from the page
try:
tmp = line.split(
'countdown">'
)[-1].split(
"<")[0]
tmp = int(tmp)
if tmp > 320:
raise LimitExceededException(
"Limit Exceeded"
)
except ValueError:
pass
else:
if tmp > 0:
wait = tmp
if 'id="downloadLinkTarget' in line:
link_file = line.split(
'<a href="'
)[1].split('"')[0]
if self.wait_func(
wait):
return self.link, None, err_msg
with URLClose(
opener.
open(
link_file,
range=
(self.
content_range,
None
)),
always_close
=False
) as sc:
try:
if sc.status == 302: #redirect error 302.
raise RedirectException(
"Redirection error"
)
except AttributeError as err: #no redirected.
source = sc
raise FileLinkFoundException(
)
else:
raise CaptchaException(
"No response from the user")
else:
raise CaptchaException(
"Captcha, max retries reached")
raise LinkNotFoundException()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
err_msg = err
except (RedirectException, FileLinkFoundException,
LinkNotFoundException, CaptchaException,
LimitExceededException) as err:
if isinstance(err, LimitExceededException):
self.set_limit_exceeded(True)
elif not isinstance(err, FileLinkFoundException):
logger.info(err)
err_msg = err
except Exception as err:
logger.exception(err)
return self.link, source, err_msg #puede ser el objeto archivo o None.
def add(self): # wait_func: wait method from thread_managed
""""""
link_file = None
err_msg = None
source = None
wait = WAITING
try:
# Remove the filename from the url
file_id = self.link.split("/file/")[1].split("/")[0]
self.link = "".join((BASE_URL, "/file/", file_id))
# file_id = link.split("/")[-1].strip("/")
cookie = cookielib.CookieJar()
opener = URLOpen(cookie) # cookielib
# form = urllib.urlencode([("checkTimeLimit", "check")]) #deprecated by fileserve
if self.wait_func(): # wait... if true: download was stopped
return self.link, None, err_msg
"""
form_action = "{0}?start=1".format(link)
it = opener.open(form_action)
form_action = "{0}?start=1".format(it.geturl()) #get redirect url
#end = form_action.split(".")[2].split("/")[0] #get .com replacement
form_action2 = "{0}/{1}?start=1".format(link, file_id)
#form_action2 = form_action2.replace(".com", end)
form = urllib.urlencode([("foo", "foo")]) #force urllib2 to do a post
"""
it = opener.open(self.link)
form_action = "{0}?start=1".format(it.geturl())
form = urllib.urlencode({})
headers = {"X-Requested-With": "XMLHttpRequest"}
with URLClose(opener.open(form_action, form, headers=headers)) as s:
# print s.read()
if self.wait_func(): # wait... if true: download was stopped
return self.link, None, err_msg
# when there is a countdown, the page need to be reloaded and search for the captcha again.
for countdown in range(3):
for line in s:
if 'Recaptcha.create("' in line:
tmp = line.split('"')[1].split('"')[0]
recaptcha_link = "http://www.google.com/recaptcha/api/challenge?k=%s" % tmp
if self.wait_func(): # wait... if true: download was stopped
return self.link, None, err_msg
c = Recaptcha(BASE_URL, recaptcha_link, self.wait_func)
for retry in range(3):
challenge, response = c.solve_captcha()
if self.wait_func(): # wait... if true: download was stopped
return self.link, None, err_msg
if response is not None:
# Submit the input to the recaptcha system
form = urllib.urlencode(
[
("recaptcha_challenge_field", challenge),
("recaptcha_response_field", response),
]
)
# recaptcha_url = "%s/checkReCaptcha.php" % BASE_URL
with URLClose(opener.open(form_action, form)) as sa:
for line in sa:
if 'downloadLink">' in line: # you get it on POST
link_file = line.split('href="')[-1].split('"')[0]
with URLClose(
opener.open(link_file, range=(self.content_range, None)),
always_close=False,
) as sd:
source = sd # ,content_range)
raise FileLinkFoundException()
else:
raise CaptchaException("No response from the user")
if retry == 2:
raise CaptchaException("Captcha, max retries reached")
# Link already there O.o
elif 'downloadLink">' in line: # you get it on POST
link_file = line.split('href="')[-1].split('"')[0]
with URLClose(
opener.open(link_file, range=(self.content_range, None)), always_close=False
) as sd:
source = sd # ,content_range)
raise FileLinkFoundException()
# waiting... ?
elif "name='tm'" in line: # you get it on POST (same as countdowndelay)
tm = line.split("value='")[-1].split("'")[0]
tm_hash = s.next().split("value='")[-1].split("'")[0]
form = urllib.urlencode([("tm", tm), ("tm_hash", tm_hash)])
# waiting...
elif "var countDownDelay" in line: # you get it on POST
wait = int(line.split("=")[-1].split(";")[0])
if wait < 80:
if self.wait_func(wait):
return self.link, None, err_msg
s = opener.open(form_action, form) # fetch the page again. but posting the tm, tm_hash
break
else:
raise LimitExceededException("Limit Exceeded")
elif "errorParallel" in line: # you get it on POST
raise LinkNotFoundException("Free users may only download 1 file at a time.")
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
err_msg = err
except (FileLinkFoundException, CaptchaException, LimitExceededException, LinkNotFoundException) as err:
if isinstance(err, LimitExceededException):
self.set_limit_exceeded(True)
err_msg = err
elif not isinstance(err, FileLinkFoundException):
logger.info(err)
err_msg = err
except Exception as err:
logger.exception(err)
err_msg = err
return self.link, source, err_msg # puede ser el objeto archivo o None.
def add(self): #wait_func: wait method from thread_managed
"""
TODO: Refactory.
"""
link_file = None
err_msg = None
source = None
wait = WAITING
try:
if "/ul.to/" in self.link:
file_id = self.link.split("/ul.to/")[-1].split("/")[0]
else:
file_id = self.link.split("/file/")[-1].split("/")[0]
self.link = BASE_URL + "/file/" + file_id
cookie = cookielib.CookieJar()
opener = URLOpen(cookie) #cookielib
with URLClose(opener.open(self.link)) as s1:
if self.wait_func():
return self.link, None, err_msg
for line in s1:
if 'class="free' in line:
try:
s1.next()
wait = int(s1.next().split("<span>")[-1].split("</span>")[0])
except Exception as err:
logger.exception(err)
wait = WAITING
break
form = urllib.urlencode({})
form_url = BASE_URL + "/io/ticket/slot/" + file_id
with URLClose(opener.open(form_url, form)) as s2:
s = s2.read()
if "succ:true" in s:
if self.wait_func(wait):
return self.link, None, err_msg
js_url = BASE_URL + "/js/download.js"
with URLClose(opener.open(js_url)) as s3:
if self.wait_func():
return self.link, None, err_msg
for line in s3:
if 'Recaptcha.create("' in line:
key = line.split('Recaptcha.create("')[-1].split('"')[0].strip()
break
print key
recaptcha_link = "http://www.google.com/recaptcha/api/challenge?k=%s" % key
c = Recaptcha(BASE_URL, recaptcha_link, self.wait_func)
for retry in range(3):
challenge, response = c.solve_captcha()
if response is not None:
form_url = BASE_URL + "/io/ticket/captcha/" + file_id
form = urllib.urlencode([("recaptcha_challenge_field", challenge), ("recaptcha_response_field", response)])
with URLClose(opener.open(form_url, form)) as s4:
if self.wait_func():
return self.link, None, err_msg
s = s4.read()
if "download" in s:
link_file = s.split("url:'")[-1].split("'")[0]
print link_file
with URLClose(opener.open(link_file, range=(self.content_range, None)), always_close=False) as s5:
source = s5
raise FileLinkFoundException()
elif "limit-dl" in s:
raise LimitExceededException("Limit Exceeded")
else: #{err:"captcha"}
print s
err_msg = "Wrong captcha"
else:
raise CaptchaException("No response from the user")
else:
LinkErrorException("Link not found")
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
err_msg = err
except (FileLinkFoundException, LimitExceededException, LinkErrorException, CaptchaException) as err:
if isinstance(err, LimitExceededException):
self.set_limit_exceeded(True)
err_msg = err
logger.info(err)
except Exception as err:
logger.exception(err)
err_msg = err
return link_file, source, err_msg
def add(self):
""""""
link_file = None
err_msg = None
source = None
wait = WAITING
max_retries = 3
try:
#Remove the filename from the url
tmp = self.link.split("/file/")[1].split("/")[0]
self.link = "%s/file/%s" % (BASE_URL, tmp)
cookie = cookielib.CookieJar()
opener = URLOpen(cookie) #cookielib
with URLClose(opener.open(self.link)) as s:
if self.wait_func():
return self.link, None, err_msg
for line in s:
if 'check:' in line:
check = line.split("check:'")[1].replace("'","").strip()
elif "Recaptcha.create" in line:
tmp = line.split('("')[1].split('"')[0]
recaptcha_link = "http://www.google.com/recaptcha/api/challenge?k=%s" % tmp
c = Recaptcha(BASE_URL, recaptcha_link, self.wait_func)
for retry in range(3):
if self.wait_func():
return self.link, None, err_msg
if retry < (max_retries + 1):
challenge, response = c.solve_captcha()
if response is not None:
#Filefactory perfoms as check on its server by doing an
#Ajax request sending the following data
form = urllib.urlencode([("recaptcha_challenge_field", challenge), ("recaptcha_response_field", response), ("recaptcha_shortencode_field", "undefined"),("check", check)])
recaptcha_link = "%s/file/checkCaptcha.php" % BASE_URL
#Getting the result back, status:{"ok"|"fail"}
with URLClose(opener.open(recaptcha_link, form)) as sa:
if self.wait_func():
return self.link, None, err_msg
for line in sa:
if 'status:"ok"' in line:
tmp = line.split('path:"')[-1].split('"')[0]
tmp_link = "%s%s" %(BASE_URL, tmp)
with URLClose(opener.open(tmp_link)) as sb:
if self.wait_func():
return self.link, None, err_msg
for line in sb:
if 'countdown">' in line:
#Try to get WAIT from the page
try:
tmp = line.split('countdown">')[-1].split("<")[0]
tmp = int(tmp)
if tmp > 320:
raise LimitExceededException("Limit Exceeded")
except ValueError:
pass
else:
if tmp > 0:
wait = tmp
if 'id="downloadLinkTarget' in line:
link_file = line.split('<a href="')[1].split('"')[0]
if self.wait_func(wait):
return self.link, None, err_msg
with URLClose(opener.open(link_file, range=(self.content_range, None)), always_close=False) as sc:
try:
if sc.status == 302: #redirect error 302.
raise RedirectException("Redirection error")
except AttributeError as err: #no redirected.
source = sc
raise FileLinkFoundException()
else:
raise CaptchaException("No response from the user")
else:
raise CaptchaException("Captcha, max retries reached")
raise LinkNotFoundException()
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
err_msg = err
except (RedirectException, FileLinkFoundException, LinkNotFoundException, CaptchaException, LimitExceededException) as err:
if isinstance(err, LimitExceededException):
self.set_limit_exceeded(True)
elif not isinstance(err, FileLinkFoundException):
logger.info(err)
err_msg = err
except Exception as err:
logger.exception(err)
return self.link, source, err_msg #puede ser el objeto archivo o None.
def add(self): #wait_func: wait method from thread_managed
"""
TODO: Refactory.
"""
link_file = None
err_msg = None
source = None
wait = WAITING
ajax_id = None
recaptcha_key = None
try:
file_id = self.link.split("/files/")[1].split("/")[0]
cookie = cookielib.CookieJar()
opener = URLOpen(cookie) #cookielib
ajax_id_url = BASE_URL + "/files-ajax/" + file_id + "/request.html"
with URLClose(opener.open(self.link)) as s1:
for line in s1:
if "var ajaxdl" in line:
ajax_id = line.split('"')[1]
elif "challenge?k=" in line:
recaptcha_key = line.split("challenge?k=")[-1].split('"')[0].strip()
if not ajax_id: #not recaptcha_key or not ajax_id:
raise LinkErrorException("Link not found.")
if self.wait_func():
return self.link, None, err_msg
#wait time.
#note: bitshare does not care for this. It can be skipped.
#headers = {"Accept:": "application/json", }
form = urllib.urlencode([("request", "generateID"), ("ajaxid", ajax_id)])
with URLClose(opener.open(ajax_id_url, form)) as s2:
response = s2.read() #may return ERROR: explanation
wait = int(response.split(":")[1]) #file:60:1
if wait > 120:
raise LimitExceededException("Limit Exceeded")
if self.wait_func(wait):
return self.link, None, err_msg
#recaptcha.
#note: bitshare does not care for this. It can be skipped.
if recaptcha_key:
recaptcha_link = "http://www.google.com/recaptcha/api/challenge?k=%s" % recaptcha_key
for retry in range(3):
c = Recaptcha(BASE_URL, recaptcha_link, self.wait_func)
challenge, response = c.solve_captcha()
if response is not None:
form = urllib.urlencode([("request", "validateCaptcha"), ("ajaxid", ajax_id), ("recaptcha_challenge_field", challenge), ("recaptcha_response_field", response)])
response_ = opener.open(ajax_id_url, form).read() #may return ERROR: explanation or SUCCESS
if not "ERROR" in response_:
break
else:
raise CaptchaException("No response from the user")
if "ERROR" in response_:
raise CaptchaException("Wrong captcha")
if self.wait_func():
return self.link, None, err_msg
#get download link
form = urllib.urlencode([("request", "getDownloadURL"), ("ajaxid", ajax_id)])
with URLClose(opener.open(ajax_id_url, form)) as s3:
response = s3.read()
link_file = response.split("http")[-1]
link_file = "http" + link_file
with URLClose(URLOpen(cookie).open(link_file, range=(self.content_range, None)), always_close=False) as sc:
source = sc
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
err_msg = err
except (LimitExceededException, CaptchaException, LinkErrorException) as err:
if isinstance(err, LimitExceededException):
self.set_limit_exceeded(True)
err_msg = err
logger.info(err)
except Exception as err:
err_msg
logger.exception(err)
return link_file, source, err_msg
def add(self): #wait_func: wait method from thread_managed
"""
TODO: Refactory.
"""
link_file = None
err_msg = None
source = None
wait = WAITING
ajax_id = None
recaptcha_key = None
try:
file_id = self.link.split("/files/")[1].split("/")[0]
cookie = cookielib.CookieJar()
opener = URLOpen(cookie) #cookielib
ajax_id_url = BASE_URL + "/files-ajax/" + file_id + "/request.html"
with URLClose(opener.open(self.link)) as s1:
for line in s1:
if "var ajaxdl" in line:
ajax_id = line.split('"')[1]
elif "challenge?k=" in line:
recaptcha_key = line.split("challenge?k=")[-1].split(
'"')[0].strip()
if not ajax_id: #not recaptcha_key or not ajax_id:
raise LinkErrorException("Link not found.")
if self.wait_func():
return self.link, None, err_msg
#wait time.
#note: bitshare does not care for this. It can be skipped.
#headers = {"Accept:": "application/json", }
form = urllib.urlencode([("request", "generateID"),
("ajaxid", ajax_id)])
with URLClose(opener.open(ajax_id_url, form)) as s2:
response = s2.read() #may return ERROR: explanation
wait = int(response.split(":")[1]) #file:60:1
if wait > 120:
raise LimitExceededException("Limit Exceeded")
if self.wait_func(wait):
return self.link, None, err_msg
#recaptcha.
#note: bitshare does not care for this. It can be skipped.
if recaptcha_key:
recaptcha_link = "http://www.google.com/recaptcha/api/challenge?k=%s" % recaptcha_key
for retry in range(3):
c = Recaptcha(BASE_URL, recaptcha_link, self.wait_func)
challenge, response = c.solve_captcha()
if response is not None:
form = urllib.urlencode([
("request", "validateCaptcha"),
("ajaxid", ajax_id),
("recaptcha_challenge_field", challenge),
("recaptcha_response_field", response)
])
response_ = opener.open(ajax_id_url, form).read(
) #may return ERROR: explanation or SUCCESS
if not "ERROR" in response_:
break
else:
raise CaptchaException("No response from the user")
if "ERROR" in response_:
raise CaptchaException("Wrong captcha")
if self.wait_func():
return self.link, None, err_msg
#get download link
form = urllib.urlencode([("request", "getDownloadURL"),
("ajaxid", ajax_id)])
with URLClose(opener.open(ajax_id_url, form)) as s3:
response = s3.read()
link_file = response.split("http")[-1]
link_file = "http" + link_file
with URLClose(URLOpen(cookie).open(link_file,
range=(self.content_range,
None)),
always_close=False) as sc:
source = sc
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
err_msg = err
except (LimitExceededException, CaptchaException,
LinkErrorException) as err:
if isinstance(err, LimitExceededException):
self.set_limit_exceeded(True)
err_msg = err
logger.info(err)
except Exception as err:
err_msg
logger.exception(err)
return link_file, source, err_msg
