def check(self, link):
""""""
name = "Unknown"
size = 0
status_msg = None
link_status = cons.LINK_ERROR
#for retry_count in range(RETRIES):
try:
with URLClose(URLOpen().open(link, time_out=10)) as s:
for line in s:
if 'class="f_arial f_14px"' in line:
name = line.split('"f_arial f_14px">')[-1].split('<')[0].strip()
name = misc.html_entities_parser(name)
tmp = s.next().split(":")[-1].split("<")[0].strip()
unit = tmp.split(" ")[-1].strip()
size = float(tmp.split(" ")[0].strip())
#convert size to bytes.
if unit.lower() == "kb":
size = size * 1024
elif unit.lower() == "mb":
size = size * 1024 * 1024
elif unit.lower() == "gb":
size = size * 1024 * 1024 * 1024
break
if size:
link_status = cons.LINK_ALIVE
else:
link_status, name, size = cons.LINK_DEAD, cons.UNKNOWN, 0
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
status_msg = "Error: {0}".format(err)
except Exception, err:
status_msg = "Error: {0}".format(err)
name, size = cons.UNKNOWN, 0
logger.exception(err)
def check(self, link):
""""""
name = "Unknown"
size = 0
status_msg = None
link_status = cons.LINK_ERROR
#for retry_count in range(RETRIES):
try:
with URLClose(URLOpen().open(link, time_out=10)) as s:
for line in s:
if 'class="f_arial f_14px"' in line:
name = line.split('"f_arial f_14px">')[-1].split(
'<')[0].strip()
name = misc.html_entities_parser(name)
tmp = s.next().split(":")[-1].split("<")[0].strip()
unit = tmp.split(" ")[-1].strip()
size = float(tmp.split(" ")[0].strip())
#convert size to bytes.
if unit.lower() == "kb":
size = size * 1024
elif unit.lower() == "mb":
size = size * 1024 * 1024
elif unit.lower() == "gb":
size = size * 1024 * 1024 * 1024
break
if size:
link_status = cons.LINK_ALIVE
else:
link_status, name, size = cons.LINK_DEAD, cons.UNKNOWN, 0
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
status_msg = "Error: {0}".format(err)
except Exception, err:
status_msg = "Error: {0}".format(err)
name, size = cons.UNKNOWN, 0
logger.exception(err)
def check(self, link):
""""""
name = cons.UNKNOWN
size = 0
status_msg = None
link_status = cons.LINK_ERROR
#for retry_count in range(RETRIES):
try:
if "/video/" in link:
link = link.replace("/video/", "/download/")
elif "/audio/" in link:
link = link.replace("/audio/", "/download/")
elif "/image/" in link:
link = link.replace("/image/", "/download/")
with URLClose(URLOpen().open(link)) as s:
for line in s:
if 'File Name:' in line:
name = s.next().split("</font>")[0].split(
'>')[-1].strip()
name = misc.html_entities_parser(name)
elif 'File Size:' in line:
tmp = line.split("</font>")[0].split('>')[-1].strip()
unit = tmp[-2:].strip()
size = float(tmp[:-2])
#convert size to bytes.
if unit.lower() == "kb":
size = size * 1024
elif unit.lower() == "mb":
size = size * 1024 * 1024
elif unit.lower() == "gb":
size = size * 1024 * 1024 * 1024
break
if size:
link_status = cons.LINK_ALIVE
else:
link_status, name, size = cons.LINK_DEAD, cons.UNKNOWN, 0
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
status_msg = "Error: {0}".format(err)
except Exception as err:
name, size = cons.UNKNOWN, 0
logger.exception(err)
return link_status, name, size, status_msg
def check(self, link):
""""""
name = cons.UNKNOWN
size = 0
status_msg = None
link_status = cons.LINK_ERROR
#for retry_count in range(RETRIES):
try:
if "/video/" in link:
link = link.replace("/video/", "/download/")
elif "/audio/" in link:
link = link.replace("/audio/", "/download/")
elif "/image/" in link:
link = link.replace("/image/", "/download/")
with URLClose(URLOpen().open(link)) as s:
for line in s:
if 'File Name:' in line:
name = s.next().split("</font>")[0].split('>')[-1].strip()
name = misc.html_entities_parser(name)
elif 'File Size:' in line:
tmp = line.split("</font>")[0].split('>')[-1].strip()
unit = tmp[-2:].strip()
size = float(tmp[:-2])
#convert size to bytes.
if unit.lower() == "kb":
size = size * 1024
elif unit.lower() == "mb":
size = size * 1024 * 1024
elif unit.lower() == "gb":
size = size * 1024 * 1024 * 1024
break
if size:
link_status = cons.LINK_ALIVE
else:
link_status, name, size = cons.LINK_DEAD, cons.UNKNOWN, 0
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
status_msg = "Error: {0}".format(err)
except Exception as err:
name, size = cons.UNKNOWN, 0
logger.exception(err)
return link_status, name, size, status_msg
def __get_filename_from_source(self, info):
""""""
file_name = None
if info.getheader("Content-Disposition", None): #Content-Disposition: Attachment; filename=name.ext
disposition = info.getheader("Content-Disposition") #get file name
if 'filename="' in disposition:
file_name = disposition.split('filename=')[-1].split('"')[1]
elif "filename='" in disposition:
file_name = disposition.split('filename=')[-1].split("'")[1]
elif 'filename=' in disposition:
file_name = disposition.split('filename=')[-1]
elif 'filename*=' in disposition:
file_name = disposition.split("'")[-1]
if not file_name: #may be an empty string or None
file_name = misc.get_filename_from_url(self.source.url)
file_name = misc.html_entities_parser(file_name)
file_name = urllib.unquote_plus(file_name)
file_name = misc.smart_decode(file_name)
file_name = misc.strip(file_name, to_strip='/\\:*?"<>|')
file_name = file_name.strip('.')
return file_name
def __get_filename_from_source(self, info):
""""""
file_name = None
if info.getheader(
"Content-Disposition",
None): #Content-Disposition: Attachment; filename=name.ext
disposition = info.getheader("Content-Disposition") #get file name
if 'filename="' in disposition:
file_name = disposition.split('filename=')[-1].split('"')[1]
elif "filename='" in disposition:
file_name = disposition.split('filename=')[-1].split("'")[1]
elif 'filename=' in disposition:
file_name = disposition.split('filename=')[-1]
elif 'filename*=' in disposition:
file_name = disposition.split("'")[-1]
if not file_name: #may be an empty string or None
file_name = misc.get_filename_from_url(self.source.url)
file_name = misc.html_entities_parser(file_name)
file_name = urllib.unquote_plus(file_name)
file_name = misc.smart_decode(file_name)
file_name = misc.strip(file_name, to_strip='/\\:*?"<>|')
file_name = file_name.strip('.')
return file_name
def check(self, link):
""""""
name = "Unknown"
size = 0
status_msg = None
link_status = cons.LINK_ERROR
try:
#strip file name
tmp = link.split("/file/")[1].split("/")[0]
link = "%s/file/%s" % (BASE_URL, tmp)
link_quoted = urllib.quote_plus(link)
with URLClose(request.get("http://www.filefactory.com/tool/links.php?func=links&links=" + link_quoted, timeout=10)) as s:
alive = False
for line in s:
if 'Available' in line:
alive = True
elif alive:
if 'class="metadata"' in line:
name = line.split('class="metadata">')[-1].split('</div>')[0].split('/')[-1].strip()
name = html_entities_parser(name)
s.next()
size_list = s.next().split("<td>")[-1].split("</td>")[0].split(" ")
#size = "".join(size_list)
size = int(float(size_list[0]))
link_status = cons.LINK_ALIVE
break
if link_status != cons.LINK_ALIVE:
link_status = cons.LINK_DEAD
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
status_msg = "Error: {0}".format(err)
logger.warning(err)
except Exception as err:
status_msg = "Error: {0}".format(err)
logger.exception(err)
return link_status, name, size, status_msg
def check(self, link):
""""""
name = cons.UNKNOWN
size = 0
status_msg = None
link_status = cons.LINK_ERROR
#for retry_count in range(RETRIES):
try:
with URLClose(URLOpen().open(link)) as s:
for line in s:
if 'name="description"' in line:
name = line.split('content="')[-1].split(
" | Free file hosting")[0]
name = misc.html_entities_parser(name)
elif "File Size:</b>" in line:
tmp = line.split("</b>")[-1].split("</div>")[0].strip()
unit = tmp[-2:]
size = float(tmp[:-2])
#convert size to bytes.
if unit == "KB":
size = size * 1024
elif unit == "MB":
size = size * 1024 * 1024
elif unit == "GB":
size = size * 1024 * 1024 * 1024
break
if size:
link_status = cons.LINK_ALIVE
else:
link_status, name, size = cons.LINK_DEAD, cons.UNKNOWN, 0
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
status_msg = "Error: {0}".format(err)
except Exception as err:
name, size = cons.UNKNOWN, 0
logger.exception(err)
return link_status, name, size, status_msg
def check(self, link):
""""""
name = cons.UNKNOWN
size = 0
status_msg = None
link_status = cons.LINK_ERROR
#for retry_count in range(RETRIES):
try:
with URLClose(URLOpen().open(link)) as s:
for line in s:
if 'name="description"' in line:
name = line.split('content="')[-1].split(" | Free file hosting")[0]
name = misc.html_entities_parser(name)
elif "File Size:</b>" in line:
tmp = line.split("</b>")[-1].split("</div>")[0].strip()
unit = tmp[-2:]
size = float(tmp[:-2])
#convert size to bytes.
if unit == "KB":
size = size * 1024
elif unit == "MB":
size = size * 1024 * 1024
elif unit == "GB":
size = size * 1024 * 1024 * 1024
break
if size:
link_status = cons.LINK_ALIVE
else:
link_status, name, size = cons.LINK_DEAD, cons.UNKNOWN, 0
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
status_msg = "Error: {0}".format(err)
except Exception as err:
name, size = cons.UNKNOWN, 0
logger.exception(err)
return link_status, name, size, status_msg
def check(self, link):
""""""
name = "Unknown"
size = 0
status_msg = None
link_status = cons.LINK_ERROR
try:
#strip file name
tmp = link.split("/file/")[1].split("/")[0]
link = "%s/file/%s" % (BASE_URL, tmp)
link_quoted = urllib.quote_plus(link)
with URLClose(URLOpen().open("http://www.filefactory.com/tool/links.php?func=links&links=" + link_quoted, time_out=10)) as s:
alive = False
for line in s:
if 'Available' in line:
alive = True
elif alive:
if 'class="metadata"' in line:
name = line.split('class="metadata">')[-1].split('</div>')[0].split('/')[-1].strip()
name = html_entities_parser(name)
s.next()
size_list = s.next().split("<td>")[-1].split("</td>")[0].split(" ")
#size = "".join(size_list)
size = int(float(size_list[0]))
link_status = cons.LINK_ALIVE
break
if link_status != cons.LINK_ALIVE:
link_status = cons.LINK_DEAD
except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
pass
except Exception as err:
status_msg = "Error: {0}".format(err)
logger.exception(err)
return link_status, name, size, status_msg