def __authFtp__(self):
"""handles ftp authentication"""
ftped = urllib2.FTPHandler()
ftpUrl = self.url.replace('ftp://', '')
req = urllib2.Request("ftp://%s:%s@%s"%(self.auth[0], self.auth[1], ftpUrl))
req.timeout = self.timeout
ftpObj = ftped.ftp_open(req)
return ftpObj
def download_request(self,request,spider):
""":return a deferred for the HTTP download"""
hander = urllib2.FTPHandler()
req = urllib2.Request(url=request.url)
opener = urllib2.build_opener(hander)
f = opener.open(req)
b = f.read()
print len(b)
respcls = Response(url=request.url,body=b,request=request)
return respcls
def _auth_ftp(self):
'''handles ftp authentication'''
self.url.username = self.username
self.url.password = self.password
req = urllib2.Request(str(self.url))
req.timeout = self.timeout
ftped = urllib2.FTPHandler()
ftp_obj = ftped.ftp_open(req)
return ftp_obj
def __init__(self, *args, **kargs):
urllib2.OpenerDirector.__init__(self, *args, **kargs)
#agregando soporte basico
self.add_handler(urllib2.ProxyHandler())
self.add_handler(urllib2.UnknownHandler())
self.add_handler(urllib2.HTTPHandler())
self.add_handler(urllib2.HTTPDefaultErrorHandler())
self.add_handler(urllib2.HTTPRedirectHandler())
self.add_handler(urllib2.FTPHandler())
self.add_handler(urllib2.FileHandler())
self.add_handler(urllib2.HTTPErrorProcessor())
#Agregar soporte para cookies. (en este momento no es necesario,
#pero uno nunca sabe si se puede llegar a nececitar)
self.cj = cookielib.CookieJar()
self.add_handler(urllib2.HTTPCookieProcessor(self.cj))
#! -*- encoding:utf-8 -*- import urllib2 handler = urllib2.FTPHandler() request = urllib2.Request(url='ftp://用户名:密码@ftp地址/') opener = urllib2.build_opener(handler) f = opener.open(request) print f.read()
req.get_method = lambda: 'HEAD'
#Create an opener that does not support local file access
opener = urllib2.OpenerDirector()
#Don't follow redirects, but don't treat them as errors either
error_nop = lambda *args, **kwargs: True
http_error_processor = urllib2.HTTPErrorProcessor()
http_error_processor.http_error_301 = error_nop
http_error_processor.http_error_302 = error_nop
http_error_processor.http_error_307 = error_nop
handlers = [
urllib2.UnknownHandler(),
urllib2.HTTPHandler(),
urllib2.HTTPDefaultErrorHandler(),
urllib2.FTPHandler(), http_error_processor
]
try:
import ssl
handlers.append(urllib2.HTTPSHandler())
except:
#Python isn't compiled with SSL support
pass
map(opener.add_handler, handlers)
if platform.python_version_tuple() >= (2, 6):
opener.open(req, timeout=10)
else:
opener.open(req)
except ValueError:
raise ValidationError(_(u'Enter a valid URL.'), code='invalid')
except: # urllib2.URLError, httplib.InvalidURL, etc.
def test():
handler = urllib2.FTPHandler()
req = urllib2.Request(url, None, headers)
req.get_method = lambda: 'HEAD'
#Create an opener that does not support local file access
opener = urllib2.OpenerDirector()
#Don't follow redirects, but don't treat them as errors either
error_nop = lambda *args, **kwargs: True
http_error_processor = urllib2.HTTPErrorProcessor()
http_error_processor.http_error_301 = error_nop
http_error_processor.http_error_302 = error_nop
http_error_processor.http_error_307 = error_nop
handlers = [urllib2.UnknownHandler(),
urllib2.HTTPHandler(),
urllib2.HTTPDefaultErrorHandler(),
urllib2.FTPHandler(),
http_error_processor]
try:
import ssl
except ImportError:
# Python isn't compiled with SSL support
pass
else:
handlers.append(urllib2.HTTPSHandler())
map(opener.add_handler, handlers)
if platform.python_version_tuple() >= (2, 6):
opener.open(req, timeout=10)
else:
opener.open(req)
except ValueError:
raise ValidationError(_(u'Enter a valid URL.'), code='invalid')
def __call__(self, value):
try:
super(RelativeURLValidator, self).__call__(value)
except ValidationError as e:
# Trivial case failed. Try for possible IDN domain
if value:
value = smart_text(value)
scheme, netloc, path, query, fragment = urlparse.urlsplit(
value)
try:
netloc = netloc.encode('idna') # IDN -> ACE
except UnicodeError: # invalid domain part
raise e
url = urlparse.urlunsplit(
(scheme, netloc, path, query, fragment))
super(RelativeURLValidator, self).__call__(url)
else:
raise
else:
url = value
if self.verify_exists:
broken_error = ValidationError(
_(u'This URL appears to be a broken link.'),
code='invalid_link')
if url.startswith('http://') or url.startswith('ftp://'):
headers = {
"Accept":
"text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5",
"Accept-Language": "en-us,en;q=0.5",
"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7",
"Connection": "close",
"User-Agent": self.user_agent,
}
url = url.encode('utf-8')
try:
req = urllib2.Request(url, None, headers)
req.get_method = lambda: 'HEAD'
#Create an opener that does not support local file access
opener = urllib2.OpenerDirector()
#Don't follow redirects, but don't treat them as errors either
error_nop = lambda *args, **kwargs: True
http_error_processor = urllib2.HTTPErrorProcessor()
http_error_processor.http_error_301 = error_nop
http_error_processor.http_error_302 = error_nop
http_error_processor.http_error_307 = error_nop
handlers = [
urllib2.UnknownHandler(),
urllib2.HTTPHandler(),
urllib2.HTTPDefaultErrorHandler(),
urllib2.FTPHandler(), http_error_processor
]
try:
import ssl
handlers.append(urllib2.HTTPSHandler())
except:
#Python isn't compiled with SSL support
pass
map(opener.add_handler, handlers)
if platform.python_version_tuple() >= (2, 6):
opener.open(req, timeout=10)
else:
opener.open(req)
except ValueError:
raise ValidationError(_(u'Enter a valid URL.'),
code='invalid')
except: # urllib2.URLError, httplib.InvalidURL, etc.
raise broken_error
else:
# Resolve the relative URL
try:
resolve(url)
except Http404:
raise broken_error
almost trivial.
"""
def find_user_password(self, realm, authuri):
creds = codetricks.stealVar("_temp_credentials")
if creds is not None:
return creds
_restrictedURLOpener = urllib2.OpenerDirector()
_restrictedURLOpener.add_handler(urllib2.HTTPRedirectHandler())
_restrictedURLOpener.add_handler(urllib2.HTTPHandler())
_restrictedURLOpener.add_handler(urllib2.HTTPSHandler())
_restrictedURLOpener.add_handler(urllib2.HTTPErrorProcessor())
_restrictedURLOpener.add_handler(
urllib2.HTTPBasicAuthHandler(_UrlopenRemotePasswordMgr()))
_restrictedURLOpener.add_handler(urllib2.FTPHandler())
_restrictedURLOpener.add_handler(urllib2.UnknownHandler())
_restrictedURLOpener.addheaders = [("user-agent", "GAVO DaCHS HTTP client")]
def urlopenRemote(url, data=None, creds=(None, None)):
"""works like urllib2.urlopen, except only http, https, and ftp URLs
are handled.
The function also massages the error messages of urllib2 a bit. urllib2
errors always become IOErrors (which is more convenient within the DC).
creds may be a pair of username and password. Those credentials
will be presented in http basic authentication to any server
that cares to ask. For both reasons, don't use any valuable credentials
here.