def add_url_param(url, **params): n=3; parts = list(urlparse.urlsplit(url)); d = dict(cgi.parse_qsl(parts[n])); # use cgi.parse_qs for list values d.update(params); parts[n]=urlencode(d); return urlparse.urlunsplit(parts);
def thumb(self, image, **kwargs):
url = image.url
if not url:
return None
if settings.THUMBOR_BASE_URL:
# If THUMBOR_BASE_URL is explicity set, use that
base = settings.THUMBOR_BASE_URL
else:
# otherwise assume that thumbor is setup behind the same
# CDN behind the `thumbor` namespace.
scheme, netloc = urlparse.urlsplit(url)[:2]
base = '{}://{}/thumbor'.format(scheme, netloc)
crypto = CryptoURL(key=settings.THUMBOR_KEY)
# just for code clarity
thumbor_kwargs = kwargs
if not 'fit_in' in thumbor_kwargs:
thumbor_kwargs['fit_in'] = False
thumbor_kwargs['image_url'] = url
path = crypto.generate(**thumbor_kwargs)
return u'{}{}'.format(base, path)
def thumb(self, image, **kwargs):
url = image.url
if not url:
return None
if settings.THUMBOR_BASE_URL:
# If THUMBOR_BASE_URL is explicity set, use that
base = settings.THUMBOR_BASE_URL
else:
# otherwise assume that thumbor is setup behind the same
# CDN behind the `thumbor` namespace.
scheme, netloc = urlparse.urlsplit(url)[:2]
base = '{}://{}/thumbor'.format(scheme, netloc)
crypto = CryptoURL(key=settings.THUMBOR_KEY)
# just for code clarity
thumbor_kwargs = kwargs
if not 'fit_in' in thumbor_kwargs:
thumbor_kwargs['fit_in'] = False
thumbor_kwargs['image_url'] = url
path = crypto.generate(**thumbor_kwargs)
return u'{}{}'.format(base, path)
def url_fix(s, charset='utf-8'):
if isinstance(s, unicode):
s = s.encode(charset, 'ignore')
scheme, netloc, path, qs, anchor = urlparse.urlsplit(s)
path = urllib.quote(path, '/%')
qs = urllib.quote_plus(qs, ':&=')
return urlparse.urlunsplit((scheme, netloc, path, qs, anchor))
def _get_encoded_url(self):
"""Convert any UTF-8 char in :obj:`File.file_path` into a url encoded ASCII string."""
sres = urllib_parse.urlsplit(self.file_path)
return urllib_parse.urlunsplit(
urllib_parse.SplitResult(sres.scheme, sres.netloc,
urllib_parse.quote(sres.path), sres.query,
sres.fragment))
def httpExists(url):
import httplib, urlparse
host, path = urlparse.urlsplit(url)[1:3]
if ":" in host:
host, port = host.split(":", 1)
try:
port = int(port)
except ValueError:
return False, url
else:
port = None
try:
connection = httplib.HTTPConnection(host, port=port)
connection.request("HEAD", path)
resp = connection.getresponse()
if resp.status == 200:
return True, url
elif resp.status == 302:
new_url = urlparse.urljoin(url, resp.getheader("location", ""))
return httpExists(new_url)
else:
return False, url
except:
return False, url
def url_fix(s, charset='utf-8'):
if isinstance(s, unicode):
s = s.encode(charset, 'ignore')
scheme, netloc, path, qs, anchor = urlparse.urlsplit(s)
path = urllib.quote(path, '/%')
qs = urllib.quote_plus(qs, ':&=')
return urlparse.urlunsplit((scheme, netloc, path, qs, anchor))
def add_url_param(url, **params):
n = 3
parts = list(urlparse.urlsplit(url))
d = dict(cgi.parse_qsl(parts[n]))
# use cgi.parse_qs for list values
d.update(params)
parts[n] = urlencode(d)
return urlparse.urlunsplit(parts)
def logout_shim(request, **kwargs):
""" log user out and redirect them to next (without domain check)
or to referring domain's root, or to own root """
auth_logout(request)
redirect_to = request.REQUEST.get('next', '')
if not redirect_to:
if 'HTTP_REFERER' in request.META:
pieces = urlparse.urlsplit(request.META['HTTP_REFERER'])
redirect_to = '%s://%s' % (pieces.scheme, pieces.netloc)
else:
redirect_to = '/'
return redirect(redirect_to)
def logout_shim(request, **kwargs):
""" log user out and redirect them to next (without domain check)
or to referring domain's root, or to own root """
auth_logout(request)
redirect_to = request.REQUEST.get('next', '')
if not redirect_to:
if 'HTTP_REFERER' in request.META:
pieces = urlparse.urlsplit(request.META['HTTP_REFERER'])
redirect_to = '%s://%s' % (pieces.scheme, pieces.netloc)
else:
redirect_to = '/'
return redirect(redirect_to)
def wrapper(self, *args, **kwargs):
# if not self.get_current_user:
if not self.session.get('loginid'):
if self.request.method in ("GET", "HEAD"):
url = self.get_login_url()
if "?" not in url:
if urlparse.urlsplit(url).scheme:
next_url = self.request.full_url()
else:
next_url = self.request.uri
url += "?" + urlencode(dict(next=next_url))
self.redirect(url)
return
raise HTTPError(403)
return method(self, *args, **kwargs)
def _login(self):
login_url = urljoin(self.base_url, '/manage_main')
# would be the propper way to login, but is not supported by geckodriver/ chromedriver yet
# self.driver.switch_to.alert.authenticate(self.login, self.password)
# Disabled because it works only in firefox
# if self.driver == 'Firefox':
# self.driver.get(login_url)
# self.driver.switch_to.alert.send_keys(self.login + Keys.TAB + self.password)
# self.driver.switch_to.alert.accept()
# else:
import urlparse
components = urlparse.urlsplit(login_url)
credentials = '%s:%s@' % (self.login, self.password)
components_with_auth = urlparse.SplitResult(
components[0], credentials + components[1], *components[2:])
self.driver.get(urlparse.urlunsplit(components_with_auth))
def download(self, custom_path=None, out=None, timeout=None):
"""
Download this file. By default, the file is saved in the current working directory with its
original filename as reported by Telegram. If a :attr:`custom_path` is supplied, it will be
saved to that path instead. If :attr:`out` is defined, the file contents will be saved to
that object using the ``out.write`` method.
Note:
`custom_path` and `out` are mutually exclusive.
Args:
custom_path (:obj:`str`, optional): Custom path.
out (:obj:`object`, optional): A file-like object. Must be opened in binary mode, if
applicable.
timeout (:obj:`int` | :obj:`float`, optional): If this value is specified, use it as
the read timeout from the server (instead of the one specified during creation of
the connection pool).
Raises:
ValueError: If both ``custom_path`` and ``out`` are passed.
"""
if custom_path is not None and out is not None:
raise ValueError('custom_path and out are mutually exclusive')
# Convert any UTF-8 char into a url encoded ASCII string.
sres = urllib_parse.urlsplit(self.file_path)
url = urllib_parse.urlunsplit(
urllib_parse.SplitResult(sres.scheme, sres.netloc,
urllib_parse.quote(sres.path), sres.query,
sres.fragment))
if out:
buf = self.bot.request.retrieve(url)
out.write(buf)
else:
if custom_path:
filename = custom_path
else:
filename = basename(self.file_path)
self.bot.request.download(url, filename, timeout=timeout)
def wrapper(self, *args, **kwargs):
skey = self.request.headers.get('skey', None)
if skey:
tbl_user = mongodb.getDB().get_collection("tb_user_profile")
tbl_user.find_one({"skey": skey})
if not self.session.get('skey'):
if self.request.method in ("GET", "HEAD"):
url = self.get_login_url()
if "?" not in url:
if urlparse.urlsplit(url).scheme:
next_url = self.request.full_url()
else:
next_url = self.request.uri
url += "?" + urlencode(dict(next=next_url))
self.redirect(url)
return
raise HTTPError(403)
return method(self, *args, **kwargs)
def wrapper(self, *args, **kwargs):
if not self.session.get('sysid'):
if self.request.method in ("GET", "HEAD"):
url = self.get_admin_login_url()
if "?" not in url:
if urlparse.urlsplit(url).scheme:
# if login url is absolute, make next absolute too
next_url = self.request.full_url()
else:
next_url = self.request.uri
url += "?" + urlencode(dict(next=next_url))
self.redirect(url)
return
raise HTTPError(403)
# else:
# if self.session['data'].get("role", "") != "superadmin":
# if self.session['data'].get("role", "admin") != "superadmin":
# raise HTTPError(403)
# self.redirect('/admin/login')
# return
return method(self, *args, **kwargs)
def parselocation(self, locstring):
'''
Returns a copyobject
c.scheme [file|http|https]
c.host <host>|None
c.port <port>|None
c.username
c.protocol
'''
self.log.debug("Parsing location %s" % locstring)
if "@" in locstring:
(username, locstring) = locstring.split('@')
sr = urlparse.urlsplit(locstring)
# scheme netloc path
scheme = sr.scheme
if scheme == '':
scheme = 'file'
netloc = sr.netloc
pathfile = sr.path
host = None
port = None
if ':' in netloc:
(host, port) = netloc.split(':')
else:
host = netloc
if scheme == 'https':
port = 443
elif scheme == 'http':
port = 80
if scheme != 'file':
password = getpass.getpass(prompt='Password for %s:%s' %
(username, host))
ce = CopyEndpoint(scheme, host, username, port, password, pathfile)
return ce
def thumb(url, **kwargs):
"""
Inspired by:
http://tech.yipit.com/2013/01/03/how-yipit-scales-thumbnailing-with-thumbor-and-cloudfront/
returns a thumbor url for 'url' with **kwargs as thumbor options.
Positional arguments:
url -- the location of the original image
Keyword arguments:
For the complete list of thumbor options
https://github.com/globocom/thumbor/wiki/Usage
and the actual implementation for the url generation
https://github.com/heynemann/libthumbor/blob/master/libthumbor/url.py
"""
THUMBOR_BASE_URL = getattr(settings, 'THUMBOR_BASE_URL', None)
THUMBOR_KEY = getattr(settings, 'THUMBOR_KEY', 'MY_SECURE_KEY')
if THUMBOR_BASE_URL:
base = THUMBOR_BASE_URL
else:
# otherwise assume that thumbor is setup behind the same
# CDN behind the `thumbor` namespace.
scheme, netloc = urlparse.urlsplit(url)[:2]
base = '{}://{}/thumbor'.format(scheme, netloc)
crypto = CryptoURL(key=THUMBOR_KEY)
# just for code clarity
thumbor_kwargs = kwargs
if not 'fit_in' in thumbor_kwargs:
thumbor_kwargs['fit_in'] = True
thumbor_kwargs['image_url'] = url
path = crypto.generate(**thumbor_kwargs)
return u'{}{}'.format(base, path)
def do_GET(self):
if self.path == 'http://weevely/':
self.send_cacert()
return
req = self
content_length = int(req.headers.get('Content-Length', 0))
req_body = self.rfile.read(content_length) if content_length else ''
if req.path[0] == '/':
if isinstance(self.connection, ssl.SSLSocket):
req.path = "https://%s%s" % (req.headers['Host'], req.path)
else:
req.path = "http://%s%s" % (req.headers['Host'], req.path)
req.headers['Content-length'] = str(len(req_body))
u = urlparse.urlsplit(req.path)
scheme, netloc, path = u.scheme, u.netloc, (u.path + '?' + u.query if u.query else u.path)
assert scheme in ('http', 'https')
if netloc:
req.headers['Host'] = netloc
setattr(req, 'headers', self.filter_headers(req.headers))
net_curl_args = [
'-X',
self.command,
'-i'
]
net_curl_args.append(self.path)
for h in req.headers:
if h.title().lower() == 'host':
host = self.headers[h]
net_curl_args += [ '-H', '%s: %s' % ( h.title(), self.headers[h] ) ]
if self.command == 'POST':
content_len = int(self.headers.getheader('content-length', 0))
net_curl_args += [ '-d', req_body ]
lock.acquire()
try:
result, headers, saved = ModuleExec(
'net_curl',
net_curl_args
).run()
finally:
lock.release()
if not headers:
log.debug('Error no headers')
self.send_error(502)
return
log.debug('> ' + '\r\n> '.join([ '%s: %s' % (h.title(), self.headers[h]) for h in self.headers ]))
log.debug('< ' + '\r\n< '.join(headers))
http_response_str = '\r\n'.join(headers) + '\r\n\r\n' + result
source = FakeSocket(http_response_str)
res = HTTPResponse(source)
res.begin()
version_table = {10: 'HTTP/1.0', 11: 'HTTP/1.1'}
setattr(res, 'headers', res.msg)
setattr(res, 'response_version', version_table[res.version])
# support streaming
if not 'Content-Length' in res.headers and 'no-store' in res.headers.get('Cache-Control', ''):
setattr(res, 'headers', self.filter_headers(res.headers))
self.relay_streaming(res)
return
try:
res_body = res.read()
except Exception as e:
log.debug(e)
self.send_error(500)
return
setattr(res, 'headers', self.filter_headers(res.headers))
self.wfile.write("%s %d %s\r\n" % (self.protocol_version, res.status, res.reason))
for line in res.headers.headers:
self.wfile.write(line)
self.end_headers()
self.wfile.write(res_body)
self.wfile.flush()
def argument_length(url):
return len(urlparse.urlsplit(url).query)
def key_value_pairs(url):
return dict(urlparse.parse_qs(urlparse.urlsplit(url).query))
def do_GET(self):
if self.path == 'http://weevely/':
self.send_cacert()
return
req = self
content_length = int(req.headers.get('Content-Length', 0))
req_body = self.rfile.read(content_length) if content_length else ''
if req.path[0] == '/':
if isinstance(self.connection, ssl.SSLSocket):
req.path = "https://%s%s" % (req.headers['Host'], req.path)
else:
req.path = "http://%s%s" % (req.headers['Host'], req.path)
req.headers['Content-length'] = str(len(req_body))
u = urlparse.urlsplit(req.path)
scheme, netloc, path = u.scheme, u.netloc, (u.path + '?' + u.query
if u.query else u.path)
assert scheme in ('http', 'https')
if netloc:
req.headers['Host'] = netloc
setattr(req, 'headers', self.filter_headers(req.headers))
net_curl_args = ['-X', self.command, '-i']
net_curl_args.append(self.path)
for h in req.headers:
if h.title().lower() == 'host':
host = self.headers[h]
net_curl_args += ['-H', '%s: %s' % (h.title(), self.headers[h])]
if self.command == 'POST':
content_len = int(self.headers.getheader('content-length', 0))
net_curl_args += ['-d', req_body]
lock.acquire()
try:
result, headers, saved = ModuleExec('net_curl',
net_curl_args).run()
finally:
lock.release()
if not headers:
log.debug('Error no headers')
self.send_error(502)
return
log.debug('> ' + '\r\n> '.join(
['%s: %s' % (h.title(), self.headers[h]) for h in self.headers]))
log.debug('< ' + '\r\n< '.join(headers))
http_response_str = '\r\n'.join(headers) + '\r\n\r\n' + result
source = FakeSocket(http_response_str)
res = HTTPResponse(source)
res.begin()
version_table = {10: 'HTTP/1.0', 11: 'HTTP/1.1'}
setattr(res, 'headers', res.msg)
setattr(res, 'response_version', version_table[res.version])
# support streaming
if not 'Content-Length' in res.headers and 'no-store' in res.headers.get(
'Cache-Control', ''):
setattr(res, 'headers', self.filter_headers(res.headers))
self.relay_streaming(res)
return
try:
res_body = res.read()
except Exception as e:
log.debug(e)
self.send_error(500)
return
setattr(res, 'headers', self.filter_headers(res.headers))
self.wfile.write("%s %d %s\r\n" %
(self.protocol_version, res.status, res.reason))
for line in res.headers.headers:
self.wfile.write(line)
self.end_headers()
self.wfile.write(res_body)
self.wfile.flush()
