def mangle_url(orig_url, url):
try:
endpoint_url = urlparse.urlparse(url)
except Exception as e:
utils.unknown("you must provide an endpoint_url in the form"
+ "<scheme>://<url>/ (%s)\n" % e)
scheme = endpoint_url.scheme
if scheme is None:
utils.unknown("you must provide an endpoint_url in the form"
+ "<scheme>://<url>/ (%s)\n" % e)
catalog_url = urlparse.urlparse(orig_url)
port = endpoint_url.port
if port is None:
if catalog_url.port is None:
port = DAEMON_DEFAULT_PORT
else:
port = catalog_url.port
netloc = "%s:%i" % (endpoint_url.hostname, port)
url = urlparse.urlunparse([scheme,
netloc,
catalog_url.path,
catalog_url.params,
catalog_url.query,
catalog_url.fragment])
return url
def auth_from_url(url):
auth = None
parsed_url = urlparse.urlparse(url).netloc
if '@' in parsed_url:
auth = parsed_url.split('@')[0].split(':')
auth = requests.auth.HTTPBasicAuth(auth[0], auth[1])
return auth
def startupagent(self, sender, **kwargs):
if not self.bind_web_address:
_log.info('Web server not started.')
return
import urlparse
parsed = urlparse.urlparse(self.bind_web_address)
hostname = parsed.hostname
port = parsed.port
_log.info('Starting web server binding to {}:{}.' \
.format(hostname, port))
self.registeredroutes.append(
(re.compile('^/discovery/$'), 'callable', self._get_discovery))
self.registeredroutes.append(
(re.compile('^/discovery/allow$'), 'callable', self._allow))
self.registeredroutes.append(
(re.compile('^/$'), 'callable', self._redirect_index))
port = int(port)
vhome = os.environ.get('VOLTTRON_HOME')
logdir = os.path.join(vhome, "log")
if not os.path.exists(logdir):
os.makedirs(logdir)
self.appContainer = WebApplicationWrapper(self, hostname, port)
svr = WSGIServer((hostname, port), self.appContainer)
self._server_greenlet = gevent.spawn(svr.serve_forever)
def check(when=time.time):
collection = connect_to_mongodb()
for data in collection.find():
ip = data.get('ip')
target_url = data.get('target_url')
ip_stamp = format_time_to_timestamp(data.get('insert_time'))
has_existed = int(when() - ip_stamp)
if has_existed > over_time:
diy_header['Host'] = urlparse.urlparse(target_url).netloc
_args = {
"url": target_url,
"diy_header": diy_header,
"time_out": 5,
"_ip": ip,
}
_id = ip + '_' + target_url
# 调用验证函数
result1, result2 = valid(_args, False)
if result1 is None:
msg = 'delete ip: [{ip}], target_url is [{target_url}]'.format(ip=ip, target_url=target_url)
logger.info(msg)
collection.delete_one({'_id': _id})
else:
msg = 'update ip: [{ip}], target_url is [{target_url}]'.format(ip=ip, target_url=target_url)
logger.info(msg)
collection.update({'_id': _id}, {'insert_time': when()})
def __init__(self,
string_proxy=None,
request_header=None,
timeout=90,
debuglevel=0,
**kwargs):
self.debuglevel = debuglevel
self.timeout = timeout
self.headers = generator_header()
# self.session = requesocks.session(headers=self.headers, timeout=timeout)
self.session = requests.session()
# self.session.headers = self.headers
if string_proxy:
socket.getaddrinfo = getaddrinfo
urlinfo = urlparse.urlparse(string_proxy)
if urlinfo.scheme == 'ssh':
self.bitvise = Bitvise(urlinfo.hostname,
urlinfo.port,
username=urlinfo.username,
password=urlinfo.password)
forwarding_ip, forwarding_port = self.bitvise.start()
string_proxy = 'socks5://%s:%s' % (forwarding_ip,
forwarding_port)
self.session.proxies = {
'http': string_proxy,
'https': string_proxy
}
self._body = None
def __init__(self, uri, consumer):
asyncore.dispatcher_with_send.__init__(self)
self.uri = uri
self.consumer = consumer
# turn the uri into a valid request
scheme, host, path, params, query, fragment = urlparse.urlparse(uri)
assert scheme == "http", "only supports HTTP requests"
try:
host, port = string.split(host, ":", 1)
port = int(port)
except (TypeError, ValueError):
port = 80 # default port
if not path:
path = "/"
if params:
path = path + ";" + params
if query:
path = path + "?" + query
self.request = "GET %s HTTP/1.0\r\nHost: %s\r\n\r\n" % (path, host)
self.host = host
self.port = port
self.status = None
self.header = None
self.data = ""
# get things going!
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect((host, port))
def step1_get_authorize_url(self, redirect_uri=None):
"""Returns a URI to redirect to the provider.
Args:
redirect_uri: string, Either the string 'urn:ietf:wg:oauth:2.0:oob' for
a non-web-based application, or a URI that handles the callback from
the authorization server. This parameter is deprecated, please move to
passing the redirect_uri in via the constructor.
Returns:
A URI as a string to redirect the user to begin the authorization flow.
"""
if redirect_uri is not None:
logger.warning((
'The redirect_uri parameter for'
'OAuth2WebServerFlow.step1_get_authorize_url is deprecated. Please'
'move to passing the redirect_uri in via the constructor.'))
self.redirect_uri = redirect_uri
if self.redirect_uri is None:
raise ValueError('The value of redirect_uri must not be None.')
query = {
'response_type': 'code',
'client_id': self.client_id,
'redirect_uri': self.redirect_uri,
'scope': self.scope,
}
query.update(self.params)
parts = list(urlparse.urlparse(self.auth_uri))
query.update(dict(parse_qsl(
parts[4]))) # 4 is the index of the query part
parts[4] = urllib.parse.urlencode(query)
return urlparse.urlunparse(parts)
def do_GET(self):
# get the query string variables
self.urlparser = urlparse.urlparse(self.path)
self.query_string = self.urlparser.query
self.query_dict = urlparse.parse_qs(self.query_string)
# get the url and output_file
self.url = self.query_dict.get('url', [
None,
])[0]
self.output_file = self.query_dict.get('output_file', [
None,
])[0]
# return error if url or output_file are missing
if not self.url or not self.output_file:
self.handle_404("url and output_file params are required")
return None
# convert all query objects from list to single items
options_dict = {}
for k, v in self.query_dict.items():
options_dict[k] = v[0]
wkhtp = WKHtmlToPdf(self.url, self.output_file, **options_dict)
output_file = wkhtp.render()
# send json response
if output_file[0]:
self.handle_200("The file has been saved", output_file[1])
else:
self.handle_500("%s - the file could not be created" %
output_file[1])
def getDetailsColumn(self, struct):
'''
This will return the contents of the details column parsed out
of the structure returned by a getAccountDetails() returned
structure.
:param struct: the data structure returned by the getAccountDetails()
method
:type struct: dictionary
:return: Returns the contents of the "description" column from the
struct that is sent
:rtype: str
'''
details = None
opkey = self.const.resourcekeys_operation
reskey = self.const.resourcekeys_Details
resdesc = self.const.resourceKeys_description
if ((self.const.resourcekeys_operation in struct) and
reskey in struct[opkey]) and \
resdesc in struct[opkey][reskey]:
# leaving this here in case the logic above was translated
# incorrectly
# if ((struct.has_key(self.const.resourcekeys_operation)) and \
# struct[opkey].has_key(reskey)) and \
# struct[opkey][reskey].has_key(resdesc):
urlFromDetails = struct[opkey][reskey][resdesc]
self.logger.debug("url details: %s", urlFromDetails)
parsed_uri = urlparse.urlparse(urlFromDetails)
details = parsed_uri.netloc
self.logger.debug("urlFromDetails: %s", details)
return details
def download(self, url, headers, proxy, num_retries, data=None):
print
'Downloading:', url
request = urllib.request.Request(url, data, headers or {})
opener = self.opener or urllib.request.build_opener()
if proxy:
proxy_params = {urlparse.urlparse(url).scheme: proxy}
opener.add_handler(urllib.ProxyHandler(proxy_params))
try:
response = opener.open(request)
html = response.read()
code = response.code
except Exception as e:
print
'Download error:', str(e)
html = ''
if hasattr(e, 'code'):
code = e.code
if num_retries > 0 and 500 <= code < 600:
# retry 5XX HTTP errors
return self._get(url, headers, proxy, num_retries - 1,
data)
else:
code = None
return {'html': html.decode(encoding="utf-8"), 'code': code}
def write(self, vals):
return_object = super(ExternalSaleOrder, self).write(vals)
# Fix
if 'shopify_landing_site' in vals:
if self.shopify_landing_site:
# get params
params = {}
parsed = urlparse.urlparse(self.shopify_landing_site)
params2 = parse_qs(parsed.query)
if len(params2) > 0:
for param2 in params2:
params[str(param2)] = str(params2[param2][0])
# landing_url
self.landing_url = parsed.path
# utm_campaign
if 'utm_campaign' in params:
self.landing_utm_campaign = params['utm_campaign']
# utm_medium
if 'utm_medium' in params:
self.landing_utm_medium = params['utm_medium']
# utm_source
if 'utm_source' in params:
self.landing_utm_source = params['utm_source']
# return
return return_object
def is_local_service(name):
"""
Determine if a service definition describes a service running on
the local node. This is true if the service URL is for localhost,
matches the machine's name, or ec2 public name
"""
if name is None:
return False
if "://" in name:
url = urlparse.urlparse(name)
if ":" in url.netloc:
name = url.netloc.split(":")[0]
else:
name = url.netloc
elif ":" in name:
name = name.split(":")[0]
if name == "localhost":
return True
if '.' in name:
name = name.split('.')[0]
node = socket.getfqdn()
if '.' in node:
node = node.split('.')[0]
if name == node:
return True
pn = public_name()
if pn is not None and pn.split(".")[0] == name:
return True
return False
def get_pg_connection() -> (Dict, Dict):
"""
Gets pg connection and cursor from postgres
Returns:
(Dict, Dict): the pg connection and cursor
"""
try:
postgres_url = os.getenv('POSTGRES_URL')
if(postgres_url):
result = urlparse.urlparse(postgres_url)
username = result.username
password = result.password
database = result.path[1:]
hostname = result.hostname
port = result.port
pg_conn = psycopg2.connect(
database = database,
user = username,
password = password,
host = hostname,
port = port
)
else:
pg_conn = connect(host='postgres-db', user='******', password='******', dbname='ai')
pg_cur = pg_conn.cursor(cursor_factory=DictCursor)
return pg_conn, pg_cur
except Exception as e:
traceback.print_exc()
logger_service.log_error('DB ERROR', traceback.format_exc())
raise e
def parse(self, response):
bs = BeautifulSoup(response.body)
category = urlparse.urlparse(response.url).path[1:-5]
for info in bs.findAll('div', {'class': 'product-item-info'}):
_id = info.find('a')['href'].split('/')[-1].replace('.html', '')
description = info.find('strong').text.strip().split(
'\n')[0].strip()
price = info.find('span', {
'data-price-type': 'finalPrice'
}).text.replace('$', '').strip()
if info.find('script'):
c = info.find('script').contents[0]
a, b = re.search(
'"priceByWeight":\{"amount":([^,]+), "weight": ([^}]+)',
c).groups()
p, w = float(a), float(b)
price = p / w
date = datetime.date.today().isoformat()
sale = len(
info.find('div', {
'class': 'product-grid-flags'
}).find_all()) == 1
# unidad = info.find('span', {'class': 'price-by'}).text.replace('$','').strip()
yield PrecioItem(price=price,
description=description,
_id=_id,
date=date,
sale_type=sale,
category=category)
def send_email_template(template, context, to, email_from=None, html=None, attachments=[]):
"""
Renders an email template with this format:
{% if subject %}Subject{% endif %}
{% if message %}Email body{% endif %}
context can be a dictionary or a template.Context instance
"""
if isinstance(context, dict):
context = Context(context)
if isinstance(to, str):
to = [to]
if not 'site' in context:
from orchestra import settings
url = urlparse.urlparse(settings.ORCHESTRA_SITE_URL)
context['site'] = {
'name': settings.ORCHESTRA_SITE_NAME,
'scheme': url.scheme,
'domain': url.netloc,
}
#subject cannot have new lines
subject = render_to_string(template, {'subject': True}, context).strip()
message = render_to_string(template, {'message': True}, context)
msg = EmailMultiAlternatives(subject, message, email_from, to, attachments=attachments)
if html:
html_message = render_to_string(html, {'message': True}, context)
msg.attach_alternative(html_message, "text/html")
msg.send()
def __init__(self, uri, consumer): asyncore.dispatcher_with_send.__init__(self) self.uri = uri self.consumer = consumer # turn the uri into a valid request scheme, host, path, params, query, fragment = urlparse.urlparse(uri) assert scheme == "http", "only supports HTTP requests" try: host, port = string.split(host, ":", 1) port = int(port) except (TypeError, ValueError): port = 80 # default port if not path: path = "/" if params: path = path + ";" + params if query: path = path + "?" + query self.request = "GET %s HTTP/1.0\r\nHost: %s\r\n\r\n" % (path, host) self.host = host self.port = port self.status = None self.header = None self.data = "" # get things going! self.create_socket(socket.AF_INET, socket.SOCK_STREAM) self.connect((host, port))
def index(request):
if request.method == 'PUT':
try:
data = (request.read().decode('utf-8'))
url = "http://192.168.0.184?"
url += data
parsed = urlparse.urlparse(url)
tmp = urlparse.parse_qs(parsed.query)['tmp'][0]
stat = "activated"
time = str(datetime.today().strftime("%Y/%m/%d %H:%M:%S"))
if int(tmp) > 28 and int(tmp) < 31:
stat = "activated"
strength = "(Weak wind mode)"
elif int(tmp) >= 31:
stat = "activated"
strength = "(Strong wind mode)"
else:
stat = "deactivated"
strength = ""
in_data = Mytable(tmp=tmp, stat=stat, strength=strength)
in_data.save()
except:
print("Put Error")
return HttpResponse("Complete")
if request.method == "GET":
try:
row = Mytable.objects.order_by('id').last()
context = {'row': row}
tmp = str(row.tmp)
strength = row.strength
stat = str(row.stat)
except:
return HttpResponse("Get Error")
return render(request, 'project/present.html', context)
def __init__(self,
username,
password,
url=URL_BASE,
validity_ms=120000,
auto_login=False,
proxies=None):
"""Log-in to the Luminoso API
username, password => (str) credentials to access the server
validity_ms => milliseconds of validity for signed messages
auto_login => remember the credentials and use them when the
connection times out (NOT IMPLEMENTED YET)
session => requests session to use for queries"""
# Store the login parameters
self._auto_login = auto_login
self.username = username if auto_login else None
self.password = password if auto_login else None
self.url = url.rstrip('/')
parsed = urlparse.urlparse(self.url)
self._host = parsed.netloc
# Store the validity parameter
self._validity_ms = validity_ms
# Initialize the requests session
self._session = requests.session()
if proxies is not None:
self._session.proxies = proxies
# Fetch session credentials
self.login(username, password)
def data_from_type(name, contenttype, content):
if contenttype == 'raw':
out = content
elif contenttype == 'base64':
if len(content) == 0:
out = ""
else:
# FIXME: this is pretty bad; if the content is long, then
# we'll be storing it all in memory
out = base64.b64decode(content)
elif contenttype == 'url':
# FIXME: this is really bad; if the file content is long, then we'll be
# storing it all in memory. We should probably download it to a
# temporary file somewhere and reference that later on.
url = urlparse.urlparse(content)
if url.scheme == "file":
with open(url.netloc + url.path) as f:
out = "".join(f.readlines())
else:
# fetch url, and put it in self.commands[name]
fp = tempfile.TemporaryFile()
oz.ozutil.http_download_file(content, fp.fileno(), False, None)
fp.seek(0)
out = "".join(fp.readlines())
fp.close()
else:
raise oz.OzException.OzException("Type for %s must be 'raw', 'url' or 'base64'" % (name))
return out
def checkValidDomain(endpoints):
validDomains = []
invalidDomains = []
validEndpoints = []
for endpoint in endpoints:
endpoint = endpoint.strip().strip("\r").strip('"').strip("'")
try:
parsedUrl = urlparse.urlparse(endpoint)
domain = parsedUrl.netloc.split(
":"
)[0] # split is to remove hosts of the following form: example.com:80
if domain in validDomains:
validEndpoints.append(endpoint)
continue
elif domain in invalidDomains:
continue
try:
socket.gethostbyname(
domain) # Will throw error if name doesn't resolve
validDomains.append(domain)
validEndpoints.append(endpoint)
except:
invalidDomains.append(domain)
except: # URL parsing error or resolving error
continue
return validEndpoints
def _get_expires_from_url(url):
if not url:
return 0
qs = urlparse.urlparse(url)
if 'expire' not in qs or not qs['expire']:
return 0
return int(qs['expire'][0])
def _login_authorize_get_token(self):
required_props = ['access_token', 'token_type']
self.login()
self._login_and_authorize()
response = self.client.get(self.redirect_url())
query = QueryDict(urlparse.urlparse(response['Location']).query)
code = query['code']
response = self.client.post(self.access_token_url(), {
'grant_type': 'authorization_code',
'client_id': self.get_client().client_id,
'client_secret': self.get_client().client_secret,
'code': code})
self.assertEqual(200, response.status_code, response.content)
token = json.loads(response.content)
for prop in required_props:
self.assertIn(prop, token, "Access token response missing "
"required property: %s" % prop)
return token
def connect(self, mongo_uri):
db_name, options = None, {}
u = urlparse.urlparse(mongo_uri)
if u.scheme == 'file':
path = u.path
self._options = urlparse.parse_qs(u.query) if u.query else {}
path = u.netloc + path
self.db = _MongoImportFileSet(path)
if 'uniq' in self._options and 'md5' in self._options['uniq']:
self._quick_uniq = _IdHashPairs(path)
elif u.scheme == 'mongodb':
if '?' in u.path and u.query == '':
# url didn't parse it properly u.path is '/dbname?options
db_name, self._options = u.path.split('?', 1)
self._options = urlparse.parse_qs(
self._options) if self._options else {}
else:
db_name = u.path
if db_name.startswith('/'):
db_name = db_name[1:]
# print('Connecting to db %s on %s with options.' % (db_name, mongo_uri, options))
try:
mongo = pymongo.Connection(mongo_uri)
except:
mongo = pymongo.MongoClient(mongo_uri)
self.db = mongo[db_name]
if 'uniq' in self._options and 'md5' in self._options['uniq']:
self._quick_uniq = False
else:
raise ValueError(
"Invalid URI scheme: '%s'. Can only accept 'file' or 'mongodb'"
% u.scheme)
def download(url, headers, proxy, num_retries, data=None):
print('Downloading:%s'%url)
request = request.Request(url, data, headers)
opener = request.build_opener()
if proxy:
proxy_params = {urlparse.urlparse(url).scheme: proxy}
opener.add_handler(request.ProxyHandler(proxy_params))
try:
response = opener.open(request)
html = response.read()
code = response.code
except request.URLError as e:
print('Download error:%s'%e.reason)
html = ''
if hasattr(e, 'code'):
code = e.code
if num_retries > 0 and 500 <= code < 600:
# retry 5XX HTTP errors
return download(url, headers, proxy, num_retries-1, data)
else:
code =None
return html
def __init__(self, username, password, url=URL_BASE,
validity_ms=120000, auto_login=False, proxies=None):
"""Log-in to the Luminoso API
username, password => (str) credentials to access the server
validity_ms => milliseconds of validity for signed messages
auto_login => remember the credentials and use them when the
connection times out (NOT IMPLEMENTED YET)
session => requests session to use for queries"""
# Store the login parameters
self._auto_login = auto_login
self.username = username if auto_login else None
self.password = password if auto_login else None
self.url = url.rstrip('/')
parsed = urlparse.urlparse(self.url)
self._host = parsed.netloc
# Store the validity parameter
self._validity_ms = validity_ms
# Initialize the requests session
self._session = requests.session()
if proxies is not None:
self._session.proxies = proxies
# Fetch session credentials
self.login(username, password)
def do_GET(self):
scheme,netloc,path,params,query,fragment = urlparse.urlparse(self.path)
upd, displ = 10, '24h'
query = parse_qs(query)
if 'update' in query: upd = int(query['update'][0])
if 'display' in query: displ = query['display'][0]
if path.endswith('.html') or path.endswith('htm'):
self.send_webpage(upd,displ)
elif path.endswith('.dat'):
f=open(datafile,'r')
self.reply('text/plain', f.read() )
f.close()
elif path.endswith('.png'):
f=open(plotfile,'rb')
self.reply('image/png', f.read() )
f.close()
elif path.startswith('/temp'):
self.reply('text/plain', '%.3f'%temp )
elif path.startswith('/max'):
self.reply('text/plain', '%.3f'%max )
elif path.startswith('/min'):
self.reply('text/plain', '%.3f'%min )
elif path.startswith('/serial'):
self.reply('text/plain', serial )
else: self.send_webpage(upd,displ)
return
def render(self, request):
"""
Render a request by forwarding it to the proxied server.
Args:
request (Request): Incoming request.
Returns:
not_done (char): Indicator to note request not yet finished.
"""
# RFC 2616 tells us that we can omit the port if it's the default port,
# but we have to provide it otherwise
request.content.seek(0, 0)
qs = urlparse.urlparse(request.uri)[4]
if qs:
rest = self.path + '?' + qs
else:
rest = self.path
clientFactory = self.proxyClientFactoryClass(
request.method, rest, request.clientproto,
request.getAllHeaders(), request.content.read(), request)
clientFactory.noisy = False
self.reactor.connectTCP(self.host, self.port, clientFactory)
# don't trigger traceback if connection is lost before request finish.
request.notifyFinish().addErrback(
lambda f: logger.log_trace("%s\nCaught errback in webserver.py:75." % f))
return NOT_DONE_YET
def fetchArgs():
if argc == 3:
seedList = sys.argv[1].strip('[]"\'').split(',')
termList = sys.argv[2].strip('[]"\'').split(',')
for i in range(len(seedList)):
seedList[i] = seedList[i].strip(' ')
if not bool(urlparse.urlparse(seedList[i]).scheme):
usage("URL list argument contains an invalid URL.")
return None, None
for i in range(len(termList)):
termList[i] = termList[i].strip(' ')
return seedList, termList
elif argc == 13: # In the case people enter 12 strings instead of 2 lists
seedList = []
termList = []
for i in range(1, 13):
arg = sys.argv[i].strip('[]"\', ')
if i in (1, 2):
seedList.append(arg)
else:
termList.append(arg)
return seedList, termList
else:
usage(
"Please enter two arguments: [list of seed URLS] | [list of ten related terms]"
)
def is_local_service(name):
"""
Determine if a service definition describes a service running on
the local node. This is true if the service URL is for localhost,
matches the machine's name, or ec2 public name
"""
if name is None:
return False
if "://" in name:
url = urlparse.urlparse(name)
if ":" in url.netloc:
name = url.netloc.split(":")[0]
else:
name = url.netloc
elif ":" in name:
name = name.split(":")[0]
if name == "localhost":
return True
if '.' in name:
name = name.split('.')[0]
node = socket.getfqdn()
if '.' in node:
node = node.split('.')[0]
if name == node:
return True
pn = public_name()
if pn is not None and pn.split(".")[0] == name:
return True
return False
def get_kabupaten(self):
for prov in self.provinsi:
raw_kab = simple_get(prov['url'])
html = bs(raw_kab, 'html.parser')
div = html.find('div', {'id': 'TabPaneCuaca1'})
table = div.find('table', {'class': 'table-prakicu-provinsi'})
for a in table.findAll('a'):
text = a.text
link = a['href']
query = urlparse.urlparse(link).query
url = os.path.join(os.path.split(CUACA_URL)[0], link)
self.kabupaten.append({
"prov":
prov['code'],
"name":
text,
"link":
link,
"url":
url,
"code":
urlparse.parse_qs(query)['AreaID'][0]
})
return self.kabupaten
def do_METHOD_Direct(self):
scheme, netloc, path, params, query, fragment = urlparse.urlparse(self.path, 'http')
try:
host, _, port = netloc.rpartition(':')
port = int(port)
except ValueError:
host = netloc
port = 80
try:
self.log_request(200)
idlecall = None
sock = self.socket_create_connection((host, port))
self.headers['Connection'] = 'close'
data = '%s %s %s\r\n' % (self.command, urlparse.urlunparse(('', '', path, params, query, '')), self.request_version)
data += ''.join('%s: %s\r\n' % (k, self.headers[k]) for k in self.headers if not k.startswith('proxy-'))
data += '\r\n'
content_length = int(self.headers.get('content-length', 0))
if content_length > 0:
data += self.rfile.read(content_length)
sock.sendall(data)
self.socket_forward(self.connection, sock, idlecall=idlecall)
except Exception as ex:
logging.exception('LocalProxyHandler.do_GET Error, %s', ex)
finally:
try:
sock.close()
del sock
except:
pass
def mirror_entity_image(self, tweet, entity_index, url):
response = requests.get(url, allow_redirects=True, timeout=15)
if response.status_code != http.client.OK:
log.warn("Failed to download image {0}", url)
return
content_type = response.headers.get('content-type')
parsed_url = urlparse.urlparse(url)
(_base, extension) = os.path.splitext(parsed_url.path)
extension = None
if not extension:
extensions = [
ext for ext in mimetypes.guess_all_extensions(content_type)
if ext != '.jpe'
]
extension = extensions[0] if extensions else ''
log.debug("Possible mime types: {0}, chose {1}", extensions,
extension)
filename = "{tweet}-{index}{extension}".format(tweet=tweet.get('id'),
index=entity_index,
extension=extension)
with NamedTemporaryFile(mode='wb', prefix='twoops',
delete=True) as fil:
fil.write(response.content)
fil.flush()
new_url = self.upload_image(fil.name, filename, content_type)
if new_url:
self.record_tweet_image(tweet, new_url)
def sanitize_redirect(host, redirect_to):
"""
Given the hostname and an untrusted URL to redirect to,
this method tests it to make sure it isn't garbage/harmful
and returns it, else returns None, similar as how's it done
on django.contrib.auth.views.
>>> print sanitize_redirect('myapp.com', None)
None
>>> print sanitize_redirect('myapp.com', '')
None
>>> print sanitize_redirect('myapp.com', {})
None
>>> print sanitize_redirect('myapp.com', 'http://notmyapp.com/path/')
None
>>> print sanitize_redirect('myapp.com', 'http://myapp.com/path/')
http://myapp.com/path/
>>> print sanitize_redirect('myapp.com', '/path/')
/path/
"""
# Quick sanity check.
if not redirect_to:
return None
# Heavier security check, don't allow redirection to a different host.
try:
netloc = urlparse.urlparse(redirect_to)[1]
except TypeError: # not valid redirect_to value
return None
if netloc and netloc != host:
return None
return redirect_to
def download(self):
url = self.imageinfo['url']
if not url.startswith('http://'):
url = 'http://' + self.site.host + url
url = urlparse.urlparse(url)
# TODO: query string
return self.site.connection.get(url[1], url[2])
def from_request(http_method, http_url, headers=None, parameters=None,
query_string=None):
"""Combines multiple parameter sources."""
if parameters is None:
parameters = {}
# Headers
if headers and 'Authorization' in headers:
auth_header = headers['Authorization']
# Check that the authorization header is OAuth.
if auth_header[:6] == 'OAuth ':
auth_header = auth_header[6:]
try:
# Get the parameters from the header.
header_params = OAuthRequest._split_header(auth_header)
parameters.update(header_params)
except Exception:
raise OAuthError('Unable to parse OAuth parameters from '
'Authorization header.')
# GET or POST query string.
if query_string:
query_params = OAuthRequest._split_url_string(query_string)
parameters.update(query_params)
# URL parameters.
param_str = urlparse.urlparse(http_url)[4] # query
url_params = OAuthRequest._split_url_string(param_str)
parameters.update(url_params)
if parameters:
return OAuthRequest(http_method, http_url, parameters)
return None
def mirror_entity_image(self, tweet, entity_index, url):
response = requests.get(url, allow_redirects=True, timeout=15)
if response.status_code != http.client.OK:
log.warn("Failed to download image {0}", url)
return
content_type = response.headers.get('content-type')
parsed_url = urlparse.urlparse(url)
(_base, extension) = os.path.splitext(parsed_url.path)
extension = None
if not extension:
extensions = [ext for ext in mimetypes.guess_all_extensions(content_type)
if ext != '.jpe']
extension = extensions[0] if extensions else ''
log.debug("Possible mime types: {0}, chose {1}", extensions, extension)
filename = "{tweet}-{index}{extension}".format(tweet=tweet.get('id'),
index=entity_index,
extension=extension)
with NamedTemporaryFile(mode='wb', prefix='twoops', delete=True) as fil:
fil.write(response.content)
fil.flush()
new_url = self.upload_image(fil.name, filename, content_type)
if new_url:
self.record_tweet_image(tweet, new_url)
def download(self):
url = self.imageinfo['url']
if not url.startswith('http://'):
url = 'http://' + self.site.host + url
url = urlparse.urlparse(url)
# TODO: query string
return self.site.connection.get(url[1], url[2])
def _get_store(self, uri):
if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir
scheme = 'file'
else:
scheme = urlparse.urlparse(uri).scheme
store_cls = self.STORE_SCHEMES[scheme]
return store_cls(uri)
def url_add_parameters(url, params):
"""Adds parameters to URL, parameter will be repeated if already present"""
if params:
fragments = list(urlparse.urlparse(url))
fragments[4] = urllib.urlencode(parse_qsl(fragments[4]) +
params.items())
url = urlparse.urlunparse(fragments)
return url
def resolveComponents(url):
parsed = urlparse.urlparse(url)
new_path = posixpath.normpath(parsed.path)
if parsed.path.endswith('/'):
# Compensate for issue1707768
new_path += '/'
cleaned = parsed._replace(path=new_path)
return cleaned.geturl()
def make_job_desc_url(self, job_post_id):
url_parts = list(urlparse.urlparse(self.url))
query = dict(urlparse.parse_qsl(url_parts[4]))
query.update({'postid': job_post_id})
url_parts[4] = urlencode(query)
return urlparse.urlunparse(url_parts)
def ZK(hosts, path):
if path.startswith("zk://"):
path = urlparse.urlparse(path)
hosts = path.netloc
path = path.path
zk = KazooClient(hosts=hosts)
zk.start()
return zk, path
def getVideoID(url):
url_data = urlparse.urlparse(url)
query = urlparse.parse_qs(url_data.query)
if not "v" in query:
print("Video Url is not valid!")
sys.exit(1)
video = query["v"][0]
return video
except ProductAlert.DoesNotExist:
messages.error(self.request, _("No alert found"))
else:
alert.cancel()
messages.success(self.request, _("Alert cancelled"))
return HttpResponseRedirect(
reverse('customer:summary')+'?tab=alerts'
)
def get_emails(self, user):
def get_normalized_http_url(self):
"""Parses the URL and rebuilds it to be scheme://host/path."""
parts = urlparse.urlparse(self.http_url)
scheme, netloc, path = parts[:3]
# Exclude default port numbers.
if scheme == 'http' and netloc[-3:] == ':80':
netloc = netloc[:-3]
elif scheme == 'https' and netloc[-4:] == ':443':
netloc = netloc[:-4]
return '%s://%s%s' % (scheme, netloc, path)
def get_callback_url(self):
if self.callback and self.verifier:
# Append the oauth_verifier.
parts = urlparse.urlparse(self.callback)
scheme, netloc, path, params, query, fragment = parts[:6]
if query:
query = '%s&oauth_verifier=%s' % (query, self.verifier)
else:
query = 'oauth_verifier=%s' % self.verifier
return urlparse.urlunparse((scheme, netloc, path, params,
query, fragment))
return self.callback
def check_url_scheme(url):
"""
Input:
url string:
a url string
Output:
True if the url contains scheme
False otherwise
"""
parsed = url_p.urlparse(url)
if not parsed.scheme:
return False
return True
def parse_link_rel(url, fn):
"""
Read through html file ``fn`` downloaded from ``url``, looking for a
link tag of the form:
<link rel="alternate"
type="application/sage"
title="currently ignored"
href=".../example.sws" />
This function reads ``fn`` looking for such tags and returns a list
of dictionaries of the form
{'title': from title field in link, 'url': absolute URL to .sws file}
for the corresponding ``.sws`` files. Naturally if there are no
appropriate link tags found, the returned list is empty. If the HTML
parser raises an HTMLParseError, we simply return an empty list.
"""
from HTMLParser import HTMLParser
class GetLinkRelWorksheets(HTMLParser):
def __init__(self):
HTMLParser.__init__(self)
self.worksheets = []
def handle_starttag(self, tag, attrs):
if (tag == 'link' and
('rel', 'alternate') in attrs and
('type', 'application/sage') in attrs):
self.worksheets.append({'title': [_ for _ in attrs if _[0] == 'title'][0][1],
'url': [_ for _ in attrs if _[0] == 'href'][0][1]})
parser = GetLinkRelWorksheets()
with open(fn) as f:
try:
parser.feed(f.read())
except HTMLParseError:
return []
ret = []
for d in parser.worksheets:
sws = d['url']
# is that link a relative URL?
if not urlparse.urlparse(sws).netloc:
# unquote-then-quote to avoid turning %20 into %2520, etc
ret.append({'url': urlparse.urljoin(url, urllib.quote(urllib.unquote(sws))),
'title': d['title']})
else:
ret.append({'url': sws, 'title': d['title']})
return ret
def add_url_params(url, new_params, concat=True, unique=True):
if isinstance(new_params, dict):
new_params = [(k, v) for k, v in new_params.iteritems()]
url_parts = list(urlparse.urlparse(url))
params = urlparse.parse_qsl(url_parts[4])
params = new_params if not concat else params + new_params
if unique:
params = dict(params)
url_parts[4] = urllib.urlencode(params)
return urlparse.urlunparse(url_parts)
def mangle_url(self, url):
# This first connection populate the structure we need inside
# the object. This does not cost anything if a connection has
# already been made.
self.check_connection()
try:
endpoint_url = urlparse.urlparse(url)
except Exception as e:
utils.unknown("you must provide an endpoint_url in the form"
+ "<scheme>://<url>/ (%s)" % e)
scheme = endpoint_url.scheme
if scheme is None:
utils.unknown("you must provide an endpoint_url in the form"
+ "<scheme>://<url>/ (%s)" % e)
catalog_url = None
try:
catalog_url = urlparse.urlparse(
self.nova_client.client.management_url)
except Exception as e:
utils.unknown("unknown error parsing the catalog url : %s" % e)
port = endpoint_url.port
if port is None:
if catalog_url.port is None:
port = self.DAEMON_DEFAULT_PORT
else:
port = catalog_url.port
netloc = "%s:%i" % (endpoint_url.hostname, port)
url = urlparse.urlunparse([scheme,
netloc,
catalog_url.path,
catalog_url.params,
catalog_url.query,
catalog_url.fragment])
self.nova_client.client.management_url = url
def clean_url(self):
# Check if we already have a valid embed url
url = self.cleaned_data['url']
if url.find('http://www.youtube.com/v/') == 0:
return url
# Parse querystring to find video ID
parsed = urlparse.urlparse(url)
qs = urlparse.parse_qs(parsed.query)
# Check if the video id exists in query string
if 'v' not in qs:
raise ValidationError('Osoitteesta ei löytynyt videotunnusta.')
# All done. Return valid url
return 'http://www.youtube.com/v/'+qs['v'][0]+'/'
def cleanup_url(self, value_url, source_url, mark):
"""
Transform relative URLs into absolute URLs if possible.
If the value_url is already absolute, or we don't know the
source_url, then return the existing value. If the value_url is
relative, and we know the source_url, then try to rewrite it.
"""
value = urlparse.urlparse(value_url)
if value.netloc or not source_url:
url = value_url
else:
url = urlparse.urljoin(source_url, value_url)
if url.startswith('//'):
url = 'http:' + url # MissingSchema fix
if mark:
url = url + mark
return url
def get_oauth_url(self):
""" Returns the URL with OAuth params """
params = {}
if "?" in self.url:
url = self.url[: self.url.find("?")]
for key, value in urlparse.parse_qsl(urlparse.urlparse(self.url).query):
params[key] = value
else:
url = self.url
params["oauth_consumer_key"] = self.consumer_key
params["oauth_timestamp"] = int(time())
params["oauth_nonce"] = HMAC(str(time() + randint(0, 99999)).encode(), "secret".encode(), sha1).hexdigest()
params["oauth_signature_method"] = "HMAC-SHA256"
params["oauth_signature"] = self.generate_oauth_signature(params, url)
query_string = urlencode(params)
return "%s?%s" % (url, query_string)
def get_back_button(context):
"""
Show back button, custom title available for different urls, for
example 'Back to search results', no back button if user came from other
site
"""
request = context.get('request', None)
if not request:
raise Exception('Cannot get request from context')
referrer = request.META.get('HTTP_REFERER', None)
if not referrer:
return None
try:
url = urlparse.urlparse(referrer)
except:
return None
if request.get_host() != url.netloc:
try:
Site.objects.get(domain=url.netloc)
except Site.DoesNotExist:
# Came from somewhere else, don't show back button:
return None
try:
match = resolve(url.path)
except Resolver404:
return None
# This dict can be extended to link back to other browsing pages
titles = {
'search:search': _('Back to search results'),
}
title = titles.get(match.view_name, None)
if title is None:
return None
return {'url': referrer, 'title': unicode(title), 'match': match}
def read_url_post(url):
'''Transform a JSON contained in a file into an equivalent
nested python dict.
Parameters
----------
url : string
where to get the json.
Returns
-------
dict
Python version of the input
Note: if the input is a bare array or literal, for example,
the output will be of the corresponding type.
'''
urlp = urlparse.urlparse(url)
main_url = urlparse.urlunsplit(
(urlp.scheme, urlp.netloc, urlp.path, '', ''))
data = json.dumps(dict(urlparse.parse_qsl(urlp.query)))
handler = urllib_request.HTTPHandler()
opener = urllib_request.build_opener(handler)
request = urllib_request.Request(main_url, data)
request.add_header("Content-Type", 'application/json')
request.get_method = lambda: 'POST'
try:
response = opener.open(request)
except Exception as e:
response = e
if response.code == 200:
json_string = response.read()
else:
json_string = response.read()
return json.loads(json_string)
def _retry_using_form_auth(self, response, args):
adapter = requests.adapters.HTTPAdapter()
request = _copy_request(response.request)
u = urlparse.urlparse(response.url)
url = urlparse.urlunparse([u.scheme, u.netloc, '/login',
None, None, None])
auth = {'username': self.username,
'password': self.password}
request2 = requests.Request('POST', url, data=auth).prepare()
response2 = adapter.send(request2, **args)
if response2.status_code == 401:
self.log.error('Login failed: Invalid username or password?')
return response
cookie = response2.headers.get('set-cookie')
if cookie is not None:
request.headers['Cookie'] = cookie
response3 = adapter.send(request, **args)
return response3
def do_GET(self):
url = urlparse.urlparse(self.path)
path = url.path
if path.endswith("/"): path += "index.html"
localpath = basedir + path
if path == "/detect":
params = urlparse.parse_qs(url.query)
text = chr(params['text'][0], 'utf-8')
json.dump(detector.detect(text), self.wfile)
elif os.path.exists(localpath):
self.send_response(200)
if path.endswith(".html"):
self.send_header("Content-Type", "text/html; charset=utf-8")
elif path.endswith(".js"):
self.send_header("Content-Type", "text/javascript; charset=utf-8")
self.end_headers()
with open(localpath, "rb") as f:
self.wfile.write(f.read())
else:
self.send_response(404, "Not Found : " + url.path)
self.send_header("Expires", "Fri, 31 Dec 2100 00:00:00 GMT")
self.end_headers()
def feed():
Meta = FrontEndMeta(Site)
Posts = sortPosts(Pages)[:10]
# Generate a unique ID for the site.
DomainName = URLParse.urlparse(Site.URIPrefix).hostname
if hasattr(Config, "SiteID"):
Meta.Site.FeedID = SiteID
else:
Meta.Site.FeedID = Site.URIPrefix + Config.SiteRoot
def urlForWithDomain(endpoint, **values):
return Site.URIPrefix + flask.url_for(endpoint, **values)
for Post in Posts:
# Generate unique IDs for each post, and generate the
# creation/update time in iso format.
CreationTime = Post["created"]
CreationTimeISO = dateTime2ISO8601Format(CreationTime)
Post.FeedID = "tag:{},{}:{}".format(DomainName, CreationTime.year, CreationTimeISO)
Post.CreatedISO = CreationTimeISO
if "updated" in Post.meta:
UpdateTime = Post["updated"]
Post.UpdatedISO = dateTime2ISO8601Format(UpdateTime)
else:
Post.UpdatedISO = ""
# Derefernece the variables in articles. We cannot use
# flask.url_for, because we need absolute URLs.
Post.body = varReplace(Post.body, {"app_root": App.config["APPLICATION_ROOT"],
"url_for": urlForWithDomain})
# Acquire the update time for the site.
Meta.Site.Updated = max([p.UpdatedISO for p in Posts] \
+ [p.CreatedISO for p in Posts])
Response = flask.make_response(flask.render_template("feed.xml", pages=Posts, meta=Meta))
Response.mimetype = "application/atom+xml"
return Response
def connect_endpoint(self, url):
"""Connect endpoint from URL.
`url`
socket.io endpoint URL.
"""
urldata = urlparse.urlparse(url)
endpoint = urldata.path
conn = self.endpoints.get(endpoint, None)
if conn is None:
conn_class = self.conn.get_endpoint(endpoint)
if conn_class is None:
logger.error('There is no handler for endpoint %s' % endpoint)
return
conn = conn_class(self, endpoint)
self.endpoints[endpoint] = conn
self.send_message(proto.connect(endpoint))
if conn.on_open(self.info) == False:
self.disconnect_endpoint(endpoint)
def do_POST(self):
"""
what to do is case of POST request
"""
parsed_path = urlparse.urlparse(self.path)
self.serve_content(parsed_path)
