def db_url_parse(url, engine=None, conn_max_age=0):
"""
Parses a database URL.
"""
if url == "sqlite://:memory:":
# urlparse will choke on :memory:
return {
"ENGINE": DATABASE_ENGINE_SCHEMES["sqlite"],
"NAME": ":memory:",
}
config = {}
url = urlparse.urlparse(url)
# split query strings from path
path = url.path[1:]
if "?" in path and not url.query:
path, query = path.split("?", 2)
else:
path, query = path, url.query
query = urlparse.parse_qs(query)
# sqlite with no path should assume :memory: (sqlalchemy behavior)
if url.scheme == "sqlite" and path == "":
path = ":memory:"
# handle postgresql percent-encoded paths
hostname = url.hostname or ""
if "%2f" in hostname.lower():
hostname = hostname.replace("%2f", "/").replace("%2F", "/")
config.update({
"NAME": urlparse.unquote(path or ""),
"USER": urlparse.unquote(url.username or ""),
"PASSWORD": urlparse.unquote(url.password or ""),
"HOST": hostname,
"PORT": url.port or "",
"CONN_MAX_AGE": conn_max_age,
})
engine = DATABASE_ENGINE_SCHEMES[url.scheme] if engine is None else engine
# pass the query string into OPTIONS
options = {}
for key, values in query.items():
if url.scheme == "mysql" and key == "ssl-ca":
options["ssl"] = {"ca": values[-1]}
continue
options[key] = values[-1]
# postgresql schema URLs
if "currentSchema" in options and engine == "django.db.backends.postgresql_psycopg2":
options["options"] = "-c search_path={0}".format(options["currentSchema"])
if options:
config["OPTIONS"] = options
if engine:
config["ENGINE"] = engine
return config
def _parse_qs(qs):
pairs = (s2 for s1 in qs.split('&') for s2 in s1.split(';'))
retval = odict()
for name_value in pairs:
if name_value is None or len(name_value) == 0:
continue
nv = name_value.split('=', 1)
if len(nv) != 2:
# Handle case of a control-name with no equal sign
nv.append(None)
name = unquote(nv[0].replace('+', ' '))
value = None
if nv[1] is not None:
value = unquote(nv[1].replace('+', ' '))
l = retval.get(name, None)
if l is None:
l = retval[name] = []
l.append(value)
return retval
def parse_url(url):
port = path = auth = userid = password = None
# parse with HTTP URL semantics
scheme = urlparse(url).scheme
parts = urlparse(url.replace("%s://" % (scheme, ), "http://"))
# The first pymongo.Connection() argument (host) can be
# a mongodb connection URI. If this is the case, don't
# use port but let pymongo get the port(s) from the URI instead.
# This enables the use of replica sets and sharding.
# See pymongo.Connection() for more info.
if scheme == 'mongodb':
# strip the scheme since it is appended automatically.
hostname = url[len('mongodb://'):]
else:
netloc = parts.netloc
if '@' in netloc:
auth, _, netloc = parts.netloc.partition('@')
userid, _, password = auth.partition(':')
hostname, _, port = netloc.partition(':')
path = parts.path or ""
if path and path[0] == '/':
path = path[1:]
port = port and int(port) or port
return dict({"hostname": unquote(hostname or "") or None,
"port": port or None,
"userid": unquote(userid or "") or None,
"password": unquote(password or "") or None,
"transport": scheme,
"virtual_host": unquote(path or "") or None},
**kwdict(dict(parse_qsl(parts.query))))
def get(self, moduleID, title, institution, teachDate, instructors, description):
account = get_account()
if account:
newCourse = dict()
newCourse["CourseURL"] = urlparse.unquote(
self.request.get('courseURL'))
newCourse["Title"] = title
newCourse["Institution"] = institution
newCourse["TeachingDate"] = teachDate
newCourse["Instructors"] = instructors
newCourse["Description"] = description
newCourse["DownloadPageLink"] = urlparse.unquote(
self.request.get('materials'))
newCourse["scoreRanking"] = 1
moduleID = int(moduleID)
match = Module.query(Module.category == moduleID).fetch()
match = match[0]
moduleCourses = match.courses
newCourse['ID'] = len(moduleCourses)
moduleCourses.append(newCourse)
match.courses = moduleCourses
match.courses = sorted(
match.courses, key=lambda k: k['scoreRanking'], reverse=True)
match.put()
response = {'success': 'Course submitted successfully.'}
else:
response = {'error': 'You are not logged in. '}
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps(response))
def __init__(self, parsed_url):
super(S3Source, self).__init__(parsed_url)
self.acccess_key_id = urlparse.unquote(parsed_url.username)
self.secret_access_key = urlparse.unquote(parsed_url.password)
self.conn = boto.connect_s3(self.acccess_key_id, self.secret_access_key)
self.bucket_name = parsed_url.hostname
self.bucket = self.conn.get_bucket(self.bucket_name, validate=False)
def getSongs(self):
format = "%Y-%m-%d %H:%M:%S"
for trackid,attributes in self.il['Tracks'].iteritems():
s = Song()
s.name = attributes.get('Name')
s.artist = attributes.get('Artist')
s.album_artist = attributes.get('Album Artist')
s.composer = attributes.get('Composer')
s.album = attributes.get('Album')
s.genre = attributes.get('Genre')
s.kind = attributes.get('Kind')
if attributes.get('Size'):
s.size = int(attributes.get('Size'))
s.total_time = attributes.get('Total Time')
s.track_number = attributes.get('Track Number')
if attributes.get('Track Count'):
s.track_count = int(attributes.get('Track Count'))
if attributes.get('Disc Number'):
s.disc_number = int(attributes.get('Disc Number'))
if attributes.get('Disc Count'):
s.disc_count = int(attributes.get('Disc Count'))
if attributes.get('Year'):
s.year = int(attributes.get('Year'))
if attributes.get('Date Modified'):
s.date_modified = time.strptime(str(attributes.get('Date Modified')),format)
if attributes.get('Date Added'):
s.date_added = time.strptime(str(attributes.get('Date Added')),format)
if attributes.get('Bit Rate'):
s.bit_rate = int(attributes.get('Bit Rate'))
if attributes.get('Sample Rate'):
s.sample_rate = int(attributes.get('Sample Rate'))
s.comments = attributes.get("Comments ")
if attributes.get('Rating'):
s.rating = int(attributes.get('Rating'))
if attributes.get('Play Count'):
s.play_count = int(attributes.get('Play Count'))
if attributes.get('Location'):
if ( self.musicPathXML is None or self.musicPathSystem is None ):
s.location = unicode(urlparse.unquote(urlparse.urlparse(attributes.get('Location')).path[1:]),"utf8")
else:
s.location = unicode(urlparse.unquote(urlparse.urlparse(attributes.get('Location')).path[1:]).replace(self.musicPathXML,self.musicPathSystem),"utf8")
s.compilation = 'Compilation' in attributes
if attributes.get('Play Date UTC'):
s.lastplayed = time.strptime(str(attributes.get('Play Date UTC')),format)
if attributes.get('Total Time'):
s.length = int(attributes.get('Total Time'))
if attributes.get('Grouping'):
s.grouping = attributes.get('Grouping')
if self.filesOnly==True and attributes.get('Track Type') == 'File':
if self.legacymode:
self.songs.append(s)
else:
self.songs[int(trackid)] = s
elif self.filesOnly==False:
if self.legacymode:
self.songs.append(s)
else:
self.songs[int(trackid)] = s
def get_form(self, form_class):
kwargs = self.get_form_kwargs()
kwargs.update({
'request': self.request
,'initial': { 'q': urlparse.unquote(self.request.GET.get('q')) if self.request.GET.get('q') else None,
'location': urlparse.unquote(self.request.GET.get('location')) if 'location' in self.request.GET or self.request.GET.get('location') else None}
}) # add the request to the form
return form_class(**kwargs)
def internalapi_testcaps(args):
indexer = urlparse.unquote(args["indexer"])
apikey = args["apikey"]
host = urlparse.unquote(args["host"])
logger.debug("Check caps for %s" % indexer)
try:
result = check_caps(host, apikey)
return jsonify({"success": True, "result": result})
except IndexerResultParsingException as e:
return jsonify({"success": False, "message": e.message})
def parse_qsl(qs, keep_blank_values=0, strict_parsing=0):
"""This was a slightly modified version of the function with the same name
that is defined in urlparse.py . I modified it, and then reverted the patch
to have different handling of '+':
- name = unquote(nv[0].replace('+', ' '))
- value = unquote(nv[1].replace('+', ' '))
+ name = unquote(nv[0])
+ value = unquote(nv[1])
Due to this [0] bug: "Proxy (and maybe others) affected by querystring +
not being decoded by URL class #9139", I reverted my changes to the function
but kept it here for better docs.
[0] https://github.com/andresriancho/w3af/issues/9139
Arguments:
qs: percent-encoded query string to be parsed
keep_blank_values: flag indicating whether blank values in
percent-encoded queries should be treated as blank strings. A
true value indicates that blanks should be retained as blank
strings. The default false value indicates that blank values
are to be ignored and treated as if they were not included.
strict_parsing: flag indicating what to do with parsing errors. If
false (the default), errors are silently ignored. If true,
errors raise a ValueError exception.
Returns a list, as G-d intended.
"""
pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
r = []
for name_value in pairs:
if not name_value and not strict_parsing:
continue
nv = name_value.split('=', 1)
if len(nv) != 2:
if strict_parsing:
raise ValueError("bad query field: %r" % name_value)
# Handle case of a control-name with no equal sign
if keep_blank_values:
nv.append('')
else:
continue
if len(nv[1]) or keep_blank_values:
name = urlparse.unquote(nv[0].replace('+', ' '))
value = urlparse.unquote(nv[1].replace('+', ' '))
r.append((name, value))
return r
def compare_urls(url_a, url_b):
if url_a.netloc != url_b.netloc or url_a.query != url_b.query:
return False
# remove / from the end of the url if required
path_a = url_a.path[:-1]\
if url_a.path.endswith('/')\
else url_a.path
path_b = url_b.path[:-1]\
if url_b.path.endswith('/')\
else url_b.path
return unquote(path_a) == unquote(path_b)
def internalapi_testcaps(args):
indexer = urlparse.unquote(args["indexer"])
apikey = args["apikey"]
host = urlparse.unquote(args["host"])
logger.debug("Check caps for %s" % indexer)
try:
ids, types = check_caps(host, apikey)
ids = sorted(list(ids))
types = sorted(list(types))
return jsonify({"success": True, "ids": ids, "types": types})
except IndexerResultParsingException as e:
return jsonify({"success": False, "message": e.message})
def handle(self, *args, **options):
for templatename in args:
# if templatename[0:1] == 't':
print templatename
f = open(templatename, "r")
lines = f.readlines()
f.close()
# print lines
attribs = []
for line in lines:
currentattrib = line.split('=',1)
attribs.append(currentattrib)
print currentattrib[0]
print urlparse.unquote(currentattrib[1]).strip()
def parse_qsl(qs, keep_blank_values=0, strict_parsing=0):
"""This is a slightly modified version of the function with the same name
that is defined in urlparse.py . I had to modify it in order to have
'+' handled in the way w3af needed it. Note that the only change is:
- name = unquote(nv[0].replace('+', ' '))
- value = unquote(nv[1].replace('+', ' '))
+ name = unquote(nv[0])
+ value = unquote(nv[1])
In other words, keep those + !
Parse a query given as a string argument.
Arguments:
qs: percent-encoded query string to be parsed
keep_blank_values: flag indicating whether blank values in
percent-encoded queries should be treated as blank strings. A
true value indicates that blanks should be retained as blank
strings. The default false value indicates that blank values
are to be ignored and treated as if they were not included.
strict_parsing: flag indicating what to do with parsing errors. If
false (the default), errors are silently ignored. If true,
errors raise a ValueError exception.
Returns a list, as G-d intended.
"""
pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
r = []
for name_value in pairs:
if not name_value and not strict_parsing:
continue
nv = name_value.split('=', 1)
if len(nv) != 2:
if strict_parsing:
raise ValueError, "bad query field: %r" % (name_value,)
# Handle case of a control-name with no equal sign
if keep_blank_values:
nv.append('')
else:
continue
if len(nv[1]) or keep_blank_values:
name = urlparse.unquote(nv[0])
value = urlparse.unquote(nv[1])
r.append((name, value))
return r
def do_GET(self):
"""Respond to GET requests"""
from urlparse import unquote
path, headers, data = self._ParseBaseRequest('GET')
if None is path: return
if ('mpd' == path[0]) and (2 == len(path)):
self._AlterMPD(unquote(path[1]), headers, data)
elif ('subtitles' == path[0]) and (3 == len(path)):
self._TranscodeSubtitle(unquote(path[1]), headers, data, path[2])
else:
Log('[PS] Invalid request received', Log.DEBUG)
self.send_error(501, 'Invalid request')
def download_request(self, request, spider):
p = urlparse_cached(request)
scheme = 'https' if request.meta.get('is_secure') else 'http'
bucket = p.hostname
path = p.path + '?' + p.query if p.query else p.path
url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path)
signed_headers = self.conn.make_request(
method=request.method,
bucket=bucket,
key=unquote(p.path),
query_args=unquote(p.query),
headers=request.headers,
data=request.body)
httpreq = request.replace(url=url, headers=signed_headers)
return self._download_http(httpreq, spider)
def score_results(results):
flat_res = filter(
None, chain.from_iterable(izip_longest(*results.values())))
flat_len = len(flat_res)
engines_len = len(results)
results = []
# deduplication + scoring
for i, res in enumerate(flat_res):
res['parsed_url'] = urlparse(res['url'])
res['host'] = res['parsed_url'].netloc
if res['host'].startswith('www.'):
res['host'] = res['host'].replace('www.', '', 1)
res['engines'] = [res['engine']]
weight = 1.0
if hasattr(engines[res['engine']], 'weight'):
weight = float(engines[res['engine']].weight)
score = int((flat_len - i) / engines_len) * weight + 1
duplicated = False
for new_res in results:
p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa
p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path # noqa
if res['host'] == new_res['host'] and\
unquote(p1) == unquote(p2) and\
res['parsed_url'].query == new_res['parsed_url'].query and\
res.get('template') == new_res.get('template'):
duplicated = new_res
break
if duplicated:
if res.get('content') > duplicated.get('content'):
duplicated['content'] = res['content']
duplicated['score'] += score
duplicated['engines'].append(res['engine'])
if duplicated['parsed_url'].scheme == 'https':
continue
elif res['parsed_url'].scheme == 'https':
duplicated['url'] = res['parsed_url'].geturl()
duplicated['parsed_url'] = res['parsed_url']
else:
res['score'] = score
results.append(res)
return sorted(results, key=itemgetter('score'), reverse=True)
def parse(self, response):
# extract item
for sel in response.css('table.DataGrid tr')[1:]:
# skip ad row
if sel.css('ins'):
continue
# ok ,extract
i = iter(sel.css('td'))
g = Proxy()
g['ip'] = re.search("[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}", urlparse.unquote(i.next().css('script::text').extract_first())).group()
g['port'] = i.next().xpath('text()').extract_first()
g['typ'] = i.next().xpath('text()').extract_first()
g['anonymity'] = i.next().xpath('text()').extract_first()
g['country'] = i.next().xpath('text()').extract_first()
g['region'] = i.next().xpath('text()').extract_first()
g['city'] = i.next().xpath('text()').extract_first()
g['uptime'] = float(i.next().xpath('text()').extract_first().rstrip('%'))
g['response'] = int(re.search("width:(.*)%", i.next().css('span::attr("style")').extract_first()).group(1))
g['transfer'] = int(re.search("width:(.*)%", i.next().css('span::attr("style")').extract_first()).group(1))
#key
g['key'] = g['ip']
yield g
#next page
if self.go('next', 'plist', True):
url = response.css('.page').xpath('a[contains(text(),"Next")]/@href').extract_first()
yield scrapy.Request(response.urljoin(url), callback=self.parse)
def clean_page_name(url):
# select part after '/' as article and unquote it (replace stuff like %20) and decode to unicode
page = bot.to_unicode(urlparse.unquote(url.split('/')[-1]))
if page.startswith('index.php') and 'title' in page:
page = page.split('?title=')[1]
print(page)
return page
def _parse_stream_map(self, data):
"""
Python's `parse_qs` can't properly decode the stream map
containing video data so we use this instead.
Keyword arguments:
data -- The parsed response from YouTube.
"""
videoinfo = {
"itag": [],
"url": [],
"quality": [],
"fallback_host": [],
"sig": [],
"type": []
}
text = data["url_encoded_fmt_stream_map"][0]
# Split individual videos
videos = text.split(",")
# Unquote the characters and split to parameters
videos = [video.split("&") for video in videos]
for video in videos:
for kv in video:
key, value = kv.split("=")
videoinfo.get(key, []).append(unquote(value))
return videoinfo
def _parse_stream_map(self, blob):
"""A modified version of ``urlparse.parse_qs`` that's able to decode
YouTube's stream map.
:param str blob:
An encoded blob of text containing the stream map data.
"""
dct = {
"itag": [],
"url": [],
"quality": [],
"fallback_host": [],
"s": [],
"type": []
}
# Split the comma separated videos.
videos = blob.split(",")
# Unquote the characters and split to parameters.
videos = [video.split("&") for video in videos]
# Split at the equals sign so we can break this key value pairs and
# toss it into a dictionary.
for video in videos:
for kv in video:
key, value = kv.split("=")
dct.get(key, []).append(unquote(value))
log.debug('decoded stream map: %s', dct)
return dct
def bbv1_compat_mode(self,options,content,query):
execute_ext=('.sh','.sh.html','.sh.htm')
execute_background_ext=('.run',)
content_ext=('.htm','.html')
content_plain_ext=('.txt',)
relative_content = content[1:]
if os.path.isfile(relative_content):
if content.startswith('.'):
content = relative_content
else:
content = './%s' %relative_content
if content.endswith(content_plain_ext):
web.header('Content-Type', 'text/plain')
return content_handler().called(options,content,query)
web.header('Content-Type', 'text/html')
execute_content=" ".join((content,unquote(self.original_qs)))
if content.endswith(execute_ext):
return execute_handler().called(options,execute_content,query)
if content.endswith(execute_background_ext):
options.append('background')
return execute_handler().called(options,execute_content,query)
if content.endswith(content_ext):
return content_handler().called(options,content,query)
#Default option
return content_handler().called(options,content,query)
def connect_uri(self, uri):
uri = urlparse(uri)
database = uri.path[1:] # ignore the leading fwd slash
arguments = [database, uri.hostname, uri.username,
uri.port, uri.password]
# unquote each URI part (they may be percent encoded)
self.connect(*list(map(lambda p: unquote(p) if p else p, arguments)))
def get_sector_id(self, redirect_uri, client_info):
"""
Pick the sector id given a number of factors
:param redirect_uri: The redirect_uri used
:param client_info: Information provided by the client in the
client registration
:return: A sector_id or None
"""
_redirect_uri = urlparse.unquote(redirect_uri)
part = urlparse.urlparse(_redirect_uri)
if part.fragment:
raise ValueError
(_base, _query) = urllib.splitquery(_redirect_uri)
sid = ""
try:
if _base in client_info["si_redirects"]:
sid = client_info["sector_id"]
except KeyError:
try:
uit = client_info["subject_type"]
if uit == "pairwise":
sid = _base
except KeyError:
pass
return sid
def unescape_ajax_request(url):
"""
?_escaped_fragment_=key1=value1%26key2=value2
^
|
V
#!key1=value1&key2=value2
Remove from the URL all tokens beginning with _escaped_fragment_= (Note especially that the = must be removed as well).
Remove from the URL the trailing ? or & (depending on whether the URL had query parameters other than _escaped_fragment_).
Add to the URL the tokens #!.
Add to the URL all tokens after _escaped_fragment_= after unescaping them.
"""
parsed = urlparse(url)
query = []
for token in parsed.query.split('&'):
if token.startswith('_escaped_fragment_='):
_dump_, fragment_params = token.split('_escaped_fragment_=')
fragment = '!' + unquote(fragment_params)
query = '&'.join(query)
return urlunparse(ParseResult(scheme=parsed.scheme,
netloc=parsed.netloc, path=parsed.path, params=parsed.params, query=query,
fragment=fragment))
def post(self):
# from urllib.parse import unquote # py3
from urlparse import unquote
input = unquote(self.request.body.decode("utf-8"))
output = self.shell.push(input)
if output is not None:
self.write("\n" + output)
def recs_debug(request):
return http.HttpResponse("disabled :(")
if request.method == "POST":
url = request.POST.get("url")
if url:
fragment = urlparse.urlparse(url).fragment
guids = json.loads(urlparse.unquote(fragment)).keys()
qs = ",".join(map(str, get_addon_ids(guids)))
return redirect(reverse("discovery.recs.debug") + "?ids=" + qs)
ctx = {"ids": request.GET.get("ids")}
if "ids" in request.GET:
ids = map(int, request.GET["ids"].split(","))
ctx["addons"] = Addon.objects.filter(id__in=ids)
synced = get_synced_collection(ids, None)
recs = synced.get_recommendations()
ctx["recommended"] = recs.addons.order_by("collectionaddon__ordering")
recs = AddonRecommendation.objects.filter(addon__in=ids)
recs_transform(recs)
ctx["recs"] = dict((k, list(v)) for k, v in itertools.groupby(recs, key=lambda x: x.addon_id))
all_recs = collections.defaultdict(int)
for rec in recs:
all_recs[rec.other_addon] += rec.score
ctx["all_recs"] = sorted([(v, k) for k, v in all_recs.items()], reverse=True)
fragment = dict((a.guid, {"type": "extension"}) for a in ctx["addons"] if a.type == amo.ADDON_EXTENSION)
ctx["fragment"] = json.dumps(fragment, separators=(",", ":"))
return jingo.render(request, "discovery/recs-debug.html", ctx)
def __getitem__(self, name):
d = self._dict()
if name not in d:
name = urlparse.unquote(name)
if name not in d and isinstance(name, str):
name = name.decode('utf-8')
return d[name]
def recs_debug(request):
return http.HttpResponse('disabled :(')
if request.method == 'POST':
url = request.POST.get('url')
if url:
fragment = urlparse.urlparse(url).fragment
guids = json.loads(urlparse.unquote(fragment)).keys()
qs = ','.join(map(str, get_addon_ids(guids)))
return redirect(reverse('discovery.recs.debug') + '?ids=' + qs)
ctx = {'ids': request.GET.get('ids')}
if 'ids' in request.GET:
ids = map(int, request.GET['ids'].split(','))
ctx['addons'] = Addon.objects.filter(id__in=ids)
synced = get_synced_collection(ids, None)
recs = synced.get_recommendations()
ctx['recommended'] = recs.addons.order_by('collectionaddon__ordering')
recs = AddonRecommendation.objects.filter(addon__in=ids)
recs_transform(recs)
ctx['recs'] = dict((k, list(v)) for k, v in
itertools.groupby(recs, key=lambda x: x.addon_id))
all_recs = collections.defaultdict(int)
for rec in recs:
all_recs[rec.other_addon] += rec.score
ctx['all_recs'] = sorted([(v, k) for k, v in all_recs.items()],
reverse=True)
fragment = dict((a.guid, {'type': 'extension'}) for a in ctx['addons']
if a.type == amo.ADDON_EXTENSION)
ctx['fragment'] = json.dumps(fragment, separators=(',', ':'))
return jingo.render(request, 'discovery/recs-debug.html', ctx)
def __init__(self, uri):
"""Create a new Connection instance using an AMQP Uri string.
e.g.
amqp://guest:guest@localhost:5672/%2F
amqps://guest:guest@localhost:5671/%2F
:param str uri: AMQP Connection string
"""
parsed = urlparse.urlparse(uri)
use_ssl = parsed.scheme == 'amqps'
hostname = parsed.hostname or 'localhost'
port = parsed.port or 5672
username = parsed.username or 'guest'
password = parsed.password or 'guest'
virtual_host = urlparse.unquote(parsed.path[1:]) or '/'
kwargs = urlparse.parse_qs(parsed.query)
heartbeat = kwargs.get('heartbeat', [60])
timeout = kwargs.get('timeout', [0])
super(UriConnection, self).__init__(hostname, username,
password, port,
virtual_host=virtual_host,
heartbeat=int(heartbeat[0]),
timeout=int(timeout[0]),
ssl=use_ssl)
def mw_info(bot, trigger, found_match=None):
"""
Retrives a snippet of the specified length from the given page on the given
server.
"""
match = found_match or trigger
say_snippet(bot, match.group(1), unquote(match.group(2)), show_url=False)
def clean_url(self, url):
# stolen from urlparse.urlsplit(), which doesn't handle
# splitting frags correctly
netloc = query = fragment = ''
i = url.find(':')
scheme = url[:i].lower()
url = url[i + 1:]
if url[:2] == '//':
delim = len(url)
for c in '/?#':
wdelim = url.find(c, 2)
if wdelim >= 0:
delim = min(delim, wdelim)
netloc, url = url[2:delim], url[delim:]
if '#' in url:
try:
url, fragment = self.get_frag_re.search(url).groups()
except:
pass
if '?' in url:
url, query = url.split('?', 1)
### now for memebots normalizing..
# make hostname lowercase and remove www
netloc = netloc.lower()
netloc = urlparse.unquote(netloc).replace('+', ' ')
if netloc.startswith('www.') and len(netloc) > 4:
netloc = netloc[4:]
if netloc.endswith('.') and len(netloc) > 1:
netloc = netloc[:-1]
# all urls have trailing slash
if url == '':
url = '/'
url = urlparse.unquote(url).replace('+', ' ')
# remove empty query settings, these are usually form artifacts
# and put them in order
try:
query = urlencode(
[item for item in sorted(parse_qsl(query)) if item[1]])
except Exception, e:
query = ''
def parse_url(url_string):
endings = ['/RS', '/RK']
endpositions = []
start = url_string.find('http', url_string.find('/RU=') + 1)
for ending in endings:
endpos = url_string.rfind(ending)
if endpos > -1:
endpositions.append(endpos)
end = 0
if len(endpositions) > 0:
end = min(endpositions)
return unquote(url_string[start:end])
def parseQueryString(query):
if not query:
return {}
dict = {}
for p in query.split("&"):
try:
k, v = p.split("=")
if v != '':
dict[k.lower()] = unquote(v.replace('+', ' '))
except ValueError:
pass
return dict
def loadFile(name):
directory = str(os.getcwd())
filepath = os.path.join(directory, name)
with codecs.open(filepath, 'r', 'utf-8') as f:
data = f.readlines()
data = list(set(data))
result = []
for d in data:
d = str(urlparse.unquote(d).encode(
'utf-8')) #converting url encoded data to simple string
result.append(d)
return result
def do_POST(self):
parsed_path = urlparse.urlparse(self.path)
path = urlparse.unquote(parsed_path.path[1:]) # strip leading /
content_len = int(self.headers.getheader('content-length', 0))
obj = json.loads(self.rfile.read(content_len))
if path == "add_project":
return self.add_project(**obj)
if path == "new_bug":
return self.new_bug(**obj)
if path == "update_review":
return self.update_review(**obj)
return self.send_error(404, "blah")
def extract_credentials(self, location):
"""
Extract the credentials from a Git store file.
See "https://git-scm.com/docs/git-credential-store" for file format.
:param location: Full path to the Git store file
:return: List of credentials founds
"""
pwd_found = []
if os.path.isfile(location):
with open(location) as f:
# One line have the following format: https://user:[email protected]
for cred in f:
if len(cred) > 0:
parts = urlparse(cred)
pwd_found.append((unquote(parts.geturl().replace(
parts.username + ":" + parts.password + "@",
"").strip()), unquote(parts.username),
unquote(parts.password)))
return pwd_found
def selectLanguage(self, path):
"""
Tries to select the best language for the current request.
"""
if translations is None:
# This project doesn't use the multi-language feature, nothing to do here
return (path)
if conf["viur.languageMethod"] == "session":
# We store the language inside the session, try to load it from there
if not session.current.getLanguage():
if "X-Appengine-Country" in self.request.headers.keys():
lng = self.request.headers["X-Appengine-Country"].lower()
if lng in conf["viur.availableLanguages"] + list(
conf["viur.languageAliasMap"].keys()):
session.current.setLanguage(lng)
self.language = lng
else:
session.current.setLanguage(
conf["viur.defaultLanguage"])
else:
self.language = session.current.getLanguage()
elif conf["viur.languageMethod"] == "domain":
host = self.request.host_url.lower()
host = host[host.find("://") + 3:].strip(" /") #strip http(s)://
if host.startswith("www."):
host = host[4:]
if host in conf["viur.domainLanguageMapping"].keys():
self.language = conf["viur.domainLanguageMapping"][host]
else: # We have no language configured for this domain, try to read it from session
if session.current.getLanguage():
self.language = session.current.getLanguage()
elif conf["viur.languageMethod"] == "url":
tmppath = urlparse.urlparse(path).path
tmppath = [
urlparse.unquote(x)
for x in tmppath.lower().strip("/").split("/")
]
if len(tmppath) > 0 and tmppath[
0] in conf["viur.availableLanguages"] + list(
conf["viur.languageAliasMap"].keys()):
self.language = tmppath[0]
return (path[len(tmppath[0]) + 1:]
) #Return the path stripped by its language segment
else: # This URL doesnt contain an language prefix, try to read it from session
if session.current.getLanguage():
self.language = session.current.getLanguage()
elif "X-Appengine-Country" in self.request.headers.keys():
lng = self.request.headers["X-Appengine-Country"].lower()
if lng in conf["viur.availableLanguages"] or lng in conf[
"viur.languageAliasMap"]:
self.language = lng
return (path)
def update(self):
"""Scan network for Daikin devices."""
entries = []
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
sock.settimeout(DISCOVERY_TIMEOUT.seconds)
sock.bind(("", UDP_SRC_PORT))
try:
sock.sendto(DISCOVERY_MSG, (DISCOVERY_ADDRESS, UDP_DST_PORT))
while True:
try:
data, (address, _) = sock.recvfrom(1024)
entry = dict([
e.split('=') for e in data.decode("UTF-8").split(',')
])
# expecting product, mac, activation code, version
if 'ret' not in entry or entry['ret'] != 'OK':
# non-OK return on response
continue
if 'mac' not in entry:
# no mac found for device"
continue
if 'type' not in entry or entry['type'] != 'aircon':
# no mac found for device"
continue
if 'name' in entry:
entry['name'] = unquote(entry['name'])
entries.append({
'id': entry['id'].encode("UTF-8"),
'name': entry['name'].encode("UTF-8"),
'ip': address,
'mac': entry['mac'].encode("UTF-8"),
'ver': entry['ver'].encode("UTF-8"),
})
except socket.timeout:
break
finally:
sock.close()
self.entries = entries
def do_POST(self):
content_len = int(self.headers['Content-Length'])
#self.queryString
self.path
s = self.rfile.read(content_len)
encoded=''
try:
s=str(s)
import urlparse
req = urlparse.parse_qs(urlparse.unquote(s))
script = req['script'][0]
arg = req['arg'][0]
if self.path.endswith('/RunCase'):
executefile = 'cr.py'
print(os.getcwd())
if os.path.exists('cr.exe') and not os.path.exists(executefile):
executefile='cr.exe'
elif self.path.endswith('/RunSuite'):
print(os.getcwd())
executefile = 'sr.py'
print(os.getcwd())
if os.path.exists('sr.exe') and not os.path.exists(executefile):
executefile='sr.exe'
script ='../..'+script
args = script+ ' '+ arg
encoded=self.RunScript(executefile, args)
print('result of '+ executefile+ ' ' + arg+ ' '+str(encoded))
except Exception as e:
print(e)
response = self.ParseFormData(s)
if response:
type, filename, data =response
encoded = self.onUploadFile(type, filename, data)
else:
encoded ='ERROR: %s, Can\'t parse Form data: %s'%(str(e),s)
encoded= encoded.encode(encoding='utf_8')
try:
requestline = self.requestline
import re
reScript=re.compile('POST\s+(.+)\s+HTTP.*', re.DOTALL)
m= re.match(reScript, requestline)
if m:
returncode =self.RunScript(m.group(1),[])
encoded ='script %s completed with return code %d!'%(m.group(1), returncode)
except Exception as e:
encoded ='can\'t run script!'
encoded = encoded.encode(encoding='utf_8', errors='strict')
def get(self, text):
# from urllib.parse import unquote # py3
from urlparse import unquote
img = res.img()
img.setQRCodeText(unquote(text.encode("utf-8"))) # f**k...
img.setQRCodeLevel(int(self.get_argument("level", 1)))
img.setQRCodeSize(int(self.get_argument("size", 10)))
img.setQRCodeVersion(int(self.get_argument("version", 0)))
img.setQRCodeForeground(self.get_argument("foreground", "000000"))
img.setQRCodeBackground(self.get_argument("background", "FFFFFF"))
self.set_header("Content-Type", "image/gif")
self.finish(
base64.b64decode(img.process()["response_params"]["image_data"]))
def get_video_info(player_config):
return {
"title":
player_config["args"]["title"],
"author":
player_config["args"]["author"],
"video_id":
player_config["args"]["video_id"],
"video_urls": [
unquote(x[x.find("url=") + 4:]) for x in player_config["args"]
["url_encoded_fmt_stream_map"].split(",")
]
}
def _parse_qs(qs):
pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
retval = defaultdict(list)
for name_value in pairs:
if name_value is None or len(name_value) == 0:
continue
nv = name_value.split('=', 1)
if len(nv) != 2:
# Handle case of a control-name with no equal sign
nv.append(None)
name = unquote(nv[0].replace('+', ' '))
value = None
if nv[1] is not None:
value = unquote(nv[1].replace('+', ' '))
retval[name].append(value)
return retval
def do_POST(self):
"""Respond to POST requests"""
from urlparse import unquote
path, headers, data = self._ParseBaseRequest('POST')
if None is path: return
if ('gpr' == path[0]) and (2 == len(path)):
self._AlterGPR(unquote(path[1]), headers, data)
else:
Log('[PS] Invalid request received', Log.DEBUG)
self.send_error(501, 'Invalid request')
def filename(self):
'''Returns the filename to be used for saving the repo file.
The filename is derived from the repo url by injecting a suffix
after the name and before the file extension. This suffix is a
partial md5 checksum of the full repourl. This avoids multiple
repos from being written to the same file.
'''
urlpath = unquote(urlsplit(self.repourl, allow_fragments=False).path)
basename = os.path.basename(urlpath)
suffix = '-' + md5(self.repourl.encode('utf-8')).hexdigest()[:5]
final_name = suffix.join(os.path.splitext(basename))
return final_name
def get(self):
result = urlfetch.fetch("https://opentdb.com/api.php?amount=10&encode=url3986").content
json_result = json.loads(result)["results"]
for q in json_result:
question = q["question"]
final_question = urlparse.unquote(question)
correct_answer = q["correct_answer"]
final_correct_answer = urlparse.unquote(correct_answer)
incorrect_answers = q["incorrect_answers"]
final_incorrect_answers = []
for answer in incorrect_answers:
final_incorrect_answers.append(urlparse.unquote(answer))
new_question = Question(question = final_question,
correct_answer = final_correct_answer,
incorrect_answers = final_incorrect_answers)
new_question.put()
self.redirect("/")
def handle_cgi_request(methods = None):
"""
Gets the JSON-RPC request from CGI environment and returns the
result to STDOUT
"""
import cgi
import cgitb
cgitb.enable()
# get response-body
request_json = sys.stdin.read()
if request_json:
# POST
request_json = urlparse.unquote(request_json)
else:
# GET
args = []
kwargs = {}
fields = cgi.FieldStorage()
jsonrpc = fields.getfirst("jsonrpc")
id = fields.getfirst("id")
method = fields.getfirst("method")
params = fields.getfirst("params")
if params:
params = rpcjson.loads(params)
if isinstance(params, list):
args = params
kwargs = {}
elif isinstance(params, dict):
args = []
kwargs = params
# Create JSON request string
request_dict = rpcrequest.create_request_dict(method, *args, **kwargs)
request_dict["jsonrpc"] = jsonrpc
request_dict["id"] = id
request_json = rpcjson.dumps(request_dict)
# Call
response_json = rpclib.JsonRpc(methods = methods).call(request_json)
# Return headers
print("Content-Type: application/json")
print("Cache-Control: no-cache")
print("Pragma: no-cache")
print()
# Return result
print(response_json)
def convert_aspectFilters(webParams, APIParams):
for k, v in webParams.items():
if k[0].isupper():
assert (len(v) == 1)
v = v[0]
del webParams[k]
aspectFilters = APIParams.get('aspectFilter', [])
aspects = urlparse.unquote(v).split('|') # todo unicode ?
aspects = [
'Not Specified' if x == '!' else xml_escape(x) for x in aspects
]
k = drop_bell(xml_escape(k))
aspectFilters.append({'aspectName': k, 'aspectValueName': aspects})
APIParams['aspectFilter'] = aspectFilters
def add_geojson():
json = urlparse.unquote(request.form['geojson'])
group_admin = sess.query(Group).filter_by(
groupid=session['groupid']).one().userid
if session['userid'] != group_admin:
return jsonify("not allowed")
cur.execute(
"INSERT INTO mapobjects (geom, groupid, userid, date) VALUES "
"(ST_Transform(ST_GeomFromText('{}',4326),3857), {}, {}, '{}');".
format(json, session['groupid'], session['userid'],
datetime.datetime.utcnow()))
pgconnect.commit()
return jsonify('success')
def get_storage_path_url(url):
"""
Returns the file tuple for an API server URL
Args:
url (unicode): API server URL
Returns:
(unicode, unicode): file tuple
"""
o = urlparse.urlsplit(url)[2]
return get_storage_path(
_sanitize_path(nidaba_cfg['storage_path'],
urlparse.unquote(os.path.join(*o.split('/')[4:]))))
def compare_urls(url_a, url_b):
# ignore www. in comparison
if url_a.netloc.startswith('www.'):
host_a = url_a.netloc.replace('www.', '', 1)
else:
host_a = url_a.netloc
if url_b.netloc.startswith('www.'):
host_b = url_b.netloc.replace('www.', '', 1)
else:
host_b = url_b.netloc
if host_a != host_b or url_a.query != url_b.query or url_a.fragment != url_b.fragment:
return False
# remove / from the end of the url if required
path_a = url_a.path[:-1]\
if url_a.path.endswith('/')\
else url_a.path
path_b = url_b.path[:-1]\
if url_b.path.endswith('/')\
else url_b.path
return unquote(path_a) == unquote(path_b)
def str_to_dict(self, s):
ret = urlparse.parse_qs(s)
for k, v in ret.iteritems():
try:
# I'm not sure about the replacement for '-', but it'll at least let it be decoded.
# For the most part it's not important since it's mostly used for the devname/ingamesn fields.
ret[k] = base64.b64decode( urlparse.unquote( v[0] ).replace("*", "=").replace("?", "/").replace(">","+").replace("-","/") )
except TypeError:
print "Could not decode following string: ret[%s] = %s" % (k, v[0])
print "url: %s" % s
ret[k] = v[0] # If you don't assign it like this it'll be a list, which breaks other code.
return ret
def preview_event():
data = json.loads(urlparse.unquote(request.args.get('data')))
current_app.logger.info(u'Preview args: {}'.format(data))
venue = api_client.get_venue_by_id(data['venue_id'])
data['venue'] = venue
return render_template('views/admin/preview.html',
images_url=current_app.config['IMAGES_URL'],
events=[data],
api_base_url=api_client.base_url,
paypal_account=current_app.config['PAYPAL_ACCOUNT'])
def normalize_url(self, url):
"""Given a URL, try to normalize it such that we can find duplicate URLs more easily"""
netloc = query = fragment = ''
i = url.find(':')
scheme = url[:i].lower()
url = url[i + 1:]
if url[:2] == '//':
delim = len(url)
for c in '/?#':
wdelim = url.find(c, 2)
if wdelim >= 0:
delim = min(delim, wdelim)
netloc, url = url[2:delim], url[delim:]
if '#' in url:
try:
url, fragment = self.fragment_re.search(url).groups()
except StandardError:
pass
if '?' in url:
url, query = url.split('?', 1)
netloc = netloc.lower()
netloc = urlparse.unquote(netloc).replace('+', ' ')
if netloc.startswith('www.') and len(netloc) > 4:
netloc = netloc[4:]
if netloc.endswith('.') and len(netloc) > 1:
netloc = netloc[:-1]
if url == '':
url = '/'
url = urlparse.unquote(url).replace('+', ' ')
url = os.path.normpath(url)
try:
query = urllib.urlencode(
[item for item in sorted(cgi.parse_qsl(query)) if item[1]])
except StandardError:
query = ''
return urlparse.urlunsplit((scheme, netloc, url, query, ''))
def create_event(self, data, data_url, user):
"""
Create an Event instance based on serialized data.
The featured image will be retrieved from the original server
and the EventDateTime instances will be attached.
"""
converted = convert(data)
items = deserialize("json", json.dumps([converted]), ignorenonexistent=True)
event = list(items)[0].object
event.id = None # Ensure new event ID
event.slug = event.generate_unique_slug()
event.site_id = current_site_id()
event.user = user
if not event.location:
event.location = data["fields"].get("location_title", "")
# Get the original featured image and save it locally
img_path = data["fields"].get("featured_image")
if img_path:
parts = urlparse(data_url)
img_url = urljoin(parts.scheme + "://" + parts.hostname, "static/media/" + img_path)
img_response = requests.get(img_url)
if img_response.status_code == requests.codes.ok:
_, filename = os.path.split(img_path)
filepath = os.path.join("uploads", "events", unquote(filename))
filepath = default_storage.save(filepath, ContentFile(img_response.content))
event.featured_image = filepath
# Discard all M2M data as it may cause integrity issues when saving
event.m2m_data = {}
# Commit the new event to the database (required before saving EventDateTimes)
event.save()
# Add EventDateTimes instances (for old Mezzanine Calendar objects)
for dt in data.get("dateandtimes", []):
dt = convert(dt)
occ = deserialize("json", json.dumps([dt]), ignorenonexistent=True)
occ = list(occ)[0].object
occ.event = event
occ.save()
# Import occurrences (if available)
for occ_data in data.get("occurrences", []):
occ = occ_data["fields"].copy()
occ.pop("event")
event.occurrences.create(**occ)
return event
def resolve (self, reply_handler):
self._reply_handler = reply_handler
bus = dbus.SystemBus ()
if not bus:
reply_handler ([])
del self._devices
del self._reply_handler
return
for uri, device in self._devices.iteritems ():
if not uri.startswith ("dnssd://"):
self._unresolved -= 1
continue
# We need to resolve the DNS-SD hostname in order to
# compare with other network devices.
try:
uri = str (uri)
except:
pass
result = urlparse.urlparse (str (uri))
hostname = result.netloc
elements = hostname.rsplit (".", 3)
if len (elements) != 4:
self._resolved ()
continue
name, stype, protocol, domain = elements
name = urlparse.unquote (name)
stype += "." + protocol # e.g. _printer._tcp
try:
obj = bus.get_object ("org.freedesktop.Avahi", "/")
server = dbus.Interface (obj,
"org.freedesktop.Avahi.Server")
self._device_uri_by_name[(name, stype, domain)] = uri
debugprint ("Resolving address for %s" % hostname)
server.ResolveService (-1, -1,
name, stype, domain,
-1, 0,
reply_handler=self._reply,
error_handler=lambda e:
self._error (uri, e))
except dbus.DBusException as e:
debugprint ("Failed to resolve address: %s" % repr (e))
self._resolved ()
def parsegff3(filename):
"""
A minimalistic GFF3 format parser.
Yields objects that contain info about a single GFF3 feature.
Supports transparent gzip decompression.
"""
# Parse with transparent decompression
openfunc = gzip.open if filename.endswith(".gz") else open
with openfunc(filename) as infile:
for line in infile:
if line.startswith("#"):
continue
parts = line.strip().split("\t")
# If this fails, the file format is not standard-compatible
assert len(parts) == len(gffInfoFields)
# Normalize data
normalizedinfo = {
"seqid":
None if parts[0] == "." else urlparse.unquote(parts[0]),
"source":
None if parts[1] == "." else urlparse.unquote(parts[1]),
"type":
None if parts[2] == "." else urlparse.unquote(parts[2]),
"start": None if parts[3] == "." else int(parts[3]),
"end": None if parts[4] == "." else int(parts[4]),
"score": None if parts[5] == "." else float(parts[5]),
"strand":
None if parts[6] == "." else urlparse.unquote(parts[6]),
"phase":
None if parts[7] == "." else urlparse.unquote(parts[7]),
"attributes": parsegffattributes(parts[8])
}
# Alternatively, you can emit the dictionary here, if you need mutability:
# yield normalizedinfo
yield GFFRecord(**normalizedinfo)
def load_all_urls_from_file(self):
try:
with open(self.file) as inFile:
lines = inFile.readlines()
for line in lines:
_ = line.split()
if len(_) == 3 and (_[2].find('^^^200') > 0
or _[2].find('^^^403') > 0):
url = urlparse.unquote(_[1])
url, depth = self._cal_depth(url)
if len(url) >= 70: continue
#print url
self._enqueue(url)
except Exception, e:
error_log('[load_all_urls_from_file Exception] %s' % e)
def test():
if not APP_DEBUG:
return jsonify({"page": "test"})
res = "<h2>Test Page (DEBUG):</h2>"
for rule in app.url_map.iter_rules():
options = {}
for arg in rule.arguments:
options[arg] = "[{0}]".format(arg)
methods = ','.join(rule.methods)
url = url_for(rule.endpoint, **options)
res += urlparse.unquote("{0:50s} {1:20s} <a href='{2}'>{2}</a><br/>".format(rule.endpoint, methods, url))
return res
def _get_proxy_info(self, scheme='http'):
if scheme not in ['http', 'https']:
return
proxy = self._settings.make_proxy_uri()
if not proxy:
return
parts = urlparse(proxy)
proxy_scheme = parts.scheme
traits = {
'http': (PROXY_TYPE_HTTP, False),
'socks5': (PROXY_TYPE_SOCKS5, False),
'socks5h': (PROXY_TYPE_SOCKS5, True),
'socks4': (PROXY_TYPE_SOCKS4, False),
'socks4a': (PROXY_TYPE_SOCKS4, True)
}
if proxy_scheme not in traits:
logger.warning('Unsupported proxy protocol.')
return
proxy_type, proxy_rdns = traits[proxy_scheme]
proxy_user, proxy_pass = parts.username, parts.password
if proxy_user:
proxy_user = unquote(proxy_user)
if proxy_pass:
proxy_pass = unquote(proxy_pass)
return httplib2.ProxyInfo(
proxy_type=proxy_type,
proxy_rdns=proxy_rdns,
proxy_host=parts.hostname,
proxy_port=parts.port,
proxy_user=proxy_user,
proxy_pass=proxy_pass,
)
def sanitize_redirect(url, valid_hosts=None):
decoded = urlparse.unquote(url).strip()
base = settings.SECURE_URL_HOST
valid_hosts = valid_hosts or [base]
# relative url
if decoded.startswith("/"):
return "/" + decoded.lstrip("/")
# absolute url, check url matches any of valid_hosts
else:
is_valid = any(map(lambda x: decoded.startswith(x), valid_hosts))
if is_valid:
return decoded
raise SuspiciousOperation('Invalid redirect url')
return url