def wrapped_fn(self, url, **kwargs):
u = URL(url)
# Fix params not appears in referer
try:
params = kwargs.pop('params')
url = '{url}?{params}'.format(url=url, params=urlencode(params))
except KeyError:
pass
if u.scheme and u.host:
# pylint: disable=protected-access
self._base_url = str(u.replace(full_path=''))
# pylint: disable=protected-access
self._scheme = u.scheme # Update scheme
if url.startswith(r'//'):
# '//example.com'
# pylint: disable=protected-access
url = '{scheme}:{where}'.format(scheme=self._scheme, where=url)
self._base_url = url # pylint: disable=protected-access
elif url.startswith(r'?'):
# '?page=rss'
url = '/' + url # -> '/?page=rss'
url = urljoin(self._base_url, url) # pylint: disable=protected-access
else:
# '/?page=rss' 'page=rss'
url = urljoin(self._base_url, url) # pylint: disable=protected-access
# pylint: disable=protected-access
u = URL(self._last_url)
# pylint: disable=protected-access
self._headers.update({
# HTTP/2 Headers lowercase only
'origin': str(u.replace(full_path='')),
'referer': self._last_url
})
self._last_url = url # pylint: disable=protected-access
return fn(self, url, **kwargs)
def test_setdefault(self):
empty = URL()
full1 = URL('scheme://user@host:80/path?query#frgment')
full2 = URL('an://oth@er:33/full?url#!!')
self.assertEqual(empty.setdefault(*full1._data), full1)
self.assertEqual(full1.setdefault(*full2._data), full1)
for idx, (field, value) in enumerate(zip(full1._fields, full1._data)):
self.assertEqual(empty.setdefault(**{field: value}),
empty.replace(**{field: value}))
self.assertEqual(empty.setdefault(**{field: value})[idx], value)
self.assertEqual(full2.setdefault(**{field: value})[idx],
full2[idx])
def test_setdefault(self):
empty = URL()
full1 = URL('scheme://user@host:80/path?query#frgment')
full2 = URL('an://oth@er:33/full?url#!!')
self.assertEqual(empty.setdefault(*full1._data), full1)
self.assertEqual(full1.setdefault(*full2._data), full1)
for idx, (field, value) in enumerate(zip(full1._fields, full1._data)):
self.assertEqual(empty.setdefault(**{field: value}),
empty.replace(**{field: value}))
self.assertEqual(empty.setdefault(**{field: value})[idx], value)
self.assertEqual(
full2.setdefault(**{field: value})[idx], full2[idx])
class MongoDBIAMConnectionString:
def __init__(self, connection_string: str, profile_name: str = 'default'):
self.creds = {}
self.url = URL(connection_string)
self.profile = profile_name
self.qs = dict(parse_qsl(self.url.query))
if self._gather_aws_creds():
self._update_url()
else:
raise InvalidAWSSession
def _gather_aws_creds(self):
if self.profile == 'default':
try:
_creds = boto_session().get_credentials(
).get_frozen_credentials()
except Exception:
return False
else:
try:
_creds = boto_session(
profile_name=self.profile).get_credentials(
).get_frozen_credentials()
except Exception:
return False
for _ in ('access_key', 'secret_key', 'token'):
if getattr(_creds, _):
self.creds.update({_: str(getattr(_creds, _))})
return True
def _dict2qs(self, d: dict):
return "&".join([f"{k}={v}" for k, v in d.items()])
def _update_url(self):
# yurl strangely parses hosts as paths, fix that
if not self.url.host and self.url.path:
self.url = self.url.replace(host=self.url.path, path='/')
# Add proper protocol if not specified
if not self.url.scheme:
self.url = self.url.replace(scheme='mongodb+srv')
# Add access and secret keys as username and password
self.url = self.url.replace(
userinfo=
f"{quote_plus(self.creds['access_key'])}:{quote_plus(self.creds['secret_key'])}"
)
# Set path to '/' if not provided
if not self.url.path:
self.url = self.url.replace(path='/')
self.qs.update({'authSource': '$external'})
self.qs.update({'authMechanism': 'MONGODB-AWS'})
_token = self.creds.get('token')
if _token:
self.qs.update({
'authMechanismProperties':
f"AWS_SESSION_TOKEN:{quote_plus(_token)}"
})
self.url = self.url.replace(query=self._dict2qs(self.qs))
def __repr__(self):
return str(self.url)
def __str__(self):
return str(self.url)
class TumblrBlog:
def __init__(self, url, session=None, **kwargs):
"""
Tumblr blog
Args:
url(URL|str): Tumblr profile URL
session(Optional[Session]): An optional custom Requests session
Keyword Args:
api_key(str): Tumblr API key
uagent(str): Custom User-Agent header
"""
self._url = url if isinstance(url, URL) else URL(url)
self._api_url = URL(scheme='https', host='api.tumblr.com', path='/v2/')
self._api_response = None # type: Response
self._api_key = kwargs.get(
'api_key', 'fuiKNFp9vQFvjLNvx4sUwti4Yb5yGutBN4Xh10LXZhhRKjWlV4')
self._uagent = kwargs.get('user_agent', 'tumdlr/{version}')
if not session:
session = Session()
session.headers.update({
'Referer':
urllib.parse.quote(self._url.as_string()),
'User-Agent':
self._uagent.format(version=__version__)
})
self.session = session
self.title = None # type: str
self.url = None # type: URL
self.name = None # type: str
self.description = None # type: str
self.is_nsfw = None # type: bool
self.likes = None # type: int|False
self.post_count = None # type: int
self.updated = None # type: int
self._posts = []
self.offset = 0
self._api_url = self._api_url.replace(
path=self._api_url.path +
'blog/{host}/posts'.format(host=self._url.host))
self._api_get()
def _api_get(self, query=None, parse=True):
"""
Execute an API query
Args:
query(Optional[dict]): Extra query parameters
parse(Optional[bool]): Parse the API response immediately
"""
# Parse extra query parameters
query_extra = []
if query:
for key, value in query.items():
query_extra.append('{key}={value}'.format(
key=urllib.parse.quote(key),
value=urllib.parse.quote(value)))
# Only prepend an ampersand if we have extra attributes, otherwise default to an empty string
if query_extra:
query_extra = '&' + '&'.join(query_extra)
else:
query_extra = ''
endpoint = self._api_url.replace(
query='api_key={api_key}&filter=text&offset={offset}{extra}'.
format(
api_key=self._api_key, offset=self.offset, extra=query_extra))
response = self.session.get(endpoint.as_string()) # type: Response
response.raise_for_status()
self._api_response = response
if parse:
self._api_parse_response()
def _api_parse_response(self):
"""
Parse an API response
"""
blog = self._api_response.json()['response']['blog']
self.title = blog['title']
self.url = URL(blog['url'])
self.name = blog['name']
self.description = blog['description']
self.is_nsfw = blog['is_nsfw']
self.likes = blog.get(
'likes', False) # Returned only if sharing of likes is enabled
self.post_count = blog['posts']
self.updated = blog['updated']
posts = self._api_response.json()['response']['posts']
for post in posts:
try:
if post['type'] in ['photo', 'link']:
self._posts.append(TumblrPhotoSet(post, self))
continue
elif post['type'] == 'video':
self._posts.append(TumblrVideoPost(post, self))
continue
self._posts.append(TumblrPost(post, self))
except TumdlrParserError:
continue
def posts(self):
"""
Yields:
TumblrPost
"""
while True:
# Out of posts?
if not self._posts:
# Do we have any more to query?
self._api_get()
if not self._posts:
# Nope, we've queried everything, break now
break
# Pop our next post and increment the offset
post = self._posts.pop(0)
self.offset += 1
yield post
class TumblrBlog:
def __init__(self, url, session=None, **kwargs):
"""
Tumblr blog
Args:
url(URL|str): Tumblr profile URL
session(Optional[Session]): An optional custom Requests session
Keyword Args:
api_key(str): Tumblr API key
uagent(str): Custom User-Agent header
"""
self._url = url if isinstance(url, URL) else URL(url)
self._api_url = URL(scheme='https', host='api.tumblr.com', path='/v2/')
self._api_response = None # type: Response
self._api_key = kwargs.get('api_key', 'fuiKNFp9vQFvjLNvx4sUwti4Yb5yGutBN4Xh10LXZhhRKjWlV4')
self._uagent = kwargs.get('user_agent', 'tumdlr/{version}')
if not session:
session = Session()
session.headers.update({
'Referer': urllib.parse.quote(self._url.as_string()),
'User-Agent': self._uagent.format(version=__version__)
})
self.session = session
self.title = None # type: str
self.url = None # type: URL
self.name = None # type: str
self.description = None # type: str
self.is_nsfw = None # type: bool
self.likes = None # type: int|False
self.post_count = None # type: int
self.updated = None # type: int
self._posts = []
self.offset = 0
self._api_url = self._api_url.replace(
path=self._api_url.path + 'blog/{host}/posts'.format(host=self._url.host)
)
self._api_get()
def _api_get(self, query=None, parse=True):
"""
Execute an API query
Args:
query(Optional[dict]): Extra query parameters
parse(Optional[bool]): Parse the API response immediately
"""
# Parse extra query parameters
query_extra = []
if query:
for key, value in query.items():
query_extra.append(
'{key}={value}'.format(
key=urllib.parse.quote(key),
value=urllib.parse.quote(value)
)
)
# Only prepend an ampersand if we have extra attributes, otherwise default to an empty string
if query_extra:
query_extra = '&' + '&'.join(query_extra)
else:
query_extra = ''
endpoint = self._api_url.replace(
query='api_key={api_key}&filter=text&offset={offset}{extra}'.format(
api_key=self._api_key, offset=self.offset, extra=query_extra
)
)
response = self.session.get(endpoint.as_string()) # type: Response
response.raise_for_status()
self._api_response = response
if parse:
self._api_parse_response()
def _api_parse_response(self):
"""
Parse an API response
"""
blog = self._api_response.json()['response']['blog']
self.title = blog['title']
self.url = URL(blog['url'])
self.name = blog['name']
self.description = blog['description']
self.is_nsfw = blog['is_nsfw']
self.likes = blog.get('likes', False) # Returned only if sharing of likes is enabled
self.post_count = blog['posts']
self.updated = blog['updated']
posts = self._api_response.json()['response']['posts']
for post in posts:
try:
if post['type'] in ['photo', 'link']:
self._posts.append(TumblrPhotoSet(post, self))
continue
elif post['type'] == 'video':
self._posts.append(TumblrVideoPost(post, self))
continue
self._posts.append(TumblrPost(post, self))
except TumdlrParserError:
continue
def posts(self):
"""
Yields:
TumblrPost
"""
while True:
# Out of posts?
if not self._posts:
# Do we have any more to query?
self._api_get()
if not self._posts:
# Nope, we've queried everything, break now
break
# Pop our next post and increment the offset
post = self._posts.pop(0)
self.offset += 1
yield post
class Scanner(object):
FINGERPRINTS = [
{
"type": "git",
"base": ".git",
"files": ["index"]
},
{
"type": "svn",
"base": ".svn",
"files": ["wc.db"]
},
{
"type": "svn_old",
"base": ".svn",
"files": ["entries"]
},
#{
# "type": "hg",
# "base": ".hg",
# "files": ["store/00manifest.i"]
#}
]
SCHEMES = ["HTTP", "HTTPS"]
def __init__(self, host):
self.host = URL(host).replace(path = "", query = "", fragment = "")
self.session = HTTP()
self.session.headers['User-Agent'] = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36"
def scan_host(self):
for scheme in self.SCHEMES:
for fingerprint in self.FINGERPRINTS:
for file in fingerprint['files']:
url = self.host.replace(path = fingerprint['base'] + "/" + file, scheme = scheme)
url = str(url)
response = self.session.get(url, verify=False)
if response.status_code == 200 and self._filter_false_positive(response.content, fingerprint['type']):
return {
"file": file,
"type": fingerprint['type'],
"scheme": scheme,
"data": response.content,
"host": self.host.replace(scheme = scheme)
}
else:
pass
if(response.status_code == 200):
pass
#print "Failed: File exists, but failed verification."
else:
pass
#print "Failed: HTTP " + str(response.status_code)
return False
def _filter_false_positive(self, data, type):
if "<html" in data and "</html>" in data:
return False
if type == "git":
if data[0:4] != "DIRC":
return False
if type == "svn_old":
if "dir" not in data or "file" not in data:
return False
if type == "svn":
if data[0:13] != "SQLite format":
return False
if type == "hg":
if not data.statswith(".hgtag"):
return False
return True
