def search_bing_api(self, query, limit=0):
url = 'https://api.cognitive.microsoft.com/bing/v7.0/search'
payload = {'q': query, 'count': 50, 'offset': 0, 'responseFilter': 'WebPages'}
headers = {'Ocp-Apim-Subscription-Key': self.get_key('bing_api')}
results = []
cnt = 0
self.verbose('Searching Bing API for: %s' % (query))
while True:
resp = self.request(url, payload=payload, headers=headers)
if resp.json == None:
raise framework.FrameworkException('Invalid JSON response.\n%s' % (resp.text))
#elif 'error' in resp.json:
elif resp.status_code == 401:
raise framework.FrameworkException('%s: %s' % (resp.json['statusCode'], resp.json['message']))
# add new results, or if there's no more, return what we have...
if 'webPages' in resp.json:
results.extend(resp.json['webPages']['value'])
else:
return results
# increment and check the limit
cnt += 1
if limit == cnt:
break
# check for more pages
# https://msdn.microsoft.com/en-us/library/dn760787.aspx
if payload['offset'] > (resp.json['webPages']['totalEstimatedMatches'] - payload['count']):
break
# set the payload for the next request
payload['offset'] += payload['count']
return results
def _get_source(self, params, query=None):
prefix = params.split()[0].lower()
if prefix in ['query', 'default']:
query = ' '.join(
params.split()[1:]) if prefix == 'query' else query
try:
results = self.query(query)
except sqlite3.OperationalError as e:
raise framework.FrameworkException(
'Invalid source query. %s %s' %
(type(e).__name__, e.message))
if not results:
sources = []
elif len(results[0]) > 1:
sources = [x[:len(x)] for x in results]
#raise framework.FrameworkException('Too many columns of data as source input.')
else:
sources = [x[0] for x in results]
elif os.path.exists(params):
sources = open(params).read().split()
else:
sources = [params]
source = [self.to_unicode(x) for x in sources]
if not source:
raise framework.FrameworkException('Source contains no input.')
return source
def search_shodan_api(self, query, limit=0):
api_key = self.get_key('shodan_api')
url = 'https://api.shodan.io/shodan/host/search'
payload = {'query': query, 'key': api_key}
results = []
cnt = 1
page = 1
self.verbose('Searching Shodan API for: %s' % (query))
while True:
resp = self.request(url, payload=payload)
if resp.json == None:
raise framework.FrameworkException(
'Invalid JSON response.\n%s' % (resp.text))
if 'error' in resp.json:
raise framework.FrameworkException(resp.json['error'])
if not resp.json['matches']:
break
# add new results
results.extend(resp.json['matches'])
# check limit
if limit == cnt:
break
cnt += 1
# next page
page += 1
payload['page'] = page
return results
def search_shodan_api(self, query, limit=0):
api_key = self.get_key('shodan_api')
url = 'https://api.shodan.io/shodan/host/search'
payload = {'query': query, 'key': api_key}
results = []
cnt = 0
page = 1
self.verbose(f"Searching Shodan API for: {query}")
while True:
time.sleep(1)
resp = self.request('GET', url, params=payload)
if resp.json() == None:
raise framework.FrameworkException(f"Invalid JSON response.{os.linesep}{resp.text}")
if 'error' in resp.json():
raise framework.FrameworkException(resp.json()['error'])
if not resp.json()['matches']:
break
# add new results
results.extend(resp.json()['matches'])
# increment and check the limit
cnt += 1
if limit == cnt:
break
# next page
page += 1
payload['page'] = page
return results
def _request_file_from_repo(self, path):
resp = self.request('GET', urljoin(self.repo_url, path))
if resp.status_code != 200:
raise framework.FrameworkException(
f"Invalid response from module repository ({resp.status_code})."
)
return resp
def search_google_api(self, query, limit=0):
api_key = self.get_key('google_api')
cse_id = self.get_key('google_cse')
url = 'https://www.googleapis.com/customsearch/v1'
payload = {
'alt': 'json',
'prettyPrint': 'false',
'key': api_key,
'cx': cse_id,
'q': query
}
results = []
cnt = 1
self.verbose('Searching Google API for: %s' % (query))
while True:
resp = None
resp = self.request(url, payload=payload)
if resp.json == None:
raise framework.FrameworkException(
'Invalid JSON response.\n%s' % (resp.text))
# add new results
if 'items' in resp.json:
results.extend(resp.json['items'])
# check limit
if limit == cnt:
break
cnt += 1
# check for more pages
if not 'nextPage' in resp.json['queries']:
break
payload['start'] = resp.json['queries']['nextPage'][0][
'startIndex']
return results
def search_bing_api(self, query, limit=0):
api_key = self.get_key('bing_api')
url = 'https://api.datamarket.azure.com/Bing/Search/Web'
payload = {'Query': "'%s'" % (query), '$format': 'json'}
results = []
cnt = 1
self.verbose('Searching Bing API for: %s' % (query))
while True:
resp = None
resp = self.request(url, payload=payload, auth=(api_key, api_key))
if resp.json == None:
raise framework.FrameworkException(
'Invalid JSON response.\n%s' % (resp.text))
# add new results
if 'results' in resp.json['d']:
results.extend(resp.json['d']['results'])
# check limit
if limit == cnt:
break
cnt += 1
# check for more pages
if not '__next' in resp.json['d']:
break
payload['$skip'] = resp.json['d']['__next'].split('=')[-1]
return results
def get_twitter_oauth_token(self):
token_name = 'twitter_token'
try:
return self.get_key(token_name)
except:
pass
twitter_key = self.get_key('twitter_api')
twitter_secret = self.get_key('twitter_secret')
url = 'https://api.twitter.com/oauth2/token'
auth = (twitter_key, twitter_secret)
headers = {
'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'
}
payload = {'grant_type': 'client_credentials'}
resp = self.request(url,
method='POST',
auth=auth,
headers=headers,
payload=payload)
if 'errors' in resp.json:
raise framework.FrameworkException(
'%s, %s' % (resp.json['errors'][0]['message'],
resp.json['errors'][0]['label']))
access_token = resp.json['access_token']
self.add_key(token_name, access_token)
return access_token
def get_twitter_oauth_token(self):
token_name = 'twitter_token'
token = self.get_key(token_name)
if token:
return token
twitter_key = self.get_key('twitter_api')
twitter_secret = self.get_key('twitter_secret')
url = 'https://api.twitter.com/oauth2/token'
auth = (twitter_key, twitter_secret)
headers = {
'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'
}
payload = {'grant_type': 'client_credentials'}
resp = self.request('POST',
url,
auth=auth,
headers=headers,
data=payload)
if 'errors' in resp.json():
raise framework.FrameworkException(
f"{resp.json()['errors'][0]['message']}, {resp.json()['errors'][0]['label']}"
)
access_token = resp.json()['access_token']
self.add_key(token_name, access_token)
return access_token
def search_google_api(self, query, limit=0):
api_key = self.get_key('google_api')
cse_id = self.get_key('google_cse')
url = 'https://www.googleapis.com/customsearch/v1'
payload = {
'alt': 'json',
'prettyPrint': 'false',
'key': api_key,
'cx': cse_id,
'q': query
}
results = []
cnt = 0
self.verbose(f"Searching Google API for: {query}")
while True:
resp = self.request('GET', url, params=payload)
if resp.json() == None:
raise framework.FrameworkException(
f"Invalid JSON response.{os.linesep}{resp.text}")
# add new results
if 'items' in resp.json():
results.extend(resp.json()['items'])
# increment and check the limit
cnt += 1
if limit == cnt:
break
# check for more pages
if not 'nextPage' in resp.json()['queries']:
break
payload['start'] = resp.json(
)['queries']['nextPage'][0]['startIndex']
return results
def _get_source(self, params, query=None):
prefix = params.split()[0].lower()
if prefix in ['query', 'default']:
query = ' '.join(params.split()[1:]) if prefix == 'query' else query
docs = self.read_doc_ES(os.path.basename(self.workspace), query)
sources = []
for doc in docs:
for value in doc['_source'].values():
sources.append(value)
elif os.path.exists(params):
sources = open(params, encoding='UTF-8').read().split()
else:
sources = [params]
if not sources:
raise framework.FrameworkException('Source contains no input.')
return sources
def search_twitter_api(self, payload, limit=False):
headers = {'Authorization': 'Bearer %s' % (self.get_twitter_oauth_token())}
url = 'https://api.twitter.com/1.1/search/tweets.json'
results = []
while True:
resp = self.request(url, payload=payload, headers=headers)
if limit:
# app auth rate limit for search/tweets is 450/15min
time.sleep(2)
jsonobj = resp.json
for item in ['error', 'errors']:
if item in jsonobj:
raise framework.FrameworkException(jsonobj[item])
results += jsonobj['statuses']
if 'next_results' in jsonobj['search_metadata']:
max_id = urlparse.parse_qs(jsonobj['search_metadata']['next_results'][1:])['max_id'][0]
payload['max_id'] = max_id
continue
break
return results
def search_twitter_api(self, payload):
headers = {
'Authorization': 'Bearer %s' % (self.get_twitter_oauth_token())
}
url = 'https://api.twitter.com/1.1/search/tweets.json'
# count causes inconsistent results when applied
#payload['count'] = 50 # api stops paginating at count=90
results = []
while True:
resp = self.request(url, payload=payload, headers=headers)
jsonobj = resp.json
for item in ['error', 'errors']:
if item in jsonobj:
raise framework.FrameworkException(jsonobj[item])
results += jsonobj['statuses']
if 'next_results' in jsonobj['search_metadata']:
max_id = urlparse.parse_qs(jsonobj['search_metadata']
['next_results'][1:])['max_id'][0]
payload['max_id'] = max_id
continue
break
return results
