def _fetch_data(from_date, to_date, page_size=20, first_page=1):
"""Fetch a chunk of vulndb"""
from_date = from_date.strftime("%Y-%m-%d")
to_date = to_date.strftime("%Y-%m-%d")
logger.info("Working on date range: {} - {}".format(from_date, to_date))
consumer = oauth2.Consumer(key=consumer_key, secret=consumer_secret)
# client = oauth2.Client(consumer)
# now get our request token
auth = OAuthFilter('*', consumer)
# initialize the page counter either at the first page or whatever page
# was requested
page_counter = first_page
finished = False
reply = dict()
reply['results'] = []
while not finished:
url = 'https://vulndb.cyberriskanalytics.com' + \
'/api/v1/vulnerabilities/find_by_date?' + \
'start_date=' + from_date + '&end_date=' + to_date + '&page=' + \
str(page_counter) + '&size=' + str(page_size) + \
'&date_type=updated_on' + \
'&nested=true'
logger.debug("Working on url: {} ".format(url))
resp = request(url, filters=[auth])
if resp.status_int == 404:
logger.warning("Could not find anything for the week " +
"begining: {}".format(from_date))
return
if resp.status_int != 200:
raise Exception("Invalid response {}.".format(resp['status']))
logger.debug("\tHTTP Response code: " + str(resp.status_int))
"""parse response and append to working set"""
page_reply = json.loads(resp.body_string())
logger.debug("Retrieving page {} of {}.".format(
page_counter, -(-page_reply['total_entries'] // page_size)))
if len(page_reply['results']) < page_size:
finished = True
reply['results'].extend(page_reply['results'])
reply['total_entries'] = page_reply['total_entries']
else:
page_counter += 1
reply['results'].extend(page_reply['results'])
logger.info("Returning {} out of {} results".format(
str(len(reply['results'])), str(reply['total_entries'])))
return reply
def rest_call(url, method, data, headers, client_auth_key, client_auth_secret):
consumer = oauth2.Consumer(key=client_auth_key, secret=client_auth_secret)
auth = OAuthFilter('*', consumer)
resource = Resource(url, filters=[auth])
response = resource.request(method,
payload=data,
headers=headers,
params=None)
json_string = response.body_string()
status = response.status
return status, json_string
def setup_oauth2(self):
# set up an OAuth Consumer
myconsumer = oauth2.Consumer(key=self.user.consumer_key,
secret=self.user.consumer_secret)
# manually update the access token/secret.
mytoken = oauth2.Token(key=self.user.oauth_token,
secret=self.user.oauth_token_secret)
# make an oauth request
self.auth = OAuthFilter('*',
consumer=myconsumer,
token=mytoken,
method=oauth2.SignatureMethod_HMAC_SHA1())
return self.auth
def __init__(self):
consumer = oauth.Consumer(key=settings.consumer_key,
secret=settings.consumer_secret)
token = oauth.Token(key=settings.token_key,
secret=settings.token_secret)
url = "http://api.twitter.com/1/"
auth = OAuthFilter('*', consumer, token)
mgr = TConnectionManager()
Resource.__init__(self,
url,
filters=[auth],
conn_manager=mgr,
client_opts={'timeout': 30})
self.remaining = 10000
def run():
o = OAuthFilter('*', self.consumer)
func(o, self.url, urllib.urlencode(self.body))
import oauth2, json
from pprint import pprint
import sys
import codecs
import locale
from datetime import date, timedelta
today = date.today()
yesterday = date.today() - timedelta(1)
sys.stdout = codecs.getwriter(locale.getpreferredencoding())(sys.stdout)
url = 'https://vulndb.cyberriskanalytics.com/api/v1/vulnerabilities/q?start_date=' + yesterday.strftime('%Y-%m-%d') + '&size=100'
consumer = oauth2.Consumer(key='CONSUMERKEY',secret='SECRET')
auth = OAuthFilter('*',consumer)
resp = request(url, filters=[auth])
print (resp.status_int)
if (resp.status_int != 200):
sys.exit(0)
data = json.loads(resp.body_string())
header = ["osvdb_id", "title", "keywords", "description", "solution", "vulndb_last_modified", "solution_date", "disclosure_date", "discovery_date", "exploit_publish_date", "vendor_informed_date", "vendor_ack_date", "third_party_solution_date",
"Nessus Script ID", "Snort Signature ID", "CVE ID", "Bugtraq ID", "Microsoft Security Bulletin", "Generic Exploit URL", "Vendor Specific Advisory URL",
"Vendor Specific Solution URL", "Keyword", "Generic Informational URL", "Related OSVDB ID", "ISS X-Force ID", "CERT", "CERT VU", "External Reference Data: RedHat RHSA",
"Secunia Advisory ID", "Vendor URL", "Microsoft Knowledge Base Article", "CIAC Advisory", "Other Advisory URL", "Other Solution URL", "US-CERT Cyber Security Alert",
"Mail List Post", "Security Tracker", "Packet Storm", "Vendor Specific News/Change Log Entry", "OVAL ID", "News Article", "VUPEN Advisory", "Nikto Item ID",
"Milw0rm", "Metasploit ID", "Exploit Database", "SCIP VulDB ID", "Tenable PVS", "DISA IAVA", "Immunity CANVAS(White Phosphorus)", "Immunity CANVAS",
"Immunity CANVAS(D2ExploitPack)", "Japan Vulnerability Notes", "vendor_0", "vendor_1", "vendor_2", "vendor_3", "product_0", "versions_0", "product_1", "versions_1", "product_2", "versions_2", "product_3", "versions_3", "score_0", "source_0", "generated_on_0", "access_vector_0" , "access_complexity_0", "authentication_0" , "confidentiality_impact_0" , "integrity_impact_0" , "availability_impact_0", "score_1", "source_1", "generated_on_1", "access_vector_1" ,"access_complexity_1", "authentication_1" , "confidentiality_impact_1" , "integrity_impact_1" , "availability_impact_1", "score_2", "source_2", "generated_on_2", "access_vector_2" , "access_complexity_2", "authentication_2" ,"confidentiality_impact_2" ,"integrity_impact_2" ,"availability_impact_2","score_3", "source_3", "generated_on_3", "access_vector_3" , "access_complexity_3", "authentication_3" , "confidentiality_impact_3" , "integrity_impact_3" , "availability_impact_3", "authors"]
filename = today.strftime('%Y-%m-%d') + '.txt'