def update_malware():
flag_splt = False
try:
response = urllib2.urlopen(paramserver + 'logreg_parameters.txt')
html = response.read()
tmp = {}
if html != None:
params = html.split()
params = map((lambda x: float(x)), params)
params = np.array(params)
np.savetxt('logreg_parameters.txt',params)
flag_splt = True
except:
flag_splt = False
flag_bd = False
try:
response = urllib2.urlopen(paramserver + 'logreg_parameters_bd.txt')
html = response.read()
tmp = {}
if html != None:
params = html.split()
params = map((lambda x: float(x)), params)
params = np.array(params)
np.savetxt('logreg_parameters_bd.txt',params)
flag_bd = True
except:
flag_bd = False
flags = {}
flags['malware_splt'] = flag_splt
flags['malware_bd'] = flag_bd
return template('admin',flags=flags)
def update_malware():
flag_splt = False
try:
response = urllib2.urlopen(paramserver + 'logreg_parameters.txt')
html = response.read()
tmp = {}
if html != None:
params = html.split()
params = map((lambda x: float(x)), params)
params = np.array(params)
np.savetxt('logreg_parameters.txt', params)
flag_splt = True
except:
flag_splt = False
flag_bd = False
try:
response = urllib2.urlopen(paramserver + 'logreg_parameters_bd.txt')
html = response.read()
tmp = {}
if html != None:
params = html.split()
params = map((lambda x: float(x)), params)
params = np.array(params)
np.savetxt('logreg_parameters_bd.txt', params)
flag_bd = True
except:
flag_bd = False
flags = {}
flags['malware_splt'] = flag_splt
flags['malware_bd'] = flag_bd
return template('admin', flags=flags)
def scrapeBBY(keywords, system):
# Best Buy requires an API key to use their web service
bbyApiKey = "crs2yyrur5c9trmtv4uggtd2"
# Request the Best Buy data
rawurl = "http://api.remix.bestbuy.com/v1/products(search=%s&categoryPath.name=video games&platform=%s&preowned in(true))?apiKey=%s" % (
keywords, system, bbyApiKey)
url = urllib.quote(rawurl, safe='/,?:=&()')
response = urllib2.urlopen(url)
html = response.read()
# Begin parsing the results
soup = bs4.BeautifulSoup("".join(html))
results = list()
for game in soup.findAll("product"):
rawName = game.find("name").text
try:
title = re.search(u'(.*)(?= \u2014)', rawName).group()
except AttributeError:
title = ""
if title != "":
title = re.sub("( 1)", " I", title)
title = re.sub("( 2)", " II", title)
title = re.sub("( 3)", " III", title)
title = re.sub(r'[^\w\s]', "", title)
title.strip
results.append(title)
return results
def sendMessage(accountName, registrationId, text):
global collapse_key
global server_account_name
authToken = getAuthToken()
if authToken == "":
return "Cannot authenticate " + server_account_name
form_fields = {
"registration_id": registrationId,
"collapse_key": str(collapse_key),
"data.message": text
}
form_data = urllib.urlencode(form_fields)
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
'Authorization': 'GoogleLogin auth=' + authToken
}
conn = httplib.HTTPSConnection("android.apis.google.com")
conn.request(method="POST",
url="/c2dm/send",
body=form_data,
headers=headers)
response = conn.getresponse()
print response
print response.status
data = response.read()
print data
collapse_key = collapse_key + 1
return data
def test_addapp(self):
"""test /addapp route"""
url = misc.URL + '/addapp'
response = urllib2.urlopen(url)
html = response.read()
self.assertEqual(response.getcode(), httplib.OK)
self.assertIn("Enter name of app", html)
def scrapeBBY(keywords, system):
# Best Buy requires an API key to use their web service
bbyApiKey = "crs2yyrur5c9trmtv4uggtd2"
# Request the Best Buy data
rawurl = "http://api.remix.bestbuy.com/v1/products(search=%s&categoryPath.name=video games&platform=%s&preowned in(true))?apiKey=%s" % (keywords, system, bbyApiKey)
url = urllib.quote(rawurl, safe='/,?:=&()')
response = urllib2.urlopen(url)
html = response.read()
# Begin parsing the results
soup = bs4.BeautifulSoup("".join(html))
results = list()
for game in soup.findAll("product"):
rawName = game.find("name").text
try:
title = re.search(u'(.*)(?= \u2014)', rawName).group()
except AttributeError:
title = ""
if title != "":
title = re.sub("( 1)", " I", title)
title = re.sub("( 2)", " II", title)
title = re.sub("( 3)", " III", title)
title = re.sub(r'[^\w\s]', "", title)
title.strip
results.append(title)
return results
def joke():
url = "https://08ad1pao69.execute-api.us-east-1.amazonaws.com/dev/random_joke"
response = urllib.urlopen(url)
data = json.loads(response.read())
setup = data["setup"]
punchline = data["punchline"]
return setup, punchline
def urllib2_rest_auth(url, method='GET', data=None, user=None, password=None):
import urllib2, base64, json
method = method.strip().upper()
data = None if method in ['GET','DELETE'] else data # just to ensure only valid request are sent
#httpHandler = urllib2.HTTPHandler()
#httpHandler.set_http_debuglevel(1)
#Instead of using urllib2.urlopen, create an opener, and pass the HTTPHandler
#and any other handlers... to it.
#opener = urllib2.build_opener(httpHandler)
#import urllib2
#urllib2.urlopen(url)
request = urllib2.Request(url, data=data)
request.get_method = lambda: method
if user and password:
base64string = base64.encodestring('%s:%s' % (user, password)).replace('\n', '')
request.add_header("Authorization", "Basic %s" % base64string)
urllib2._opener.handlers[1].set_http_debuglevel(100)
#urllib2.install_opener(urllib2.build_opener(urllib2.HTTPHandler(debuglevel=1)))
response = urllib2.urlopen(request)
data = response.read()
return response.getcode(), response.headers, data
def sendMessage( accountName, registrationId, text ):
global collapse_key
global server_account_name
authToken = getAuthToken()
if authToken == "":
return "Cannot authenticate " + server_account_name
form_fields = {
"registration_id": registrationId,
"collapse_key": str(collapse_key),
"data.message": text
}
form_data = urllib.urlencode(form_fields)
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
'Authorization': 'GoogleLogin auth='+authToken
}
conn = httplib.HTTPSConnection("android.apis.google.com")
conn.request(method="POST", url="/c2dm/send", body=form_data, headers=headers)
response = conn.getresponse()
print response
print response.status
data = response.read()
print data
collapse_key = collapse_key + 1
return data
def getJsonRequest(): credential = '[email protected]&password=qazwsxedcrfvtgb&query=' login_url = 'https://www.space-track.org/ajaxauth/login' print "REQUEST :" + request.body.read() json_request = json.loads(request.body.read()) if "controller" not in json_request: return {"error":"invalid: no controller"} if "action" not in json_request: return {"error":"invalid: no action"} if "class" not in json_request: return {"error":"invalid: no class"} result = getQueryUrlAndCacheFileName(json_request) query_url = result["url"] cache_file = result["cache"] if(os.path.exists(cache_file)): print "LOAD FROM CACHE" return open(cache_file) req = urllib2.Request(login_url) response = urllib2.urlopen(req, credential + query_url) json_response = json.loads(response.read()) print "LOAD FROM SERVER" print "RESULT: " + json.dumps(json_response) with open(cache_file, "w") as cache: json.dump(json_response,cache) return json.dumps(json_response)
def getJsonRequest():
credential = '[email protected]&password=qazwsxedcrfvtgb&query='
login_url = 'https://www.space-track.org/ajaxauth/login'
print "REQUEST :" + request.body.read()
json_request = json.loads(request.body.read())
if "controller" not in json_request:
return {"error": "invalid: no controller"}
if "action" not in json_request:
return {"error": "invalid: no action"}
if "class" not in json_request:
return {"error": "invalid: no class"}
result = getQueryUrlAndCacheFileName(json_request)
query_url = result["url"]
cache_file = result["cache"]
if (os.path.exists(cache_file)):
print "LOAD FROM CACHE"
return open(cache_file)
req = urllib2.Request(login_url)
response = urllib2.urlopen(req, credential + query_url)
json_response = json.loads(response.read())
print "LOAD FROM SERVER"
print "RESULT: " + json.dumps(json_response)
with open(cache_file, "w") as cache:
json.dump(json_response, cache)
return json.dumps(json_response)
def get_mp3_link(self, video_id):
'''get mp3 link from a youtube video id using youtube2mp3 service'''
response = urllib2.urlopen(YOUTUBE2_MP3_LINK1+video_id)
data = response.read()
#ignore this reponse and send the second request
response = urllib2.urlopen(YOUTUBE2_MP3_LINK2+video_id)
data = response.read()
result = re.search("[\w\d]{32}", data)
#the video was converted
if result is not None:
hash_value = result.group(0)
final_download_link = YOUTUBE2_MP3_LINK3 % (video_id, hash_value)
self.songIdCache[final_download_link] = video_id
self.songLinkCache[video_id] = final_download_link
return final_download_link
else:
return None
def send_getrequest(url):
request = urllib2.Request(url, None, {'Content-Type':'application/x-www-form-urlencoded; charset=utf-8'})
response = urllib2.urlopen(request)
data = response.read()
response.close()
return data.replace('\xef\xbb\xbf','')
def busqueda():
provincia = bottle.request.forms.get("provincia")
if len(provincia) > 1:
provincia = "province=%s" % provincia
else:
provincia = ""
categoria = bottle.request.forms.get("categoria")
if len(categoria) > 1:
categoria = "&subcategory=%s" % categoria
else:
categoria = ""
contratos = bottle.request.forms.get("contratos")
if len(contratos) > 1:
contratos = "&contractType=%s" % contratos
else:
contratos = ""
formacion = bottle.request.forms.get("formacion")
if len(formacion) > 1:
formacion = "&study=%s" % formacion
else:
formacion = ""
clave = 'ZDAyMjUyODYwZjQwNDQzYjhhZmI2OGFkMjYxMzdmM2M6RlRtbGQzUTRxMTY3djNlOFpZMnlaYXIvR1JsOEpnbTFRS2k0SVM3dk5mVFFrN0U2WVA='
headers = {"Authorization" : "Basic %s" % clave}
conn = httplib.HTTPConnection("api.infojobs.net")
conn.request("GET", "/api/1/offer?%s%s%s%s" % (provincia,categoria,contratos,formacion), headers=headers)
response = conn.getresponse()
data = response.read()
jsondata = json.loads(data)
conn.close()
listatitulos = []
listaciudad = []
listanombreempresa = []
listaexperiencia = []
listajornada = []
listaestudios = []
listarequisitosmin = []
listalink = []
ofertas = jsondata["offers"]
for oferta in ofertas:
titulo = oferta["title"]
ciudad = oferta["city"]
nombreempresa = oferta["author"]["name"]
experienciaminima = oferta["experienceMin"]["value"]
jornada = oferta["workDay"]["value"]
estudios = oferta["study"]["value"]
requisitosmin = oferta["requirementMin"]
link = oferta["link"]
listatitulos.append(titulo)
listaciudad.append(ciudad)
listanombreempresa.append(nombreempresa)
listaexperiencia.append(experienciaminima)
listajornada.append(jornada)
listaestudios.append(estudios)
listarequisitosmin.append(requisitosmin)
listalink.append(link)
return template('respuesta.tpl', {'ofertas':ofertas,'titulo':listatitulos,'ciudad':listaciudad,'nombreempresa':listanombreempresa,'experienciaminima':listaexperiencia,'jornada':listajornada,'estudios':listaestudios,'requisitosmin':listarequisitosmin,'link':listalink})
def auth(username, password):
"""Registration of user"""
url = "http://{}?action=user-kis&login={}&passwd={}".format(
auth_service, username, password)
req = urllib2.Request(url)
response = urllib2.urlopen(req)
d = json.loads(response.read())
if d['result'] == u'ok':
return True
else:
return False
def parse_url():
url = request.query.url
response = urlopen(url)
raw = response.read()
parsed_dict = utils.parse_markdown(raw)
return parsed_dict
def get_application_jwt(username,password):
try:
url = '%s%s' % (server_config["app_rest_base_url"], server_config["app_login_url"])
values = {'email' : username,'pass' : password}
req = urllib2.Request(url, json.dumps(values), headers={'Content-type': 'application/json', 'Accept': 'application/json'})
response = urllib2.urlopen(req)
token = response.read()
json_data = json.loads(token)
return json_data["token"]
except urllib2.URLError:
print 'URL Invalid Cant Connect to Backend...'
sys.exit()
def update_all_devices():
req = urllib.request.Request(WIFI_PAGE)
with urllib.request.urlopen(req) as response:
the_page = response.read().decode(encoding="UTF-8")
# print(the_page)
data = json.loads(the_page)
print("READ NEW PAGE WITH", len(data['features']), "PHONES")
for dev in data['features']:
prop = dev["properties"]
id = prop["name"]
# if name in devices:
# oldp = devices[name]
# update(oldp, prop)
devices[id] = prop
def get_xml(request):
req = urllib.request.Request(request)
try:
with urllib.request.urlopen(req) as response:
data = response.read()
except urllib.error.URLError as err:
print ("Couldn't reach the Daily Mail's RSS feed")
print(err.reason)
except urllib.error.HTTPError as err:
print("The server failed to complete the request with error ")
print(err.code)
xmlData = xmltodict.parse(data)
return xmlData
def compare(remoteurl):
import json
compareurl = remoteurl + "/api/info"
response = urllib.request.urlopen(compareurl)
strangerinfo = json.loads(response.read())
owninfo = info()
#add_known_server(compareto)
artists = {}
for a in owninfo["artists"]:
artists[a.lower()] = {"name":a,"self":int(owninfo["artists"][a]*1000),"other":0}
for a in strangerinfo["artists"]:
artists[a.lower()] = artists.setdefault(a.lower(),{"name":a,"self":0})
artists[a.lower()]["other"] = int(strangerinfo["artists"][a]*1000)
for a in artists:
common = min(artists[a]["self"],artists[a]["other"])
artists[a]["self"] -= common
artists[a]["other"] -= common
artists[a]["common"] = common
best = sorted((artists[a]["name"] for a in artists),key=lambda x: artists[x.lower()]["common"],reverse=True)
result = {
"unique_self":sum(artists[a]["self"] for a in artists if artists[a]["common"] == 0),
"more_self":sum(artists[a]["self"] for a in artists if artists[a]["common"] != 0),
"common":sum(artists[a]["common"] for a in artists),
"more_other":sum(artists[a]["other"] for a in artists if artists[a]["common"] != 0),
"unique_other":sum(artists[a]["other"] for a in artists if artists[a]["common"] == 0)
}
total = sum(result[c] for c in result)
for r in result:
result[r] = (result[r],result[r]/total)
return {
"result":result,
"info":{
"ownname":owninfo["name"],
"remotename":strangerinfo["name"]
},
"commonartist":best[0]
}
def get_jsonparsed_data(url):
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"}
req = urllib.request.Request(url, headers=headers)
try:
with urllib.request.urlopen(req) as response:
data = response.read()
except urllib.error.HTTPError as err:
print("The server failed to complete the request with error ")
print(err.code)
except urllib.error.URLError as err:
print('We failed to reach the server.')
print('Reason: ', err.reason)
returnedJson = json.loads(data)
return returnedJson
def get_login_role(username):
global auth_token
global server_config
try:
url = '%s%s?email=eq.%s' % (server_config['app_rest_base_url'],server_config['app_ldap_db_user_table_url'],username)
req = urllib2.Request(url=url)
req.add_header('Authorization','Bearer ' + auth_token)
response = urllib2.urlopen(req)
role_ = response.read()
json_data = json.loads(role_)
return json_data[0]['role']
except urllib2.HTTPError, e:
print 'HTTPError = ' + str(e.code)
def get_youtube_data(video_id):
try:
base_url = 'http://gdata.youtube.com/feeds/api/videos/'
youtube_url = base_url + video_id
response = urllib2.urlopen(youtube_url)
xml = BeautifulSoup(response.read())
description = xml.findAll('media:description')[0].string
title = xml.findAll('media:title')[0].string
try:
duration = int(xml.findAll('yt:duration')[0].split('"')[1])
except:
duration = 0
return {
'description':description,
'title':title,
'duration':duration
}
except:
raise
def make_gcm_summary(data, response):
"""
Helper function to display the result of a /send request.
"""
json_string = response.read()
json_response = json.loads(json_string)
html = """
<html>
<head>
<title>GCM send result</title>
</head>
<body>
<h2>Request</h2>
<pre>%s</pre>
<h2>Response</h2>
<pre>%s</pre>
<h3>Per device</h3>
<ol>""" % (repr(data), json_string)
reg_id_list = data['registration_ids']
for i in xrange(len(reg_id_list)):
reg_id = reg_id_list[i]
result = json_response['results'][i]
html += """
<li>
reg_id: <code>%s</code><br/>
<pre>%s</pre>
</li>""" % (reg_id, json.dumps(result))
html += """
</ol>
<a href="/">Back</a>
</body>
</html>"""
return html
def processStatus(self, status):
log("Received message <{0}> from @{1}".format(status.text, status.author.screen_name), self.twitterAPI.api.me().screen_name)
# Detect file in tweet
if "media" in status.entities:
req = urlopen(status.entities["media"][0]["media_url_https"])
arr = np.asarray(bytearray(req.read()), dtype=np.uint8)
img = cv2.imdecode(arr, -1) # 'load it as it is'
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
faces = self.faceCascade.detectMultiScale(
gray,
scaleFactor=1.1,
minNeighbors=5,
minSize=(30, 30)
# flags = cv2.CV_HAAR_SCALE_IMAGE
)
req.close()
temp = tempfile.TemporaryFile()
if len(faces) >= 1:
req = urlopen(status.entities["media"][0]["media_url_https"])
with Image(file=req) as bg_img:
with Image(filename='asunaxan.png') as waifu:
bg_img.composite(waifu, left=faces[0][0]+40, top=faces[0][1]+20)
temp = tempfile.TemporaryFile()
temp.write(bg_img.make_blob(format="jpeg"))
temp.seek(0)
self.twitterAPI.api.update_with_media("Example.jpg", status="@{0} ojalá tú y yo juntos bb".format(self.boyfriendNEW.twitter.screen_name), file=temp, in_reply_to_status_id=status.id)
temp.close()
# draw(w)
display(bg_img)
req.close()
# Check for mention
elif "@"+self.me.upper() in status.text.upper():
if status.author.id == self.boyfriendNEW.twitter.id:
if "¿Qué temperatura hay?" in status.text and (self.forecast.limit - self.forecast.current) > 0:
message = "Ahora mismo hay {0}ºC en tu zona".format(self.forecast.getForecast(self.boyfriendNEW.city).currently().temperature)
self.answer(message, status, status.user.screen_name)
elif "holi" in status.text.lower():
message = "Holaaaa mi amooor {0}".format(self.getRandomWord(self.happyExpressions))
self.answer(message, status, status.user.screen_name)
elif "quien es tu senpai" in status.text.lower():
message = "Tu eres mi senpai {0}".format(self.getRandomWord(self.happyExpressions))
self.answer(message, status, status.user.screen_name)
elif "donde estoy" in status.text.lower():
if status.place != None:
place = status.place.full_name
message = "Ahora mismo te encuentras en {0}".format(place)
else:
message = "Lo siento, pero no dispongo de datos de geolocalizacion"
self.answer(message, status, status.user.screen_name)
elif "recomiendame un anime" in status.text.lower():
log("Buscando un anime", self.me)
params = urlencode({'client': "watashiwaifu", 'clientver': 1, 'protover': 1, "request": "hotanime"})
url = "http://api.anidb.net:9001/httpapi?%s" % params
request = Request(url, None, FAKE_HEADERS)
with contextlib.closing(urlopen(request)) as response:
if response.info().get('Content-Encoding') == 'gzip':
buf = BytesIO(response.read())
f = gzip.GzipFile(fileobj=buf)
data = f.read()
else:
data = response.read()
root = ElementTree.XML(data)
animes = root.findall("anime")
rand = random.randint(0, len(animes)-1)
anime = animes[rand]
rank = ""
name = ""
for title in anime.findall("title"):
name = title.text
for picture in anime.findall("picture"):
photo = "http://img7.anidb.net/pics/anime/" + picture.text
portada = urlopen(photo).read()
for rating in anime.findall("ratings"):
for temporary in rating.findall("temporary"):
rank = temporary.text
temp = tempfile.TemporaryFile()
temp.write(portada)
temp.seek(0)
self.twitterAPI.api.update_with_media("Example.jpg", status="@{0} {1} {2}/10".format(status.author.screen_name, name, rank), file=temp, in_reply_to_status_id=status.id)
temp.close()
else:
message = "Lo siento, no entiendo lo que me pides {0}".format(self.getRandomWord(self.sadExpressions))
self.answer(message, status, status.user.screen_name)
else:
message = "¿Tú qué haces hablandome? {0}".format(self.getRandomWord(self.angryExpressions))
self.answer(message, status, status.user.screen_name)
for task in subgroup.imap(get_swift_containers,threads):
list.extend(task)
print list
return list
def get_swift_containers(arg_list):
"""
Get a list of containers for a specific object-store service.
"""
token,tenant_id,region,url = arg_list
request = urllib2.Request(url,None, {'X-Auth-Token':token})
try:
response = urllib2.urlopen(request)
except urllib2.HTTPError, e:
return []
file_list = response.read().splitlines()
print file_list
list = []
list.append({"name":region,"value":""})
for file in file_list:
list.append({"name":file,"value":file})
return list
def get_all_nova_element_list(token,tenant_id,service_catalog,sub_function):
"""
Get a list of items from every compute service the user has access to.
"""
list = []
threads = []
subgroup = Group()
def fetchHtml(url):
print "Open ", url, "\n\n"
response = urllib2.urlopen(url)
html = response.read()
print html
def locateBBY(title, system, zip, root):
# Best Buy requires an API key to use their web service
bbyApiKey = "crs2yyrur5c9trmtv4uggtd2"
# Request the Best Buy data
rawurl = "http://api.remix.bestbuy.com/v1/products(search=%s&categoryPath.name=video games&platform=%s&preowned in(true))+stores(area(%s,10))?apiKey=%s" % (title, system, zip, bbyApiKey)
url = urllib.quote(rawurl, safe='/,?:=&()')
response = urllib2.urlopen(url)
html = response.read()
# Begin parsing the results
soup = bs4.BeautifulSoup("".join(html))
results = list()
for game in soup.findAll("product"):
rawName = game.find("name").text
try:
titleOpt = re.search(u'(.*)(?= \u2014)', rawName).group()
except AttributeError:
titleOpt = ""
if titleOpt != "":
titleOpt = re.sub("( 1)", " I", titleOpt)
titleOpt = re.sub("( 2)", " II", titleOpt)
titleOpt = re.sub("( 3)", " III", titleOpt)
titleOpt = re.sub(r'[^\w\s]', "", titleOpt)
titleOpt = str(titleOpt)
titleOpt = titleOpt.strip()
titleOpt = titleOpt.lower()
price = "Unknown"
if titleOpt == title:
# grab onSale. if text is true, grab salePrice else grab regularPrice
if game.find("onsale").text == "true":
price = game.find("saleprice").text
else:
price = game.find("regularprice").text
# grab every store in stores
for store in game.findAll("store"):
# grab longName, address, city, region, postalCode, lat, lng, phone
# and add everything to the tree
locElem = etree.Element('location')
storeElem = etree.Element('store')
storeElem.text = 'BestBuy'
locElem.append(storeElem)
nameElem = etree.Element('name')
nameElem.text = store.find("longname").text
locElem.append(nameElem)
add1Elem = etree.Element('address1')
add1Elem.text = store.find("address").text
locElem.append(add1Elem)
cityElem = etree.Element('city')
cityElem.text = store.find("city").text
locElem.append(cityElem)
stateElem = etree.Element('state')
stateElem.text = store.find("region").text
locElem.append(stateElem)
zipElem = etree.Element('zip')
zipElem.text = store.find("postalcode").text
locElem.append(zipElem)
phoneElem = etree.Element('phone')
phoneElem.text = store.find("phone").text
locElem.append(phoneElem)
latElem = etree.Element('lat')
latElem.text = store.find("lat").text
locElem.append(latElem)
longElem = etree.Element('long')
longElem.text = store.find("lng").text
locElem.append(longElem)
priceElem = etree.Element('price')
priceElem.text = price
locElem.append(priceElem)
root.append(locElem)
return root
return root
def get_all_regions_from_mongo(mongodb, mongodbOld):
regions_entity = {
"_links": {
"self": {
"href": "/monitoring/regions"
}
},
"_embedded": {
"regions": [
]
},
"basicUsers": 0,
"trialUsers": 0,
"communityUsers": 0,
"totalUsers": 0,
"total_nb_users": 0,
"totalCloudOrganizations": 0,
"totalUserOrganizations": 0,
"total_nb_organizations": 0,
#
"total_nb_cores": 0,
"total_nb_cores_enabled": 0,
"total_nb_ram": 0,
"total_nb_disk": 0,
"total_nb_vm": 0,
"total_ip_assigned": 0,
"total_ip_allocated": 0,
"total_ip": 0
}
pool = Pool(processes=1)
async_result = pool.apply_async(get_all_regions_from_js, ()) # Start thread for async http call
new_regions = app.config["main_config"]["regionNew"]
region_list = {}
for region_id, is_new in new_regions.iteritems():
if str2bool(is_new):
region = get_region_from_mongo(mongodb, region_id)
if region is not None:
region_list[region_id] = region
elif not str2bool(is_new):
response = make_request("/monitoring/regions/" + region_id, request=None, regionid=region_id)
if response.getcode() == 200:
region_list[region_id] = json.loads(response.read())
for region in region_list.iteritems():
region = region[1]
region_item = {"id": {}, "_links": {"self": {"href": {}}}}
region_item["id"] = region["id"]
region_item["_links"]["self"]["href"] = "/monitoring/regions/" + region["id"]
regions_entity["_embedded"]["regions"].append(copy.deepcopy(region_item))
# sum resources form each region entity
if region["nb_cores"] != '':
regions_entity["total_nb_cores"] += int(region["nb_cores"])
regions_entity["total_nb_cores_enabled"] += int(region["nb_cores"])
if region["nb_ram"] != '':
regions_entity["total_nb_ram"] += int(region["nb_ram"])
if region["nb_disk"] != '':
regions_entity["total_nb_disk"] += int(region["nb_disk"])
if region["nb_vm"] != '':
regions_entity["total_nb_vm"] += int(region["nb_vm"])
if region["measures"][0]["ipAssigned"] != '':
regions_entity["total_ip_assigned"] += int(decimal.Decimal(region["measures"][0]["ipAssigned"]).normalize())
if region["measures"][0]["ipAllocated"] != '':
regions_entity["total_ip_allocated"] += int(
decimal.Decimal(region["measures"][0]["ipAllocated"]).normalize())
if region["measures"][0]["ipTot"] != '':
regions_entity["total_ip"] += int(decimal.Decimal(region["measures"][0]["ipTot"]).normalize())
# get IDM infos from oldMonitoring
try:
regions_tmp = async_result.get(10) # get the return value from thread
regions_entity["basicUsers"] = regions_tmp["basicUsers"]
regions_entity["trialUsers"] = regions_tmp["trialUsers"]
regions_entity["communityUsers"] = regions_tmp["communityUsers"]
regions_entity["totalUsers"] = regions_tmp["totalUsers"]
regions_entity["total_nb_users"] = regions_tmp["total_nb_users"]
regions_entity["totalCloudOrganizations"] = regions_tmp["totalCloudOrganizations"]
regions_entity["totalUserOrganizations"] = regions_tmp["totalUserOrganizations"]
regions_entity["total_nb_organizations"] = regions_tmp["total_nb_organizations"]
except TimeoutError:
print("HTTP call to JS monitoringAPI to retrieve IDM info did not respond in 10 seconds. No IDM data returned")
finally:
pool.close()
pool.join()
return regions_entity
def scrapeGS(keywords, system):
# Get the code for the system we're searching
# Also why doesn't Python have switch statements sadface
if system == "xbox 360":
system = "1385"
sysPrefix = "/xbox-360"
elif system == "playstation 3":
system = "138d"
sysPrefix = "/playstation-3"
elif system == "nintendo wii u":
system = "131b0"
sysPrefix = "/nintendo-wii-u"
elif system == "nintendo wii":
system = "138a"
sysPrefix = "/nintendo-wii"
elif system == "nintendo 3ds":
system = "131a2"
sysPrefix = "/nintendo-3ds"
elif system == "nintendo ds":
system = "1386"
sysPrefix = "/nintendo-ds"
elif system == "ps vita":
system = "131af"
sysPrefix = "/ps-vita"
else:
system = ""
sysPrefix = ""
condition = "-50" # For now we are only doing Used games
# Get the code for used/new/both (both will have no code)
#if condition == "Used":
# condition = "-50"
#elif condition == "New":
# condition = "-4f"
#else:
# condition = ""
# Add keywords the the gamestop url
rawurl = 'http://www.gamestop.com/browse%s?nav=16k-3-%s,28zu0,%s%s' % (sysPrefix, keywords, system, condition)
# Ensure that we don't encode the commas
url = urllib.quote(rawurl, safe='/,?:=&')
# Request the gamestop page
response = urllib2.urlopen(url)
html = response.read()
# Begin parsing the results
soup = bs4.BeautifulSoup("".join(html))
# Get the viewstate
vsTag = soup.find("input", attrs={"name":"__VIEWSTATE"})
viewstate = vsTag['value']
# Get the sections containg each game and locate the ID for each
# which we will need when checking the stores later
# For each ID, add to a dict mapping the game's title to the ID
results = list()
games = soup.findAll("div", attrs={"class":"product preowned_product"}) # This will need to be changed to handle new products when conditions are introduced
for game in games:
title = game.find("a", attrs={"id":re.compile("hypTitle")}).text
title = re.sub("( 1)", " I", title)
title = re.sub("( 2)", " II", title)
title = re.sub("( 3)", " III", title)
title = re.sub(r'[^\w\s]', "", title)
title = str(title)
title = title.strip()
results.append(title)
return results
# Function to get json from a URL
def get_jsonparsed_data(url):
try:
response = urlopen(url)
except HTTPError, err:
if err.code == 404:
return "Page not found!"
elif err.code == 403:
return "Access denied!"
else:
return "Something happened! Error code", err.code
except URLError, err:
return "Some other error happened:", err.reason
data = str(response.read())
return json.loads(data);
# URL for LivefyreAPI
urlHottest = "https://bskyb.bootstrap.fyre.co/api/v3.0/hottest/?site=360818&number=10"
#Maximum number of tries
done = 0
maxTries = 5000
while done < maxTries:
# Get the 'hottest' stories for comments
jsonDataStories = get_jsonparsed_data(urlHottest)
list.extend(task)
print list
return list
def get_swift_containers(arg_list):
"""
Get a list of containers for a specific object-store service.
"""
token, tenant_id, region, url = arg_list
request = urllib2.Request(url, None, {'X-Auth-Token': token})
try:
response = urllib2.urlopen(request)
except urllib2.HTTPError, e:
return []
file_list = response.read().splitlines()
print file_list
list = []
list.append({"name": region, "value": ""})
for file in file_list:
list.append({"name": file, "value": file})
return list
def get_all_nova_element_list(token, tenant_id, service_catalog, sub_function):
"""
Get a list of items from every compute service the user has access to.
"""
list = []
threads = []
subgroup = Group()
def locateGS(title, system, zip, root):
# Get the code for the system we're searching
# Also why doesn't Python have switch statements sadface
if system == "xbox 360":
system = "1385"
sysPrefix = "/xbox-360"
elif system == "playstation 3":
system = "138d"
sysPrefix = "/playstation-3"
elif system == "nintendo wii u":
system = "131b0"
sysPrefix = "/nintendo-wii-u"
elif system == "nintendo wii":
system = "138a"
sysPrefix = "/nintendo-wii"
elif system == "nintendo 3ds":
system = "131a2"
sysPrefix = "/nintendo-3ds"
elif system == "nintendo ds":
system = "1386"
sysPrefix = "/nintendo-ds"
elif system == "ps vita":
system = "131af"
sysPrefix = "/ps-vita"
else:
system = ""
sysPrefix = ""
condition = "-50" # For now we are only doing Used games
# Get the code for used/new/both (both will have no code)
#if condition == "Used":
# condition = "-50"
#elif condition == "New":
# condition = "-4f"
#else:
# condition = ""
# Let it be known that I am in too much of a hurry right now to carefully
# parse out the PostBack, so I am just setting it manually here. It may
# become stale if GS changes their site in the future.
eventtarget = "ctl00$mainContentPlaceHolder$dynamicContent$ctl00$RepeaterResultFoundTemplate$ctl01$ResultFoundPlaceHolder$ctl00$ctl00$ctl03$StandardPlaceHolderTop$ctl00$rptResults$ctl00$res$btnPickupInStore"
# Get the viewstate, gameID, and price
url = "http://www.gamestop.com/browse%s?nav=16k-3-%s,28zu0,%s%s" % (
sysPrefix, title, system, condition)
url = urllib.quote(url, safe='/,?:=&()')
# Request the gamestop page
response = urllib2.urlopen(url)
html = response.read()
# Begin parsing the results
soup = bs4.BeautifulSoup("".join(html))
# Get the viewstate
vsTag = soup.find("input", attrs={"name": "__VIEWSTATE"})
if not vsTag:
return root
viewstate = vsTag['value']
games = soup.findAll("div", attrs={"class": "product preowned_product"})
gameID = 0
price = "Unknown"
for gameOpt in games:
titleOpt = gameOpt.find("a", attrs={"id": re.compile("hypTitle")}).text
titleOpt = re.sub("( 1)", " I", titleOpt)
titleOpt = re.sub("( 2)", " II", titleOpt)
titleOpt = re.sub("( 3)", " III", titleOpt)
titleOpt = re.sub(r'[^\w\s]', "", titleOpt)
titleOpt = str(titleOpt)
titleOpt = titleOpt.strip()
titleOpt = titleOpt.lower()
if titleOpt == title:
gameIDTag = gameOpt.find(
"a", attrs={"onclick": re.compile("(WishListItemAdded)(.*)")})
gameIDTagValue = gameIDTag['onclick']
matches = re.findall("(?<=')([0-9]*?)(?=')", gameIDTagValue)
i = 0
for gameOptID in matches:
if i == 1:
gameID = gameOptID
i = i + 1
price = gameOpt.find("p", attrs={"class": "pricing"}).text
break
# Now that we have viewstate, do a POST instead of a GET so that we
# can get the full VIEWSTATE, which does not appear to be title nor
# request/session specific
data = urllib.urlencode({
"__EVENTTARGET": eventtarget,
"__EVENTARGUMENT": "",
"__LASTFOCUS": "",
"__VIEWSTATE": viewstate,
"header": "$ctl00$",
"searchtext": ""
})
req = urllib2.Request(url)
req.add_data(data)
response = urllib2.urlopen(req)
html = response.read()
# Begin parsing the results
soup = bs4.BeautifulSoup("".join(html))
# Get the viewstate
vsTag = soup.find("input", attrs={"name": "__VIEWSTATE"})
if not vsTag:
return root
viewstate = vsTag['value']
# Now that we have the full viewstate, do a POST to the store search
# Normally we'd need to parse out the other form fields from the response
# above, but right now I am in too much of a hurry to be so thorough and
# careful. Instead I am just assigning them manually. They may become stale
# if GS changes their site in the future.
eventtarget = "ctl00$ctl00$BaseContentPlaceHolder$mainContentPlaceHolder$StoreSearchControl$FindZipButton"
scriptField = "ctl00$ctl00$ScriptManager1" # Or should this be encoded? Does urllib encode the fields as well as the values?
scriptValue = "ctl00$ctl00$ScriptManager1|ctl00$ctl00$BaseContentPlaceHolder$mainContentPlaceHolder$StoreSearchControl$FindZipButton"
searchTextField = "ctl00$ctl00$BaseContentPlaceHolder$cHeader$ctl00$searchtext"
zipField = "ctl00$ctl00$BaseContentPlaceHolder$mainContentPlaceHolder$StoreSearchControl$EnterZipTextBox"
storeSavedField = "ctl00$ctl00$BaseContentPlaceHolder$mainContentPlaceHolder$StoreSearchControl$StoreSavedModalPopup$PopupTargetControl"
noStoresField = "ctl00$ctl00$BaseContentPlaceHolder$mainContentPlaceHolder$StoreSearchControl$NoStoresFoundModalPopup$PopupTargetControl"
url = "http://www.gamestop.com/Browse/StoreSearch.aspx?sku=%s" % gameID
data = urllib.urlencode({
scriptField: scriptValue,
searchTextField: "",
zipField: zip,
"__EVENTTARGET": eventtarget,
"__EVENTARGUMENT": "",
"__LASTFOCUS": "",
"__VIEWSTATE": viewstate,
storeSavedField: "",
noStoresField: "",
"ASYNCPOST": "true&"
})
req = urllib2.Request(url)
req.add_data(data)
response = urllib2.urlopen(req)
html = response.read()
# Begin parsing the results
soup = bs4.BeautifulSoup("".join(html))
stores = dict()
markers = re.findall(re.compile('setMarker(.*)'), html)
for marker in markers:
latlongs = re.findall(re.compile('(-{0,}[0-9]{1,}\.[0-9]{1,})'),
marker)
storeInfos = re.findall(re.compile('\'(.+?)\''), marker)
stores[storeInfos[1]] = (storeInfos[0], latlongs[0], latlongs[1])
# Example: 6212 -> ( Willow Lawn , 37.582845 , -77.498765 )
add1RegEx = re.compile('.*Address1.*')
cityRegex = re.compile('.*CityLabel')
stateRegex = re.compile('.*StateLabel')
zipRegex = re.compile('.*ZipLabel')
phoneRegex = re.compile('.*PhoneLabel')
storeTable = soup.find(class_="map_results", id="searchResults")
for key in stores:
info = storeTable.find("tr", id=key)
if info:
locElem = etree.Element('location')
storeElem = etree.Element('store')
storeElem.text = 'GameStop'
locElem.append(storeElem)
nameElem = etree.Element('name')
nameElem.text = stores[key][0]
locElem.append(nameElem)
add1Elem = etree.Element('address1')
add1 = info.find("span", id=add1RegEx)
add1Elem.text = add1.text
locElem.append(add1Elem)
cityElem = etree.Element('city')
city = info.find("span", id=cityRegex)
cityElem.text = city.text
locElem.append(cityElem)
stateElem = etree.Element('state')
state = info.find("span", id=stateRegex)
stateElem.text = state.text
locElem.append(stateElem)
zipElem = etree.Element('zip')
zip = info.find("span", id=zipRegex)
zipElem.text = zip.text
locElem.append(zipElem)
phoneElem = etree.Element('phone')
phone = info.find("span", id=phoneRegex)
phoneElem.text = phone.text
locElem.append(phoneElem)
latElem = etree.Element('lat')
latElem.text = stores[key][1]
locElem.append(latElem)
longElem = etree.Element('long')
longElem.text = stores[key][2]
locElem.append(longElem)
priceElem = etree.Element('price')
priceElem.text = price
locElem.append(priceElem)
root.append(locElem)
return root
from bottle import route, run, request, response
import urllib2
import shutil
import os
import sys
response = urllib2.urlopen('https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-gambling-p**n-social/hosts')
data = response.read()
filename = "hosts"
hostfile= open(filename, 'w')
hostfile.write(data)
hostfile.close()
response.close()
@route('/hello')
def hello():
blocktheshit()
def blocktheshit():
os.system ("copy %s %s" % ('\windows\system32\drivers\etc\hosts', 'C:\AdBlock\hosts_backup'))
os.system ("copy %s %s" % ('C:\AdBlock\hosts', '\windows\system32\drivers\etc'))
count()
@route('/unblock')
def unblock():
os.system("echo.")
os.system("echo Working on it....")
os.system("timeout /t 45 /nobreak")
os.system ("del /f \windows\system32\drivers\etc\hosts")
os.system ("copy %s %s" % ('C:\AdBlock\hosts_backup\orignal\hosts', '\windows\system32\drivers\etc'))
count()
@route('/count')
def count():
count = 0
with open('\windows\system32\drivers\etc\hosts') as fileObj:
import boto
import boto.dynamodb
import urllib2
import csv
from boto.dynamodb.condition import *
import bottle
from bottle import route, request, response, template, get, run, HTTPResponse
import json
#bottle = Bottle()
#establish connection to dynamodb
conn = boto.dynamodb.connect_to_region('us-west-2')
response = urllib2.urlopen('http://www.fhfa.gov/DataTools/Downloads/Documents/HPI/HPI_master.csv')
#Read the file data
myfile = response.read()
#Create a file named myfile.csv
filename = open('myfile.csv','w')
#write previously fetched filedata into newly created file
filename.write(myfile)
#Close the file
filename.close()
#function to create table
def create_table():
try:
#create schema with hash key as 'id'
myschema=conn.create_schema(hash_key_name='id',hash_key_proto_value=str)
#create table using schema
table=conn.create_table(name='data', schema=myschema, read_units=1, write_units=1)
print "Table created successfully"
def fetch_url(query):
req = urllib2.Request(query)
response = urllib2.urlopen(req)
return response.read()
def locateGS(title, system, zip, root):
# Get the code for the system we're searching
# Also why doesn't Python have switch statements sadface
if system == "xbox 360":
system = "1385"
sysPrefix = "/xbox-360"
elif system == "playstation 3":
system = "138d"
sysPrefix = "/playstation-3"
elif system == "nintendo wii u":
system = "131b0"
sysPrefix = "/nintendo-wii-u"
elif system == "nintendo wii":
system = "138a"
sysPrefix = "/nintendo-wii"
elif system == "nintendo 3ds":
system = "131a2"
sysPrefix = "/nintendo-3ds"
elif system == "nintendo ds":
system = "1386"
sysPrefix = "/nintendo-ds"
elif system == "ps vita":
system = "131af"
sysPrefix = "/ps-vita"
else:
system = ""
sysPrefix = ""
condition = "-50" # For now we are only doing Used games
# Get the code for used/new/both (both will have no code)
#if condition == "Used":
# condition = "-50"
#elif condition == "New":
# condition = "-4f"
#else:
# condition = ""
# Let it be known that I am in too much of a hurry right now to carefully
# parse out the PostBack, so I am just setting it manually here. It may
# become stale if GS changes their site in the future.
eventtarget = "ctl00$mainContentPlaceHolder$dynamicContent$ctl00$RepeaterResultFoundTemplate$ctl01$ResultFoundPlaceHolder$ctl00$ctl00$ctl03$StandardPlaceHolderTop$ctl00$rptResults$ctl00$res$btnPickupInStore"
# Get the viewstate, gameID, and price
url ="http://www.gamestop.com/browse%s?nav=16k-3-%s,28zu0,%s%s" % (sysPrefix, title, system, condition)
url = urllib.quote(url, safe='/,?:=&()')
# Request the gamestop page
response = urllib2.urlopen(url)
html = response.read()
# Begin parsing the results
soup = bs4.BeautifulSoup("".join(html))
# Get the viewstate
vsTag = soup.find("input", attrs={"name":"__VIEWSTATE"})
if not vsTag:
return root
viewstate = vsTag['value']
games = soup.findAll("div", attrs={"class":"product preowned_product"})
gameID = 0
price = "Unknown"
for gameOpt in games:
titleOpt = gameOpt.find("a", attrs={"id":re.compile("hypTitle")}).text
titleOpt = re.sub("( 1)", " I", titleOpt)
titleOpt = re.sub("( 2)", " II", titleOpt)
titleOpt = re.sub("( 3)", " III", titleOpt)
titleOpt = re.sub(r'[^\w\s]', "", titleOpt)
titleOpt = str(titleOpt)
titleOpt = titleOpt.strip()
titleOpt = titleOpt.lower()
if titleOpt == title:
gameIDTag = gameOpt.find("a", attrs={"onclick":re.compile("(WishListItemAdded)(.*)")})
gameIDTagValue = gameIDTag['onclick']
matches = re.findall("(?<=')([0-9]*?)(?=')", gameIDTagValue)
i = 0
for gameOptID in matches:
if i == 1:
gameID = gameOptID
i = i + 1
price = gameOpt.find("p", attrs={"class":"pricing"}).text
break
# Now that we have viewstate, do a POST instead of a GET so that we
# can get the full VIEWSTATE, which does not appear to be title nor
# request/session specific
data = urllib.urlencode({"__EVENTTARGET":eventtarget, "__EVENTARGUMENT":"", "__LASTFOCUS":"", "__VIEWSTATE":viewstate, "header":"$ctl00$", "searchtext":""})
req = urllib2.Request(url)
req.add_data(data)
response = urllib2.urlopen(req)
html = response.read()
# Begin parsing the results
soup = bs4.BeautifulSoup("".join(html))
# Get the viewstate
vsTag = soup.find("input", attrs={"name":"__VIEWSTATE"})
if not vsTag:
return root
viewstate = vsTag['value']
# Now that we have the full viewstate, do a POST to the store search
# Normally we'd need to parse out the other form fields from the response
# above, but right now I am in too much of a hurry to be so thorough and
# careful. Instead I am just assigning them manually. They may become stale
# if GS changes their site in the future.
eventtarget = "ctl00$ctl00$BaseContentPlaceHolder$mainContentPlaceHolder$StoreSearchControl$FindZipButton"
scriptField = "ctl00$ctl00$ScriptManager1" # Or should this be encoded? Does urllib encode the fields as well as the values?
scriptValue = "ctl00$ctl00$ScriptManager1|ctl00$ctl00$BaseContentPlaceHolder$mainContentPlaceHolder$StoreSearchControl$FindZipButton"
searchTextField = "ctl00$ctl00$BaseContentPlaceHolder$cHeader$ctl00$searchtext"
zipField = "ctl00$ctl00$BaseContentPlaceHolder$mainContentPlaceHolder$StoreSearchControl$EnterZipTextBox"
storeSavedField = "ctl00$ctl00$BaseContentPlaceHolder$mainContentPlaceHolder$StoreSearchControl$StoreSavedModalPopup$PopupTargetControl"
noStoresField = "ctl00$ctl00$BaseContentPlaceHolder$mainContentPlaceHolder$StoreSearchControl$NoStoresFoundModalPopup$PopupTargetControl"
url = "http://www.gamestop.com/Browse/StoreSearch.aspx?sku=%s" % gameID
data = urllib.urlencode(
{ scriptField:scriptValue, searchTextField:"", zipField:zip, "__EVENTTARGET":eventtarget, "__EVENTARGUMENT":"",
"__LASTFOCUS":"", "__VIEWSTATE":viewstate, storeSavedField:"", noStoresField:"", "ASYNCPOST":"true&" })
req = urllib2.Request(url)
req.add_data(data)
response = urllib2.urlopen(req)
html = response.read()
# Begin parsing the results
soup = bs4.BeautifulSoup("".join(html))
stores = dict()
markers = re.findall(re.compile('setMarker(.*)'), html)
for marker in markers:
latlongs = re.findall(re.compile('(-{0,}[0-9]{1,}\.[0-9]{1,})'), marker)
storeInfos = re.findall(re.compile('\'(.+?)\''), marker)
stores[storeInfos[1]] = (storeInfos[0], latlongs[0], latlongs[1])
# Example: 6212 -> ( Willow Lawn , 37.582845 , -77.498765 )
add1RegEx = re.compile('.*Address1.*')
cityRegex = re.compile('.*CityLabel')
stateRegex = re.compile('.*StateLabel')
zipRegex = re.compile('.*ZipLabel')
phoneRegex = re.compile('.*PhoneLabel')
storeTable = soup.find(class_="map_results", id="searchResults")
for key in stores:
info = storeTable.find("tr", id=key)
if info:
locElem = etree.Element('location')
storeElem = etree.Element('store')
storeElem.text = 'GameStop'
locElem.append(storeElem)
nameElem = etree.Element('name')
nameElem.text = stores[key][0]
locElem.append(nameElem)
add1Elem = etree.Element('address1')
add1 = info.find("span", id=add1RegEx)
add1Elem.text = add1.text
locElem.append(add1Elem)
cityElem = etree.Element('city')
city = info.find("span", id=cityRegex)
cityElem.text = city.text
locElem.append(cityElem)
stateElem = etree.Element('state')
state = info.find("span", id=stateRegex)
stateElem.text = state.text
locElem.append(stateElem)
zipElem = etree.Element('zip')
zip = info.find("span", id=zipRegex)
zipElem.text = zip.text
locElem.append(zipElem)
phoneElem = etree.Element('phone')
phone = info.find("span", id=phoneRegex)
phoneElem.text = phone.text
locElem.append(phoneElem)
latElem = etree.Element('lat')
latElem.text = stores[key][1]
locElem.append(latElem)
longElem = etree.Element('long')
longElem.text = stores[key][2]
locElem.append(longElem)
priceElem = etree.Element('price')
priceElem.text = price
locElem.append(priceElem)
root.append(locElem)
return root
import boto
import boto.dynamodb
import urllib2
import csv
from boto.dynamodb.condition import *
import bottle
from bottle import route, request, response, template, get, run, HTTPResponse
import json
#bottle = Bottle()
#establish connection to dynamodb
conn = boto.dynamodb.connect_to_region('us-west-2')
response = urllib2.urlopen(
'http://www.fhfa.gov/DataTools/Downloads/Documents/HPI/HPI_master.csv')
#Read the file data
myfile = response.read()
#Create a file named myfile.csv
filename = open('myfile.csv', 'w')
#write previously fetched filedata into newly created file
filename.write(myfile)
#Close the file
filename.close()
#function to create table
def create_table():
try:
#create schema with hash key as 'id'
myschema = conn.create_schema(hash_key_name='id',
hash_key_proto_value=str)
#create table using schema
def scrapeGS(keywords, system):
# Get the code for the system we're searching
# Also why doesn't Python have switch statements sadface
if system == "xbox 360":
system = "1385"
sysPrefix = "/xbox-360"
elif system == "playstation 3":
system = "138d"
sysPrefix = "/playstation-3"
elif system == "nintendo wii u":
system = "131b0"
sysPrefix = "/nintendo-wii-u"
elif system == "nintendo wii":
system = "138a"
sysPrefix = "/nintendo-wii"
elif system == "nintendo 3ds":
system = "131a2"
sysPrefix = "/nintendo-3ds"
elif system == "nintendo ds":
system = "1386"
sysPrefix = "/nintendo-ds"
elif system == "ps vita":
system = "131af"
sysPrefix = "/ps-vita"
else:
system = ""
sysPrefix = ""
condition = "-50" # For now we are only doing Used games
# Get the code for used/new/both (both will have no code)
#if condition == "Used":
# condition = "-50"
#elif condition == "New":
# condition = "-4f"
#else:
# condition = ""
# Add keywords the the gamestop url
rawurl = 'http://www.gamestop.com/browse%s?nav=16k-3-%s,28zu0,%s%s' % (
sysPrefix, keywords, system, condition)
# Ensure that we don't encode the commas
url = urllib.quote(rawurl, safe='/,?:=&')
# Request the gamestop page
response = urllib2.urlopen(url)
html = response.read()
# Begin parsing the results
soup = bs4.BeautifulSoup("".join(html))
# Get the viewstate
vsTag = soup.find("input", attrs={"name": "__VIEWSTATE"})
viewstate = vsTag['value']
# Get the sections containg each game and locate the ID for each
# which we will need when checking the stores later
# For each ID, add to a dict mapping the game's title to the ID
results = list()
games = soup.findAll(
"div", attrs={"class": "product preowned_product"}
) # This will need to be changed to handle new products when conditions are introduced
for game in games:
title = game.find("a", attrs={"id": re.compile("hypTitle")}).text
title = re.sub("( 1)", " I", title)
title = re.sub("( 2)", " II", title)
title = re.sub("( 3)", " III", title)
title = re.sub(r'[^\w\s]', "", title)
title = str(title)
title = title.strip()
results.append(title)
return results
from datetime import datetime import time import mysql.connector import json import pyrebase from jose import jwt import urllib, json import credentials from bson import json_util target_audience = "collaborative-mapping-proto" bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 * 1024 #(or whatever you want) certificate_url = 'https://www.googleapis.com/robot/v1/metadata/x509/[email protected]' response = urllib.urlopen(certificate_url) certs = response.read() certs = json.loads(certs) #credentials firebaseConfig = credentials.firebaseConfig config = credentials.config firebase = pyrebase.initialize_app(firebaseConfig) auth = firebase.auth() app = Bottle() @app.route('/getposts/<postnumber:int>', method='GET') def returnPage(postnumber): cnx = mysql.connector.connect(**config)
def getinfo():
url = request.query.url # this is url-decoded already
endpoint = 'http://' + cdx_server + '/web/timemap/cdx'
endpoint += '?' + urllib.parse.urlencode({ 'url': url })
req = urllib.request.Request(endpoint)
req.add_header('Cookie', 'cdx_auth_token='+cdx_secret)
with urllib.request.urlopen(req) as response:
lines = response.read().decode('utf-8').splitlines() # yeah this is really 7-bit ascii with % encoding
table = []
for line in lines:
fields = line.split(' ')
surt = fields[0]
date = fields[1]
orig = fields[2]
rec = fields[3]
code = fields[4]
sha1 = fields[5]
length = fields[8] # this is the lenth of the content plus the WARC header!
item, filename = fields[10].split('/', maxsplit=1)
# I should probably include these as 'no change' XXX
if rec == 'warc/revisit':
continue
# begone: 302 redirs from www->non and non->www, 301 redirs for foo to foo/
if code.startswith('3'):
if surt == makesurt(orig):
continue
row = { 'date': date, 'code': code, 'sha1': sha1, 'length': length, 'item': item }
table.append(row)
table = sorted(table, key=itemgetter('date'))
item_to_collection = {}
with open(os.path.expanduser('~/.item_to_collection'), 'r') as f:
for line in f:
[i, c] = line.rstrip().split(sep=' ', maxsplit=1)
item_to_collection[i] = c
i2c_changed = 0
for row in table:
if row['item'] not in item_to_collection:
resp = get_item(row['item'])
collections = resp.metadata.get('collection', [])
collection = pick_collection(collections)
if not collection:
collection = 'unknown'
item_to_collection[row['item']] = collection
i2c_changed = 1
row['why'] = item_to_collection[row['item']]
if i2c_changed:
with open(os.path.expanduser('~/.item_to_collection.new'), 'w') as f:
for k in item_to_collection:
f.write(k + ' ' + item_to_collection[k] + '\n')
os.rename(os.path.expanduser('~/.item_to_collection.new'), os.path.expanduser('~/.item_to_collection'))
last_200_sha1 = ''
last_200_length = 0
for row in table:
if row['code'] == '404':
change = '404'
elif row['code'].startswith('3'):
change = 'redir'
elif row['sha1'] != last_200_sha1:
change = 'minor'
# note -- lengths bounce around because they include the WARC header.
try:
diff = abs(int(last_200_length) - int(row['length']))
except:
diff = 10000
# when in doubt, use the worst available algorithm!
if diff > 500:
change = 'major'
last_200_sha1 = row['sha1']
last_200_length = row['length']
else:
change = 'none'
last_200_sha1 = row['sha1'] # should be identical
last_200_length = row['length'] # might vary a little due to WARC headers
row['change'] = change
captures = []
for row in table:
outrow = {}
for k in ['date', 'why', 'change']:
outrow[k] = row[k]
captures.append(outrow)
return { 'captures': captures }
def locateBBY(title, system, zip, root):
# Best Buy requires an API key to use their web service
bbyApiKey = "crs2yyrur5c9trmtv4uggtd2"
# Request the Best Buy data
rawurl = "http://api.remix.bestbuy.com/v1/products(search=%s&categoryPath.name=video games&platform=%s&preowned in(true))+stores(area(%s,10))?apiKey=%s" % (
title, system, zip, bbyApiKey)
url = urllib.quote(rawurl, safe='/,?:=&()')
response = urllib2.urlopen(url)
html = response.read()
# Begin parsing the results
soup = bs4.BeautifulSoup("".join(html))
results = list()
for game in soup.findAll("product"):
rawName = game.find("name").text
try:
titleOpt = re.search(u'(.*)(?= \u2014)', rawName).group()
except AttributeError:
titleOpt = ""
if titleOpt != "":
titleOpt = re.sub("( 1)", " I", titleOpt)
titleOpt = re.sub("( 2)", " II", titleOpt)
titleOpt = re.sub("( 3)", " III", titleOpt)
titleOpt = re.sub(r'[^\w\s]', "", titleOpt)
titleOpt = str(titleOpt)
titleOpt = titleOpt.strip()
titleOpt = titleOpt.lower()
price = "Unknown"
if titleOpt == title:
# grab onSale. if text is true, grab salePrice else grab regularPrice
if game.find("onsale").text == "true":
price = game.find("saleprice").text
else:
price = game.find("regularprice").text
# grab every store in stores
for store in game.findAll("store"):
# grab longName, address, city, region, postalCode, lat, lng, phone
# and add everything to the tree
locElem = etree.Element('location')
storeElem = etree.Element('store')
storeElem.text = 'BestBuy'
locElem.append(storeElem)
nameElem = etree.Element('name')
nameElem.text = store.find("longname").text
locElem.append(nameElem)
add1Elem = etree.Element('address1')
add1Elem.text = store.find("address").text
locElem.append(add1Elem)
cityElem = etree.Element('city')
cityElem.text = store.find("city").text
locElem.append(cityElem)
stateElem = etree.Element('state')
stateElem.text = store.find("region").text
locElem.append(stateElem)
zipElem = etree.Element('zip')
zipElem.text = store.find("postalcode").text
locElem.append(zipElem)
phoneElem = etree.Element('phone')
phoneElem.text = store.find("phone").text
locElem.append(phoneElem)
latElem = etree.Element('lat')
latElem.text = store.find("lat").text
locElem.append(latElem)
longElem = etree.Element('long')
longElem.text = store.find("lng").text
locElem.append(longElem)
priceElem = etree.Element('price')
priceElem.text = price
locElem.append(priceElem)
root.append(locElem)
return root
return root