def do_GET(self):
# self.send_header("Access-Control-Allow-Origin", "*")
out = bytes('Generic Error/ Not authenticated', 'utf-16')
# auto register first UUID
if not cache.read(['config', 'secure-gets']):
pass
elif 'X-API-Auth' in self.headers.keys(
) and self.headers['X-API-Auth'] in [
cache.read(['config'])['password'],
cache.read(['config'])['master_key']
]:
pass
elif not cache.read(['auth_uuid']):
args = dictify(self.path.split('?')[1:])
if 'auth' in args.keys():
cache.write(['auth_uuid'], args['auth'])
status = 200
try: # try to execute a function from get_routes.py that has the same name as the first sub-directory
if not self.path[1:].split('?')[0].startswith('_'):
function = getattr(
get_routes,
self.path[1:].split('?')[0]) # gets function from module
data = {}
if len(self.path.split(
'?')) > 1: # if there are GET parameters:
data = dictify(
self.path.split('?')[1:]) # convert GETs to dict
if not cache.read(['config', 'secure-gets']) or (
'auth' in data.keys()
and cache.read(['auth_uuid']) == data['auth']) or (
'X-API-Auth' in self.headers.keys()
and self.headers['X-API-Auth'] in [
cache.read(['config'])['password'],
cache.read(['config'])['master_key']
]):
out = function(data)
out = bytes(
str(out),
'utf-16') # convert string to returnable format
else:
raise AttributeError('cannot call function starting with _')
except AttributeError: # No dynamic route found, try loading a static file
try:
with open('../Frontend/' + self.path[1:],
'rb') as f: # open the file
out = f.read() # and read the content
except: # there is no file?
status = 404 # set the correct status code
out = bytes(
"No dynamic or static route is linked to this path",
'utf-16') # Notify the user
except Exception as e: # the loaded function fails:
status = 500 # set the correct status code
out = bytes(traceback.format_exc(),
'utf-16') # return the traceback
self.send_response(status)
self.end_headers()
self.wfile.write(out)
def get(url):
c = conn.cursor()
c.execute("SELECT * FROM pages WHERE url = ?", (url, ))
row = c.fetchone()
utcnow = int(datetime.datetime.utcnow().strftime("%s"))
if row is not None and row[1] > utcnow:
c.execute("DELETE FROM pages WHERE url = ?", (url, ))
conn.commit()
cache.remove(url)
row = None
if row is None:
response = urllib2.urlopen(url)
html = response.read()
expiry = response.info().getheader("Expires")
expiry = datetime.datetime(*(eut.parsedate(expiry)[0:7]))
expiry = int(expiry.strftime("%s"))
c.execute("INSERT INTO pages VALUES (?,?)", (url, expiry))
cache.write(url, html)
conn.commit()
else:
html = cache.get(url)
return BeautifulSoup(html, "html5lib")
def lookup(searchDate):
slug = 'giantbomb-'+searchDate+'-v11'
cacheContent = cache.read(slug)
if cacheContent != None:
return cacheContent
print("Hitting the GiantBomb API for "+searchDate)
query = "http://www.giantbomb.com/api/releases/?api_key={}&format=json&filter=release_date:{},region:1".format(
config.get().GiantBombApiKey,
searchDate)
headers = {
'User-Agent': 'Olava Data Grabber',
'From': '*****@*****.**'
}
# rate limit to 1 query per second
time.sleep(1)
response = requests.get(
query,
headers=headers
)
if response == None:
print("Response wasn't populated for [{}].", query)
import pprint
pprint.pprint(vars(response))
return {}
entries = {}
try:
entries = response.json()
except ValueError as e:
print("An error occurred while grabbing JSON.")
import pprint
pprint.pprint(vars(response))
return {}
cache.write(slug, entries)
return entries
def commit(self):
def getCommitDate(date):
return date[:4] + '-' + date[4:6] + '-' + date[6:8] + ' ' + \
date[9:11] + ':' + date[11:13] + ':' + date[13:15]
def getUserName(user):
return str(user).split(' <')[0]
def getUserEmail(user):
email = search('<.*@.*>', str(user))
if email == None:
return '<%s@%s>' % (user.lower().replace(' ','.').replace("'", ''), mailSuffix)
else:
return email.group(0)
files = []
for file in self.files:
files.append(file.file)
for file in self.files:
file.add(files)
cache.write()
env = os.environ
user = users.get(self.user, self.user)
env['GIT_AUTHOR_DATE'] = env['GIT_COMMITTER_DATE'] = str(getCommitDate(self.date))
env['GIT_AUTHOR_NAME'] = env['GIT_COMMITTER_NAME'] = getUserName(user)
env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = str(getUserEmail(user))
comment = self.comment if self.comment.strip() != "" else "<empty message>"
try:
git_exec(['commit', '-m', comment.encode(ENCODING)], env=env)
except Exception as e:
if search('nothing( added)? to commit', e.args[0]) == None:
raise
def simulate(cache, trace):
global misses
global hits
for line in trace:
splitLine = line.split()
if (len(splitLine) == 3):
trash, op, address = splitLine
if op == 'R':
result = cache.read(address)
if (result == 0):
misses += 1
cache.load(address)
cache.read(address)
else:
hits += 1
else:
result = cache.write(address)
if (result == 0):
misses += 1
cache.load(address)
cache.write(address)
else:
hits += 1
print_results(misses, hits)
def lookup_many(user_ids):
""" """
method_name = 'twitter.users.lookup'
new_users = []
cached_users = []
for user in user_ids:
if cache.has(method_name, user):
cached_users.append(user)
else:
new_users.append(user)
data = {}
pp("Retrieving user metadata from cache")
for user in cached_users:
data[user] = lookup(user)
if type(lookup(user)) is list:
pp('Watch out for %s' % user)
pp("Using API to lookup data from %d new users" % len(new_users))
for user_slice in [
new_users[x:x + 100] for x in xrange(0, len(new_users), 100)
]:
query = ",".join([str(x) for x in user_slice])
logger.debug("Query is %s" % query)
try:
pp("Requesting user slice...")
metadatas = call_api(twitter.users.lookup,
{id_or_sn(query): query})
for user_data in metadatas:
screen_name = user_data['screen_name']
cache.write(method_name, screen_name, user_data)
data[screen_name] = user_data
pp("Successfully wrote slice to cache")
except TwitterHTTPError as e:
## is there something more graceful to be done with 404's here?
## maybe...if we handle unauthorized requests better.
print e
logger.error(e)
return data
def lookup_many(user_ids):
""" """
method_name = 'twitter.users.lookup'
new_users = []
cached_users = []
for user in user_ids:
if cache.has(method_name,user):
cached_users.append(user)
else:
new_users.append(user)
data = {}
pp("Retrieving user metadata from cache")
for user in cached_users:
data[user] = lookup(user)
if type(lookup(user)) is list:
pp('Watch out for %s' % user)
pp("Using API to lookup data from %d new users" % len(new_users))
for user_slice in [new_users[x:x+100]
for x
in xrange(0,len(new_users),100)]:
query = ",".join([str(x) for x in user_slice])
logger.debug("Query is %s" % query)
try:
pp("Requesting user slice...")
metadatas = call_api(twitter.users.lookup,
{id_or_sn(query):query})
for user_data in metadatas:
screen_name = user_data['screen_name']
cache.write(method_name,screen_name,user_data)
data[screen_name] = user_data
pp("Successfully wrote slice to cache")
except TwitterHTTPError as e:
## is there something more graceful to be done with 404's here?
## maybe...if we handle unauthorized requests better.
print e
logger.error(e)
return data
def call_api_with_cache(user_id, method, method_name,parameters={}):
# first check the cache
if cache.has(method_name,user_id):
logger.debug("%s %s exists in cache." % (method_name, user_id))
return cache.read(method_name,user_id)
else:
# if not in cache call the API
logger.debug("%s %s does not exists in cache. Will retrieve it from web." % (method_name, user_id))
try:
data = call_api(method,
dict([(id_or_sn(user_id),user_id)]
+ parameters.items()))
cache.write(method_name,user_id,data)
return data
except TwitterHTTPError as e:
logger.error(e)
#hack to prevent crawling this
return {'error': e}
def call_api_with_cache(user_id, method, method_name, parameters={}):
# first check the cache
if cache.has(method_name, user_id):
logger.debug("%s %s exists in cache." % (method_name, user_id))
return cache.read(method_name, user_id)
else:
# if not in cache call the API
logger.debug(
"%s %s does not exists in cache. Will retrieve it from web." %
(method_name, user_id))
try:
data = call_api(
method,
dict([(id_or_sn(user_id), user_id)] + parameters.items()))
cache.write(method_name, user_id, data)
return data
except TwitterHTTPError as e:
logger.error(e)
#hack to prevent crawling this
return {'error': e}
def lookup_many(user_ids):
""" """
method_name = 'twitter.users.lookup'
new_users = []
cached_users = []
for user in user_ids:
if cache.has(method_name,user):
cached_users.append(user)
else:
new_users.append(user)
data = {}
for user in cached_users:
data[user] = lookup(user)
if type(lookup(user)) is list:
pp('Watch out for %s' % user)
for user_slice in [new_users[x:x+100]
for x
in xrange(0,len(new_users),100)]:
query = ",".join([str(x) for x in user_slice])
logger.debug("Query is %s" % query)
try:
metadatas = call_api(twitter.users.lookup,
{id_or_sn(query):query})
for user_data in metadatas:
screen_name = user_data['screen_name']
cache.write(method_name,screen_name,user_data)
data[screen_name] = user_data
except TwitterHTTPError as e:
print e
logger.error(e)
return data
def game_info(gameApiUrl):
slug = gameApiUrl.replace("http://www.", '')
slug = gameApiUrl.replace('/', '-')
slug += 'v11'
cacheContent = cache.read(slug)
if cacheContent != None:
return cacheContent
print("Hitting GiantBomb API for "+slug)
query = gameApiUrl+"?api_key={}&format=json".format(
config.get().GiantBombApiKey)
headers = {
'User-Agent': 'Olava Data Grabber',
'From': '*****@*****.**'
}
# rate limit to 1 query per second
time.sleep(1)
response = requests.get(
query,
headers=headers
)
if response == None:
print("Response wasn't populated for [{}].", query)
import pprint
pprint.pprint(vars(response))
return {}
game = {}
try:
game = response.json()
except ValueError as e:
print("An error occurred while grabbing JSON.")
import pprint
pprint.pprint(vars(response))
return {}
cache.write(slug, game)
return game
def commit(self):
def getCommitDate(date):
return date[:4] + '-' + date[4:6] + '-' + date[6:8] + ' ' + \
date[9:11] + ':' + date[11:13] + ':' + date[13:15]
def getUserName(user):
return str(user).split(' <')[0]
def getUserEmail(user):
email = search('<.*@.*>', str(user))
if email == None:
return '<%s@%s>' % (user.lower().replace(' ', '.').replace(
"'", ''), mailSuffix)
else:
return email.group(0)
files = []
for file in self.files:
files.append(file.file)
for file in self.files:
file.add(files)
cache.write()
env = os.environ
user = users.get(self.user, self.user)
env['GIT_AUTHOR_DATE'] = env['GIT_COMMITTER_DATE'] = str(
getCommitDate(self.date))
env['GIT_AUTHOR_NAME'] = env['GIT_COMMITTER_NAME'] = getUserName(user)
env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = str(
getUserEmail(user))
comment = self.comment if self.comment.strip(
) != "" else "<empty message>"
try:
git_exec(['commit', '-m', comment.encode(ENCODING)], env=env)
except Exception as e:
if search('nothing( added)? to commit', e.args[0]) == None:
raise
def parse_item(self, response):
body = response.body
soup = BeautifulSoup(body, 'html.parser')
form = soup.find(id = 'file_down')
data = dict((dom.get('name'), dom.get('value'))
for dom in form.find_all('input')
)
req = FormRequest.from_response(
response = response,
formid = 'file_down',
formdata = data,
callback = lambda response: write(response, HistDataSpider.name)
)
return req
# Determine center pixel from angular coordinates
direction[i] = famousModel.ang_to_cart(theta[i], phi[i])
center_id[i] = find(direction[i], pixels)
# If simulation fell into an pixel, continue
if center_id[i] == None:
n_tot[i] = np.nan
continue
# Count the number of photons which have hit the found pixel
for point in points:
if pixels[center_id[i]].contains_point(point):
n_pix[i] += 1
# Cache
if len(files) > 0:
print "Writing cache ..."
cache.write(path + "/cache_light_loss.npy", (files, n, theta, phi, n_pix, n_tot, direction, center_id))
# Integrate efficiency per pixel.
pixel_effs = np.zeros(len(pixels))
pixel_effs_num = np.zeros(len(pixels))
for i, dir in enumerate(direction):
if center_id[i] == None:
continue
pixel_effs[center_id[i]] += n_pix[i] / n[i]
pixel_effs_num[center_id[i]] += 1
pixel_effs = 100.0 * pixel_effs / pixel_effs_num
# Compute efficiency of the central pixel.
eff = ma.fix_invalid(n_pix / n, fill_value=0.0).data * 100.0
def set_uuid(*args):
cache.write(['auth_uuid'], args[0]['auth'])
def write_to_cache(url, hostname, conf):
return cache.write(url, hostname, conf)
def update_title(*args):
cache.write(['title'], urllib.parse.unquote(args[0]['title']))
return 'OK'
def update_grid(*args):
cache.write(['grid', 'rows'], int(args[0]['rows']))
cache.write(['grid', 'columns'], int(args[0]['columns']))
return "OK"
def write_to_cache(the_id, response):
global CACHE_NAME
lines = "%s\n%s" % (the_id, response)
cache.write(CACHE_NAME, lines)
def get_ads(base_url):
c = conn.cursor()
page = download.get(base_url + "/search/cpg")
for p in page.select(".row"):
pid = p['data-pid']
a_tag = p.find('a', class_='hdrlnk')
ad_href = a_tag['href']
ad_title = a_tag.text
dt = p.find('time')['datetime']
dt = datetime.datetime.strptime(dt, "%Y-%m-%d %H:%M")
dt = int(dt.strftime("%s"))
c.execute("SELECT * FROM ad WHERE id = ?", (pid, ))
row = c.fetchone()
if row is None:
url = ad_href
if not ad_href.startswith('http'):
url = base_url + ad_href
time.sleep(0.5)
ad = download.get(url)
print url
ad_text = ad.find(id='postingbody')
if ad_text is None:
if ad.find(id='has_been_removed'):
continue
else:
raise "malformed body"
ad_text = ad_text.text.strip()
ad_text = filter(lambda x: x in string.printable, ad_text)
nilsimsa = Nilsimsa(ad_text)
lshash = nilsimsa.hexdigest()
# c.execute("SELECT * FROM ad")
# row = c.fetchone()
# while row:
# diff = nilsimsa.compare(row[4], True)
# if diff < 10:
# print diff
# print cache.get("text:" + row[0])
# print "----"
# print ad_text
# sys.exit()
seen = generate_word_counts(ad_text)
cache.write("text:" + pid, ad_text)
row = (pid, url, ad_title, dt, lshash)
c.execute(
"INSERT INTO ad (id, url, title, posted, lshash) " +
" VALUES (?,?,?,?,?)", row)
for word in seen:
if word not in stopwords:
row = (pid, word, seen[word])
c.execute(
"INSERT INTO tf (id, word, cnt) " + "VALUES (?,?,?)",
row)
conn.commit()
def write_to_cache(the_id, response):
global CACHE_NAME
lines = "%s\n%s" % (the_id, response)
cache.write(CACHE_NAME, lines)
httpd.serve_forever()
# Setup cache structure
cache.hard_set({
'node_html': [], # Node html, can have multiple per node, cycles
'html_index': [], # currently displayed node HTML index
'update_timer':
[], # internal timer, if value hits zero: reset to update_delay
'update_delay': [],
'force_update':
False, # If there is only one item, do not update unless this flag is set
'auth_uuid': '',
'title': 'NodeBoard v0.25',
'grid': {
"rows": 2,
"columns": 3
}
})
import os
if os.getcwd() == "/home/pi":
os.chdir("/home/pi/Schreibtisch/magicdashboard/Backend/")
# Load data from config.json
with open('config.json', 'r') as f:
cache.write(['config'], json.loads(f.read()))
# Start server
server()
def force_update(*args):
cache.write(['force_update'], True)
return "OK"
