Exemplo n.º 1
0
    def do_GET(self):
        # self.send_header("Access-Control-Allow-Origin", "*")
        out = bytes('Generic Error/ Not authenticated', 'utf-16')
        # auto register first UUID
        if not cache.read(['config', 'secure-gets']):
            pass
        elif 'X-API-Auth' in self.headers.keys(
        ) and self.headers['X-API-Auth'] in [
                cache.read(['config'])['password'],
                cache.read(['config'])['master_key']
        ]:
            pass
        elif not cache.read(['auth_uuid']):
            args = dictify(self.path.split('?')[1:])
            if 'auth' in args.keys():
                cache.write(['auth_uuid'], args['auth'])

        status = 200
        try:  # try to execute a function from get_routes.py that has the same name as the first sub-directory
            if not self.path[1:].split('?')[0].startswith('_'):
                function = getattr(
                    get_routes,
                    self.path[1:].split('?')[0])  # gets function from module
                data = {}
                if len(self.path.split(
                        '?')) > 1:  # if there are GET parameters:
                    data = dictify(
                        self.path.split('?')[1:])  # convert GETs to dict
                if not cache.read(['config', 'secure-gets']) or (
                        'auth' in data.keys()
                        and cache.read(['auth_uuid']) == data['auth']) or (
                            'X-API-Auth' in self.headers.keys()
                            and self.headers['X-API-Auth'] in [
                                cache.read(['config'])['password'],
                                cache.read(['config'])['master_key']
                            ]):
                    out = function(data)
                    out = bytes(
                        str(out),
                        'utf-16')  # convert string to returnable format
            else:
                raise AttributeError('cannot call function starting with _')
        except AttributeError:  # No dynamic route found, try loading a static file
            try:
                with open('../Frontend/' + self.path[1:],
                          'rb') as f:  # open the file
                    out = f.read()  # and read the content
            except:  # there is no file?
                status = 404  # set the correct status code
                out = bytes(
                    "No dynamic or static route is linked to this path",
                    'utf-16')  # Notify the user
        except Exception as e:  # the loaded function fails:
            status = 500  # set the correct status code
            out = bytes(traceback.format_exc(),
                        'utf-16')  # return the traceback

        self.send_response(status)
        self.end_headers()
        self.wfile.write(out)
Exemplo n.º 2
0
def get(url):
    c = conn.cursor()
    c.execute("SELECT * FROM pages WHERE url = ?", (url, ))
    row = c.fetchone()

    utcnow = int(datetime.datetime.utcnow().strftime("%s"))
    if row is not None and row[1] > utcnow:
        c.execute("DELETE FROM pages WHERE url = ?", (url, ))
        conn.commit()
        cache.remove(url)
        row = None

    if row is None:
        response = urllib2.urlopen(url)
        html = response.read()

        expiry = response.info().getheader("Expires")
        expiry = datetime.datetime(*(eut.parsedate(expiry)[0:7]))
        expiry = int(expiry.strftime("%s"))

        c.execute("INSERT INTO pages VALUES (?,?)", (url, expiry))

        cache.write(url, html)

        conn.commit()
    else:
        html = cache.get(url)
    return BeautifulSoup(html, "html5lib")
Exemplo n.º 3
0
def lookup(searchDate):
    slug = 'giantbomb-'+searchDate+'-v11'
    cacheContent = cache.read(slug)
    if cacheContent != None:
        return cacheContent
    print("Hitting the GiantBomb API for "+searchDate)
    query = "http://www.giantbomb.com/api/releases/?api_key={}&format=json&filter=release_date:{},region:1".format(
        config.get().GiantBombApiKey,
        searchDate)
    headers = {
        'User-Agent': 'Olava Data Grabber',
        'From': '*****@*****.**'
    }
    # rate limit to 1 query per second
    time.sleep(1)
    response = requests.get(
        query,
        headers=headers
    )
    if response == None:
        print("Response wasn't populated for [{}].", query)
        import pprint
        pprint.pprint(vars(response))
        return {}
    entries = {}
    try:
        entries = response.json()
    except ValueError as e:
        print("An error occurred while grabbing JSON.")
        import pprint
        pprint.pprint(vars(response))
        return {}
    cache.write(slug, entries)
    return entries
Exemplo n.º 4
0
 def commit(self):
     def getCommitDate(date):
         return date[:4] + '-' + date[4:6] + '-' + date[6:8] + ' ' + \
                date[9:11] + ':' + date[11:13] + ':' + date[13:15]
     def getUserName(user):
         return str(user).split(' <')[0]
     def getUserEmail(user):
         email = search('<.*@.*>', str(user))
         if email == None:
             return '<%s@%s>' % (user.lower().replace(' ','.').replace("'", ''), mailSuffix)
         else:
             return email.group(0)
     files = []
     for file in self.files:
         files.append(file.file)
     for file in self.files:
         file.add(files)
     cache.write()
     env = os.environ
     user = users.get(self.user, self.user)
     env['GIT_AUTHOR_DATE'] = env['GIT_COMMITTER_DATE'] = str(getCommitDate(self.date))
     env['GIT_AUTHOR_NAME'] = env['GIT_COMMITTER_NAME'] = getUserName(user)
     env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = str(getUserEmail(user))
     comment = self.comment if self.comment.strip() != "" else "<empty message>"
     try:
         git_exec(['commit', '-m', comment.encode(ENCODING)], env=env)
     except Exception as e:
         if search('nothing( added)? to commit', e.args[0]) == None:
             raise
Exemplo n.º 5
0
def simulate(cache, trace):
    global misses
    global hits
    for line in trace:
        splitLine = line.split()
        if (len(splitLine) == 3):
            trash, op, address = splitLine
            if op == 'R':
                result = cache.read(address)
                if (result == 0):
                    misses += 1
                    cache.load(address)
                    cache.read(address)
                else:
                    hits += 1

            else:
                result = cache.write(address)
                if (result == 0):
                    misses += 1
                    cache.load(address)
                    cache.write(address)
                else:
                    hits += 1
    print_results(misses, hits)
Exemplo n.º 6
0
def lookup_many(user_ids):
    """ """
    method_name = 'twitter.users.lookup'

    new_users = []
    cached_users = []

    for user in user_ids:
        if cache.has(method_name, user):
            cached_users.append(user)
        else:
            new_users.append(user)

    data = {}

    pp("Retrieving user metadata from cache")

    for user in cached_users:
        data[user] = lookup(user)
        if type(lookup(user)) is list:
            pp('Watch out for %s' % user)

    pp("Using API to lookup data from %d new users" % len(new_users))

    for user_slice in [
            new_users[x:x + 100] for x in xrange(0, len(new_users), 100)
    ]:

        query = ",".join([str(x) for x in user_slice])
        logger.debug("Query is %s" % query)

        try:
            pp("Requesting user slice...")
            metadatas = call_api(twitter.users.lookup,
                                 {id_or_sn(query): query})
            for user_data in metadatas:
                screen_name = user_data['screen_name']
                cache.write(method_name, screen_name, user_data)
                data[screen_name] = user_data

            pp("Successfully wrote slice to cache")

        except TwitterHTTPError as e:
            ## is there something more graceful to be done with 404's here?
            ## maybe...if we handle unauthorized requests better.
            print e
            logger.error(e)

    return data
Exemplo n.º 7
0
def lookup_many(user_ids):
    """ """
    method_name = 'twitter.users.lookup'

    new_users = []
    cached_users = []

    for user in user_ids:
        if cache.has(method_name,user):
            cached_users.append(user)
        else:
            new_users.append(user)

    data = {}

    pp("Retrieving user metadata from cache")

    for user in cached_users:
        data[user] = lookup(user)
        if type(lookup(user)) is list:
            pp('Watch out for %s' % user)

    pp("Using API to lookup data from %d new users" % len(new_users))
    
    for user_slice in [new_users[x:x+100]
                       for x
                       in xrange(0,len(new_users),100)]:

        query = ",".join([str(x) for x in user_slice])
        logger.debug("Query is %s" % query)

        try:
            pp("Requesting user slice...")
            metadatas = call_api(twitter.users.lookup,
                                 {id_or_sn(query):query})
            for user_data in metadatas:
                screen_name = user_data['screen_name']
                cache.write(method_name,screen_name,user_data)
                data[screen_name] = user_data

            pp("Successfully wrote slice to cache")

        except TwitterHTTPError as e:
            ## is there something more graceful to be done with 404's here?
            ## maybe...if we handle unauthorized requests better.
            print e
            logger.error(e)

    return data
Exemplo n.º 8
0
def call_api_with_cache(user_id, method, method_name,parameters={}):
    # first check the cache
    if cache.has(method_name,user_id):
        logger.debug("%s %s exists in cache." % (method_name, user_id))
        return cache.read(method_name,user_id)
    else:
    # if not in cache call the API
        logger.debug("%s %s does not exists in cache. Will retrieve it from web." % (method_name, user_id))
        try:
            data = call_api(method,
                            dict([(id_or_sn(user_id),user_id)]
                            + parameters.items()))
            cache.write(method_name,user_id,data)
            return data
        except TwitterHTTPError as e:
            logger.error(e)
            #hack to prevent crawling this
            return {'error': e}
Exemplo n.º 9
0
def call_api_with_cache(user_id, method, method_name, parameters={}):
    # first check the cache
    if cache.has(method_name, user_id):
        logger.debug("%s %s exists in cache." % (method_name, user_id))
        return cache.read(method_name, user_id)
    else:
        # if not in cache call the API
        logger.debug(
            "%s %s does not exists in cache. Will retrieve it from web." %
            (method_name, user_id))
        try:
            data = call_api(
                method,
                dict([(id_or_sn(user_id), user_id)] + parameters.items()))
            cache.write(method_name, user_id, data)
            return data
        except TwitterHTTPError as e:
            logger.error(e)
            #hack to prevent crawling this
            return {'error': e}
Exemplo n.º 10
0
def lookup_many(user_ids):
    """ """
    method_name = 'twitter.users.lookup'

    new_users = []
    cached_users = []

    for user in user_ids:
        if cache.has(method_name,user):
            cached_users.append(user)
        else:
            new_users.append(user)

    data = {}

    for user in cached_users:
        data[user] = lookup(user)
        if type(lookup(user)) is list:
            pp('Watch out for %s' % user)

    for user_slice in [new_users[x:x+100]
                       for x
                       in xrange(0,len(new_users),100)]:

        query = ",".join([str(x) for x in user_slice])
        logger.debug("Query is %s" % query)

        try:
            metadatas = call_api(twitter.users.lookup,
                                 {id_or_sn(query):query})
            for user_data in metadatas:
                screen_name = user_data['screen_name']
                cache.write(method_name,screen_name,user_data)
                data[screen_name] = user_data

        except TwitterHTTPError as e:
            print e
            logger.error(e)

    return data
Exemplo n.º 11
0
def game_info(gameApiUrl):
    slug = gameApiUrl.replace("http://www.", '')
    slug = gameApiUrl.replace('/', '-')
    slug += 'v11'

    cacheContent = cache.read(slug)
    if cacheContent != None:
        return cacheContent

    print("Hitting GiantBomb API for "+slug)
    query = gameApiUrl+"?api_key={}&format=json".format(
        config.get().GiantBombApiKey)
    headers = {
        'User-Agent': 'Olava Data Grabber',
        'From': '*****@*****.**'
    }
    # rate limit to 1 query per second
    time.sleep(1)
    response = requests.get(
        query,
        headers=headers
    )
    if response == None:
        print("Response wasn't populated for [{}].", query)
        import pprint
        pprint.pprint(vars(response))
        return {}
    game = {}
    try:
        game = response.json()
    except ValueError as e:
        print("An error occurred while grabbing JSON.")
        import pprint
        pprint.pprint(vars(response))
        return {}
    cache.write(slug, game)
    return game
Exemplo n.º 12
0
    def commit(self):
        def getCommitDate(date):
            return date[:4] + '-' + date[4:6] + '-' + date[6:8] + ' ' + \
                   date[9:11] + ':' + date[11:13] + ':' + date[13:15]

        def getUserName(user):
            return str(user).split(' <')[0]

        def getUserEmail(user):
            email = search('<.*@.*>', str(user))
            if email == None:
                return '<%s@%s>' % (user.lower().replace(' ', '.').replace(
                    "'", ''), mailSuffix)
            else:
                return email.group(0)

        files = []
        for file in self.files:
            files.append(file.file)
        for file in self.files:
            file.add(files)
        cache.write()
        env = os.environ
        user = users.get(self.user, self.user)
        env['GIT_AUTHOR_DATE'] = env['GIT_COMMITTER_DATE'] = str(
            getCommitDate(self.date))
        env['GIT_AUTHOR_NAME'] = env['GIT_COMMITTER_NAME'] = getUserName(user)
        env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = str(
            getUserEmail(user))
        comment = self.comment if self.comment.strip(
        ) != "" else "<empty message>"
        try:
            git_exec(['commit', '-m', comment.encode(ENCODING)], env=env)
        except Exception as e:
            if search('nothing( added)? to commit', e.args[0]) == None:
                raise
Exemplo n.º 13
0
    def parse_item(self, response):
        body  = response.body
        soup  = BeautifulSoup(body, 'html.parser')
        form  = soup.find(id = 'file_down')
        data  = dict((dom.get('name'), dom.get('value'))
            for dom in form.find_all('input')
        )

        req   = FormRequest.from_response(
            response = response,
            formid   = 'file_down',
            formdata = data,
            callback = lambda response: write(response, HistDataSpider.name)
        )

        return req
Exemplo n.º 14
0
    # Determine center pixel from angular coordinates
    direction[i] = famousModel.ang_to_cart(theta[i], phi[i])
    center_id[i] = find(direction[i], pixels)
    # If simulation fell into an pixel, continue
    if center_id[i] == None:
        n_tot[i] = np.nan
        continue
    # Count the number of photons which have hit the found pixel
    for point in points:
        if pixels[center_id[i]].contains_point(point):
            n_pix[i] += 1

# Cache
if len(files) > 0:
    print "Writing cache ..."
    cache.write(path + "/cache_light_loss.npy", (files, n, theta, phi, n_pix, n_tot, direction, center_id))
    
# Integrate efficiency per pixel.
pixel_effs = np.zeros(len(pixels))
pixel_effs_num = np.zeros(len(pixels))
for i, dir in enumerate(direction):
    if center_id[i] == None:
        continue
    pixel_effs[center_id[i]] += n_pix[i] / n[i]
    pixel_effs_num[center_id[i]] += 1
    
pixel_effs = 100.0 * pixel_effs / pixel_effs_num

# Compute efficiency of the central pixel.       
eff = ma.fix_invalid(n_pix / n, fill_value=0.0).data * 100.0
def set_uuid(*args):
    cache.write(['auth_uuid'], args[0]['auth'])
Exemplo n.º 16
0
def write_to_cache(url, hostname, conf):
    return cache.write(url, hostname, conf)
def update_title(*args):
    cache.write(['title'], urllib.parse.unquote(args[0]['title']))
    return 'OK'
def update_grid(*args):
    cache.write(['grid', 'rows'], int(args[0]['rows']))
    cache.write(['grid', 'columns'], int(args[0]['columns']))
    return "OK"
Exemplo n.º 19
0
def write_to_cache(the_id, response):
    global CACHE_NAME
    lines = "%s\n%s" % (the_id, response)
    cache.write(CACHE_NAME, lines)
Exemplo n.º 20
0
def get_ads(base_url):
    c = conn.cursor()

    page = download.get(base_url + "/search/cpg")

    for p in page.select(".row"):
        pid = p['data-pid']

        a_tag = p.find('a', class_='hdrlnk')
        ad_href = a_tag['href']
        ad_title = a_tag.text

        dt = p.find('time')['datetime']
        dt = datetime.datetime.strptime(dt, "%Y-%m-%d %H:%M")
        dt = int(dt.strftime("%s"))

        c.execute("SELECT * FROM ad WHERE id = ?", (pid, ))
        row = c.fetchone()

        if row is None:
            url = ad_href
            if not ad_href.startswith('http'):
                url = base_url + ad_href

            time.sleep(0.5)
            ad = download.get(url)

            print url
            ad_text = ad.find(id='postingbody')
            if ad_text is None:
                if ad.find(id='has_been_removed'):
                    continue
                else:
                    raise "malformed body"
            ad_text = ad_text.text.strip()

            ad_text = filter(lambda x: x in string.printable, ad_text)
            nilsimsa = Nilsimsa(ad_text)
            lshash = nilsimsa.hexdigest()

            # c.execute("SELECT * FROM ad")
            # row = c.fetchone()
            # while row:
            #     diff = nilsimsa.compare(row[4], True)
            #     if diff < 10:
            #         print diff
            #         print cache.get("text:" + row[0])
            #         print "----"
            #         print ad_text
            #         sys.exit()

            seen = generate_word_counts(ad_text)

            cache.write("text:" + pid, ad_text)

            row = (pid, url, ad_title, dt, lshash)
            c.execute(
                "INSERT INTO ad (id, url, title, posted, lshash) " +
                " VALUES (?,?,?,?,?)", row)

            for word in seen:
                if word not in stopwords:
                    row = (pid, word, seen[word])
                    c.execute(
                        "INSERT INTO tf (id, word, cnt) " + "VALUES (?,?,?)",
                        row)
            conn.commit()
Exemplo n.º 21
0
def write_to_cache(the_id, response):
    global CACHE_NAME
    lines = "%s\n%s" % (the_id, response)
    cache.write(CACHE_NAME, lines)
Exemplo n.º 22
0
    httpd.serve_forever()


# Setup cache structure
cache.hard_set({
    'node_html': [],  # Node html, can have multiple per node, cycles
    'html_index': [],  # currently displayed node HTML index
    'update_timer':
    [],  # internal timer, if value hits zero: reset to update_delay
    'update_delay': [],
    'force_update':
    False,  # If there is only one item, do not update unless this flag is set
    'auth_uuid': '',
    'title': 'NodeBoard v0.25',
    'grid': {
        "rows": 2,
        "columns": 3
    }
})

import os

if os.getcwd() == "/home/pi":
    os.chdir("/home/pi/Schreibtisch/magicdashboard/Backend/")

# Load data from config.json
with open('config.json', 'r') as f:
    cache.write(['config'], json.loads(f.read()))
# Start server
server()
def force_update(*args):
    cache.write(['force_update'], True)
    return "OK"