Example #1
0
    def _fetch(self, url, method="GET", data=None, auth=None):
        if type(data) == dict:
            data = urlencode(data)

        request_data = dict(url=url, headers={}, method=method, body=data)

        if auth is not None:
            try:
                passhash = b64encode(':'.join(auth).encode('ascii'))
            except TypeError as e:
                raise TokenError(
                    'Missing credentials (client_id:client_secret)', str(e))
            request_data['headers']['Authorization'] = 'Basic %s' % passhash

        request_data.update(self._http_options)
        request = HTTPRequest(**request_data)
        logging.info(
            'request:%s %s\n%s\n%s' %
            (request.method, request.url, request.headers, request.body))
        try:
            response = yield self._http_client.fetch(request)
        except HTTPError as e:
            raise TokenError('Failed to request token', str(e))
        result = json.loads(response.body)
        raise gen.Return(result)
Example #2
0
def get_entities_(url):
    data = geturl('http://api.evri.com/v1/media/entities.json?uri=' + urllib.quote(url))
    data = json.loads(data)
    graph = data['evriThing']['graph']
    def getfacet(item):
        try:
            return item['facets']['facet']['$']
        except KeyError: pass
        return ''

    try:
        category_list=mklist(graph['categories']['category'])
        categories=[item['$'] for item in category_list]
    except KeyError:
        categories=[]

    try:
        entity_list = mklist(graph['entities']['entity'])
    except KeyError:
        entity_list = []

    return dict(
        categories=categories,
        entities=[dict(
                       name=getname(item),
                       facet=getfacet(item),
                       ref=gethref(item)
                      )
                  for item in entity_list],
    )
Example #3
0
def get_entity_references_(url):
    data = geturl('http://api.evri.com/v1/media/related.json?includeTopEntities=true&entityURI=' + urllib.quote(url))
    data = json.loads(data)
    data = data['evriThing']['mediaResult']['articleList']['article']
    def get_path(article):
        link = article.get('link', {})
        try:
            return 'http://%s%s' % (link['@hostName'], link['@path'])
        except KeyError:
            return ''

    def getentities(article):
        try:
            entities = mklist(article['topEntities']['entity'])
        except KeyError: return []
        return [dict(
                     name=getname(entity),
                     ref=gethref(entity),
                    ) for entity in entities]
        
    return [dict(
                 author=article.get('author', {}).get('$', ''),
                 content=article.get('content', {}).get('$', ''),
                 title=article.get('title', {}).get('$', ''),
                 url=get_path(article),
                 entities=getentities(article),
            ) for article in data]
Example #4
0
 def get(self):
     search = self.request.get('search', '')
     gurl = ('http://content.guardianapis.com/search?q=%s&format=json' %
             urllib.quote(search))
     results = geturl(gurl)
     results = json.loads(results)
     try:
         results = results['response']['results']
     except KeyError:
         results = []
     context = dict(search=search)
     fullresults = []
     count = 0
     for result in results:
         url = result['webUrl']
         if count < 1:
             entities = getentities.get_entities(url)
         else:
             entities = {}
         fullresults.append(dict(url=url.encode('utf8'),
                                 entities=entities.get('entities'),
                                 categories=entities.get('categories'),
                                 title=result['webTitle'],
                                 author='Guardian',
                            ))
         count += 1
     context['results'] = fullresults
     self.render("topic.html", context)
Example #5
0
    def _fetch(self, url, method="GET", data=None, auth=None):
        if type(data) == dict:
            data = urlencode(data)

        request_data = dict(
            url=url,
            headers={},
            method=method,
            body=data
        )

        if auth is not None:
            try:
                passhash = b64encode(':'.join(auth).encode('ascii'))
            except TypeError as e:
                raise TokenError('Missing credentials (client_id:client_secret)', str(e))
            request_data['headers']['Authorization'] = 'Basic %s' % passhash

        request_data.update(self._http_options)
        request = HTTPRequest(**request_data)
        logging.info('request:%s %s\n%s\n%s' % (request.method, request.url, request.headers, request.body))
        try:
            response = yield self._http_client.fetch(request)
        except HTTPError as e:
            raise TokenError('Failed to request token', str(e))
        result = json.loads(response.body)
        raise gen.Return(result)
Example #6
0
    def _fetch(self, url, method="GET", data=None, auth=None):
        if type(data) == dict:
            data = urlencode(data)

        request_data = dict(url=url, headers={}, method=method, body=data)

        if auth is not None:
            try:
                passhash = b64encode(':'.join(auth).encode('ascii'))
            except TypeError as e:
                raise TokenError(
                    'Missing credentials (client_id:client_secret)', str(e))

            request_data['headers']['Authorization'] = (
                'Basic %s' % passhash.decode('utf-8'))

        request_data.update(self._http_options)
        request = HTTPRequest(**request_data)

        logger.debug('Request: %s %s', request.method, request.url)
        for header in request.headers:
            logger.debug('Header %s: %s', header, request.headers[header])

        try:
            response = yield self._http_client.fetch(request)
        except HTTPError as e:
            raise TokenHTTPError('Failed to request token', e.response)

        result = json.loads(response.body.decode("utf-8"))
        raise gen.Return(result)
Example #7
0
def list_job_ids(host="http://localhost", port=6800, project="default"):
    command = "curl %s:%d/listjobs.json?project=%s" % (host, port, project)
    command_result = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE).stdout.readline()
    running_ids = json.loads(command_result).get("running")
    ids = []
    for i in running_ids:
        ids.append(i["id"])
    return ids
    def parse_dummy_songs(cls, dummy_songs, stringify=True):
        if stringify:
            dummy_songs = json.loads(cls.js_context.JSON.stringify(dummy_songs))

        songs = []
        for s in dummy_songs:
            song = parse_to_dsong(s)
            if song:
                songs.append(song)
        return songs
    def parse_dummy_songs(cls, dummy_songs, stringify=True):
        if stringify:
            dummy_songs = json.loads(
                cls.js_context.JSON.stringify(dummy_songs))

        songs = []
        for s in dummy_songs:
            song = parse_to_dsong(s)
            if song:
                songs.append(song)
        return songs
Example #10
0
    def on_reports_new(self, request):
        info = json.loads(request.data)
        self.logger.debug('request: %r', info)
        info['uuid'] = gen_uuid()

        self.queue.enqueue(generate_report, info)

        values = {
            'status' : 'ok',
            'report_id' : info['uuid'],
        }
        return values
Example #11
0
    def unserialise(self, value):
        """Load the schema from json.

        Overrides any existing configuration in the schema.

        """
        schema = json.loads(value)
        if schema['version'] != SCHEMA_VERSION:
            raise RuntimeError("Can't handle this version of the schema (got "
                               "version %s - I understand version %s" %
                               (schema['version'], SCHEMA_VERSION))
        self.types = dict(schema['fieldtypes'])
        self.modified = False
Example #12
0
    def _fetch(self, url, method="GET", data=None, auth=None):
        if type(data) == dict:
            data = urlencode(data)

        request_data = dict(
            url=url,
            headers={},
            method=method,
            body=data
        )

        if auth is not None:
            try:
                passhash = b64encode(':'.join(auth).encode('ascii'))
            except TypeError as e:
                raise TokenError(
                    'Missing credentials (client_id:client_secret)', str(e)
                )

            request_data['headers']['Authorization'] = (
                'Basic %s' % passhash.decode('utf-8')
            )

        request_data.update(self._http_options)
        request = HTTPRequest(**request_data)

        logger.debug('Request: %s %s', request.method, request.url)
        for header in request.headers:
            logger.debug('Header %s: %s', header, request.headers[header])

        try:
            response = yield self._http_client.fetch(request)
        except HTTPError as http_err:
            err = TokenHTTPError('Failed to request token', http_err.response)
            logger.error(
                'Could not request a token for client id: %s, error: %s',
                self._client_id, err)
            raise err

        result = json.loads(response.body.decode("utf-8"))
        raise gen.Return(result)
		else:
			print "%s irc updated!" % repo_name

if __name__ == "__main__":
	request = urllib2.Request("https://github.com/api/v2/json/repos/show/%s" % GITHUB_USERNAME)
	#request = urllib2.Request("https://api.github.com/users/%s/repos" % GITHUB_USERNAME)
	request.add_header("Accept", "application/json")

	try:
		response = urllib2.urlopen(request)
	except urllib2.HTTPError, e:
		print "Server returned error %d" % e.code
	except urllib2.URLError, e:
		print "Could not reach the server: %s" % e.reason
	else:
		data = json.loads(response.read())

		for repo in data['repositories']:
		#for repo in data:
			repo_name = repo['name']
			repo_private = repo['private']
			owner_login = repo['owner']
			#owner_login = repo['owner']['login']

			if repo_name not in GITHUB_IGNORE_REPOS:
				if repo_private == True:
					print "Skipping '%s'" % repo_name
					continue

				try:
					update_twitter_settings(owner_login, repo_name)
Example #14
0
 def loads(self, body):
     """Load from body
     """
     return json.loads(body)
Example #15
0
        except urllib2.URLError, e:
            if str(e.reason) == 'timed out':
                retries = retries + 1
                timeout += 5

        except urllib2.HTTPError, e:
            print "Error code: %s" % e.code
            raise

    if retries == 3:
        exit(1)

    if re.match(r'^2[0-9][0-9]$', str(urlobj.code).strip()):
        body = urlobj.read()
        content = json.loads(body)

    return content


def is_mounted(device_name, basepath='/srv/node'):
    """
    Check if a device is mounted

    :param device_name: The name of the device to check. We assume that the
                        mount point has the same name.
    :param basepath: The path where swift drives are mounted.
    :returns: A boolean value that reflects the result of the operation.
    """
    return os.path.ismount(os.path.join(basepath, device_name))
Example #16
0
]

same = []
changed = []
for currency in currencies:
    # print '\n' + currency

    model = 'models/{0}.pkl'.format(currency.upper())
    rfc = sk.externals.joblib.load(model)

    pathData = realpath('data/{0}'.format(currency.upper()))
    onlyfiles = sorted([f for f in listdir(pathData) if isfile(join(pathData, f))])
    data = {}
    for file in onlyfiles:
        with open(pathData + '/' + file, 'rb') as f:
            fileData = json.loads(f.read())
            data = dict(data.items() + fileData.items())
    print 'data loaded'

    features = extractFeatures(data)
    print len(features), 'features extracted'
    print json.dumps(features[-1], indent=4)

    rewards = calcRewards(data)
    print len(rewards), 'rewards calculated'
    print json.dumps(rewards[-10:-5], indent=4)

    # split sets
    X_train, X_test, y_train, y_test = sk.cross_validation.train_test_split(
        features,
        rewards,
def unmarshal(request_obj):
    # TODO: ensure json, support xml
    json_raw = json.loads(request_obj.data)
    return json_raw
 def wrapper(jsobject):
     pyobject = json.loads(obj.js_context.JSON.stringify(jsobject))
     return self.func(obj, pyobject)
Example #19
0
class TokenManager(object):
    def __init__(self, token_endpoint, client_id, client_secret):
        self._token_endpoint = token_endpoint
        self._client_id = client_id
        self._client_secret = client_secret
        self._token = Token()
        self._http_client = AsyncHTTPClient()

    def _has_token(self):
        return self._token.is_valid()

    @gen.coroutine
    def get_token(self):
        if not self._has_token():
            yield self._update_token()
        raise gen.Return(self._token.access_token)

    @gen.coroutine
    def _get_token_data(self):
        token_data = yield self._request_token()
        raise gen.Return(token_data)

    def reset_token(self):
        self._token = Token()

    @gen.coroutine
    def _update_token(self):
        token_data = yield self._get_token_data()
        self._token = Token(token_data.get('access_token', ''),
                            token_data.get('expires_in', 0))

    @gen.coroutine
    def _request_token(self):
        token_data = yield self._fetch(
            url=self._token_endpoint,
            method="POST",
            auth=(self._client_id, self._client_secret),
            data={'grant_type': 'client_credentials'})

        raise gen.Return(token_data)

    @gen.coroutine
    def _fetch(self, url, method="GET", data=None, auth=None):
        if type(data) == dict:
            data = urlencode(data)

        request_data = dict(
            url=url,
            # headers={'Content-Type': 'application/x-www-form-urlencoded'},
            headers={},
            method=method,
            body=data)

        if auth is not None:
            passhash = b64encode(':'.join(auth))
            request_data['headers']['Authorization'] = 'Basic %s' % passhash

        request = HTTPRequest(**request_data)
        print "request:%s %s\n%s\n%s" % (request.method, request.url,
                                         request.headers, request.body)
        try:
            response = yield self._http_client.fetch(request)
        except HTTPError, e:
            raise TokenError('Failed to request token', e)
        result = json.loads(response.body)
        raise gen.Return(result)
Example #20
0
    #'action' : {
    #    'smtp' : {
    #        'from' : '*****@*****.**',
    #        'to' : ['*****@*****.**',],
    #        'subject' : 'Report',
    #    }
    #},
    #'action' : {
    #    'dropbox' : {
    #    }
    #},
    'data' : {
        'guid' : gen_uuid(),
        'name' : 'Stephane Wirtel',
        'date' : '15/09/1980',
        'time' : time.strftime('%Y-%m-%d %H:%M:%S'),
        'lines' : [
            {'firstname' : 'Stephane',
             'lastname' : 'Wirtel',
             'login' : 'stw',
            }
        ]
    }
}
jsonified = json.dumps(values)
headers = {'Content-Type' : 'application/json'}

query = requests.post(url, headers=headers, data=jsonified)
if query.ok:
    print json.loads(query.content)
 def wrapper(jsobject):
     pyobject = json.loads(obj.js_context.JSON.stringify(jsobject))
     return self.func(obj, pyobject)