def _fetch(self, url, method="GET", data=None, auth=None): if type(data) == dict: data = urlencode(data) request_data = dict(url=url, headers={}, method=method, body=data) if auth is not None: try: passhash = b64encode(':'.join(auth).encode('ascii')) except TypeError as e: raise TokenError( 'Missing credentials (client_id:client_secret)', str(e)) request_data['headers']['Authorization'] = 'Basic %s' % passhash request_data.update(self._http_options) request = HTTPRequest(**request_data) logging.info( 'request:%s %s\n%s\n%s' % (request.method, request.url, request.headers, request.body)) try: response = yield self._http_client.fetch(request) except HTTPError as e: raise TokenError('Failed to request token', str(e)) result = json.loads(response.body) raise gen.Return(result)
def get_entities_(url): data = geturl('http://api.evri.com/v1/media/entities.json?uri=' + urllib.quote(url)) data = json.loads(data) graph = data['evriThing']['graph'] def getfacet(item): try: return item['facets']['facet']['$'] except KeyError: pass return '' try: category_list=mklist(graph['categories']['category']) categories=[item['$'] for item in category_list] except KeyError: categories=[] try: entity_list = mklist(graph['entities']['entity']) except KeyError: entity_list = [] return dict( categories=categories, entities=[dict( name=getname(item), facet=getfacet(item), ref=gethref(item) ) for item in entity_list], )
def get_entity_references_(url): data = geturl('http://api.evri.com/v1/media/related.json?includeTopEntities=true&entityURI=' + urllib.quote(url)) data = json.loads(data) data = data['evriThing']['mediaResult']['articleList']['article'] def get_path(article): link = article.get('link', {}) try: return 'http://%s%s' % (link['@hostName'], link['@path']) except KeyError: return '' def getentities(article): try: entities = mklist(article['topEntities']['entity']) except KeyError: return [] return [dict( name=getname(entity), ref=gethref(entity), ) for entity in entities] return [dict( author=article.get('author', {}).get('$', ''), content=article.get('content', {}).get('$', ''), title=article.get('title', {}).get('$', ''), url=get_path(article), entities=getentities(article), ) for article in data]
def get(self): search = self.request.get('search', '') gurl = ('http://content.guardianapis.com/search?q=%s&format=json' % urllib.quote(search)) results = geturl(gurl) results = json.loads(results) try: results = results['response']['results'] except KeyError: results = [] context = dict(search=search) fullresults = [] count = 0 for result in results: url = result['webUrl'] if count < 1: entities = getentities.get_entities(url) else: entities = {} fullresults.append(dict(url=url.encode('utf8'), entities=entities.get('entities'), categories=entities.get('categories'), title=result['webTitle'], author='Guardian', )) count += 1 context['results'] = fullresults self.render("topic.html", context)
def _fetch(self, url, method="GET", data=None, auth=None): if type(data) == dict: data = urlencode(data) request_data = dict( url=url, headers={}, method=method, body=data ) if auth is not None: try: passhash = b64encode(':'.join(auth).encode('ascii')) except TypeError as e: raise TokenError('Missing credentials (client_id:client_secret)', str(e)) request_data['headers']['Authorization'] = 'Basic %s' % passhash request_data.update(self._http_options) request = HTTPRequest(**request_data) logging.info('request:%s %s\n%s\n%s' % (request.method, request.url, request.headers, request.body)) try: response = yield self._http_client.fetch(request) except HTTPError as e: raise TokenError('Failed to request token', str(e)) result = json.loads(response.body) raise gen.Return(result)
def _fetch(self, url, method="GET", data=None, auth=None): if type(data) == dict: data = urlencode(data) request_data = dict(url=url, headers={}, method=method, body=data) if auth is not None: try: passhash = b64encode(':'.join(auth).encode('ascii')) except TypeError as e: raise TokenError( 'Missing credentials (client_id:client_secret)', str(e)) request_data['headers']['Authorization'] = ( 'Basic %s' % passhash.decode('utf-8')) request_data.update(self._http_options) request = HTTPRequest(**request_data) logger.debug('Request: %s %s', request.method, request.url) for header in request.headers: logger.debug('Header %s: %s', header, request.headers[header]) try: response = yield self._http_client.fetch(request) except HTTPError as e: raise TokenHTTPError('Failed to request token', e.response) result = json.loads(response.body.decode("utf-8")) raise gen.Return(result)
def list_job_ids(host="http://localhost", port=6800, project="default"): command = "curl %s:%d/listjobs.json?project=%s" % (host, port, project) command_result = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE).stdout.readline() running_ids = json.loads(command_result).get("running") ids = [] for i in running_ids: ids.append(i["id"]) return ids
def parse_dummy_songs(cls, dummy_songs, stringify=True): if stringify: dummy_songs = json.loads(cls.js_context.JSON.stringify(dummy_songs)) songs = [] for s in dummy_songs: song = parse_to_dsong(s) if song: songs.append(song) return songs
def parse_dummy_songs(cls, dummy_songs, stringify=True): if stringify: dummy_songs = json.loads( cls.js_context.JSON.stringify(dummy_songs)) songs = [] for s in dummy_songs: song = parse_to_dsong(s) if song: songs.append(song) return songs
def on_reports_new(self, request): info = json.loads(request.data) self.logger.debug('request: %r', info) info['uuid'] = gen_uuid() self.queue.enqueue(generate_report, info) values = { 'status' : 'ok', 'report_id' : info['uuid'], } return values
def unserialise(self, value): """Load the schema from json. Overrides any existing configuration in the schema. """ schema = json.loads(value) if schema['version'] != SCHEMA_VERSION: raise RuntimeError("Can't handle this version of the schema (got " "version %s - I understand version %s" % (schema['version'], SCHEMA_VERSION)) self.types = dict(schema['fieldtypes']) self.modified = False
def _fetch(self, url, method="GET", data=None, auth=None): if type(data) == dict: data = urlencode(data) request_data = dict( url=url, headers={}, method=method, body=data ) if auth is not None: try: passhash = b64encode(':'.join(auth).encode('ascii')) except TypeError as e: raise TokenError( 'Missing credentials (client_id:client_secret)', str(e) ) request_data['headers']['Authorization'] = ( 'Basic %s' % passhash.decode('utf-8') ) request_data.update(self._http_options) request = HTTPRequest(**request_data) logger.debug('Request: %s %s', request.method, request.url) for header in request.headers: logger.debug('Header %s: %s', header, request.headers[header]) try: response = yield self._http_client.fetch(request) except HTTPError as http_err: err = TokenHTTPError('Failed to request token', http_err.response) logger.error( 'Could not request a token for client id: %s, error: %s', self._client_id, err) raise err result = json.loads(response.body.decode("utf-8")) raise gen.Return(result)
else: print "%s irc updated!" % repo_name if __name__ == "__main__": request = urllib2.Request("https://github.com/api/v2/json/repos/show/%s" % GITHUB_USERNAME) #request = urllib2.Request("https://api.github.com/users/%s/repos" % GITHUB_USERNAME) request.add_header("Accept", "application/json") try: response = urllib2.urlopen(request) except urllib2.HTTPError, e: print "Server returned error %d" % e.code except urllib2.URLError, e: print "Could not reach the server: %s" % e.reason else: data = json.loads(response.read()) for repo in data['repositories']: #for repo in data: repo_name = repo['name'] repo_private = repo['private'] owner_login = repo['owner'] #owner_login = repo['owner']['login'] if repo_name not in GITHUB_IGNORE_REPOS: if repo_private == True: print "Skipping '%s'" % repo_name continue try: update_twitter_settings(owner_login, repo_name)
def loads(self, body): """Load from body """ return json.loads(body)
except urllib2.URLError, e: if str(e.reason) == 'timed out': retries = retries + 1 timeout += 5 except urllib2.HTTPError, e: print "Error code: %s" % e.code raise if retries == 3: exit(1) if re.match(r'^2[0-9][0-9]$', str(urlobj.code).strip()): body = urlobj.read() content = json.loads(body) return content def is_mounted(device_name, basepath='/srv/node'): """ Check if a device is mounted :param device_name: The name of the device to check. We assume that the mount point has the same name. :param basepath: The path where swift drives are mounted. :returns: A boolean value that reflects the result of the operation. """ return os.path.ismount(os.path.join(basepath, device_name))
] same = [] changed = [] for currency in currencies: # print '\n' + currency model = 'models/{0}.pkl'.format(currency.upper()) rfc = sk.externals.joblib.load(model) pathData = realpath('data/{0}'.format(currency.upper())) onlyfiles = sorted([f for f in listdir(pathData) if isfile(join(pathData, f))]) data = {} for file in onlyfiles: with open(pathData + '/' + file, 'rb') as f: fileData = json.loads(f.read()) data = dict(data.items() + fileData.items()) print 'data loaded' features = extractFeatures(data) print len(features), 'features extracted' print json.dumps(features[-1], indent=4) rewards = calcRewards(data) print len(rewards), 'rewards calculated' print json.dumps(rewards[-10:-5], indent=4) # split sets X_train, X_test, y_train, y_test = sk.cross_validation.train_test_split( features, rewards,
def unmarshal(request_obj): # TODO: ensure json, support xml json_raw = json.loads(request_obj.data) return json_raw
def wrapper(jsobject): pyobject = json.loads(obj.js_context.JSON.stringify(jsobject)) return self.func(obj, pyobject)
class TokenManager(object): def __init__(self, token_endpoint, client_id, client_secret): self._token_endpoint = token_endpoint self._client_id = client_id self._client_secret = client_secret self._token = Token() self._http_client = AsyncHTTPClient() def _has_token(self): return self._token.is_valid() @gen.coroutine def get_token(self): if not self._has_token(): yield self._update_token() raise gen.Return(self._token.access_token) @gen.coroutine def _get_token_data(self): token_data = yield self._request_token() raise gen.Return(token_data) def reset_token(self): self._token = Token() @gen.coroutine def _update_token(self): token_data = yield self._get_token_data() self._token = Token(token_data.get('access_token', ''), token_data.get('expires_in', 0)) @gen.coroutine def _request_token(self): token_data = yield self._fetch( url=self._token_endpoint, method="POST", auth=(self._client_id, self._client_secret), data={'grant_type': 'client_credentials'}) raise gen.Return(token_data) @gen.coroutine def _fetch(self, url, method="GET", data=None, auth=None): if type(data) == dict: data = urlencode(data) request_data = dict( url=url, # headers={'Content-Type': 'application/x-www-form-urlencoded'}, headers={}, method=method, body=data) if auth is not None: passhash = b64encode(':'.join(auth)) request_data['headers']['Authorization'] = 'Basic %s' % passhash request = HTTPRequest(**request_data) print "request:%s %s\n%s\n%s" % (request.method, request.url, request.headers, request.body) try: response = yield self._http_client.fetch(request) except HTTPError, e: raise TokenError('Failed to request token', e) result = json.loads(response.body) raise gen.Return(result)
#'action' : { # 'smtp' : { # 'from' : '*****@*****.**', # 'to' : ['*****@*****.**',], # 'subject' : 'Report', # } #}, #'action' : { # 'dropbox' : { # } #}, 'data' : { 'guid' : gen_uuid(), 'name' : 'Stephane Wirtel', 'date' : '15/09/1980', 'time' : time.strftime('%Y-%m-%d %H:%M:%S'), 'lines' : [ {'firstname' : 'Stephane', 'lastname' : 'Wirtel', 'login' : 'stw', } ] } } jsonified = json.dumps(values) headers = {'Content-Type' : 'application/json'} query = requests.post(url, headers=headers, data=jsonified) if query.ok: print json.loads(query.content)