def __init__(self, db, collections, username, password, host='localhost', port=8091, clear=False, **entries): auth = restkit.BasicAuth(username, password) server = restkit.Resource('http://%s:%s' % (host, port), filters=[auth]) def make_conn(bucket): self._ensure_bucket(server, bucket) return Couchbase.connect(bucket=bucket, host=host, port=port, timeout=5.0) if clear: all_buckets = self._all_buckets(server) for coll in collections: if coll in all_buckets: self._flush_bucket(server, coll) update(self, server=server, connections=dict(('docs_%s' % coll, make_conn(coll)) for coll in collections))
def routerEndpoints(): # ideally this would all be in the same rdf store, with int and # ext versions of urls txt = open("/my/site/magma/tomato_config.js").read().replace('\n', '') knownMacAddr = jsValue(txt, 'knownMacAddr') tomatoUrl = jsValue(txt, 'tomatoUrl') from rdflib.Graph import Graph g = Graph() g.parse("/my/proj/openid_proxy/access.n3", format="n3") repl = {'/tomato1/': None, '/tomato2/': None} for k in repl: rows = list( g.query(''' PREFIX p: <http://bigasterisk.com/openid_proxy#> SELECT ?prefix WHERE { [ p:requestPrefix ?public; p:proxyUrlPrefix ?prefix ] }''', initBindings={Variable("public"): Literal(k)})) repl[k] = str(rows[0][0]) routers = [] for url in tomatoUrl: for k, v in repl.items(): url = url.replace(k, v) routers.append(restkit.Resource(url, timeout=2)) return routers, knownMacAddr
def _resource(self): """ Accessor to restkit Resource @param self: the object itself @return: the resource """ return restkit.Resource(self._uri)
def hasShortUrlSync(longUri): try: result = json.loads( restkit.Resource(networking.shortenerRoot()).get( "shortLinkTest", long=longUri).body_string()) except restkit.ResourceNotFound: return None return pat % result['short']
def plainCallSync(agent, name, uri): if not uri: raise ValueError("no uri for service call to %s" % name) t1 = time.time() log.debug("serviceCall: %s %s", name, uri) svc = restkit.Resource(networking.serviceUrl(name) + uri) rsp = svc.get(headers={'x-foaf-agent' : str(agent)}) log.info("timing: service call %r in %.01f ms", name, 1000 * (time.time() - t1)) return rsp.body_string()
def put(self, path): assert path == "/2/level", path val = max(0, min(99, int(99*float(self.request.body.strip())))) restkit.Resource("http://dash:9999").post( "valuepost.html", payload="2-SWITCH MULTILEVEL-user-byte-1-0=%d" % val, headers={"content-type":"application/x-www-form-urlencoded"}, ) self.write("done")
def sizeAttrs_by_http(foafUser, uri, sizeName): innerUri = localSite(uri) + '/size' site = restkit.Resource('http://bang:8086/') # restkit.get would hang in this twisted process return json.loads( site.get(path=innerUri, size=sizeName, headers={ 'x-foaf-agent': foafUser }).body_string())
def ingest(fileObj, mock=False, newImageCb=lambda uri: None): #f = open("/my/mail/drewp/cur/1283804816.32729_0.bang:2,S") msg = maillib.Message.from_file(fileObj) attachments = findAttachments(msg) if not attachments: log.info("no attachments with image extensions") return uri = URIRef("mid:" + urllib.quote(msg.headers['Message-ID'].strip('<>'))) msgDate = msg.date.date().isoformat() now = Literal(datetime.datetime.now(tzlocal())) sesameImport = restkit.Resource("http://bang:9042/", timeout=5) stmts = emailStatements(uri, msg) errs = [] for filename, content in attachments: outDir = "/my/pic/email/%s" % msgDate if not os.path.isdir(outDir): os.mkdir(outDir) for outPath in filenameAttempts(filename, outDir): img = uriOfFilename(rootUri=URIRef(SITE.rstrip('/')), root='/my/pic', filename=outPath) if os.path.exists(outPath) and open(outPath).read() == content: log.info("already wrote this to %s" % outPath) else: try: if not mock: writeExcl(outPath, content) # jhead -autorot might be good to run on the result except OSError, e: log.error(e) continue # next suffix attempt stmts.extend([ (uri, DCTERMS['hasPart'], img), (img, PHO['filenameInEmail'], Literal(filename)), (img, PHO['emailReadTime'], now), ]) log.info(" described new image: %s", img) try: newImageCb(img) except Exception, e: log.error("newImageCb failed on %r: %s" % (img, e)) if not mock: log.debug(" post to sesameImport") sesameImport.post(file=outPath) break
def scanPic(self, uri): mr = MediaResource(graph, uri) jpg, mtime = mr.getImageAndMtime(1000) mat = cv.CreateMatHeader(1, len(jpg), cv.CV_8UC1) cv.SetData(mat, jpg, len(jpg)) img = cv.DecodeImage(mat) grayscale = cv.CreateImage((img.width, img.height), 8, 1) cv.CvtColor(img, grayscale, cv.CV_RGB2GRAY) cv.EqualizeHist(grayscale, grayscale) storage = cv.CreateMemStorage(0) faces = cv.HaarDetectObjects( grayscale, self.cascade, storage, 1.2, # scaleFactor between scans 3, # minNeighbors cv.CV_HAAR_DO_CANNY_PRUNING, (20, 20) # min window size ) size = cv.GetSize(grayscale) for f, neighbors in faces: desc = { 'source': str(uri), 'types': [PHO.Crop], 'tag': 'face', 'x1': f[0] / size[0], 'y1': f[1] / size[1], 'x2': (f[0] + f[2]) / size[0], 'y2': (f[1] + f[3]) / size[1], # this ought to have a padded version for showing, and # also the face coords inside that padded version, for # recognition. Note that the padded one may run into # the margins 'neighbors': neighbors, } alt = restkit.Resource( uri.replace('http://photo.bigasterisk.com/', 'http://bang:8031/') + "/alt") resp = alt.post(payload=json.dumps(desc), headers={ 'content-type': 'application/json', 'x-foaf-agent': 'http://bigasterisk.com/tool/scanFace' }) print resp.status, resp.body_string()
def __init__(self, serverUrl, initNs=None, sendSourceLine=False, resultFormat='json'): _RemoteGraph.__init__(self, serverUrl, initNs, sendSourceLine, resultFormat) self.serverUrl = serverUrl self.resource = restkit.Resource(serverUrl, headers={ "Accept": "application/sparql-results+xml", })
def arrayForImage(uri): res = 15 hueStrength = 1.0 satStrength = .5 valStrength = .4 au = absoluteSite(uri) jpg = restkit.Resource(au).get(size='thumb').body_string() i = Image.open(StringIO(jpg)) i = Image.blend(i, ImageOps.autocontrast(i, cutoff=5), .8) i = i.resize((res, int(res * 3 / 4)), Image.ANTIALIAS) ar = numpy.asarray(i, dtype='f') / 255 ar.shape = i.size[1], i.size[0], 3 ar = hsv_from_rgb(ar) * [hueStrength / 360, satStrength, valStrength] return ar
def onNewImage(self, uri, extra=''): c3po = restkit.Resource('http://bang:9040/') for u in [ 'http://bigasterisk.com/foaf.rdf#drewp', 'http://bigasterisk.com/kelsi/foaf.rdf#kelsi', ]: c3po.post( path='', payload={ 'user': u, 'msg': 'new image from email: %s/page%s' % (uri, extra), 'mode': 'email' }, headers={'content-type': 'application/x-www-form-urlencoded'})
def __init__(self, direct_uri, username, password, **kwargs): super().__init__() import restkit self.restkit = restkit import json self.json = json if username or password: filters = kwargs.get('filters', []) filters.append(restkit.BasicAuth(username, password)) kwargs['filters'] = filters self.username = username self.session = restkit.Resource(uri=direct_uri, **kwargs)
def table_list_cache(self): if(self._table_list_cache is None): url = self.oauth['instance_url'] + query.API_STUB + '/sobjects/' headers = dict() headers['Authorization'] = 'OAuth %s' % self.oauth['access_token'] headers['Content-Type'] = 'application/json' salesforce_timeout = getattr(settings, 'SALESFORCE_QUERY_TIMEOUT', 3) resource = restkit.Resource(url, timeout=salesforce_timeout) log.debug('Request API URL: %s' % url) response = query.handle_api_exceptions(url, resource.get, headers=headers) body = response.body_string() jsrc = force_unicode(body).encode(settings.DEFAULT_CHARSET) self._table_list_cache = json.loads(jsrc) return self._table_list_cache
def table_description_cache(self, table): if(table not in self._table_description_cache): url = self.oauth['instance_url'] + query.API_STUB + ('/sobjects/%s/describe/' % table) headers = dict() headers['Authorization'] = 'OAuth %s' % self.oauth['access_token'] headers['Content-Type'] = 'application/json' salesforce_timeout = getattr(settings, 'SALESFORCE_QUERY_TIMEOUT', 3) resource = restkit.Resource(url, timeout=salesforce_timeout) log.debug('Request API URL: %s' % url) response = query.handle_api_exceptions(url, resource.get, headers=headers) body = response.body_string() jsrc = force_unicode(body).encode(settings.DEFAULT_CHARSET) self._table_description_cache[table] = json.loads(jsrc) assert self._table_description_cache[table]['fields'][0]['type'] == 'id' assert self._table_description_cache[table]['fields'][0]['name'] == 'Id' del self._table_description_cache[table]['fields'][0] return self._table_description_cache[table]
def __init__(self, repoUrl, repoName, initNs={}): """ repoUrl ends with /repositories repoName is the repo to create/use initNs = dict of namespace prefixes to use on all queries """ self.root = restkit.Resource(repoUrl) self.repoName = repoName self.initNs = initNs self.sparqlHeader = ''.join('PREFIX %s: <%s>\n' % (p, f) for p,f in initNs.items()) self.sendNamespaces() if 'openrdf-sesame' in repoUrl: pass else: allegroCall(self.root.post, id=self.repoName, directory='/tmp/agraph-catalog', **{'if-exists' : 'open'})
def placeName(long, lat): geonames = restkit.Resource('http://ws.geonames.org/') log.info("going to ws.geonames.org for %s" % ((long, lat), )) addr = json.loads( geonames.get("findNearestAddressJSON", lat=lat, lng=long).body_string()) if 'address' in addr: addr = addr['address'] return "%s %s, %s %s" % (addr['streetNumber'], addr['street'] or addr['adminName2'], addr['placename'], addr['adminCode1']) else: pl = json.loads( geonames.get("findNearbyPlaceNameJSON", lat=lat, lng=long, style='short').body_string()) if 'geonames' in pl: return pl['geonames'][0]['name'] else: return "no geo %r" % pl
def sendSms(d): name = describeLocation( config, d['longitude'], d['latitude'], d.get('horizAccuracy', d.get('accuracy', 0)), d['user'] # this could be redone for each -recipient- user ) c3po = restkit.Resource('http://bang:9040/') tellUsers = set(config['tellUsers']) if d['user'] in tellUsers: #tellUsers.discard(d['user']) now = time.time() for u in tellUsers: if u in timeOfLastSms: if now < timeOfLastSms[u] + 4.9 * 60: continue timeOfLastSms[u] = now c3po.post( path='', payload={ 'user': u, 'msg': '%s position is %s %s' % (foafName(d['user']), name, config['smsUrl']), 'mode': 'xmpp' }, headers={'content-type': 'application/x-www-form-urlencoded'}) # "eta home in 15 mins" return jsonlib.dumps({'posName': name})
def loadFromGoogle(id): report = {} for m in loadMappings(): if m['id'] == id: if 'kml' in m: url = m['kml'] else: url = "https://maps.google.com/maps/ms?ie=UTF8&hl=en&vps=1&jsv=200b&msa=0&output=kml&msid=%s" % m[ 'msid'] break else: raise ValueError("no matches for id %r" % id) t1 = time.time() report['mapSource'] = url feed = restkit.Resource(url).get().body_string() log.info("load google map data from %s, %s bytes in %s sec" % (url, len(feed), time.time() - t1)) feed = maybeUnzip(feed) root = etree.fromstring(feed.encode('utf8')) #return parseGeoRss(root) return parseKml(root) or parseKmlOld(root), report
#!/usr/bin/python """ read /var/lib/mpd/tag_cache and fill an elasticsearch database """ import restkit, jsonlib index = restkit.Resource("http://plus:9200/") try: index.delete('/mpd/') except restkit.errors.RequestFailed: pass index.put("mpd/") index.put( "mpd/_mapping", payload=jsonlib.write({ "song": { # this was meant to make highlighting work, but i can't use HL # with fuzzy matching, it seems "_all": { "type": "string", "store": "yes", "term_vector": "with_positions_offsets" }, "properties": { "title": { "type": "string", "store": "yes", "term_vector": "with_positions_offsets"
""" https://github.com/binarybucks/mqttitude phone app sends to mosquitto broker we subscribe on that broker and send http to update.py """ import json, restkit, logging import paho.mqtt.client logging.basicConfig(level=logging.INFO, format="%(asctime)-15s %(message)s") log = logging.getLogger() config = json.load(open('priv.json')) updateServer = restkit.Resource("http://bang:9033/update") client = paho.mqtt.client.Client("map-receiver") client.connect("localhost") log.info('connected to %s', client._host) # need more auth here, to be able to read client.subscribe("/mqttitude/#", 0) def on_message(mosq, obj, msg): payload = json.loads(msg.payload) log.info("got message %r %r", msg.topic, payload) try: userFromTopic = config['mqttTopic'][msg.topic] except KeyError: log.warn("ignoring unknown topic") return
) timeOfLastSms = {} # todo: compute this from the db instead class updateTrails(object): def POST(self): """new client is here; send everyone fresh trails""" updateWebClients() return '<div>ok</div>' GET = POST # webos is sending get for my $.post() map3 = restkit.Resource(config['postUpdates']) def updateWebClients(movingUser=None): map3.post(path='', payload=json.dumps(getUpdateMsg(movingUser)), headers={"content-type": "application/json"}) class history(object): def GET(self): web.header('Content-type', 'application/xhtml+xml; charset=UTF-8') rows = mongo.find().sort([("timestamp", -1)]).limit(50) def closest(row):
def _setRoot(self, rootUrl): self._resource = restkit.Resource(rootUrl)
def _resource(self): return restkit.Resource(self._uri)
#!bin/python """ learn about updates from phones. send occasional c3po announcements about them. forward the pings to the web site """ from bottle import route, run, request import json, restkit from describelocation import describeLocation site = restkit.Resource("http://*****:*****@route("/newUpdate", method="POST") def newUpdate(): """ this update in the payload has just been added to mongodb """ d = json.load(request.body) #sendSms(d) pingSite(d) def pingSite(d): site.post("/gotNewTrails", payload=json.dumps(d)) def sendSms(d): name = describeLocation( config, d['longitude'],
No c3po announcements here. """ from bottle import route, run, request import json, time, restkit, traceback, logging from dateutil.parser import parse from dateutil.tz import tzlocal from pymongo import Connection logging.basicConfig() log = logging.getLogger() config = json.loads(open("priv.json").read()) m = config['mongo'] mongo = Connection(m['host'], m['port'])[m['db']][m['collection']] notifier = restkit.Resource("http://localhost:9098/") def finish(d): """ after gathering the attributes into the dict, call this to write it and ping notifier """ print "writing to mongo %s" % json.dumps(d) print mongo.insert(d, safe=True) print " assigned id", d['_id'] found = mongo.find_one(d['_id']) if found['user'] != d['user']: raise ValueError("new record doesn't match sent data") # if you get this far, the primary request was a success. Further # errors are only logged. try:
def get_resource(url): salesforce_timeout = getattr(settings, 'SALESFORCE_QUERY_TIMEOUT', 3) resource = restkit.Resource(url, timeout=salesforce_timeout) log.debug('Request API URL: %s' % url) return resource
#!/usr/bin/python import web, json, restkit elastic = restkit.Resource("http://bang:9200/main") class search(object): def GET(self): params = web.input() elasticResponse = json.loads(elastic.get( "doc/_search", payload=json.dumps({ "query" : { "filtered" : { "query" : {"prefix" : {"text" : params['q']}}, "filter" : {"term" : {"source" : params['source']}} } }, "highlight" : { "pre_tags" : ["<em>"], "post_tags" : ["</em>"], "fields" : {"text" : {"number_of_fragments" : 2}}, } })).body_string()) web.header("Content-Type", "application/json") ret = [] for hit in elasticResponse['hits']['hits']: s = hit['_source'] ret.append(dict(uri=hit['_id'], text=' '.join(hit['highlight']['text']), source=s['source']))
#!/usr/bin/python """ web app to query an elasticsearch database, edit mpd playlist """ import restkit, jsonlib, logging, web from web.contrib.template import render_genshi logging.basicConfig() log = logging.getLogger() render = render_genshi('.', auto_reload=True) elastic = restkit.Resource("http://plus:9200/") urls = ( r"/", "index", r"/search", "search", r"/jquery-1.4.2.min.js", "jquery", ) class index(object): def GET(self): web.header("Content-Type", "application/xhtml+xml") return render.searchelastic() class search(object):
def hiresArray(uri): jpg = restkit.Resource(absoluteSite(uri)).get(size='screen').body_string() i = Image.open(StringIO(jpg)).convert('L') ar = numpy.asarray(i, dtype='f') / 255 ar.shape = i.size[1], i.size[0] return ar