def __init__(self,
db,
collections,
username,
password,
host='localhost',
port=8091,
clear=False,
**entries):
auth = restkit.BasicAuth(username, password)
server = restkit.Resource('http://%s:%s' % (host, port),
filters=[auth])
def make_conn(bucket):
self._ensure_bucket(server, bucket)
return Couchbase.connect(bucket=bucket,
host=host,
port=port,
timeout=5.0)
if clear:
all_buckets = self._all_buckets(server)
for coll in collections:
if coll in all_buckets:
self._flush_bucket(server, coll)
update(self,
server=server,
connections=dict(('docs_%s' % coll, make_conn(coll))
for coll in collections))
def routerEndpoints():
# ideally this would all be in the same rdf store, with int and
# ext versions of urls
txt = open("/my/site/magma/tomato_config.js").read().replace('\n', '')
knownMacAddr = jsValue(txt, 'knownMacAddr')
tomatoUrl = jsValue(txt, 'tomatoUrl')
from rdflib.Graph import Graph
g = Graph()
g.parse("/my/proj/openid_proxy/access.n3", format="n3")
repl = {'/tomato1/': None, '/tomato2/': None}
for k in repl:
rows = list(
g.query('''
PREFIX p: <http://bigasterisk.com/openid_proxy#>
SELECT ?prefix WHERE {
[
p:requestPrefix ?public;
p:proxyUrlPrefix ?prefix
]
}''',
initBindings={Variable("public"): Literal(k)}))
repl[k] = str(rows[0][0])
routers = []
for url in tomatoUrl:
for k, v in repl.items():
url = url.replace(k, v)
routers.append(restkit.Resource(url, timeout=2))
return routers, knownMacAddr
def _resource(self):
"""
Accessor to restkit Resource
@param self: the object itself
@return: the resource
"""
return restkit.Resource(self._uri)
def hasShortUrlSync(longUri):
try:
result = json.loads(
restkit.Resource(networking.shortenerRoot()).get(
"shortLinkTest", long=longUri).body_string())
except restkit.ResourceNotFound:
return None
return pat % result['short']
def plainCallSync(agent, name, uri):
if not uri:
raise ValueError("no uri for service call to %s" % name)
t1 = time.time()
log.debug("serviceCall: %s %s", name, uri)
svc = restkit.Resource(networking.serviceUrl(name) + uri)
rsp = svc.get(headers={'x-foaf-agent' : str(agent)})
log.info("timing: service call %r in %.01f ms", name, 1000 * (time.time() - t1))
return rsp.body_string()
def put(self, path):
assert path == "/2/level", path
val = max(0, min(99, int(99*float(self.request.body.strip()))))
restkit.Resource("http://dash:9999").post(
"valuepost.html",
payload="2-SWITCH MULTILEVEL-user-byte-1-0=%d" % val,
headers={"content-type":"application/x-www-form-urlencoded"},
)
self.write("done")
def sizeAttrs_by_http(foafUser, uri, sizeName):
innerUri = localSite(uri) + '/size'
site = restkit.Resource('http://bang:8086/')
# restkit.get would hang in this twisted process
return json.loads(
site.get(path=innerUri,
size=sizeName,
headers={
'x-foaf-agent': foafUser
}).body_string())
def ingest(fileObj, mock=False, newImageCb=lambda uri: None):
#f = open("/my/mail/drewp/cur/1283804816.32729_0.bang:2,S")
msg = maillib.Message.from_file(fileObj)
attachments = findAttachments(msg)
if not attachments:
log.info("no attachments with image extensions")
return
uri = URIRef("mid:" + urllib.quote(msg.headers['Message-ID'].strip('<>')))
msgDate = msg.date.date().isoformat()
now = Literal(datetime.datetime.now(tzlocal()))
sesameImport = restkit.Resource("http://bang:9042/", timeout=5)
stmts = emailStatements(uri, msg)
errs = []
for filename, content in attachments:
outDir = "/my/pic/email/%s" % msgDate
if not os.path.isdir(outDir):
os.mkdir(outDir)
for outPath in filenameAttempts(filename, outDir):
img = uriOfFilename(rootUri=URIRef(SITE.rstrip('/')),
root='/my/pic',
filename=outPath)
if os.path.exists(outPath) and open(outPath).read() == content:
log.info("already wrote this to %s" % outPath)
else:
try:
if not mock:
writeExcl(outPath, content)
# jhead -autorot might be good to run on the result
except OSError, e:
log.error(e)
continue # next suffix attempt
stmts.extend([
(uri, DCTERMS['hasPart'], img),
(img, PHO['filenameInEmail'], Literal(filename)),
(img, PHO['emailReadTime'], now),
])
log.info(" described new image: %s", img)
try:
newImageCb(img)
except Exception, e:
log.error("newImageCb failed on %r: %s" % (img, e))
if not mock:
log.debug(" post to sesameImport")
sesameImport.post(file=outPath)
break
def scanPic(self, uri):
mr = MediaResource(graph, uri)
jpg, mtime = mr.getImageAndMtime(1000)
mat = cv.CreateMatHeader(1, len(jpg), cv.CV_8UC1)
cv.SetData(mat, jpg, len(jpg))
img = cv.DecodeImage(mat)
grayscale = cv.CreateImage((img.width, img.height), 8, 1)
cv.CvtColor(img, grayscale, cv.CV_RGB2GRAY)
cv.EqualizeHist(grayscale, grayscale)
storage = cv.CreateMemStorage(0)
faces = cv.HaarDetectObjects(
grayscale,
self.cascade,
storage,
1.2, # scaleFactor between scans
3, # minNeighbors
cv.CV_HAAR_DO_CANNY_PRUNING,
(20, 20) # min window size
)
size = cv.GetSize(grayscale)
for f, neighbors in faces:
desc = {
'source': str(uri),
'types': [PHO.Crop],
'tag': 'face',
'x1': f[0] / size[0],
'y1': f[1] / size[1],
'x2': (f[0] + f[2]) / size[0],
'y2': (f[1] + f[3]) / size[1],
# this ought to have a padded version for showing, and
# also the face coords inside that padded version, for
# recognition. Note that the padded one may run into
# the margins
'neighbors': neighbors,
}
alt = restkit.Resource(
uri.replace('http://photo.bigasterisk.com/',
'http://bang:8031/') + "/alt")
resp = alt.post(payload=json.dumps(desc),
headers={
'content-type':
'application/json',
'x-foaf-agent':
'http://bigasterisk.com/tool/scanFace'
})
print resp.status, resp.body_string()
def __init__(self,
serverUrl,
initNs=None,
sendSourceLine=False,
resultFormat='json'):
_RemoteGraph.__init__(self, serverUrl, initNs, sendSourceLine,
resultFormat)
self.serverUrl = serverUrl
self.resource = restkit.Resource(serverUrl,
headers={
"Accept":
"application/sparql-results+xml",
})
def arrayForImage(uri):
res = 15
hueStrength = 1.0
satStrength = .5
valStrength = .4
au = absoluteSite(uri)
jpg = restkit.Resource(au).get(size='thumb').body_string()
i = Image.open(StringIO(jpg))
i = Image.blend(i, ImageOps.autocontrast(i, cutoff=5), .8)
i = i.resize((res, int(res * 3 / 4)), Image.ANTIALIAS)
ar = numpy.asarray(i, dtype='f') / 255
ar.shape = i.size[1], i.size[0], 3
ar = hsv_from_rgb(ar) * [hueStrength / 360, satStrength, valStrength]
return ar
def onNewImage(self, uri, extra=''):
c3po = restkit.Resource('http://bang:9040/')
for u in [
'http://bigasterisk.com/foaf.rdf#drewp',
'http://bigasterisk.com/kelsi/foaf.rdf#kelsi',
]:
c3po.post(
path='',
payload={
'user': u,
'msg': 'new image from email: %s/page%s' % (uri, extra),
'mode': 'email'
},
headers={'content-type': 'application/x-www-form-urlencoded'})
def __init__(self, direct_uri, username, password, **kwargs):
super().__init__()
import restkit
self.restkit = restkit
import json
self.json = json
if username or password:
filters = kwargs.get('filters', [])
filters.append(restkit.BasicAuth(username, password))
kwargs['filters'] = filters
self.username = username
self.session = restkit.Resource(uri=direct_uri, **kwargs)
def table_list_cache(self):
if(self._table_list_cache is None):
url = self.oauth['instance_url'] + query.API_STUB + '/sobjects/'
headers = dict()
headers['Authorization'] = 'OAuth %s' % self.oauth['access_token']
headers['Content-Type'] = 'application/json'
salesforce_timeout = getattr(settings, 'SALESFORCE_QUERY_TIMEOUT', 3)
resource = restkit.Resource(url, timeout=salesforce_timeout)
log.debug('Request API URL: %s' % url)
response = query.handle_api_exceptions(url, resource.get, headers=headers)
body = response.body_string()
jsrc = force_unicode(body).encode(settings.DEFAULT_CHARSET)
self._table_list_cache = json.loads(jsrc)
return self._table_list_cache
def table_description_cache(self, table):
if(table not in self._table_description_cache):
url = self.oauth['instance_url'] + query.API_STUB + ('/sobjects/%s/describe/' % table)
headers = dict()
headers['Authorization'] = 'OAuth %s' % self.oauth['access_token']
headers['Content-Type'] = 'application/json'
salesforce_timeout = getattr(settings, 'SALESFORCE_QUERY_TIMEOUT', 3)
resource = restkit.Resource(url, timeout=salesforce_timeout)
log.debug('Request API URL: %s' % url)
response = query.handle_api_exceptions(url, resource.get, headers=headers)
body = response.body_string()
jsrc = force_unicode(body).encode(settings.DEFAULT_CHARSET)
self._table_description_cache[table] = json.loads(jsrc)
assert self._table_description_cache[table]['fields'][0]['type'] == 'id'
assert self._table_description_cache[table]['fields'][0]['name'] == 'Id'
del self._table_description_cache[table]['fields'][0]
return self._table_description_cache[table]
def __init__(self, repoUrl, repoName, initNs={}):
"""
repoUrl ends with /repositories
repoName is the repo to create/use
initNs = dict of namespace prefixes to use on all queries
"""
self.root = restkit.Resource(repoUrl)
self.repoName = repoName
self.initNs = initNs
self.sparqlHeader = ''.join('PREFIX %s: <%s>\n' % (p, f)
for p,f in initNs.items())
self.sendNamespaces()
if 'openrdf-sesame' in repoUrl:
pass
else:
allegroCall(self.root.post, id=self.repoName,
directory='/tmp/agraph-catalog',
**{'if-exists' : 'open'})
def placeName(long, lat):
geonames = restkit.Resource('http://ws.geonames.org/')
log.info("going to ws.geonames.org for %s" % ((long, lat), ))
addr = json.loads(
geonames.get("findNearestAddressJSON", lat=lat,
lng=long).body_string())
if 'address' in addr:
addr = addr['address']
return "%s %s, %s %s" % (addr['streetNumber'], addr['street']
or addr['adminName2'], addr['placename'],
addr['adminCode1'])
else:
pl = json.loads(
geonames.get("findNearbyPlaceNameJSON",
lat=lat,
lng=long,
style='short').body_string())
if 'geonames' in pl:
return pl['geonames'][0]['name']
else:
return "no geo %r" % pl
def sendSms(d):
name = describeLocation(
config,
d['longitude'],
d['latitude'],
d.get('horizAccuracy', d.get('accuracy', 0)),
d['user'] # this could be redone for each -recipient- user
)
c3po = restkit.Resource('http://bang:9040/')
tellUsers = set(config['tellUsers'])
if d['user'] in tellUsers:
#tellUsers.discard(d['user'])
now = time.time()
for u in tellUsers:
if u in timeOfLastSms:
if now < timeOfLastSms[u] + 4.9 * 60:
continue
timeOfLastSms[u] = now
c3po.post(
path='',
payload={
'user':
u,
'msg':
'%s position is %s %s' %
(foafName(d['user']), name, config['smsUrl']),
'mode':
'xmpp'
},
headers={'content-type': 'application/x-www-form-urlencoded'})
# "eta home in 15 mins"
return jsonlib.dumps({'posName': name})
def loadFromGoogle(id):
report = {}
for m in loadMappings():
if m['id'] == id:
if 'kml' in m:
url = m['kml']
else:
url = "https://maps.google.com/maps/ms?ie=UTF8&hl=en&vps=1&jsv=200b&msa=0&output=kml&msid=%s" % m[
'msid']
break
else:
raise ValueError("no matches for id %r" % id)
t1 = time.time()
report['mapSource'] = url
feed = restkit.Resource(url).get().body_string()
log.info("load google map data from %s, %s bytes in %s sec" %
(url, len(feed), time.time() - t1))
feed = maybeUnzip(feed)
root = etree.fromstring(feed.encode('utf8'))
#return parseGeoRss(root)
return parseKml(root) or parseKmlOld(root), report
#!/usr/bin/python
"""
read /var/lib/mpd/tag_cache and fill an elasticsearch database
"""
import restkit, jsonlib
index = restkit.Resource("http://plus:9200/")
try:
index.delete('/mpd/')
except restkit.errors.RequestFailed:
pass
index.put("mpd/")
index.put(
"mpd/_mapping",
payload=jsonlib.write({
"song": {
# this was meant to make highlighting work, but i can't use HL
# with fuzzy matching, it seems
"_all": {
"type": "string",
"store": "yes",
"term_vector": "with_positions_offsets"
},
"properties": {
"title": {
"type": "string",
"store": "yes",
"term_vector": "with_positions_offsets"
"""
https://github.com/binarybucks/mqttitude
phone app sends to mosquitto broker
we subscribe on that broker and send http to update.py
"""
import json, restkit, logging
import paho.mqtt.client
logging.basicConfig(level=logging.INFO, format="%(asctime)-15s %(message)s")
log = logging.getLogger()
config = json.load(open('priv.json'))
updateServer = restkit.Resource("http://bang:9033/update")
client = paho.mqtt.client.Client("map-receiver")
client.connect("localhost")
log.info('connected to %s', client._host)
# need more auth here, to be able to read
client.subscribe("/mqttitude/#", 0)
def on_message(mosq, obj, msg):
payload = json.loads(msg.payload)
log.info("got message %r %r", msg.topic, payload)
try:
userFromTopic = config['mqttTopic'][msg.topic]
except KeyError:
log.warn("ignoring unknown topic")
return
)
timeOfLastSms = {} # todo: compute this from the db instead
class updateTrails(object):
def POST(self):
"""new client is here; send everyone fresh trails"""
updateWebClients()
return '<div>ok</div>'
GET = POST # webos is sending get for my $.post()
map3 = restkit.Resource(config['postUpdates'])
def updateWebClients(movingUser=None):
map3.post(path='',
payload=json.dumps(getUpdateMsg(movingUser)),
headers={"content-type": "application/json"})
class history(object):
def GET(self):
web.header('Content-type', 'application/xhtml+xml; charset=UTF-8')
rows = mongo.find().sort([("timestamp", -1)]).limit(50)
def closest(row):
def _setRoot(self, rootUrl):
self._resource = restkit.Resource(rootUrl)
def _resource(self):
return restkit.Resource(self._uri)
#!bin/python
"""
learn about updates from phones. send occasional c3po announcements about them. forward the pings to the web site
"""
from bottle import route, run, request
import json, restkit
from describelocation import describeLocation
site = restkit.Resource("http://*****:*****@route("/newUpdate", method="POST")
def newUpdate():
"""
this update in the payload has just been added to mongodb
"""
d = json.load(request.body)
#sendSms(d)
pingSite(d)
def pingSite(d):
site.post("/gotNewTrails", payload=json.dumps(d))
def sendSms(d):
name = describeLocation(
config,
d['longitude'],
No c3po announcements here.
"""
from bottle import route, run, request
import json, time, restkit, traceback, logging
from dateutil.parser import parse
from dateutil.tz import tzlocal
from pymongo import Connection
logging.basicConfig()
log = logging.getLogger()
config = json.loads(open("priv.json").read())
m = config['mongo']
mongo = Connection(m['host'], m['port'])[m['db']][m['collection']]
notifier = restkit.Resource("http://localhost:9098/")
def finish(d):
"""
after gathering the attributes into the dict, call this to write
it and ping notifier
"""
print "writing to mongo %s" % json.dumps(d)
print mongo.insert(d, safe=True)
print " assigned id", d['_id']
found = mongo.find_one(d['_id'])
if found['user'] != d['user']:
raise ValueError("new record doesn't match sent data")
# if you get this far, the primary request was a success. Further
# errors are only logged.
try:
def get_resource(url):
salesforce_timeout = getattr(settings, 'SALESFORCE_QUERY_TIMEOUT', 3)
resource = restkit.Resource(url, timeout=salesforce_timeout)
log.debug('Request API URL: %s' % url)
return resource
#!/usr/bin/python
import web, json, restkit
elastic = restkit.Resource("http://bang:9200/main")
class search(object):
def GET(self):
params = web.input()
elasticResponse = json.loads(elastic.get(
"doc/_search",
payload=json.dumps({
"query" : {
"filtered" : {
"query" : {"prefix" : {"text" : params['q']}},
"filter" : {"term" : {"source" : params['source']}}
}
},
"highlight" : {
"pre_tags" : ["<em>"],
"post_tags" : ["</em>"],
"fields" : {"text" : {"number_of_fragments" : 2}},
}
})).body_string())
web.header("Content-Type", "application/json")
ret = []
for hit in elasticResponse['hits']['hits']:
s = hit['_source']
ret.append(dict(uri=hit['_id'],
text=' '.join(hit['highlight']['text']),
source=s['source']))
#!/usr/bin/python
"""
web app to query an elasticsearch database, edit mpd playlist
"""
import restkit, jsonlib, logging, web
from web.contrib.template import render_genshi
logging.basicConfig()
log = logging.getLogger()
render = render_genshi('.', auto_reload=True)
elastic = restkit.Resource("http://plus:9200/")
urls = (
r"/",
"index",
r"/search",
"search",
r"/jquery-1.4.2.min.js",
"jquery",
)
class index(object):
def GET(self):
web.header("Content-Type", "application/xhtml+xml")
return render.searchelastic()
class search(object):
def hiresArray(uri):
jpg = restkit.Resource(absoluteSite(uri)).get(size='screen').body_string()
i = Image.open(StringIO(jpg)).convert('L')
ar = numpy.asarray(i, dtype='f') / 255
ar.shape = i.size[1], i.size[0]
return ar
