class LocalTouristClassifier(): def __init__(self): self.tweets = Connection().tweetsDB.tweetsCollection self.tweetUsers = Connection().tweetsDB.tweetUsersCollection self.tweetUsers.ensure_index( [("loc", GEO2D )] ) self.photos = Connection().flickrDB.flickrCollection self.linked = Connection().linkedDB.linkedCollection API_KEY = 'dj0yJmk9UUY5TWxNMXBRb0M3JmQ9WVdrOVV6RlVOWFEzTjJzbWNHbzlNVGMzTVRBNE5EazJNZy0tJnM9Y29uc3VtZXJzZWNyZXQmeD0zYQ--' SHARED_SECRET = '92a96753c369996f18b6a2ef4a6b1b9c85de04f5' self.y = yql.TwoLegged(API_KEY, SHARED_SECRET) self.yqlCache = {} def unescape_html_chars(self, item): return item.replace("&", "&").replace(">", ">").replace("<", "<").replace(""", "\"") def classifyTwitter(self): for tweet in self.tweets.find({"place":self.place}): if tweet['fromUserID'] is not None: tweetUser = self.tweetUsers.find_one({'id':tweet['fromUserID']}) if tweetUser is not None: tweetUserLocation = tweetUser['location'] if tweetUserLocation is not None and tweetUser['loc'] is None: tweetUserLocation = tweetUserLocation.encode('utf-8') #print "%s || %s" % (tweetUserLocation, self.place) # we use the yqlCache local dictionary to use as few calls as possible if self.yqlCache.get(tweetUserLocation) is not None and self.yqlCache[tweetUserLocation] != 0: tweetUser['loc'] = self.yqlCache[tweetUserLocation] print 'cacheSuccess: %20s %15s %s' % (tweetUserLocation, tweetUser['id'], tweetUser['loc']) else: # send request out to YQL yqlQuery = 'select * from geo.placefinder where text="%s";' % tweetUserLocation try: yqlResult = self.y.execute(yqlQuery) if yqlResult.rows == []: # yql couldn't figure out where this is, so don't save a loc self.yqlCache[tweetUserLocation] = 0 print 'fail: %20s %s' % (tweetUserLocation, tweetUser['id']) else: # yql found a lat and lon, so let's tag it loc = [float(yqlResult.rows[0].get('latitude')), float(yqlResult.rows[0].get('longitude'))] tweetUser['loc'] = loc self.yqlCache[tweetUserLocation] = loc print 'success: %20s %15s %s' % (tweetUserLocation, tweetUser['id'], loc) except: print "Exception Detected:", sys.exc_info()[0] # ready to save user self.tweetUsers.save(tweetUser)
from pymongo import Connection import cyclone.web, cyclone.websocket from twisted.internet import reactor from describelocation import describeLocationFull, metersFromHome logging.basicConfig(level=logging.INFO) log = logging.getLogger() MAP = Namespace("http://bigasterisk.com/map#") XSD = Namespace("http://www.w3.org/2001/XMLSchema#") sys.path.append("/my/site/magma") from stategraph import StateGraph config = json.loads(open("priv.json").read()) m = config['mongo'] mongo = Connection(m['host'], m['port'])[m['db']][m['collection']] mongo.ensure_index([('recv_time', 1)]) TIME_SORT = ('timestamp', -1) def pt_sec(pt): return pt['timestamp'] if 0: # owntracks is stalling on the 'tst' time value, but sending mostly ok data TIME_SORT = ('recv_time', -1) def pt_sec(pt): return pt['recv_time'] class Trails(cyclone.web.RequestHandler): def get(self): q = json.loads(self.get_argument('q')) self.write(getUpdateMsg(query=q))
proc_col.ensure_index('most_recent_event_timestamp') # For time range searches! This multi-key index ensures fast # searches for creation_time alone too! proc_col.ensure_index([('creation_time', ASCENDING), ('exit_time', ASCENDING)]) proc_col.ensure_index('phases.name') proc_col.ensure_index('phases.start_time') proc_col.ensure_index('phases.files_read.timestamp') proc_col.ensure_index('phases.files_written.timestamp') proc_col.ensure_index('phases.files_renamed.timestamp') # index all collections by session_tag: for c in all_cols: c.ensure_index('session_tag') # one-shot mode is useful for debugging or running on archival logs if options.one_shot: do_incremental_index() sys.exit(0) atexit.register(exit_handler) signal(SIGTERM, lambda signum, frame: exit(1)) # trigger the atexit function to run # this loop can only be interrupted by exit_handler() while True: # sleep first so that we can give the logs some time to build up at # the beginning of a login session ... time.sleep(INDEXING_PERIOD_SEC)
proc_col.ensure_index('exited') proc_col.ensure_index('most_recent_event_timestamp') # For time range searches! This multi-key index ensures fast # searches for creation_time alone too! proc_col.ensure_index([('creation_time', ASCENDING), ('exit_time', ASCENDING)]) proc_col.ensure_index('phases.name') proc_col.ensure_index('phases.start_time') proc_col.ensure_index('phases.files_read.timestamp') proc_col.ensure_index('phases.files_written.timestamp') proc_col.ensure_index('phases.files_renamed.timestamp') # index all collections by session_tag: for c in all_cols: c.ensure_index('session_tag') # one-shot mode is useful for debugging or running on archival logs if options.one_shot: do_incremental_index() sys.exit(0) atexit.register(exit_handler) signal(SIGTERM, lambda signum,frame: exit(1)) # trigger the atexit function to run # this loop can only be interrupted by exit_handler() while True: # sleep first so that we can give the logs some time to build up at # the beginning of a login session ...
'-o', psFile.name]) svgFile = NamedTemporaryFile(suffix='.svg') check_call(['pstoedit', '-f', 'plot-svg', '-yshift', '580', '-xshift', '20', psFile.name, svgFile.name]) lines = open(svgFile.name).readlines() return ''.join(lines[2:]) def codeElem(s): return '<div class="bc">%s</div>' % makeCode(s) mpdPaths = Connection("bang", 27017)['barcodePlayer']['mpdPaths'] # {mpdPath:"music/path/to/album/or/song", "_id":12} mpdPaths.ensure_index([('mpdPath', 1)]) def idForMpdPath(p): match = mpdPaths.find_one({"mpdPath" : p}) if match: return match['_id'] top = list(mpdPaths.find().sort([('_id', -1)]).limit(1)) newId = top[0]['_id'] + 1 if top else 0 mpdPaths.insert({"mpdPath" : p, "_id" : newId}) return newId out = open("out.xhtml", "w") out.write("""<?xml version="1.0" encoding="utf-8"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">