def launch_lsh_calc(self): # store tweets and kick off run_lsh tw_from = self.cursor tw_till = len(self.tweets) dui = ndb.Key(urlsafe=self.duik).get() dui = dui.extend_tweets(self.tweets[tw_from:tw_till]) self.cursor = len(self.tweets) if not self.matrix: Matrix._initialize() MatrixRow._initialize() self.matrix = Matrix.create(filename=dui.filename(), source='tweets', file_key=self.duik) if self.matrix: dui = dui.set_ds_key(self.matrix.ds_key) if self.matrix: timestamp = datetime.datetime.utcnow().isoformat() deferred.defer(run_lsh, self.duik, self.tweets[tw_from:tw_till], self.matrix.ds_key, tw_from, timestamp) else: frameinfo = getframeinfo(currentframe()) logging.error('file %s, line %s Matrix is missing', frameinfo.filename, frameinfo.lineno + 1)
def post(self): logging.info('TwitterAgent.post()') try: logging.info('TwitterAgent have twitter token: %s', 'yes' if 'tw_auth' in self.session else 'no') if not ('tw_auth' in self.session): logging.error('Not logged in into twitter') raise NotLoggedIn("Not logged in into twitter") # auth = self.session['tw_auth'] auth = tweepy.OAuthHandler(APP_KEY, APP_SECRET) authkey, authsec = (self.session['auth.access_token.key'], self.session['auth.access_token.secret'],) auth.set_access_token(authkey, authsec) api = tweepy.API(auth) self.session['tw_auth'] = auth if not api: logging.error('API not found') self.session['tw_logged_in'] = False self.session['tw_auth'] = None raise NotLoggedIn("Not logged in into twitter") u = users.get_current_user() old_dui = DemoUserInfo.latest_for_user(u) old_duik = old_dui.key.urlsafe() dui = DemoUserInfo(user_id = u.user_id(), email = u.email(), nickname = u.nickname()) duikey = dui.put() self.session['duik'] = duikey.urlsafe() frameinfo = getframeinfo(currentframe()) logging.info('file %s, line %s auth %s %s', frameinfo.filename, frameinfo.lineno+1, authkey, authsec) ################################ deferred.defer(get_tweets, authkey, authsec, self.session['duik'], old_duik) ################################ self.redirect('/') # ################################ # got_tweets = TwitterStatusListener.get_tweets(authkey, authsec, self.session['duik'], old_duik) # ################################ # frameinfo = getframeinfo(currentframe()) # logging.info('file %s, line %s got_tweets %s', frameinfo.filename, frameinfo.lineno+1, len(got_tweets)) # dui = duikey.get() # self.session['tweets'] = dui.tweets # logging.info('TwitterAgent completed, NumTweets = %d', len(self.session['tweets'])) # self.redirect('/') except: frameinfo = getframeinfo(currentframe()) logging.info('file %s, line %s Exception', frameinfo.filename, frameinfo.lineno+1) self.session['tw_auth'] = None self.redirect('/') return self.redirect('/')
def lsh_zipfile(LineFormat, zip_reader, source, filename, file_key = ''): infolist = zip_reader.infolist() dummydoc = MatrixRow.create() # force the creation of the table dataset = Matrix.create(source, filename, file_key) # force the creation of the table and filling it with a row dataset = Matrix.find(dataset.ds_key) start = time.time() all_stats = defaultdict(float) new_docs_count = 0 docs_cache = Cache(max_size = 15) for info in infolist: with zip_reader.open(info) as text_reader: logging.debug('Reading file %s', info.filename) deferred.defer(lsh_text, LineFormat, zip_reader, filename, matrix_key = dataset.ds_key, text_filename = info.filename) return
def launch_lsh_calc(self): # store tweets and kick off run_lsh tw_from = self.cursor tw_till = len(self.tweets) dui = ndb.Key(urlsafe = self.duik).get() dui = dui.extend_tweets(self.tweets[tw_from:tw_till]) self.cursor = len(self.tweets) if not self.matrix: Matrix._initialize() MatrixRow._initialize() self.matrix = Matrix.create(filename = dui.filename(), source = 'tweets', file_key = self.duik) if self.matrix: dui = dui.set_ds_key(self.matrix.ds_key) if self.matrix: timestamp = datetime.datetime.utcnow().isoformat() deferred.defer(run_lsh, self.duik, self.tweets[tw_from:tw_till], self.matrix.ds_key, tw_from, timestamp) else: frameinfo = getframeinfo(currentframe()) logging.error('file %s, line %s Matrix is missing', frameinfo.filename, frameinfo.lineno+1)
def post(self): try: this_app = AppOpenLSH.get_or_insert('KeyOpenLSH') auth = tweepy.OAuthHandler(this_app.twitter_consumer_key, this_app.twitter_consumer_secret) auth.set_access_token(this_app.twitter_access_token_key, this_app.twitter_access_token_secret) api = tweepy.API(auth) if api: self.session['tw_logged_in'] = True # this_app.twitter_access_token_key = self.session['auth.access_token.key'] # this_app.twitter_access_token_secret = self.session['auth.access_token.secret'] # this_app.put() else: logging.error('API not found') self.session['tw_logged_in'] = False this_app.twitter_access_token() raise NotLoggedIn("Not logged in into twitter") u = users.get_current_user() old_dui = DemoUserInteraction.latest_for_user(u) if old_dui: old_duik = old_dui.key.urlsafe() else: old_duik = None demo_user_key = ndb.Key(DemoUser, u.user_id()) demo_user = demo_user_key.get() demo_user = demo_user.refresh() dui = DemoUserInteraction(parent=demo_user_key) duikey = dui.put() self.session['duik'] = duikey.urlsafe() if dui else None authkey = this_app.twitter_access_token_key authsec = this_app.twitter_access_token_secret # frameinfo = getframeinfo(currentframe()) # logging.info('file %s, line %s auth %s %s', frameinfo.filename, frameinfo.lineno+1, authkey, authsec) ################################ deferred.defer(get_tweets, self.session['duik'], old_duik) ################################ start = time.time() fetching = False for i in xrange(5): time.sleep( (1 + i)**2 ) # Time for the fetches to get going, exponential back off dui = dui.refresh() if dui.fetching: fetching = True logging.info( 'Fetch began within %6.2f seconds, proceeding...', time.time() - start) break if not fetching: logging.error('Fetch did not begin within %6.2f seconds, ???', time.time() - start) self.redirect('/') except: frameinfo = getframeinfo(currentframe()) logging.error('file %s, line %s Exception', frameinfo.filename, frameinfo.lineno + 1) self.redirect('/') return self.redirect('/')
def post(self): try: this_app = AppOpenLSH.get_or_insert('KeyOpenLSH') auth = tweepy.OAuthHandler(this_app.twitter_consumer_key, this_app.twitter_consumer_secret) auth.set_access_token(this_app.twitter_access_token_key, this_app.twitter_access_token_secret) api = tweepy.API(auth) if api: self.session['tw_logged_in'] = True # this_app.twitter_access_token_key = self.session['auth.access_token.key'] # this_app.twitter_access_token_secret = self.session['auth.access_token.secret'] # this_app.put() else: logging.error('API not found') self.session['tw_logged_in'] = False this_app.twitter_access_token() raise NotLoggedIn("Not logged in into twitter") u = users.get_current_user() old_dui = DemoUserInteraction.latest_for_user(u) if old_dui: old_duik = old_dui.key.urlsafe() else: old_duik = None demo_user_key = ndb.Key(DemoUser, u.user_id()) demo_user = demo_user_key.get() demo_user = demo_user.refresh() dui = DemoUserInteraction(parent = demo_user_key) duikey = dui.put() self.session['duik'] = duikey.urlsafe() if dui else None authkey = this_app.twitter_access_token_key authsec = this_app.twitter_access_token_secret # frameinfo = getframeinfo(currentframe()) # logging.info('file %s, line %s auth %s %s', frameinfo.filename, frameinfo.lineno+1, authkey, authsec) ################################ deferred.defer(get_tweets, self.session['duik'], old_duik) ################################ start = time.time() fetching = False for i in xrange(5): time.sleep((1+i)**2) # Time for the fetches to get going, exponential back off dui = dui.refresh() if dui.fetching: fetching = True logging.info('Fetch began within %6.2f seconds, proceeding...', time.time()-start) break if not fetching: logging.error('Fetch did not begin within %6.2f seconds, ???', time.time()-start) self.redirect('/') except: frameinfo = getframeinfo(currentframe()) logging.error('file %s, line %s Exception', frameinfo.filename, frameinfo.lineno+1) self.redirect('/') return self.redirect('/')