def launch_lsh_calc(self):
# store tweets and kick off run_lsh
tw_from = self.cursor
tw_till = len(self.tweets)
dui = ndb.Key(urlsafe=self.duik).get()
dui = dui.extend_tweets(self.tweets[tw_from:tw_till])
self.cursor = len(self.tweets)
if not self.matrix:
Matrix._initialize()
MatrixRow._initialize()
self.matrix = Matrix.create(filename=dui.filename(),
source='tweets',
file_key=self.duik)
if self.matrix:
dui = dui.set_ds_key(self.matrix.ds_key)
if self.matrix:
timestamp = datetime.datetime.utcnow().isoformat()
deferred.defer(run_lsh, self.duik, self.tweets[tw_from:tw_till],
self.matrix.ds_key, tw_from, timestamp)
else:
frameinfo = getframeinfo(currentframe())
logging.error('file %s, line %s Matrix is missing',
frameinfo.filename, frameinfo.lineno + 1)
def post(self):
logging.info('TwitterAgent.post()')
try:
logging.info('TwitterAgent have twitter token: %s', 'yes' if 'tw_auth' in self.session else 'no')
if not ('tw_auth' in self.session):
logging.error('Not logged in into twitter')
raise NotLoggedIn("Not logged in into twitter")
# auth = self.session['tw_auth']
auth = tweepy.OAuthHandler(APP_KEY, APP_SECRET)
authkey, authsec = (self.session['auth.access_token.key'], self.session['auth.access_token.secret'],)
auth.set_access_token(authkey, authsec)
api = tweepy.API(auth)
self.session['tw_auth'] = auth
if not api:
logging.error('API not found')
self.session['tw_logged_in'] = False
self.session['tw_auth'] = None
raise NotLoggedIn("Not logged in into twitter")
u = users.get_current_user()
old_dui = DemoUserInfo.latest_for_user(u)
old_duik = old_dui.key.urlsafe()
dui = DemoUserInfo(user_id = u.user_id(), email = u.email(), nickname = u.nickname())
duikey = dui.put()
self.session['duik'] = duikey.urlsafe()
frameinfo = getframeinfo(currentframe())
logging.info('file %s, line %s auth %s %s', frameinfo.filename, frameinfo.lineno+1, authkey, authsec)
################################
deferred.defer(get_tweets, authkey, authsec, self.session['duik'], old_duik)
################################
self.redirect('/')
# ################################
# got_tweets = TwitterStatusListener.get_tweets(authkey, authsec, self.session['duik'], old_duik)
# ################################
# frameinfo = getframeinfo(currentframe())
# logging.info('file %s, line %s got_tweets %s', frameinfo.filename, frameinfo.lineno+1, len(got_tweets))
# dui = duikey.get()
# self.session['tweets'] = dui.tweets
# logging.info('TwitterAgent completed, NumTweets = %d', len(self.session['tweets']))
# self.redirect('/')
except:
frameinfo = getframeinfo(currentframe())
logging.info('file %s, line %s Exception', frameinfo.filename, frameinfo.lineno+1)
self.session['tw_auth'] = None
self.redirect('/')
return
self.redirect('/')
def lsh_zipfile(LineFormat, zip_reader, source, filename, file_key = ''):
infolist = zip_reader.infolist()
dummydoc = MatrixRow.create() # force the creation of the table
dataset = Matrix.create(source, filename, file_key) # force the creation of the table and filling it with a row
dataset = Matrix.find(dataset.ds_key)
start = time.time()
all_stats = defaultdict(float)
new_docs_count = 0
docs_cache = Cache(max_size = 15)
for info in infolist:
with zip_reader.open(info) as text_reader:
logging.debug('Reading file %s', info.filename)
deferred.defer(lsh_text, LineFormat, zip_reader, filename, matrix_key = dataset.ds_key, text_filename = info.filename)
return
def launch_lsh_calc(self):
# store tweets and kick off run_lsh
tw_from = self.cursor
tw_till = len(self.tweets)
dui = ndb.Key(urlsafe = self.duik).get()
dui = dui.extend_tweets(self.tweets[tw_from:tw_till])
self.cursor = len(self.tweets)
if not self.matrix:
Matrix._initialize()
MatrixRow._initialize()
self.matrix = Matrix.create(filename = dui.filename(), source = 'tweets', file_key = self.duik)
if self.matrix:
dui = dui.set_ds_key(self.matrix.ds_key)
if self.matrix:
timestamp = datetime.datetime.utcnow().isoformat()
deferred.defer(run_lsh, self.duik, self.tweets[tw_from:tw_till], self.matrix.ds_key, tw_from, timestamp)
else:
frameinfo = getframeinfo(currentframe())
logging.error('file %s, line %s Matrix is missing', frameinfo.filename, frameinfo.lineno+1)
def post(self):
try:
this_app = AppOpenLSH.get_or_insert('KeyOpenLSH')
auth = tweepy.OAuthHandler(this_app.twitter_consumer_key,
this_app.twitter_consumer_secret)
auth.set_access_token(this_app.twitter_access_token_key,
this_app.twitter_access_token_secret)
api = tweepy.API(auth)
if api:
self.session['tw_logged_in'] = True
# this_app.twitter_access_token_key = self.session['auth.access_token.key']
# this_app.twitter_access_token_secret = self.session['auth.access_token.secret']
# this_app.put()
else:
logging.error('API not found')
self.session['tw_logged_in'] = False
this_app.twitter_access_token()
raise NotLoggedIn("Not logged in into twitter")
u = users.get_current_user()
old_dui = DemoUserInteraction.latest_for_user(u)
if old_dui:
old_duik = old_dui.key.urlsafe()
else:
old_duik = None
demo_user_key = ndb.Key(DemoUser, u.user_id())
demo_user = demo_user_key.get()
demo_user = demo_user.refresh()
dui = DemoUserInteraction(parent=demo_user_key)
duikey = dui.put()
self.session['duik'] = duikey.urlsafe() if dui else None
authkey = this_app.twitter_access_token_key
authsec = this_app.twitter_access_token_secret
# frameinfo = getframeinfo(currentframe())
# logging.info('file %s, line %s auth %s %s', frameinfo.filename, frameinfo.lineno+1, authkey, authsec)
################################
deferred.defer(get_tweets, self.session['duik'], old_duik)
################################
start = time.time()
fetching = False
for i in xrange(5):
time.sleep(
(1 + i)**2
) # Time for the fetches to get going, exponential back off
dui = dui.refresh()
if dui.fetching:
fetching = True
logging.info(
'Fetch began within %6.2f seconds, proceeding...',
time.time() - start)
break
if not fetching:
logging.error('Fetch did not begin within %6.2f seconds, ???',
time.time() - start)
self.redirect('/')
except:
frameinfo = getframeinfo(currentframe())
logging.error('file %s, line %s Exception', frameinfo.filename,
frameinfo.lineno + 1)
self.redirect('/')
return
self.redirect('/')
def post(self):
try:
this_app = AppOpenLSH.get_or_insert('KeyOpenLSH')
auth = tweepy.OAuthHandler(this_app.twitter_consumer_key, this_app.twitter_consumer_secret)
auth.set_access_token(this_app.twitter_access_token_key, this_app.twitter_access_token_secret)
api = tweepy.API(auth)
if api:
self.session['tw_logged_in'] = True
# this_app.twitter_access_token_key = self.session['auth.access_token.key']
# this_app.twitter_access_token_secret = self.session['auth.access_token.secret']
# this_app.put()
else:
logging.error('API not found')
self.session['tw_logged_in'] = False
this_app.twitter_access_token()
raise NotLoggedIn("Not logged in into twitter")
u = users.get_current_user()
old_dui = DemoUserInteraction.latest_for_user(u)
if old_dui:
old_duik = old_dui.key.urlsafe()
else:
old_duik = None
demo_user_key = ndb.Key(DemoUser, u.user_id())
demo_user = demo_user_key.get()
demo_user = demo_user.refresh()
dui = DemoUserInteraction(parent = demo_user_key)
duikey = dui.put()
self.session['duik'] = duikey.urlsafe() if dui else None
authkey = this_app.twitter_access_token_key
authsec = this_app.twitter_access_token_secret
# frameinfo = getframeinfo(currentframe())
# logging.info('file %s, line %s auth %s %s', frameinfo.filename, frameinfo.lineno+1, authkey, authsec)
################################
deferred.defer(get_tweets, self.session['duik'], old_duik)
################################
start = time.time()
fetching = False
for i in xrange(5):
time.sleep((1+i)**2) # Time for the fetches to get going, exponential back off
dui = dui.refresh()
if dui.fetching:
fetching = True
logging.info('Fetch began within %6.2f seconds, proceeding...', time.time()-start)
break
if not fetching:
logging.error('Fetch did not begin within %6.2f seconds, ???', time.time()-start)
self.redirect('/')
except:
frameinfo = getframeinfo(currentframe())
logging.error('file %s, line %s Exception', frameinfo.filename, frameinfo.lineno+1)
self.redirect('/')
return
self.redirect('/')
