def handle(self, components):
if len(components) > 2 and self.is_admin():
logging.info("Trying to reparse %s" % components[2])
account = model.Account.query(
model.Account.twitter_handle == components[2]).get()
else:
account = model.account_for(self.current_account())
if account is None:
self.redirect("/")
logging.info("Reparsing %s" % account.twitter_handle)
key_str = account.key.urlsafe()
existing = memcache.get("parse_%s" % key_str)
if existing is not None:
self.redirect("/")
memcache.set("parse_%s" % key_str, model.now(), 36000)
account.update_status(model.STATUS_PARSE_URLS)
raw = account.newest_raw_data()
taskqueue.add(queue_name='default',
url='/tasks/parse',
params={
'key': raw.key.urlsafe(),
'clean_urls': self.request.get("clean_urls"),
'clean_all': self.request.get("clean_all"),
})
self.redirect("/me")
def handle(self, components):
account = model.account_for(self.current_account())
if account is not None:
self.redirect("/me")
self.template = 'home.html'
return True
def handle(self, components):
account = model.account_for(self.current_account())
if account is None:
self.redirect("/")
time.sleep(
random.random()) #prevent double fetches, which happen locally
existing = memcache.get("fetch_%s" % account.key.urlsafe())
if existing is not None:
self.redirect("/")
memcache.set("fetch_%s" % account.key.urlsafe(), model.now(), 36000)
first_time = account.status == model.STATUS_CREATED
account.update_status(model.STATUS_FETCH_INITIATED)
raw = model.RawData(account_key=account.key, source="twitter", data=[])
raw.data = []
raw.put()
taskqueue.add(queue_name='default',
url='/tasks/fetch',
params={
'key': raw.key.urlsafe(),
'send_email': str(first_time)
})
self.redirect("/profile?new=true") if first_time else self.redirect(
"me")
def get(self):
session = gaesessions.get_current_session()
account = model.account_for(session.get("account"))
if account is None:
self.redirect("/")
upload_url = blobstore.create_upload_url('/upload')
page_values = {
'handle': account.twitter_handle,
'upload_url': upload_url,
}
path = os.path.join(os.path.dirname(__file__), "html/upload.html")
to_write = template.render(path, page_values)
self.response.out.write(to_write)
def post(self):
account = model.account_for(self.current_account())
if self.is_admin() and len(self.request.get("handle")) > 0:
account = model.Account.query(model.Account.twitter_handle ==
self.request.get("handle")).get()
logging.info("Saving changes")
if account is not None:
account.privacy = 1 if len(self.request.get("privacy")) > 0 else 0
account.name = self.request.get("name")
account.email = self.request.get("email")
if self.is_admin():
account.status = int(self.request.get("status"))
account.put()
self.redirect("/")
def handle(self, components):
account = model.account_for(self.current_account())
if account is None:
self.redirect("/")
key_str = account.key.urlsafe()
existing = memcache.get("unpack_%s" % key_str)
if existing is not None:
return "Already unpacking, started at %s" % existing
memcache.set("unpack_%s" % key_str, model.now(), 36000)
upload = account.newest_upload()
account.update_status(model.STATUS_FILE_UPLOADED)
taskqueue.add(queue_name='unpacker',
url='/tasks/unpack',
params={'key': upload.key.urlsafe()})
self.redirect("/me")
def post(self):
session = gaesessions.get_current_session()
account = model.account_for(session.get("account"))
if account is None:
self.redirect("/")
upload_files = self.get_uploads(
'file') # 'file' is file upload field in the form
blob_info = upload_files[0]
upload = model.UploadedFile(account_key=account.key,
source="twitter",
blob_key=blob_info.key())
upload.put()
taskqueue.add(queue_name='unpacker',
url='/tasks/unpack',
params={'key': upload.key.urlsafe()})
self.response.out.write("success")
def handle(self, components):
account = model.account_for(self.current_account())
if self.is_admin() and len(self.request.get("handle")) > 0:
account = model.Account.query(model.Account.twitter_handle ==
self.request.get("handle")).get()
if account is None:
self.redirect("/")
template_values = {
'account': account,
'first_time': self.request.get("new") == "true",
'handle': self.request.get("handle"),
'admin': self.is_admin(),
}
self.page_values.update(template_values)
self.template = 'profile.html'
return True
def get(self):
self.response.headers['Content-Type'] = 'application/json'
session = gaesessions.get_current_session()
key = session.get("account")
if key is None:
self.response.out.write('{"success":false,"error":"No session"}')
return
status = memcache.get("%s_status" % key)
if status is None:
account = model.account_for(key)
status = account.status
terminal = status == model.STATUS_CREATED or status % 1000 == model.STATUS_COMPLETE
results = {
'success': True,
'terminal': terminal,
'status': model.status_string(status)
}
self.response.out.write(json.dumps(results))
def post(self):
raw_key = self.request.get("key")
send_email = self.request.get("send_email")
raw = ndb.Key(urlsafe = raw_key).get()
account = model.account_for(raw.account_key.urlsafe())
statuses = raw.data
logging.info("parsing %s statuses" % len(statuses))
if self.request.get("clean_all")=="true":
account.urls = {}
account.keywords = []
account.timeline = []
account.update_status(model.STATUS_PARSE_URLS);
new_timeline = self.build_timeline (statuses)
account.timeline = self.merge_timeline(account.timeline, new_timeline)
new_urls = reverse_dict(self.flattened_dict_for(statuses, 'urls'))
account.urls = self.merge_dicts(account.urls, new_urls)
authors = self.authors_for(statuses, account.twitter_handle)
keywords = self.flattened_dict_for(statuses, 'keywords')
hashtags = self.flattened_dict_for(statuses, 'hashtags')
mentions = self.flattened_dict_for(statuses, 'mentions')
sites = {}
urls = reverse_dict(account.urls)
for key in urls:
site = self.url_to_site(key)
sites[site] = sites[site] + urls[key] if site in sites else urls[key]
#paths = map(self.path_to_words, urls)
collected = [['author',authors], ['hashtag',hashtags], ['keyword',keywords], ['mention',mentions], ['site', sites]]
existing = account.keywords
if existing is not None and len(existing)>0:
for collection in collected:
if collection[0]=='site': next
previous = filter(lambda x:x[0]==collection[0], existing)
if len(previous)>0:
previous = previous[0][1]
collection[1] = self.merge_dicts(previous, collection[1])
account.keywords = collected
urlphase = self.request.get("clean_all")=="true" or self.request.get("clean_urls")=="true"
account.update_status(model.STATUS_EXPANDING_URLS if urlphase else model.STATUS_COMPLETE);
account.put()
if urlphase:
logging.info("Urlphase complete")
taskqueue.add(queue_name='expander',
url='/tasks/expand',
params={'key' : raw_key, 'send_email' : send_email})
else:
memcache.delete("parse_%s" % account.key.urlsafe())
logging.info("Parsing complete")
if "True"==self.request.get('send_email'):
self.send_email_to(account)
def handle(self, components):
(handle, year, has_year, month, has_month, category,
term) = ('me', None, False, None, False, None, None)
args = self.parse_components(components)
if len(components) > 1:
handle = components[1]
if len(components) > 2 and components[2].isdigit():
year = int(components[2])
has_year = True
if has_year and len(components) > 3 and components[3].isdigit():
month = int(components[3])
has_month = True
if len(components) > 3 and not has_year:
category = components[2]
term = components[3]
if len(components) > 4 and has_year and not has_month:
category = components[3]
term = components[4]
if len(components) > 5 and has_year and has_month:
category = components[4]
term = components[5]
year = int(components[2])
if handle == 'me':
if len(self.request.get("v")) > 0:
key = memcache.get(self.request.get("v"))
self.set_account(key)
memcache.delete(self.request.get("v"))
self.redirect("/refetch")
account = model.account_for(self.current_account())
if account is None:
logging.info("redirect for handle=me, components %s" %
components)
self.redirect("/")
else:
account = model.Account.query(
model.Account.twitter_handle == handle).get()
if account is not None and account.key.urlsafe(
) != self.current_account() and account.is_private():
return "That user's Vortext is private."
if account is None:
return "That user has not yet activated their Vortext."
status = memcache.get("%s_status" % account.key.urlsafe())
status = account.status if status is None else status
terminal_status = status is None or status == model.STATUS_CREATED or status % 1000 == model.STATUS_COMPLETE
template_values = {
'user_status': model.status_string(status),
'fetching': status is None or status < model.STATUS_FETCH_COMPLETE,
'terminal_status': 'true' if terminal_status else 'false',
'tweets_per_scroll': 10,
'autodata': '[]',
'status_json': '[]',
'args': args,
}
self.page_values.update(template_values)
if account.timeline is not None and account.keywords is not None:
timeline = account.timeline
keywords = account.keywords
if len(timeline) == 0:
statuses = []
else:
if term is not None:
term = urllib.unquote(term)
collections = keywords if category == "all" or category == "search" else filter(
lambda x: x[0] == category, keywords)
term_tweets = []
for collection in collections:
vals = collection[1]
#handle multiple entries with different case
if category == "search":
terms = filter(
lambda x: x.lower().startswith(term.lower()),
vals.keys())
else:
terms = filter(lambda x: x.lower() == term.lower(),
vals.keys())
category_tweets = [vals[t] for t in terms]
category_tweets = reduce(list.__add__, category_tweets,
[])
term_tweets += category_tweets
if year is not None:
months = filter(lambda x: x['number'] == year,
timeline)[0]['months']
if month is not None:
month_ids = filter(lambda x: x['number'] == month,
months)[0]
time_tweets = month_ids['ids']
else:
month_ids = [m['ids'] for m in months]
time_tweets = reduce(list.__add__, month_ids, [])
if year is None and term is None:
month_ids = [m['ids'] for m in timeline[0]['months']]
statuses = reduce(list.__add__, month_ids, [])
elif term is None and year is not None:
statuses = time_tweets
elif year is None and term is not None:
statuses = term_tweets
else:
statuses = list(
set(term_tweets).intersection(set(time_tweets)))
if args.mytweets or args.retweets:
authors = filter(lambda x: x[0] == "author", keywords)[0][1]
authored = [
tweetid for author in authors.values()
for tweetid in author
]
if args.mytweets:
statuses = list(set(statuses).difference(set(authored)))
else:
statuses = list(set(statuses).intersection(set(authored)))
statuses.sort(key=lambda x: long(x))
if args.reverse:
statuses.reverse()
keywords = self.collect(keywords)
template_values = {
'self':
self.current_account() == account.key.urlsafe(),
'handle':
account.twitter_handle,
'timeline':
timeline,
'autodata':
json.dumps(keywords),
'default_year':
timeline[0]['number'] if len(timeline) > 0 else 2015,
'selected_year':
year,
'selected_month':
month,
'month_name':
'' if month is None else model.MONTHS[month],
'category':
category,
'term':
term,
'statuses':
statuses,
'status_json':
json.dumps(statuses),
'show_args':
args.reverse or args.cards or args.conversations
or args.mytweets or args.retweets,
}
self.page_values.update(template_values)
self.template = 'main.html'
return True