def init_author(self, name, insert):
name = name.lower()
if insert:
self.author = Author.get_or_insert(name)
else:
self.author = Author.get_by_id(name)
if self.author.status == Status.suppressed:
raise RequestAborted('author is suppressed')
self.author_dirty = False
def test_delete_missing_author(self):
author = Author(id='test')
author.put()
self.respond_to_github('https://api.github.com/users/test', {'status': 404})
response = self.app.get(util.update_author_task('TEST'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
author = Author.get_by_id('test')
self.assertIsNone(author)
def test_delete_missing_author(self):
author = Author(id='test')
author.put()
self.respond_to_github('https://api.github.com/users/test', {'status': 404})
response = self.app.get(util.update_author_task('TEST'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
author = Author.get_by_id('test')
self.assertIsNone(author)
def init_author(self, name, insert):
name = name.lower()
if insert:
self.author = Author.get_or_insert(name)
else:
self.author = Author.get_by_id(name)
if self.author.status == Status.suppressed:
raise RequestAborted('author is suppressed')
self.author_dirty = False
def test_update_suppressed_is_noop(self):
author = Author(id='test', status=Status.suppressed)
author.put()
response = self.app.get('/task/update/test', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual(len(tasks), 0)
author = author.key.get()
self.assertEqual(author.status, Status.suppressed)
def test_update_suppressed_is_noop(self):
author = Author(id='test', status=Status.suppressed)
author.put()
response = self.app.get('/task/update/test', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual(len(tasks), 0)
author = author.key.get()
self.assertEqual(author.status, Status.suppressed)
def test_update_all(self):
library_key = Library(id='owner/repo').put()
author_key = Author(id='owner').put()
response = self.app.get('/manage/update-all', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.update_library_task(library_key.id()),
util.update_author_task(author_key.id()),
], [task.url for task in tasks])
def test_update_all(self):
library_key = Library(id='owner/repo').put()
author_key = Author(id='owner').put()
response = self.app.get('/manage/update-all', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.update_library_task(library_key.id()),
util.update_author_task(author_key.id()),
], [task.url for task in tasks])
def handle_get(self):
queue = taskqueue.Queue('update')
if queue.fetch_statistics().tasks > 0:
self.response.write('update already in progress')
return
query = Library.query()
cursor = None
more = True
task_count = 0
while more:
keys, cursor, more = query.fetch_page(50,
keys_only=True,
start_cursor=cursor)
for key in keys:
task_count = task_count + 1
task_url = util.update_library_task(key.id())
util.new_task(task_url, target='manage', queue_name='update')
logging.info('triggered %d library updates', task_count)
query = Author.query()
cursor = None
more = True
task_count = 0
while more:
keys, cursor, more = query.fetch_page(50,
keys_only=True,
start_cursor=cursor)
for key in keys:
task_count = task_count + 1
task_url = util.update_author_task(key.id())
util.new_task(task_url, target='manage', queue_name='update')
logging.info('triggered %d author updates', task_count)
def handle_get(self):
while True:
deleted_something = False
for library_key in Library.query().fetch(keys_only=True, limit=10):
delete_library(library_key, response_for_logging=self.response)
deleted_something = True
for author_key in Author.query().fetch(keys_only=True, limit=10):
delete_author(author_key, response_for_logging=self.response)
deleted_something = True
if not deleted_something:
break
# Delete any remaining entries in the search index.
index = search.Index('repo')
while True:
docs = [
document.doc_id for document in index.get_range(ids_only=True)
]
if not docs:
break
self.response.write('search docs: %s\n' + repr(docs))
index.delete(docs)
self.response.write('Finished')
def handle_get(self):
keys = (
Library.query().filter(Library.kind == 'element')
# pylint: disable=singleton-comparison
.filter(Library.shallow_ingestion == False).filter(
Library.status == Status.ready).fetch(
keys_only=True, read_policy=ndb.EVENTUAL_CONSISTENCY))
elements = Sitemap(id='elements')
elements.pages = [key.id() for key in keys]
elements.put()
logging.info('%d elements', len(elements.pages))
keys = (
Library.query().filter(Library.kind == 'collection')
# pylint: disable=singleton-comparison
.filter(Library.shallow_ingestion == False).filter(
Library.status == Status.ready).fetch(
keys_only=True, read_policy=ndb.EVENTUAL_CONSISTENCY))
collections = Sitemap(id='collections')
collections.pages = [key.id() for key in keys]
collections.put()
logging.info('%d collections', len(collections.pages))
keys = Author.query().fetch(keys_only=True,
read_policy=ndb.EVENTUAL_CONSISTENCY)
authors = Sitemap(id='authors')
authors.pages = [key.id() for key in keys]
authors.put()
logging.info('%d authors', len(authors.pages))
def handle_get(self):
while True:
deleted_something = False
for library_key in Library.query().fetch(keys_only=True, limit=10):
delete_library(library_key, response_for_logging=self.response)
deleted_something = True
for author_key in Author.query().fetch(keys_only=True, limit=10):
delete_author(author_key, response_for_logging=self.response)
deleted_something = True
if not deleted_something:
break
# Delete any remaining entries in the search index.
index = search.Index('repo')
while True:
docs = [
document.doc_id
for document
in index.get_range(ids_only=True)]
if not docs:
break
self.response.write('search docs: %s\n' + repr(docs))
index.delete(docs)
self.response.write('Finished')
def handle_get(self):
queue = taskqueue.Queue('update')
if queue.fetch_statistics().tasks > 0:
self.response.write('update already in progress')
return
query = Library.query()
cursor = None
more = True
task_count = 0
while more:
keys, cursor, more = query.fetch_page(50, keys_only=True, start_cursor=cursor)
for key in keys:
task_count = task_count + 1
task_url = util.update_library_task(key.id())
util.new_task(task_url, target='manage', queue_name='update')
logging.info('triggered %d library updates', task_count)
query = Author.query()
cursor = None
more = True
task_count = 0
while more:
keys, cursor, more = query.fetch_page(50, keys_only=True, start_cursor=cursor)
for key in keys:
task_count = task_count + 1
task_url = util.update_author_task(key.id())
util.new_task(task_url, target='manage', queue_name='update')
logging.info('triggered %d author updates', task_count)
def handle_get(self):
keys = (Library.query()
.filter(Library.kind == 'element')
# pylint: disable=singleton-comparison
.filter(Library.shallow_ingestion == False)
.filter(Library.status == Status.ready)
.fetch(keys_only=True, read_policy=ndb.EVENTUAL_CONSISTENCY))
elements = Sitemap(id='elements')
elements.pages = [key.id() for key in keys]
elements.put()
logging.info('%d elements', len(elements.pages))
keys = (Library.query()
.filter(Library.kind == 'collection')
# pylint: disable=singleton-comparison
.filter(Library.shallow_ingestion == False)
.filter(Library.status == Status.ready)
.fetch(keys_only=True, read_policy=ndb.EVENTUAL_CONSISTENCY))
collections = Sitemap(id='collections')
collections.pages = [key.id() for key in keys]
collections.put()
logging.info('%d collections', len(collections.pages))
keys = Author.query().fetch(keys_only=True, read_policy=ndb.EVENTUAL_CONSISTENCY)
authors = Sitemap(id='authors')
authors.pages = [key.id() for key in keys]
authors.put()
logging.info('%d authors', len(authors.pages))
def get(self, author):
self.response.headers['Access-Control-Allow-Origin'] = '*'
author_object = Author.get_by_id(author.lower())
if author_object is None or author_object.status != Status.ready:
self.response.set_status(404)
return
metadata = json.loads(author_object.metadata)
result = {
'type': metadata['type'],
'login': metadata['login'],
'name': metadata['name'],
'company': metadata['company'],
'blog': metadata['blog'],
'location': metadata['location'],
'email': metadata['email'],
'bio': metadata['bio'],
'avatar_url': metadata['avatar_url'],
'followers': metadata['followers'],
'following': metadata['following'],
'public_gists': metadata['public_gists'],
'public_repos': metadata['public_repos'],
}
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps(result))
def test_ensure_when_present(self):
Author(id='author').put()
response = self.app.get(util.ensure_author_task('author'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([], [task.url for task in tasks])
def test_ingest_author(self):
metadata = '{HI}'
self.respond_to_github('https://api.github.com/users/name', metadata)
response = self.app.get(util.ingest_author_task('NAME'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
author = Author.get_by_id('name')
self.assertIsNone(author.error)
self.assertEqual(author.status, 'ready')
self.assertEqual(author.metadata, metadata)
def test_ingest_author(self):
metadata = '{HI}'
self.respond_to_github('https://api.github.com/users/name', metadata)
response = self.app.get(util.ingest_author_task('NAME'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
author = Author.get_by_id('name')
self.assertIsNone(author.error)
self.assertEqual(author.status, 'ready')
self.assertEqual(author.metadata, metadata)
def handle_get(self, name):
author = Author.get_by_id(name.lower())
if author is None:
task_url = util.ingest_author_task(name)
util.new_task(task_url, target='manage')
def handle_get(self, name):
author = Author.get_by_id(name.lower())
if author is None:
task_url = util.ingest_author_task(name)
util.new_task(task_url, target='manage')