def trigger_analysis(self, tag, sha, transactional=False):
analysis_sha = None
if self.library.kind == 'collection':
analysis_sha = sha
version_key = ndb.Key(Library, self.library.key.id(), Version, tag)
content = Content.get_by_id('analysis', parent=version_key)
if content is None or content.status == Status.error:
Content(id='analysis', parent=version_key,
status=Status.pending).put()
task_url = util.ingest_analysis_task(self.owner, self.repo, tag,
analysis_sha)
util.new_task(task_url,
target='analysis',
transactional=transactional,
queue_name='analysis')
def test_analyze_resets_error_content_when_reanalyzing(self):
library_key = Library(id='owner/repo').put()
version_key = Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put()
content = Content(id='analysis', parent=version_key, status=Status.pending)
content.status = Status.error
content.put()
response = self.app.get('/task/analyze/owner/repo', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
content = Content.get_by_id('analysis', parent=version_key)
self.assertEqual(content.status, Status.pending)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('owner', 'repo', 'v1.1.1'),
], [task.url for task in tasks])
def test_compressed(self):
library_key = Library(id='owner/repo').put()
version_key = Version(id='v1.1.1',
parent=library_key,
sha='sha',
status='ready').put()
content = Content(id='analysis',
parent=version_key,
status=Status.pending)
content.json = dict({"analyzerData": "some data"})
content.status = Status.ready
content.put()
response = self.app.get(
'/api/docs/owner/repo/v1.1.1?use_analyzer_data')
self.assertEqual(response.status_int, 200)
self.assertEqual(
json.loads(response.normal_body).get('analysis'), "some data")
def update_bower(self):
response = urlfetch.fetch(util.content_url(self.owner, self.repo,
self.sha, 'bower.json'),
validate_certificate=True)
if response.status_code == 200:
try:
bower_json = json.loads(response.content)
except ValueError:
return self.error("could not parse bower.json")
Content(parent=self.version_key,
id='bower',
content=response.content,
status=Status.ready,
etag=response.headers.get('ETag', None)).put()
return bower_json
elif response.status_code == 404:
return self.error("missing bower.json")
else:
return self.retry('could not access bower.json (%d)' %
response.status_code)
def test_update_indexes(self):
metadata = """{
"full_name": "full-name"
}"""
collection_library_key = Library(id='my/collection',
status=Status.ready,
kind='collection',
metadata=metadata).put()
collection_version_key = Version(id='v1.0.0',
parent=collection_library_key,
sha='sha',
status=Status.ready).put()
Content(id='bower',
parent=collection_version_key,
content="""{"dependencies": {
"a": "org/element-1#1.0.0",
"b": "org/element-2#1.0.0"
}}""").put()
VersionCache.update(collection_library_key)
response = self.app.get(util.update_indexes_task('my', 'collection'),
headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
# Triggers ingestions
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ensure_library_task('org', 'element-1'),
util.ensure_library_task('org', 'element-2'),
], [task.url for task in tasks])
# Ensures collection references
ref1 = CollectionReference.get_by_id(id="my/collection/v1.0.0",
parent=ndb.Key(
Library, "org/element-1"))
self.assertIsNotNone(ref1)
ref2 = CollectionReference.get_by_id(id="my/collection/v1.0.0",
parent=ndb.Key(
Library, "org/element-2"))
self.assertIsNotNone(ref2)
def update_pages(self):
bower = Content.get_by_id('bower', parent=self.version_key)
if bower is None:
return
bower_json = json.loads(bower.content)
for _, path in bower_json.get('pages', {}).iteritems():
response = util.github_get('repos',
self.owner,
self.repo,
'contents/' + path,
params={'ref': self.sha})
if response.status_code == 200:
response_json = json.loads(response.content)
markdown = None
# Ensure a file was returned
if isinstance(response_json,
dict) and response_json.get('type') == 'file':
markdown = base64.b64decode(response_json.get('content'))
elif response.status_code == 404:
markdown = None
else:
return self.retry('error fetching page %s (%d)' %
(path, response.status_code))
if markdown is not None:
response = util.github_markdown(markdown)
if response.status_code == 200:
Content(parent=self.version_key,
id='page-' + path,
content=response.content,
status=Status.ready,
etag=response.headers.get('ETag', None)).put()
else:
return self.retry(
'error converting page to markdown %s (%d)' %
(path, response.status_code))
def test_analyzer_index(self):
metadata = """{
"full_name": "full-name"
}"""
library_key = Library(id='owner/repo', metadata=metadata).put()
version_key = Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put()
content = Content(id='analysis', parent=version_key, status=Status.pending)
data = {
"analyzerData": {
"elements": [{"tagname": "my-element"}, {"classname": "another-element"}],
"metadata": {
"polymer": {"behaviors": [{"name": "polymer-behavior"}]}
}
}
}
content.json = data
content.status = Status.ready
content.put()
VersionCache.update(library_key)
response = self.app.get(util.update_indexes_task('owner', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
index = search.Index('repo')
document = index.get('owner/repo')
self.assertIsNotNone(document)
self.assertTrue(len(document.fields) > 0)
elements = [field for field in document.fields if field.name == 'element']
self.assertEqual(len(elements), 1)
self.assertEqual(len(elements[0].value.split(' ')), 2)
behaviors = [field for field in document.fields if field.name == 'behavior']
self.assertEqual(len(behaviors), 1)
self.assertEqual(len(behaviors[0].value.split(' ')), 1)