def CreateRepoAsync(repo_url, html_url, name, description, open_files):
"""Asynchronously create a repo."""
repo = GetRepo(repo_url)
if not repo:
user = GetTemplateOwner()
repo = Repo(id=repo_url, html_url=html_url, name=name,
description=description, open_files=open_files,
namespace=settings.PLAYGROUND_NAMESPACE)
elif repo.in_progress_task_name:
shared.w('ignoring recreation of {} which is already executing in task {}'
.format(repo_url, repo.in_progress_task_name))
return
task = taskqueue.add(queue_name='repo',
url='/_playground_tasks/populate_repo',
params={'repo_url': repo_url})
shared.i('task {} added to populate repo {}'.format(task.name, repo_url))
repo.in_progress_task_name = task.name
if repo.project:
SetProjectOwningTask(repo.project, task.name)
else:
project = CreateProject(user=user,
template_url=repo_url,
html_url=html_url,
project_name=name,
project_description=description,
open_files=open_files,
in_progress_task_name=task.name)
repo.project = project.key
repo.put()
return repo
def FixProject(project):
"""Fix or update a project entity."""
shared.w(project.key.id())
dirty = False
if not project.access_key:
project.access_key = secret.GenerateRandomString()
dirty = True
if project._properties.has_key('end_user_url'):
project._properties.pop('end_user_url')
dirty = True
if dirty:
project.put()
shared.w('fixed {}'.format(project.key))
def _GetRepoCollections():
repo_collections = []
for uri, description in REPO_COLLECTIONS:
repo_collection = model.GetOrInsertRepoCollection(uri, description)
task = taskqueue.add(queue_name='repo',
url='/_playground_tasks/populate_repo_collection',
params={
'repo_collection_url':
repo_collection.key.id(),
})
shared.w('adding task {} to populate repo collection {!r}'.format(
task.name, uri))
repo_collections.append(repo_collection)
ndb.put_multi(repo_collections)
return repo_collections
def post(self):
assert self.request.environ[common.HTTP_X_APPENGINE_QUEUENAME]
query = model.PlaygroundProject.query(
namespace=settings.PLAYGROUND_NAMESPACE)
cursor = self.request.get('cursor', None)
if cursor:
cursor = Cursor(urlsafe=cursor)
projects, next_cursor, more = query.fetch_page(_CURSOR_PAGE_SIZE,
start_cursor=cursor)
if next_cursor:
taskqueue.add(url='/playground/fix/project',
params={'cursor': next_cursor.urlsafe()})
for project in projects:
FixProject(project)
if not next_cursor:
shared.w('REACHED END OF QUERY CURSOR, '
'ALTHOUGH OTHER TASKS MAY STILL BE EXECUTING')
def add_files(dirname):
for path in os.listdir(os.path.join(repo_url, dirname)):
if path == _PLAYGROUND_SETTINGS_FILENAME:
continue
if common.GetExtension(path) in settings.SKIP_EXTENSIONS:
continue
relpath = os.path.join(dirname, path)
fullpath = os.path.join(repo_url, dirname, path)
if os.path.isdir(fullpath):
add_files(relpath)
else:
try:
with open(fullpath, 'rb') as f:
shared.i('- {0}'.format(relpath))
tree.SetFile(relpath, f.read())
except IOError:
# file access may be disallowed due to app.yaml skip_files
shared.w('skipping {}'.format(relpath))
def add_files(dirname):
for path in os.listdir(os.path.join(repo_url, dirname)):
if path == _PLAYGROUND_SETTINGS_FILENAME:
continue
if common.GetExtension(path) in settings.SKIP_EXTENSIONS:
continue
relpath = os.path.join(dirname, path)
fullpath = os.path.join(repo_url, dirname, path)
if os.path.isdir(fullpath):
add_files(relpath)
else:
try:
with open(fullpath, 'rb') as f:
shared.i('- {0}'.format(relpath))
tree.SetFile(relpath, f.read())
except IOError:
# file access may be disallowed due to app.yaml skip_files
shared.w('skipping {}'.format(relpath))
def _CheckResponse(self):
if self.response:
return
self.response = self.rpc.get_result()
shared.i('{} {}'.format(self.response.status_code, self.url))
if self.response.content_was_truncated:
raise FetchError(self.url, self.response)
if self.response.status_code == httplib.NOT_MODIFIED:
return
if self.response.status_code == httplib.OK:
self.response_content = self.response.content
self.etag = self.response.headers['ETag']
model.PutResource(self.url, self.etag, self.response_content)
return
if self.etag and self.response_content:
shared.w('using existing content etag={}, url={}'.format(
self.etag, self.url))
return
raise FetchError(self.url, self.response)
def PopulateRepos(self):
shared.EnsureRunningInTask() # gives us automatic retries
baseurl = self.repo_collection.key.id()
fetched = fetcher.Fetcher(baseurl, follow_redirects=True)
page = fetched.content
candidate_repos = self._GetChildPaths(page)
fetches = []
# we found a project in the root directory
if 'app.yaml' in candidate_repos:
candidate_repos.insert(0, '')
if common.IsDevMode():
# fetch fewer repos during development
candidate_repos = candidate_repos[:1]
for c in candidate_repos:
if c and not c.endswith('/'):
continue
project_url = '{0}{1}'.format(baseurl, c)
app_yaml_url = '{0}app.yaml'.format(project_url)
fetched = fetcher.Fetcher(app_yaml_url, follow_redirects=True)
fetches.append((c, project_url, app_yaml_url, fetched))
repos = []
for c, project_url, app_yaml_url, fetched in fetches:
try:
content = fetched.content
shared.i('found app.yaml: {}'.format(app_yaml_url))
name = c.rstrip('/') or project_url
description = 'Sample code from {0}'.format(project_url)
model.CreateRepoAsync(repo_url=project_url,
html_url=project_url,
name=name,
description=description,
open_files=[])
except urlfetch_errors.Error:
exc_info = sys.exc_info()
formatted_exception = traceback.format_exception(
exc_info[0], exc_info[1], exc_info[2])
shared.w('skipping {0}'.format(project_url))
for line in [line for line in formatted_exception if line]:
shared.w(line)
def _CheckResponse(self):
if self.response:
return
self.response = self.rpc.get_result()
shared.i('{} {}'.format(self.response.status_code, self.url))
if self.response.content_was_truncated:
raise FetchError(self.url, self.response)
if self.response.status_code == httplib.NOT_MODIFIED:
return
if self.response.status_code == httplib.OK:
self.response_content = self.response.content
self.etag = self.response.headers['ETag']
model.PutResource(self.url, self.etag, self.response_content)
return
if self.etag and self.response_content:
shared.w('using existing content etag={}, url={}'
.format(self.etag, self.url))
return
raise FetchError(self.url, self.response)
def PopulateRepos(self):
shared.EnsureRunningInTask() # gives us automatic retries
baseurl = self.repo_collection.key.id()
fetched = fetcher.Fetcher(baseurl, follow_redirects=True)
page = fetched.content
candidate_repos = self._GetChildPaths(page)
fetches = []
# we found a project in the root directory
if 'app.yaml' in candidate_repos:
candidate_repos.insert(0, '')
if common.IsDevMode():
# fetch fewer repos during development
candidate_repos = candidate_repos[:1]
for c in candidate_repos:
if c and not c.endswith('/'):
continue
project_url = '{0}{1}'.format(baseurl, c)
app_yaml_url = '{0}app.yaml'.format(project_url)
fetched = fetcher.Fetcher(app_yaml_url, follow_redirects=True)
fetches.append((c, project_url, app_yaml_url, fetched))
repos = []
for c, project_url, app_yaml_url, fetched in fetches:
try:
content = fetched.content
shared.i('found app.yaml: {}'.format(app_yaml_url))
name = c.rstrip('/') or project_url
description = 'Sample code from {0}'.format(project_url)
model.CreateRepoAsync(repo_url=project_url, html_url=project_url,
name=name, description=description, open_files=[])
except urlfetch_errors.Error:
exc_info = sys.exc_info()
formatted_exception = traceback.format_exception(exc_info[0],
exc_info[1],
exc_info[2])
shared.w('skipping {0}'.format(project_url))
for line in [line for line in formatted_exception if line]:
shared.w(line)
def CreateProjectTreeFromRepo(self, tree, repo):
# e.g. https://github.com/GoogleCloudPlatform/appengine-guestbook-python
# e.g. https://github.com/GoogleCloudPlatform/appengine-guestbook-python/tree/part6-staticfiles
html_url = repo.key.id()
info = GetInfo(html_url)
# e.g. https://api.github.com/repos/GoogleCloudPlatform/appengine-guestbook-python/branches/part6-staticfiles
branches_url = info.branches_url()
fetched = FetchAsyncWithAuth(branches_url)
data = fetched.json_content
# see http://developer.github.com/v3/git/trees/
tree_url = data['commit']['commit']['tree']['url'] + '?recursive=1'
fetched = FetchAsyncWithAuth(tree_url)
data = fetched.json_content
entries = [entry for entry in data['tree'] if entry['type'] == 'blob']
fetches = []
for entry in entries:
fetched = FetchAsyncWithAuth(entry['url'])
fetches.append((entry, fetched))
for entry, fetched in fetches:
try:
data = fetched.json_content
base64_content = data['content']
decoded_content = base64.b64decode(base64_content)
tree.SetFile(entry['path'], decoded_content)
except urlfetch_errors.Error:
exc_info = sys.exc_info()
formatted_exception = traceback.format_exception(exc_info[0],
exc_info[1],
exc_info[2])
shared.w('skipping {0} {1}'.format(entry['path'], entry['url']))
for line in [line for line in formatted_exception if line]:
shared.w(line)
def _GetAppEnginePythonRepos(self, data):
"""Get list of App Engine Python repos.
Given a list of repos, return those repo names which appear to be
Python App Engine repos, and which are not in _PROJECT_URL_SKIP_LIST.
This function can parse the JSON parsed contents of:
https://api.github.com/users/GoogleCloudPlatform/repos
Args:
page: the JSON response returned by
https://api.github.com/users/{user}/repos
Returns:
A list of repos.
"""
# keys we care about:
# - html_url, branches_url, name, description, master_branch, owner.login
repos = [entry for entry in data
if self._IsAppEnginePythonRepo(entry['name'])
and entry['html_url'] not in _PROJECT_URL_SKIP_LIST]
# fetch master_branch url for each repo
candidates1 = []
for repo in repos:
# only proceed with repos which look like App Engine Python projects
if not self._IsAppEnginePythonRepo(repo['name']):
shared.w('skipping non App Engine Python repo {}'
.format(repo['html_url']))
continue
info = Info(user=repo['owner']['login'], repo=repo['name'],
branch=repo['master_branch'])
branches_url = info.branches_url()
fetched = FetchAsyncWithAuth(branches_url)
candidates1.append((repo, fetched))
# fetch tree url for each repo
candidates2 = []
for repo, fetched in candidates1:
try:
data = fetched.json_content
except fetcher.FetchError:
continue
# see http://developer.github.com/v3/git/trees/
tree_url = data['commit']['commit']['tree']['url'] + '?recursive=1'
fetched = FetchAsyncWithAuth(tree_url)
candidates2.append((repo, fetched))
# filter for trees containing 'app.yaml'
candidates3 = []
for repo, fetched in candidates2:
data = fetched.json_content
contains_app_yaml = False
app_yaml_urls = [
entry['url'] for entry in data['tree']
if entry['path'] == 'app.yaml' and entry['type'] == 'blob'
]
if not app_yaml_urls:
shared.w('skipping repo due to missing app.yaml: {}'
.format(repo['html_url']))
continue
fetched = FetchAsyncWithAuth(app_yaml_urls[0])
candidates3.append((repo, fetched))
# filter repos whose app.yaml does not contain 'runtime: python27'
candidates4 = []
for repo, fetched in candidates3:
data = fetched.json_content
base64_content = data['content']
decoded_content = base64.b64decode(base64_content)
config = yaml.safe_load(decoded_content)
runtime = config.get('runtime')
if runtime != 'python27':
shared.w('skipping repo due to "runtime: {}" app {}'
.format(runtime, repo['html_url']))
continue
repos.append(repo)
return repos