def setUp(self, *args, **kwargs):
if utils.is_appengine():
from google.appengine.ext import testbed
self.testbed = testbed.Testbed()
self.testbed.activate()
self.testbed.init_datastore_v3_stub()
self.testbed.init_memcache_stub()
def setUp(self, *args, **kwargs):
if utils.is_appengine():
from google.appengine.ext import testbed
self.testbed = testbed.Testbed()
self.testbed.activate()
self.testbed.init_datastore_v3_stub()
self.testbed.init_memcache_stub()
def test_diff(self):
my_index = indexes.Index.create({
'/file.txt':
rendered_document.RenderedDocument('/file.txt', 'test'),
'/file2.txt':
rendered_document.RenderedDocument('/file2.txt', 'test'),
'/foo/file.txt':
rendered_document.RenderedDocument('/foo/file.txt', 'test'),
})
their_index = indexes.Index.create({
'/file2.txt':
rendered_document.RenderedDocument('/file2.txt', 'change'),
'/foo/file.txt':
rendered_document.RenderedDocument('/foo/file.txt', 'test'),
'/bar/new.txt':
rendered_document.RenderedDocument('/bar/new.txt', 'test'),
})
expected = messages.DiffMessage(
adds=[messages.FileMessage(path='/file.txt')],
edits=[messages.FileMessage(path='/file2.txt')],
deletes=[messages.FileMessage(path='/bar/new.txt')],
nochanges=[messages.FileMessage(path='/foo/file.txt')],
)
if utils.is_appengine():
self.assertRaises(utils.UnavailableError, indexes.Diff.create,
my_index, their_index)
else:
diff = indexes.Diff.create(my_index, their_index)
self.assertFilePathsEqual(expected.adds, diff.adds)
def get_or_create_credentials(scope, storage_key=DEFAULT_STORAGE_KEY):
key_file = os.getenv('AUTH_KEY_FILE')
# If AUTH_KEY_FILE is unset, use default auth key file if it exists.
if not key_file and os.path.exists(DEFAULT_AUTH_KEY_FILE):
key_file = DEFAULT_AUTH_KEY_FILE
if key_file:
key_file = os.path.expanduser(key_file)
return (
service_account.ServiceAccountCredentials.from_json_keyfile_name(
key_file, scope))
if appengine and utils.is_appengine():
return appengine.AppAssertionCredentials(scope)
credentials, storage = get_credentials_and_storage(scope,
storage_key=storage_key)
if credentials is None:
parser = tools.argparser
if os.getenv('INTERACTIVE_AUTH'):
args = []
else:
args = ['--noauth_local_webserver']
flags, _ = parser.parse_known_args(args)
flow = client.OAuth2WebServerFlow(CLIENT_ID,
CLIENT_SECRET,
scope,
redirect_uri=REDIRECT_URI)
credentials = tools.run_flow(flow, storage, flags)
# run_flow changes the logging level, so change it back.
logging.getLogger().setLevel(getattr(logging, 'INFO'))
return credentials
def test_diff(self):
my_index = indexes.Index.create({
'/file.txt': 'test',
'/file2.txt': 'test',
'/foo/file.txt': 'test',
'/foo/file.txt': 'test',
})
their_index = indexes.Index.create({
'/file2.txt': 'change',
'/foo/file.txt': 'test',
'/foo/file.txt': 'test',
'/bar/new.txt': 'test',
})
expected = messages.DiffMessage(
adds=[messages.FileMessage(path='/file.txt')],
edits=[messages.FileMessage(path='/file2.txt')],
deletes=[messages.FileMessage(path='/bar/new.txt')],
nochanges=[messages.FileMessage(path='/foo/file.txt')],
)
if utils.is_appengine():
self.assertRaises(utils.UnavailableError,
indexes.Diff.create, my_index, their_index)
else:
diff = indexes.Diff.create(my_index, their_index)
self.assertFilePathsEqual(expected.adds, diff.adds)
def get_or_create_credentials(scope, storage_key=DEFAULT_STORAGE_KEY):
key_file = os.getenv('AUTH_KEY_FILE')
# If AUTH_KEY_FILE is unset, use default auth key file if it exists.
if not key_file and os.path.exists(DEFAULT_AUTH_KEY_FILE):
key_file = DEFAULT_AUTH_KEY_FILE
if key_file:
key_file = os.path.expanduser(key_file)
return (service_account.
ServiceAccountCredentials.from_json_keyfile_name(key_file, scope))
if appengine and utils.is_appengine():
return appengine.AppAssertionCredentials(scope)
credentials, storage = get_credentials_and_storage(scope,
storage_key=storage_key)
if credentials is None:
parser = tools.argparser
if os.getenv('INTERACTIVE_AUTH'):
args = []
else:
args = ['--noauth_local_webserver']
flags, _ = parser.parse_known_args(args)
flow = client.OAuth2WebServerFlow(CLIENT_ID, CLIENT_SECRET, scope,
redirect_uri=REDIRECT_URI)
credentials = tools.run_flow(flow, storage, flags)
# run_flow changes the logging level, so change it back.
logging.getLogger().setLevel(getattr(logging, 'INFO'))
return credentials
def test_deploy_local(self):
if utils.is_appengine():
text = 'Skipping Git destination test on GAE.'
raise skip.SkipTest(text)
import git
path = tempfile.mkdtemp()
git.Repo.init(path)
self._test_deploy(path)
def setUp(self, *args, **kwargs):
self.is_appengine = utils.is_appengine()
if self.is_appengine:
# pylint: disable=import-error
from google.appengine.ext import testbed
self.testbed = testbed.Testbed()
self.testbed.activate()
self.testbed.init_datastore_v3_stub()
self.testbed.init_memcache_stub()
def get_storage(key, username):
"""Returns the Storage class compatible with the current environment."""
if appengine and utils.is_appengine():
return appengine.StorageByKeyName(
appengine.CredentialsModel, username, 'credentials')
key = utils.slugify(key)
file_name = os.path.expanduser('~/.config/grow/{}_{}'.format(key, username))
dir_name = os.path.dirname(file_name)
if not os.path.exists(dir_name):
os.makedirs(dir_name)
return oauth_file.Storage(file_name)
def get_storage(key, username):
"""Returns the Storage class compatible with the current environment."""
if appengine and utils.is_appengine():
return appengine.StorageByKeyName(appengine.CredentialsModel, username,
'credentials')
key = utils.slugify(key)
file_name = os.path.expanduser('~/.config/grow/{}_{}'.format(
key, username))
dir_name = os.path.dirname(file_name)
if not os.path.exists(dir_name):
os.makedirs(dir_name)
return oauth_file.Storage(file_name)
def test_run(self):
pod = testing.create_pod()
fields = {
'preprocessors': [{
'name': 'sass',
'kind': 'sass',
'sass_dir': '/source/sass/',
'out_dir': '/dist/css/',
}],
}
pod.write_yaml('/podspec.yaml', fields)
content = 'body\n color: red'
pod.write_file('/source/sass/main.sass', content)
if utils.is_appengine():
return
pod.preprocess()
result = pod.read_file('/dist/css/main.min.css')
expected = 'body{color:red}\n'
self.assertEqual(expected, result)
def test_run(self):
pod = testing.create_pod()
fields = {
'preprocessors': [{
'name': 'sass',
'kind': 'sass',
'sass_dir': '/source/sass/',
'out_dir': '/dist/css/',
}],
}
pod.write_yaml('/podspec.yaml', fields)
content = 'body\n color: red'
pod.write_file('/source/sass/main.sass', content)
if utils.is_appengine():
self.assertRaises(utils.UnavailableError, pod.preprocess)
return
pod.preprocess()
result = pod.read_file('/dist/css/main.min.css')
expected = 'body{color:red}\n'
self.assertEqual(expected, result)
def get_or_create_credentials(scope, storage_key=DEFAULT_STORAGE_KEY):
key_file = os.getenv('AUTH_KEY_FILE')
# If AUTH_KEY_FILE is unset, use default auth key file if it exists.
if not key_file and os.path.exists(DEFAULT_AUTH_KEY_FILE):
key_file = DEFAULT_AUTH_KEY_FILE
if key_file:
key_file = os.path.expanduser(key_file)
return (
service_account.ServiceAccountCredentials.from_json_keyfile_name(
key_file, scope))
if appengine and utils.is_appengine():
return appengine.AppAssertionCredentials(scope)
credentials, storage = get_credentials_and_storage(scope,
storage_key=storage_key)
if credentials is None:
parser = tools.argparser
if os.getenv('INTERACTIVE_AUTH'):
args = []
else:
args = ['--noauth_local_webserver']
flags, _ = parser.parse_known_args(args)
flow = client.OAuth2WebServerFlow(CLIENT_ID,
CLIENT_SECRET,
scope,
redirect_uri=REDIRECT_URI)
credentials = tools.run_flow(flow, storage, flags)
# run_flow changes the logging level, so change it back.
logging.getLogger().setLevel(getattr(logging, 'INFO'))
# Avoid logspam by logging the email address only once.
if hasattr(credentials, 'id_token') and credentials.id_token:
email = credentials.id_token.get('email')
global _LAST_LOGGED_EMAIL
if email and _LAST_LOGGED_EMAIL != email:
logging.info('Authorizing using -> {}'.format(email))
_LAST_LOGGED_EMAIL = email
return credentials
"""Renderer for performing render operations for the pod."""
import sys
from grow.common import utils
from grow.pods import errors
if utils.is_appengine():
# pylint: disable=invalid-name
ThreadPool = None # pragma: no cover
else:
from multiprocessing.dummy import Pool as ThreadPool
class Error(Exception):
"""Base renderer error."""
pass
class RenderError(Error):
"""Errors that occured during the rendering."""
def __init__(self, message, err, err_tb):
super(RenderError, self).__init__(message)
self.err = err
self.err_tb = err_tb
class RenderNotStartedError(Error):
"""Rendering was not started."""
pass
def cache(self):
if utils.is_appengine():
return werkzeug_cache.MemcachedCache(default_timeout=0)
return werkzeug_cache.SimpleCache(default_timeout=0)
def cache(self):
if utils.is_appengine():
return werkzeug_cache.MemcachedCache(default_timeout=0)
return werkzeug_cache.SimpleCache(default_timeout=0)
"""Indexes for tracking the changes to pods across deployments and builds."""
import datetime
import hashlib
import logging
import ConfigParser
import progressbar
import texttable
from grow.common import progressbar_non
from grow.common import utils as common_utils
from protorpc import protojson
from . import messages
from . import utils
if common_utils.is_appengine():
# pylint: disable=invalid-name
pool = None
else:
from multiprocessing import pool
class Error(Exception):
pass
class CorruptIndexError(Error):
pass
class Diff(object):
POOL_SIZE = 100 # Thread pool size for applying a diff.
"""Renderer for performing render operations for the pod."""
import sys
from grow.common import utils
if utils.is_appengine():
# pylint: disable=invalid-name
ThreadPool = None # pragma: no cover
else:
from multiprocessing.dummy import Pool as ThreadPool
class Error(Exception):
"""Base renderer error."""
pass
class RenderError(Error):
"""Errors that occured during the rendering."""
def __init__(self, message, err, err_tb):
super(RenderError, self).__init__(message)
self.err = err
self.err_tb = err_tb
class RenderNotStartedError(Error):
"""Rendering was not started."""
pass
def render_func(batch, tick=None):
"""Threaded loader that forces a list of docs to be loaded from filesystem."""
from grow.common import utils as common_utils
from grow.documents import document_front_matter
if common_utils.is_appengine():
# pylint: disable=invalid-name
ThreadPool = None
else:
from multiprocessing.dummy import Pool as ThreadPool
# pylint: disable=too-few-public-methods
class DocsLoader(object):
"""Loader that threads the docs' file system reads."""
MAX_POOL_SIZE = 100
MIN_POOL_COUNT = 50
POOL_RATIO = 0.02
@classmethod
def load(cls, docs, ignore_errors=False):
"""Force load the provided docs to read from file system."""
if not docs:
return
pod = docs[0].pod
def load_func(doc):
"""Force the doc to read the source file."""
try:
def tearDown(self, *args, **kwargs):
if utils.is_appengine():
self.testbed.deactivate()
def tearDown(self, *args, **kwargs):
if utils.is_appengine():
self.testbed.deactivate()