def get_build_urls_list(bucket_path, reverse=True):
"""Returns a sorted list of build urls from a bucket path."""
if not bucket_path:
return []
base_url = os.path.dirname(bucket_path)
if environment.is_running_on_app_engine():
build_urls = list(storage.list_blobs(base_url))
else:
keys_directory = environment.get_value('BUILD_URLS_DIR')
keys_filename = '%s.list' % utils.string_hash(bucket_path)
keys_file_path = os.path.join(keys_directory, keys_filename)
# For one task, keys file that is cached locally should be re-used.
# Otherwise, we do waste lot of network bandwidth calling and getting the
# same set of urls (esp for regression and progression testing).
if not os.path.exists(keys_file_path):
# Get url list by reading the GCS bucket.
with open(keys_file_path, 'w') as f:
for path in storage.list_blobs(base_url):
f.write(path + '\n')
content = utils.read_data_from_file(keys_file_path, eval_data=False)
if not content:
return []
build_urls = content.splitlines()
return _sort_build_urls_by_revision(build_urls, bucket_path, reverse)
def get_stacktrace(testcase, stack_attribute='crash_stacktrace'):
"""Returns the stacktrace for a test case.
This may require a blobstore read.
"""
result = getattr(testcase, stack_attribute)
if not result or not result.startswith(data_types.BLOBSTORE_STACK_PREFIX):
return result
# For App Engine, we can't write to local file, so use blobs.read_key instead.
if environment.is_running_on_app_engine():
key = result[len(data_types.BLOBSTORE_STACK_PREFIX):]
return unicode(blobs.read_key(key), errors='replace')
key = result[len(data_types.BLOBSTORE_STACK_PREFIX):]
tmpdir = environment.get_value('BOT_TMPDIR')
tmp_stacktrace_file = os.path.join(tmpdir, 'stacktrace.tmp')
blobs.read_blob_to_disk(key, tmp_stacktrace_file)
try:
handle = open(tmp_stacktrace_file)
result = handle.read()
handle.close()
except:
logs.log_error(
'Unable to read stacktrace for testcase %d.' % testcase.key.id())
result = ''
shell.remove_file(tmp_stacktrace_file)
return result
def _check_commits(testcase, bisect_type, old_commit, new_commit):
"""Check old and new commit validity."""
if old_commit != new_commit or build_manager.is_custom_binary():
return old_commit, new_commit
# Something went wrong during bisection for the same commit to be chosen for
# both the start and end range.
# Get the bisection infrastructure to re-bisect.
if environment.is_running_on_app_engine():
bucket_path = data_handler.get_value_from_job_definition(
testcase.job_type, 'RELEASE_BUILD_BUCKET_PATH')
else:
bucket_path = build_manager.get_primary_bucket_path()
revision_list = build_manager.get_revisions_list(bucket_path)
last_tested_revision = testcase.get_metadata('last_tested_crash_revision')
known_crash_revision = last_tested_revision or testcase.crash_revision
if bisect_type == 'fixed':
# Narrowest range: last crashing revision up to the latest build.
return _get_commits(
str(known_crash_revision) + ':' + str(revision_list[-1]),
testcase.job_type)
if bisect_type == 'regressed':
# Narrowest range: first build to the first crashing revision.
return _get_commits(
str(revision_list[0]) + ':' + str(testcase.crash_revision),
testcase.job_type)
raise ValueError('Invalid bisection type: ' + bisect_type)
def read_blob_to_disk(blob_key, local_file):
"""Copy data stored in the blobstore to a local file."""
assert not environment.is_running_on_app_engine()
directory = os.path.dirname(local_file)
if not os.path.exists(directory):
os.makedirs(directory)
gcs_path = get_gcs_path(blob_key)
return storage.copy_file_from(gcs_path, local_file)
def decorator(func):
"""Decorates the given function."""
if environment.is_running_on_app_engine():
# multiprocessing doesn't work on App Engine.
return func
@functools.wraps(func)
def _wrapper(*args, **kwargs):
"""Wrapper."""
# FIXME: Weird exceptions in imports, might be something relating to our
# reload module. Needs furthur investigation, try this as a temporary fix.
import multiprocessing.pool
import threading
# Fix for Python < 2.7.2.
if not hasattr(threading.current_thread(), '_children'):
# pylint: disable=protected-access
threading.current_thread(
)._children = weakref.WeakKeyDictionary()
global THREAD_POOL
if THREAD_POOL is None:
THREAD_POOL = multiprocessing.pool.ThreadPool(processes=3)
try:
async_result = THREAD_POOL.apply_async(func,
args=args,
kwds=kwargs)
return async_result.get(timeout=duration)
except multiprocessing.TimeoutError:
# Sleep for some minutes in order to wait for flushing metrics.
time.sleep(120)
# If we don't exit here, we will cause threads to pile up and leading to
# out-of-memory. Safe to just exit here.
logs.log_fatal_and_exit((
'Exception occurred in function {0}: args: {1}, kwargs: {2}'
' exception: {3}').format(func, args, kwargs,
sys.exc_info()[1]))
return _wrapper
def _chunk_size(self):
if environment.is_running_on_app_engine():
# To match App Engine URLFetch's request size limit.
return 10 * 1024 * 1024 # 10 MiB.
return None
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""NDB importer."""
from system import environment
if (not environment.is_running_on_app_engine()
and not environment.get_value('PY_UNITTESTS')):
# Override the default google-cloud-datastore retry parameters to include
# INTERNAL errors.
# See https://github.com/googleapis/google-cloud-python/issues/6119.
from google.cloud.datastore_v1.gapic import datastore_client_config
datastore_config = datastore_client_config.config['interfaces'][
'google.datastore.v1.Datastore']
retry_codes = datastore_config['retry_codes']['idempotent']
if 'INTERNAL' not in retry_codes:
retry_codes.append('INTERNAL')
import ndb_patcher
ndb_patcher.patch_ndb()
from google.appengine.ext.ndb import * # pylint:disable=wildcard-import
