def run(self, job_name, input_files):
empty = True
for filename in input_files:
if files.stat(filename).st_size > 0:
empty = False
break
if empty:
self.complete([])
return
shard_number = len(input_files)
output_files = []
for i in range(shard_number):
blob_file_name = (job_name + "-shuffle-output-" + str(i))
file_name = files.blobstore.create(
_blobinfo_uploaded_filename=blob_file_name)
output_files.append(file_name)
self.fill(self.outputs._output_files, output_files)
target = modules.get_current_version_name()
module_name = modules.get_current_module_name()
if module_name != "default":
target = "%s.%s." % (target, module_name)
files.shuffler.shuffle(
"%s-%s" % (job_name, int(time.time())), input_files, output_files,
{
"url": self.get_callback_url(),
"method": "GET",
"queue": self.queue_name,
"version": target,
})
def _old_request_ended(self, shard_state):
"""Whether previous slice retry has ended according to Logs API.
Args:
shard_state: shard state.
Returns:
True if the request of previous slice retry has ended. False if it has
not or unknown.
"""
assert shard_state.slice_start_time is not None
assert shard_state.slice_request_id is not None
request_ids = [shard_state.slice_request_id]
try:
logs = list(logservice.fetch(request_ids=request_ids))
except logservice.InvalidArgumentError:
global using_modules
if using_modules:
logs = list(logservice.fetch(
request_ids=request_ids,
module_versions=[(modules.get_current_module_name(),
modules.get_current_version_name())]))
else:
logs = list(logservice.fetch(
request_ids=request_ids,
server_versions=[(servers.get_current_server_name(),
servers.get_current_version_name())]))
if not logs or not logs[0].finished:
return False
return True
def invite_send():
"""Sends an email to invite the user at the email address given."""
name = request.values.get('displayname')
person_obj = model.Person.find_or_create_by_name(name)
url = ''.join([
'http://',
modules.get_hostname(version=modules.get_current_version_name()),
'/register/',
person_obj.key.urlsafe()
])
email = request.values.get('email')
message = request.values.get('message')
sender = 'invite@' + app_identity.get_application_id() + ".appspotmail.com"
logging.debug(sender)
logging.debug(url)
# logging.debug(render_template("invite_message.txt", join_url=url, personal_message=message))
mail.send_mail(sender=sender,
to=email,
subject="You have been invited to a Book Club!",
bcc='*****@*****.**',
body=render_template("invite_message.txt",
join_url=url,
personal_message=message))
flash("Email sent to %s" % email)
return redirect(url_for('admin.invite_form'))
def get_doc(self, service, version):
cache_key = 'discovery_doc/%s/%s/%s' % (
modules.get_current_version_name(), service, version)
cached = memcache.get(cache_key)
if cached:
return cached[0]
logging.info('Fetching actual discovery document')
doc_url = '%s://%s/_ah/api/discovery/v1/apis/%s/%s/rest' % (
self.request.scheme, # Needed for local devserver.
self.request.host,
service,
version)
try:
doc = net.json_request(url=doc_url, deadline=45)
logging.info('Fetched actual discovery document')
except net.NotFoundError:
doc = None
if doc:
for key in ('baseUrl', 'basePath', 'rootUrl'):
url = urlparse.urlparse(doc.get(key))
if url.path.startswith('/_ah/'):
url = url._replace(path=url.path[len('/_ah'):])
doc[key] = urlparse.urlunparse(url)
if 'batchPath' in doc:
del doc['batchPath']
memcache.add(cache_key, (doc, ))
return doc
class Configuration(ndb.Model):
_CACHE_TIME = timedelta(minutes=5)
_VERSION = get_current_version_name()
_INSTANCE = None
_INSTANCE_AGE = None
client_id = ndb.StringProperty()
auth_header = ndb.StringProperty()
base_uri = ndb.StringProperty()
redirect_uri = ndb.StringProperty()
scopes = ndb.StringProperty()
srp_admins = ndb.IntegerProperty(repeated=True)
srp_payers = ndb.IntegerProperty(repeated=True)
super_admins = ndb.IntegerProperty(repeated=True)
pos_admins = ndb.IntegerProperty(repeated=True)
alliance_id = ndb.IntegerProperty()
@classmethod
def get_instance(cls):
now = datetime.now()
if not cls._INSTANCE or cls._INSTANCE_AGE + cls._CACHE_TIME < now or cls._INSTANCE.client_id is None:
_log.info('Loading configuration from datastore.')
cls._INSTANCE = cls.get_or_insert(cls._VERSION)
cls._INSTANCE_AGE = now
return cls._INSTANCE
def addTask(queues, func, *args, **kwargs):
""" Enqueue a task to execute the specified function call later from the task queue.
Handle exceptions and dispatching to the right queue.
@param queues: List of queues names. We will randomly select one to push the task into.
Can be 'default' to use default queues.
@param func: The function to execute later
@param _countdown: seconds to wait before calling the function
@param _eta: timestamp defining when to call the function
@param _name: Name to give the Task; if not specified, a name will be
auto-generated when added to a queue and assigned to this object.
Must match the _TASK_NAME_PATTERN regular expression: ^[a-zA-Z0-9_-]{1,500}$
@param _target: specific version and/or module the task should execute on
@param _raiseIfExists: if set to True, we raise the eventual TaskAlreadyExistsError
@param _transactional: to make sure task in enqueued in the task queue
@param _retry_options: task queue retry options
@param _parent: ndb Key instance, if provided, function payload will be stored in data store entity under this
parent if the size of payload exceeds 100KB. Max size of payload could be 1MB otherwise data store will throw error
@return: A Future that will yield True if the task could be enqueued.
@rtype: ndb.Future
"""
if not isinstance(queues, list):
queues = DEFAULT_QUEUES
_raiseIfExists = kwargs.pop('_raiseIfExists', False)
taskName = kwargs.pop('_name', None)
countdown = kwargs.pop('_countdown', None)
eta = kwargs.pop('_eta', None)
target = kwargs.pop('_target', None)
transactional = kwargs.pop('_transactional', False)
retry_options = kwargs.pop('_retry_options', None)
parent = kwargs.pop('_parent', None)
if not target and BACKGROUND_MODULE:
# Tasks from the default module are executed into the background module.
# Tasks from other modules (stage, background) stays inside their module.
if modules.get_current_module_name() == 'default':
# Target mirror of current version to avoid compatibility issues
# If that version does not exist, it will fall back to the default background version.
target = modules.get_current_version_name() + '.' + BACKGROUND_MODULE
success = False
try:
yield _defer(queues, func, args, kwargs, countdown, eta, taskName, target, transactional, retry_options, parent)
success = True
except (taskqueue.TaskAlreadyExistsError, taskqueue.TombstonedTaskError):
# TaskAlreadyExistsError: a task with same name is in the queue
# TombstonedTaskError: a task with same name has been in the queue recently
if taskName:
# If we specified a name it's to avoid duplicated so this error is expected
logging.info("TaskAlreadyExistsError: task with name %s already enqueued.", taskName)
if _raiseIfExists:
raise
else:
logging.exception("Could not enqueue the task")
except:
logging.exception("Could not enqueue the task")
raise ndb.Return(success)
def info_route():
app_id = app_identity.get_application_id()
current_version = modules.get_current_version_name()
config_docid = get_config_docid()
commit_head = get_commit_head()
found_dict = scan_config('all')
return render_template('info.html', id=app_id, version=current_version, docid=config_docid, commit=commit_head, cache_keys=cache_dict, scan_results=found_dict)
def get(self):
service = app_identity.get_application_id()
version = modules.get_current_version_name()
instance_id = hash(modules.get_current_instance_id()) % 10
endpoint = MONACQ_ENDPOINT
config.initialize(job_name=version, instance=instance_id,
service_name=service, endpoint=endpoint)
self.response.set_status(200, 'Initialized instance of ts_mon.')
def namespace_manager_default_namespace_for_request():
if MULTITENANCY:
try:
return modules.get_current_version_name()
except:
return namespace_manager.google_apps_namespace()
else:
return namespace_manager.google_apps_namespace()
def _ResetAnalysis(self, master_name, builder_name, build_number): analysis = WfAnalysis.Get(master_name, builder_name, build_number) analysis.pipeline_status_path = self.pipeline_status_path() analysis.status = wf_analysis_status.ANALYZING analysis.result_status = None analysis.start_time = datetime.utcnow() analysis.version = modules.get_current_version_name() analysis.end_time = None analysis.put()
def render_response(self, template_name, **context):
self.response.headers['Content-Type'] = 'text/html'
template = JINJA_ENVIRONMENT.get_template(template_name)
context['user'] = users.get_current_user()
context['application_id'] = app_identity.get_application_id()
context['module_id'] = modules.get_current_module_name()
context['version_id'] = modules.get_current_version_name()
context['IS_DEVSERVER'] = appengine_config.IS_DEVSERVER
self.response.write(template.render(**context))
def render_response(self, template_name, **context): self.response.headers['Content-Type'] = 'text/html' template = JINJA_ENVIRONMENT.get_template(template_name) context['user'] = users.get_current_user() context['application_id'] = app_identity.get_application_id() context['module_id'] = modules.get_current_module_name() context['version_id'] = modules.get_current_version_name() context['IS_DEVSERVER'] = appengine_config.IS_DEVSERVER self.response.write(template.render(**context))
def _ResetAnalysis(self, master_name, builder_name, build_number):
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
analysis.pipeline_status_path = self.pipeline_status_path()
analysis.status = wf_analysis_status.ANALYZING
analysis.result_status = None
analysis.start_time = datetime.utcnow()
analysis.version = modules.get_current_version_name()
analysis.end_time = None
analysis.put()
def get(self):
module = modules.get_current_module_name()
instance = modules.get_current_instance_id()
version = modules.get_current_version_name()
self.complete('respond', {
'module': module,
'instance': instance,
'version': version
})
def get(self):
module = modules.get_current_module_name()
instance = modules.get_current_instance_id()
version = modules.get_current_version_name()
self.complete('respond', {
'module': module,
'instance': instance,
'version': version
})
def _VersionBaseURL(request):
"""Return a version-specific URL that we use to load static assets."""
if settings.local_mode:
version_base = '%s://%s' % (request.scheme, request.host)
else:
version_base = '%s://%s-dot-%s' % (
request.scheme, modules.get_current_version_name(),
app_identity.get_default_version_hostname())
return version_base
def get_url_for_instance(instance_id):
"""Return a full url of the current instance.
Args:
A string to represent an VM instance.
Returns:
URL string for the instance.
"""
hostname = app_identity.get_default_version_hostname()
return 'http://{}.{}.{}'.format(
instance_id, modules.get_current_version_name(), hostname)
def get(self):
# Setting webhook.
version = modules.get_current_version_name()
host_name = app_identity.get_default_version_hostname()
host_url = 'https://{}-dot-{}{}'.format(version, host_name, BOT_URL)
BOT.setWebhook(host_url)
# Returning info.
self.response.headers['Content-Type'] = 'text/plain'
self.response.write(BOT.getMe())
self.response.write('\nWEBHOOK set to: ' + host_url)
def get(self):
# self.response.headers['Content-Type'] = 'text/plain'
self.response.write('Hello, World!')
self.response.write("<p>Version (os.environ['CURRENT_VERSION_ID']): " +
os.environ['CURRENT_VERSION_ID'] + '</p>')
self.response.write("<p>modules.get_current_version_name(): " +
modules.get_current_version_name() + "</p>")
self.response.write("<p>Request Host: " +
self.request.headers['Host'] + "</p>")
self.response.write("<p>Custom environment variable: DEVELOPERTIPS_CUSTOM_ENV='" +
os.environ['DEVELOPERTIPS_CUSTOM_ENV'] + "'</p>" )
def landing_page(request):
if debug:
print >> sys.stderr,'Called '+sys._getframe().f_code.co_name
print >> sys.stderr,'App Version: '+modules.get_current_version_name()
try:
print >> sys.stderr,'App BACKEND_ID: '+os.getenv('BACKEND_ID')
except:
print >> sys.stderr,"Printing os.getenv('BACKEND_ID') Failed"
return render(request, 'GenespotRE/landing.html',
{'request': request})
def get_deferred_target():
"""Return the name of an App Engine module or version for running a
deferred task.
"""
settings_key = 'WORKER_MODULE_NAME'
if hasattr(settings, settings_key):
target = getattr(settings, settings_key)
else:
target = modules.get_current_version_name()
return target
def get_url_for_instance(instance_id):
"""Return a full url of the guestbook running on a particular instance.
Args:
A string to represent an VM instance.
Returns:
URL string for the guestbook form on the instance.
"""
hostname = app_identity.get_default_version_hostname()
return 'https://{}-dot-{}-dot-{}/guestbook'.format(
instance_id, modules.get_current_version_name(), hostname)
def get(self):
self.response.headers['Content-Type'] = 'application/json'
current_module = get_current_module_name()
self.response.out.write(
json.dumps({
'modules': get_modules(),
'current_module_versions': get_versions(current_module),
'default_version': get_default_version(current_module),
'current_module': current_module,
'current_version': get_current_version_name(),
'current_instance_id': get_current_instance_id()
}))
def get(self):
# self.response.headers['Content-Type'] = 'text/plain'
self.response.write('Hello, World!')
self.response.write("<p>Version (os.environ['CURRENT_VERSION_ID']): " +
os.environ['CURRENT_VERSION_ID'] + '</p>')
self.response.write("<p>modules.get_current_version_name(): " +
modules.get_current_version_name() + "</p>")
self.response.write("<p>Request Host: " +
self.request.headers['Host'] + "</p>")
self.response.write(
"<p>Custom environment variable: DEVELOPERTIPS_CUSTOM_ENV='" +
os.environ['DEVELOPERTIPS_CUSTOM_ENV'] + "'</p>")
def get_deferred_target():
"""Return the name of an App Engine module or version for running a
deferred task.
"""
settings_key = 'WORKER_MODULE_NAME'
if hasattr(settings, settings_key):
target = getattr(settings, settings_key)
else:
target = modules.get_current_version_name()
return target
class PastyVersionMiddleware(object):
"""Adds a X-Pasty-Version header to the response."""
key = 'X-Pasty-Version'
version = modules.get_current_version_name()
def __init__(self, get_response):
self.get_response = get_response
def __call__(self, request):
response = self.get_response(request)
response[self.key] = self.version
return response
def render(self, page, values):
"""
Add module info to template values
and render template
"""
values['module'] = modules.get_current_module_name()
values['instance'] = modules.get_current_instance_id()
values['version'] = modules.get_current_version_name()
values['handler'] = self.handlerName
template = self.jinjaEnvironment.get_template(self.handlerName + '/' + page + '.html')
self.response.out.write(template.render(values))
def get_settings_name():
"""
Gets the correct module to define Django's settings
"""
# application_id -> settings file
version = get_current_version_name()
logging.info("Using version: {0}".format(version))
settings_module = 'greenday_core.settings.local'
if version is not None:
if version == 'qa':
settings_module = 'greenday_core.settings.qa'
else:
settings_module = 'greenday_core.settings.live'
logging.info("Using settings: {0}".format(settings_module))
return settings_module
def run(self, job_name, input_files):
# Return immediately if we have no content to shuffle.
# Big shuffler can not handle no input.
empty = True
for filename in input_files:
if files.stat(filename).st_size > 0:
empty = False
break
if empty:
self.complete([])
return
shard_number = len(input_files)
output_files = []
for i in range(shard_number):
blob_file_name = (job_name + "-shuffle-output-" + str(i))
file_name = files.blobstore.create(
_blobinfo_uploaded_filename=blob_file_name)
output_files.append(file_name)
self.fill(self.outputs._output_files, output_files)
# Support shuffler callbacks going to specific modules and
# specific non-default versions of those modules.
target = modules.get_current_version_name()
module_name = modules.get_current_module_name()
if module_name != "default":
# NOTE(user): The final dot is necessary here because old versions
# of the shuffler library would put "myversion.12345678" in this field,
# expecting the admin-shuffler app to remove the timestamp suffix.
target = "%s.%s." % (target, module_name)
files.shuffler.shuffle(
"%s-%s" % (job_name, int(time.time())),
input_files,
output_files,
{
"url": self.get_callback_url(),
# NOTE(user): This is always GET because of
# how the admin_shuffler app adds the callback
# task with additional URL params.
"method": "GET",
"queue": self.queue_name,
"version": target,
})
def landing_page(request):
if debug:
print >> sys.stderr,'Called '+sys._getframe().f_code.co_name
print >> sys.stderr,'App Version: '+modules.get_current_version_name()
try:
print >> sys.stderr,'App BACKEND_ID: '+os.getenv('BACKEND_ID')
except:
print >> sys.stderr,"Printing os.getenv('BACKEND_ID') Failed"
if request.user :
if request.user.id :
redirect_url = reverse('dashboard')
return redirect(redirect_url)
else :
return render(request, 'GenespotRE/marketing_landing.html',
{'request': request, 'full_width': 'true'})
else :
return render(request, 'GenespotRE/marketing_landing.html',
{'request': request, 'full_width': 'true'})
def run(self, job_name, input_files):
# Return immediately if we have no content to shuffle.
# Big shuffler can not handle no input.
empty = True
for filename in input_files:
if files.stat(filename).st_size > 0:
empty = False
break
if empty:
self.complete([])
return
shard_number = len(input_files)
output_files = []
for i in range(shard_number):
blob_file_name = (job_name + "-shuffle-output-" + str(i))
file_name = files.blobstore.create(
_blobinfo_uploaded_filename=blob_file_name)
output_files.append(file_name)
self.fill(self.outputs._output_files, output_files)
# Support shuffler callbacks going to specific modules and
# specific non-default versions of those modules.
target = modules.get_current_version_name()
module_name = modules.get_current_module_name()
if module_name != "default":
# NOTE(user): The final dot is necessary here because old versions
# of the shuffler library would put "myversion.12345678" in this field,
# expecting the admin-shuffler app to remove the timestamp suffix.
target = "%s.%s." % (target, module_name)
files.shuffler.shuffle("%s-%s" % (job_name, int(time.time())),
input_files,
output_files,
{
"url": self.get_callback_url(),
# NOTE(user): This is always GET because of
# how the admin_shuffler app adds the callback
# task with additional URL params.
"method": "GET",
"queue": self.queue_name,
"version": target,
})
def ScheduleAnalysisIfNeeded(
master_name, builder_name, build_number, failed_steps=None, force=False, queue_name="default"
):
"""Schedules an analysis if needed and returns the build analysis.
When the build failure was already analyzed and a new analysis is scheduled,
the returned WfAnalysis will still have the result of last completed analysis.
Args:
master_name (str): The master name of the failed build.
builder_name (str): The builder name of the failed build.
build_number (int): The build number of the failed build.
failed_steps (list): The names of all failed steps reported for the build.
force (bool): If True, a fresh new analysis will be triggered even when an
old one was completed already; otherwise bail out.
queue_name (str): The task queue to be used for pipeline tasks.
Returns:
A WfAnalysis instance.
"""
if NeedANewAnalysis(master_name, builder_name, build_number, failed_steps, force):
pipeline_job = analyze_build_failure_pipeline.AnalyzeBuildFailurePipeline(
master_name, builder_name, build_number
)
# Explicitly run analysis in the backend module "build-failure-analysis".
# Note: Just setting the target in queue.yaml does NOT work for pipeline
# when deployed to App Engine, but it does work in dev-server locally.
# A possible reason is that pipeline will pick a default target if none is
# specified explicitly, and the default target is used rather than the one
# in the queue.yaml file, but this contradicts the documentation in
# https://cloud.google.com/appengine/docs/python/taskqueue/tasks#Task.
pipeline_job.target = "%s.build-failure-analysis" % modules.get_current_version_name()
pipeline_job.start(queue_name=queue_name)
logging.info(
"An analysis was scheduled for build %s, %s, %s: %s",
master_name,
builder_name,
build_number,
pipeline_job.pipeline_status_path(),
)
return WfAnalysis.Get(master_name, builder_name, build_number)
def ScheduleAnalysisIfNeeded(master_name,
builder_name,
build_number,
failed_steps=None,
force=False,
queue_name='default'):
"""Schedules an analysis if needed and returns the build analysis.
When the build failure was already analyzed and a new analysis is scheduled,
the returned WfAnalysis will still have the result of last completed analysis.
Args:
master_name (str): The master name of the failed build.
builder_name (str): The builder name of the failed build.
build_number (int): The build number of the failed build.
failed_steps (list): The names of all failed steps reported for the build.
force (bool): If True, a fresh new analysis will be triggered even when an
old one was completed already; otherwise bail out.
queue_name (str): The task queue to be used for pipeline tasks.
Returns:
A WfAnalysis instance.
"""
if NeedANewAnalysis(master_name, builder_name, build_number, failed_steps,
force):
pipeline_job = analyze_build_failure_pipeline.AnalyzeBuildFailurePipeline(
master_name, builder_name, build_number)
# Explicitly run analysis in the backend module "build-failure-analysis".
# Note: Just setting the target in queue.yaml does NOT work for pipeline
# when deployed to App Engine, but it does work in dev-server locally.
# A possible reason is that pipeline will pick a default target if none is
# specified explicitly, and the default target is used rather than the one
# in the queue.yaml file, but this contradicts the documentation in
# https://cloud.google.com/appengine/docs/python/taskqueue/tasks#Task.
pipeline_job.target = ('%s.build-failure-analysis' %
modules.get_current_version_name())
pipeline_job.start(queue_name=queue_name)
logging.info('An analysis was scheduled for build %s, %s, %s: %s',
master_name, builder_name, build_number,
pipeline_job.pipeline_status_path())
return WfAnalysis.Get(master_name, builder_name, build_number)
def run(self, job_name, input_files):
empty = True
for filename in input_files:
if files.stat(filename).st_size > 0:
empty = False
break
if empty:
self.complete([])
return
shard_number = len(input_files)
output_files = []
for i in range(shard_number):
blob_file_name = (job_name + "-shuffle-output-" + str(i))
file_name = files.blobstore.create(
_blobinfo_uploaded_filename=blob_file_name)
output_files.append(file_name)
self.fill(self.outputs._output_files, output_files)
target = modules.get_current_version_name()
module_name = modules.get_current_module_name()
if module_name != "default":
target = "%s.%s." % (target, module_name)
files.shuffler.shuffle("%s-%s" % (job_name, int(time.time())),
input_files,
output_files,
{
"url": self.get_callback_url(),
"method": "GET",
"queue": self.queue_name,
"version": target,
})
def initialize(service_name):
is_local_unittest = ('expect_tests' in sys.argv[0])
if is_local_unittest:
appengine_name = 'unittest'
service_name = 'unittest'
hostname = 'unittest'
else: # pragma: no cover
appengine_name = app_identity.get_application_id()
hostname = '%s, %s' % (modules.get_current_module_name(),
modules.get_current_version_name())
# Only send events if we are running on the actual AppEngine.
if os.environ.get('SERVER_SOFTWARE', '').startswith('Google App Engine'):
run_type = 'prod' # pragma: no cover
else:
run_type = 'dry'
config.setup_monitoring(run_type, hostname, service_name, appengine_name)
logging.info(
'Initialized event_mon with run_type=%s, hostname=%s, service_name=%s, '
'appengine_name=%s', run_type, hostname, service_name, appengine_name)
def delete_many_builds(bucket, status, tags=None, created_by=None):
if status not in (model.BuildStatus.SCHEDULED, model.BuildStatus.STARTED):
raise errors.InvalidInputError(
'status can be STARTED or SCHEDULED, not %s' % status)
if not acl.can_delete_scheduled_builds(bucket):
raise current_identity_cannot('delete scheduled builds of %s', bucket)
# Validate created_by prior scheduled a push task.
created_by = parse_identity(created_by)
deferred.defer(
_task_delete_many_builds,
bucket,
status,
tags=tags,
created_by=created_by,
# Schedule it on the backend module of the same version.
# This assumes that both frontend and backend are uploaded together.
_target='%s.backend' % modules.get_current_version_name(),
# Retry immediatelly.
_retry_options=taskqueue.TaskRetryOptions(
min_backoff_seconds=0,
max_backoff_seconds=1,
),
)
def delete_many_builds(bucket_id, status, tags=None, created_by=None):
if status not in (model.BuildStatus.SCHEDULED, model.BuildStatus.STARTED):
raise errors.InvalidInputError(
'status can be STARTED or SCHEDULED, not %s' % status)
if not user.can_delete_scheduled_builds_async(bucket_id).get_result():
raise user.current_identity_cannot('delete builds of %s', bucket_id)
# Validate created_by prior scheduled a push task.
created_by = user.parse_identity(created_by)
deferred.defer(
_task_delete_many_builds,
bucket_id,
status,
tags=tags,
created_by=created_by,
# Schedule it on the backend module of the same version.
# This assumes that both frontend and backend are uploaded together.
_target='%s.backend' % modules.get_current_version_name(),
# Retry immediatelly.
_retry_options=taskqueue.TaskRetryOptions(
min_backoff_seconds=0,
max_backoff_seconds=1,
),
)
def save_to_file_for_elk(self, request_id, end_time, request_log):
start_time = request_log.startTime
start_time_ms = float(start_time) / 1000
end_time_ms = float(end_time) / 1000
# Render app logs:
app_logs_str = '\n'.join([
'{} {} {}'.format(
LEVELS[log.level],
datetime.utcfromtimestamp(
log.time / 1000000).strftime('%Y-%m-%d %H:%M:%S'),
log.message) for log in request_log.appLogs
])
request_info = {
'generated_id': '{}-{}'.format(start_time, request_id),
'serviceName': get_current_module_name(),
'versionName': get_current_version_name(),
'startTime': start_time_ms,
'endTime': end_time_ms,
'latency': int(end_time_ms - start_time_ms),
'level': max(0, 0, *[log.level for log in request_log.appLogs]),
'appId': request_log.appId,
'appscale-host': os.environ['MY_IP_ADDRESS'],
'port': int(os.environ['MY_PORT']),
'ip': request_log.ip,
'method': request_log.method,
'requestId': request_id,
'resource': request_log.resource,
'responseSize': request_log.responseSize,
'status': request_log.status,
'userAgent': request_log.userAgent,
'appLogs': app_logs_str
}
self._requests_logger.write(request_info)
INTERNAL_IPS = (
'127.0.0.1',
'localhost',
'::1'
)
DEBUG = not APPENGINE_PRODUCTION
TESTING = False
try:
APP_ID = app_identity.get_application_id()
HOME_LINK = app_identity.get_default_version_hostname()
GCS_BUCKET = app_identity.get_default_gcs_bucket_name()
CURRENT_VERSION = modules.get_current_version_name()
CURRENT_MODULE = modules.get_current_module_name()
except AttributeError:
APP_ID = 'greenday-project-v02-local'
HOME_LINK = 'http://localhost:8080/'
GCS_BUCKET = 'greenday-project-v02-local.appspot.com'
CURRENT_VERSION = 'local'
CURRENT_MODULE = ''
ALLOWED_HOSTS = [ # Django 1.5 requirement when DEBUG = False
"localhost",
"montage.storyful.com",
app_identity.get_default_version_hostname(),
# non-default versions
"{0}.{1}".format(CURRENT_VERSION, HOME_LINK),
def get_app_version(): """Returns currently running version (not necessary a default one).""" # Sadly, this causes an RPC and when called too frequently, throws quota # errors. return modules.get_current_version_name()
import werkzeug.urls
import socket
import rotoken
import flask
from flask import Flask, request
import jinja2
from google.appengine.api import app_identity
from google.appengine.api import modules
from google.appengine.ext import ndb
app = Flask(__name__)
app.config['DEBUG'] = True
# Where my rotating tokens are kept.
purse = rotoken.Rotoken(modules.get_current_version_name())
JINJA_ENVIRONMENT = jinja2.Environment(
loader=jinja2.FileSystemLoader(os.path.join(os.path.dirname(__file__),
'templates')),
extensions=['jinja2.ext.autoescape'],
autoescape=True)
TOPIC = "shout-requests"
SUBSCRIPTION = "shout-request-workers"
TIMEOUT_SECONDS = 90
APP_ID = app_identity.get_application_id()
RANDOM_ID_LEN = 43 # Equivalent to 256 bits of randomness.
###############################################################################
# Data model.
def namespace_manager_default_namespace_for_request():
"""Set the namespace to current version before every request."""
return modules.get_current_version_name().lower()
def get_current_version_name_safe():
"""Returns the current version of the app, or None if there is no current version found."""
try:
return modules.get_current_version_name()
except KeyError:
return None
def HandleGet(self):
"""Responses the deployed version of this app."""
self.response.write(modules.get_current_version_name())
def get_current_version():
return modules.get_current_version_name()
def _get_target():
version_name = modules.get_current_version_name()
if version_name in ('prod', 'm', 'api', 'batchops', 'mapreducer'):
return 'batchops'
return version_name
def instance_key_id():
return '%s.%s.%s' % (
modules.get_current_instance_id(),
modules.get_current_version_name(),
modules.get_current_module_name())
def instance_key_id():
return '%s.%s.%s' % (modules.get_current_instance_id(),
modules.get_current_version_name(),
modules.get_current_module_name())
def addTask(queues, func, *args, **kwargs):
""" Enqueue a task to execute the specified function call later from the task queue.
Handle exceptions and dispatching to the right queue.
@param queues: List of queues names. We will randomly select one to push the task into.
Can be 'default' to use default queues.
@param func: The function to execute later
@param _countdown: seconds to wait before calling the function
@param _eta: timestamp defining when to call the function
@param _name: Name to give the Task; if not specified, a name will be
auto-generated when added to a queue and assigned to this object.
Must match the _TASK_NAME_PATTERN regular expression: ^[a-zA-Z0-9_-]{1,500}$
@param _target: specific version and/or module the task should execute on
@param _raiseIfExists: if set to True, we raise the eventual TaskAlreadyExistsError
@return: A Future that will yield True if the task could be enqueued.
@rtype: ndb.Future
"""
if not isinstance(queues, list):
queues = DEFAULT_QUEUES
_raiseIfExists = kwargs.pop('_raiseIfExists', False)
taskName = kwargs.pop('_name', None)
countdown = kwargs.pop('_countdown', None)
eta = kwargs.pop('_eta', None)
target = kwargs.pop('_target', None)
if not target and BACKGROUND_MODULE:
# Tasks from the default module are executed into the background module.
# Tasks from other modules (stage, background) stays inside their module.
if modules.get_current_module_name() == 'default':
# Target mirror of current version to avoid compatibility issues
# If that version does not exist, it will fall back to the default background version.
target = modules.get_current_version_name(
) + '.' + BACKGROUND_MODULE
success = False
try:
yield _defer(queues, func, args, kwargs, countdown, eta, taskName,
target)
success = True
except (taskqueue.TaskAlreadyExistsError, taskqueue.TombstonedTaskError):
# TaskAlreadyExistsError: a task with same name is in the queue
# TombstonedTaskError: a task with same name has been in the queue recently
if taskName:
# If we specified a name it's to avoid duplicated so this error is expected
logging.info(
"TaskAlreadyExistsError: task with name %s already enqueued.",
taskName)
if _raiseIfExists:
raise
else:
logging.exception("Could not enqueue the task")
except:
logging.exception("Could not enqueue the task")
raise ndb.Return(success)
def version_name(self):
from google.appengine.api import modules
return modules.get_current_version_name()
def get_app_version(): """Returns currently running version (not necessary a default one).""" # Sadly, this causes an RPC and when called too frequently, throws quota # errors. return modules.get_current_version_name()
import requests
import requests_toolbelt.adapters.appengine
# Use the App Engine Requests adapter. This makes sure that Requests uses
# URLFetch.
requests_toolbelt.adapters.appengine.monkeypatch()
from urllib import quote, urlencode, urlopen
from flask import Flask, flash, render_template, request, redirect, url_for
from admin import admin_blueprint
BASEURL = 'https://' + modules.get_hostname(
module=modules.get_current_module_name(),
version=modules.get_current_version_name())
ADD_SCAN_REDIR = quote(BASEURL + '/add/{CODE}')
BORROW_SCAN_REDIR = quote(BASEURL + '/borrow/{CODE}')
RESULTS_PER_PAGE = 40
BOOK_DATA_FAILURE_MSG = ("Failed to fetch book data.\n" +
"Please fill in manually.")
app = Flask(__name__)
app.secret_key = "Some_Secret_Key"
app.register_blueprint(admin_blueprint, url_prefix="/admin")
@app.route('/register/<userkey>', methods=['GET', 'POST'])
def process_user_invite(userkey):
if request.method == 'GET':
def get(self):
from google.appengine.api import modules
self.html_response("I'm %s!!" % modules.get_current_version_name())
from google.appengine.api import modules
from google.appengine.api import app_identity
import telegram
import telegram_token
from service import WordCountService
# Creating the bot and getting basic info of it.
BOT = telegram.Bot(token=telegram_token.TOKEN)
BOT_ME = BOT.getMe()
if not appengine_config.DEBUG:
# Setting the webhook (callback).
VERSION = modules.get_current_version_name()
HOST_NAME = app_identity.get_default_version_hostname()
HOST_URL = 'https://{}-dot-{}/{}'.format(VERSION, HOST_NAME, telegram_token.TOKEN)
BOT.setWebhook(HOST_URL)
# Text to send back after substitution.
RESPONSE_TEXT = """Letters: %(letters)s
Words: %(words)s
Lines: %(lines)s
"""
RESPONSE_NO_TEXT = "Please, sent me some text. " + telegram.Emoji.PAGE_FACING_UP
class MainPage(webapp2.RequestHandler):
def get_deferred_target():
"""Return the name of an App Engine module for running a deferred task."""
current_module = modules.get_current_version_name()
target = getattr(settings, 'WORKER_MODULE_NAME', current_module)
return target
def initialize(app=None,
is_enabled_fn=None,
cron_module='default',
is_local_unittest=None):
"""Instruments webapp2 `app` with gae_ts_mon metrics.
Instruments all the endpoints in `app` with basic metrics.
Args:
app (webapp2 app): the app to instrument.
is_enabled_fn (function or None): a function returning bool if ts_mon should
send the actual metrics. None (default) is equivalent to lambda: True.
This allows apps to turn monitoring on or off dynamically, per app.
cron_module (str): the name of the module handling the
/internal/cron/ts_mon/send endpoint. This allows moving the cron job
to any module the user wants.
is_local_unittest (bool or None): whether we are running in a unittest.
"""
if is_local_unittest is None: # pragma: no cover
# Since gae_ts_mon.initialize is called at module-scope by appengine apps,
# AppengineTestCase.setUp() won't have run yet and none of the appengine
# stubs will be initialized, so accessing Datastore or even getting the
# application ID will fail.
is_local_unittest = ('expect_tests' in sys.argv[0])
if is_enabled_fn is not None:
interface.state.flush_enabled_fn = is_enabled_fn
if app is not None:
instrument_wsgi_application(app)
if is_local_unittest or modules.get_current_module_name(
) == cron_module:
instrument_wsgi_application(handlers.app)
# Use the application ID as the service name and the module name as the job
# name.
if is_local_unittest: # pragma: no cover
service_name = 'unittest'
job_name = 'unittest'
hostname = 'unittest'
else:
service_name = app_identity.get_application_id()
job_name = modules.get_current_module_name()
hostname = modules.get_current_version_name()
runtime.set_shutdown_hook(_shutdown_hook)
interface.state.target = targets.TaskTarget(service_name,
job_name,
shared.REGION,
hostname,
task_num=-1)
interface.state.flush_mode = 'manual'
interface.state.last_flushed = datetime.datetime.utcnow()
# Don't send metrics when running on the dev appserver.
if (is_local_unittest or os.environ.get('SERVER_SOFTWARE',
'').startswith('Development')):
logging.info('Using debug monitor')
interface.state.global_monitor = monitors.DebugMonitor()
else:
logging.info('Using https monitor %s with %s',
shared.PRODXMON_ENDPOINT,
shared.PRODXMON_SERVICE_ACCOUNT_EMAIL)
interface.state.global_monitor = monitors.HttpsMonitor(
shared.PRODXMON_ENDPOINT,
monitors.DelegateServiceAccountCredentials(
shared.PRODXMON_SERVICE_ACCOUNT_EMAIL,
monitors.AppengineCredentials()))
interface.state.use_new_proto = True
interface.register_global_metrics([shared.appengine_default_version])
interface.register_global_metrics_callback(shared.INTERNAL_CALLBACK_NAME,
_internal_callback)
# We invoke global callbacks once for the whole application in the cron
# handler. Leaving this set to True would invoke them once per task.
interface.state.invoke_global_callbacks_on_flush = False
standard_metrics.init()
logging.info(
'Initialized ts_mon with service_name=%s, job_name=%s, '
'hostname=%s', service_name, job_name, hostname)
def get_current_version():
return modules.get_current_version_name()
def initialize(app=None, is_enabled_fn=None, cron_module='default',
is_local_unittest=None):
"""Instruments webapp2 `app` with gae_ts_mon metrics.
Instruments all the endpoints in `app` with basic metrics.
Args:
app (webapp2 app): the app to instrument.
is_enabled_fn (function or None): a function returning bool if ts_mon should
send the actual metrics. None (default) is equivalent to lambda: True.
This allows apps to turn monitoring on or off dynamically, per app.
cron_module (str): the name of the module handling the
/internal/cron/ts_mon/send endpoint. This allows moving the cron job
to any module the user wants.
is_local_unittest (bool or None): whether we are running in a unittest.
"""
if is_local_unittest is None: # pragma: no cover
# Since gae_ts_mon.initialize is called at module-scope by appengine apps,
# AppengineTestCase.setUp() won't have run yet and none of the appengine
# stubs will be initialized, so accessing Datastore or even getting the
# application ID will fail.
is_local_unittest = ('expect_tests' in sys.argv[0])
if is_enabled_fn is not None:
interface.state.flush_enabled_fn = is_enabled_fn
if app is not None:
instrument_wsgi_application(app)
if is_local_unittest or modules.get_current_module_name() == cron_module:
instrument_wsgi_application(handlers.app)
# Use the application ID as the service name and the module name as the job
# name.
if is_local_unittest: # pragma: no cover
service_name = 'unittest'
job_name = 'unittest'
hostname = 'unittest'
else:
service_name = app_identity.get_application_id()
job_name = modules.get_current_module_name()
hostname = modules.get_current_version_name()
runtime.set_shutdown_hook(_shutdown_hook)
interface.state.target = targets.TaskTarget(
service_name, job_name, shared.REGION, hostname, task_num=-1)
interface.state.flush_mode = 'manual'
interface.state.last_flushed = datetime.datetime.utcnow()
# Don't send metrics when running on the dev appserver.
if (is_local_unittest or
os.environ.get('SERVER_SOFTWARE', '').startswith('Development')):
logging.info('Using debug monitor')
interface.state.global_monitor = monitors.DebugMonitor()
else:
logging.info('Using pubsub monitor %s/%s', shared.PUBSUB_PROJECT,
shared.PUBSUB_TOPIC)
interface.state.global_monitor = monitors.PubSubMonitor(
monitors.APPENGINE_CREDENTIALS, shared.PUBSUB_PROJECT,
shared.PUBSUB_TOPIC)
shared.register_global_metrics([shared.appengine_default_version])
shared.register_global_metrics_callback(
shared.INTERNAL_CALLBACK_NAME, _internal_callback)
logging.info('Initialized ts_mon with service_name=%s, job_name=%s, '
'hostname=%s', service_name, job_name, hostname)
