def api_get_identity(): session = get_current_session() try: auth_plugin = get_auth_plugin() except PluginNotFoundException: google_user = users.get_current_user() if google_user: url = u'https://console.cloud.google.com/iam-admin/iam?project=%s' % app_identity.get_application_id() app_identity.get_application_id() modules = [] for plugin in get_plugins(): plugin_modules = plugin.get_modules() if plugin_modules: modules.extend(plugin_modules) module_strings = [m.name for m in sorted(modules, key=lambda m: m.sort_order)] return IdentityTO(google_user.email(), [], module_strings, [], u'en', u'/logout', url) raise HttpUnAuthorizedException(u'no_authenticated_plugin') try: if not session or not auth_plugin.validate_session(session): raise HttpUnAuthorizedException(data={'login_url': get_base_url() + auth_plugin.get_login_url()}) language = auth_plugin.get_user_language() or get_browser_language() modules = auth_plugin.get_visible_modules() logout_url = auth_plugin.get_logout_url() profile_url = auth_plugin.get_profile_url() permissions = [] for plugin in get_plugins(): permissions.extend(plugin.get_permissions()) return IdentityTO(session.user_id, session.scopes, modules, permissions, language, logout_url, profile_url) except HttpException: raise except Exception: logging.exception('Exception while trying to get identity') raise HttpUnAuthorizedException(data={'login_url': get_base_url() + auth_plugin.get_login_url()})
def mapping_query(table_name, hpo_ids, dataset_id=None, project_id=None): """ Get query used to generate new ids for a CDM table :param table_name: name of CDM table :param hpo_ids: identifies the HPOs :param dataset_id: identifies the BQ dataset containing the input table :param project_id: identifies the GCP project containing the dataset :return: the query """ if dataset_id is None: dataset_id = bq_utils.get_dataset_id() if project_id is None: project_id = app_identity.get_application_id() subqueries = _mapping_subqueries(table_name, hpo_ids, dataset_id, project_id) union_all_query = UNION_ALL.join(subqueries) return ''' WITH all_{table_name} AS ( {union_all_query} ) SELECT src_table_id, src_{table_name}_id, ROW_NUMBER() OVER () AS {table_name}_id FROM all_{table_name} '''.format(union_all_query=union_all_query, table_name=table_name)
def most_common_heel_errors(app_id=None, dataset_id=None, hpo_ids=None): """ :param app_id: Application Id :param dataset_id: Dataset Id :param hpo_ids: list of Hpo_ids :return: None """ heel_errors = list() if app_id is None: app_id = app_identity.get_application_id() if dataset_id is None: dataset_id = bq_utils.get_dataset_id() if not os.path.exists(HEEL_ERRORS_JSON) and not os.path.exists( HEEL_ERRORS_CSV): for hpo_id in hpo_ids: if bq_utils.table_exists( table_id='{hpo_id}_achilles_heel_results'.format( hpo_id=hpo_id), dataset_id=dataset_id): query = heel_error_query.format(app_id=app_id, dataset_id=dataset_id, hpo_id=hpo_id) query_job = bq_utils.query(query) result = bq_utils.response2rows(query_job) heel_errors.extend(result) with open(HEEL_ERRORS_JSON, 'w') as fp: json.dump(heel_errors, fp, sort_keys=True, indent=4) parse_json_csv()
def get(self): access_token = self.get_oauth_token('https://www.googleapis.com/auth/datastore') app_id = app_identity.get_application_id() timestamp = datetime.datetime.now().strftime('%Y-%m-%d') backup_entities = self.get_backup_entities() backup_bucket = self.get_backup_bucket() output_url_prefix = "gs://{}/{}".format(backup_bucket, timestamp) entity_filter = { 'kinds': backup_entities, } request = { 'project_id': app_id, 'output_url_prefix': output_url_prefix, 'entity_filter': entity_filter } headers = { 'Content-Type': 'application/json', 'Authorization': 'Bearer ' + access_token } url = 'https://datastore.googleapis.com/v1beta1/projects/%s:export' % app_id status, _ = self.fetch_url( url=url, payload=json.dumps(request), method=urlfetch.POST, headers=headers) status_sitevar = Sitevar.get_by_id('apistatus') if status == 200 and status_sitevar and 'backup' in status_sitevar.contents: status_sitevar.contents['backup']['db_export'] = timestamp status_sitevar.put()
class GoogleCloudStorage(object): storage = google.cloud.storage.Client(app_identity.get_application_id()) @webapp2.cached_property def bucket(self): return self.storage.get_bucket('{}.appspot.com'.format( app_identity.get_application_id(), ))
def get(self): access_token = self.get_oauth_token( 'https://www.googleapis.com/auth/datastore') app_id = app_identity.get_application_id() timestamp = datetime.datetime.now().strftime('%Y-%m-%d') backup_entities = self.get_backup_entities() backup_bucket = self.get_backup_bucket() output_url_prefix = "gs://{}/{}".format(backup_bucket, timestamp) entity_filter = { 'kinds': backup_entities, } request = { 'project_id': app_id, 'output_url_prefix': output_url_prefix, 'entity_filter': entity_filter } headers = { 'Content-Type': 'application/json', 'Authorization': 'Bearer ' + access_token } url = 'https://datastore.googleapis.com/v1beta1/projects/%s:export' % app_id status, _ = self.fetch_url(url=url, payload=json.dumps(request), method=urlfetch.POST, headers=headers) status_sitevar = Sitevar.get_by_id('apistatus') if status == 200 and status_sitevar and 'backup' in status_sitevar.contents: status_sitevar.contents['backup']['db_export'] = timestamp status_sitevar.put()
def _stream_metadata(self, insert_all_data, dataset_id, table_id): partition = datetime.datetime.now().strftime("%Y%m%d") return self.service.tabledata().insertAll( projectId=app_identity.get_application_id(), datasetId=dataset_id, tableId='{}${}'.format(table_id, partition), body=insert_all_data).execute(num_retries=3)
def get_most_recent(app_id=None, drc_bucket=None, report_for=None): """ Query audit logs for paths to the most recent datasources.json files in the DRC bucket. Note: Results are cached in a local json file to avoid unnecessary queries. :param app_id: identifies the GCP project :param drc_bucket: identifies the DRC bucket :param report_for: denotes which query to use b/w achilles and results :return: list of dict with keys `file_path`, `upload_timestamp` """ if app_id is None: app_id = app_identity.get_application_id() if drc_bucket is None: drc_bucket = gcs_utils.get_drc_bucket() if report_for == common.REPORT_FOR_ACHILLES: if not os.path.exists(common.LATEST_REPORTS_JSON): query = LATEST_REPORTS_QUERY.format(app_id=app_id, drc_bucket=drc_bucket, year=common.LOG_YEAR) query_job = bq_utils.query(query) result = bq_utils.response2rows(query_job) with open(common.LATEST_REPORTS_JSON, 'w') as fp: json.dump(result, fp, sort_keys=True, indent=4) with open(common.LATEST_REPORTS_JSON, 'r') as fp: return json.load(fp) elif report_for == common.REPORT_FOR_RESULTS: if not os.path.exists(common.LATEST_RESULTS_JSON): query = LATEST_RESULTS_QUERY.format(app_id=app_id, drc_bucket=drc_bucket, year=common.LOG_YEAR) query_job = bq_utils.query(query) result = bq_utils.response2rows(query_job) with open(common.LATEST_RESULTS_JSON, 'w') as fp: json.dump(result, fp, sort_keys=True, indent=4) with open(common.LATEST_RESULTS_JSON, 'r') as fp: return json.load(fp)
def __init__(self): self.app_id = app_identity.get_application_id() self.service = googleapiclient.discovery.build( 'datastore', 'v1', credentials=self.__create_credentials(), http=self.__create_http(), )
def sync_query(self, query, timeout=30000, use_legacy_sql=False): query_data = { 'query': query, 'timeoutMs': timeout, 'useLegacySql': use_legacy_sql } return self.service.jobs().query( projectId=app_identity.get_application_id(), body=query_data).execute(num_retries=3)
def send_emails_to_support(subject, body): cfg = get_config(NAMESPACE) sender = 'no-reply@%s.appspotmail.com' % app_identity.get_application_id() logging.debug('Sending email to support: %s\n %s', subject, body) for email in cfg.support_emails: logging.debug('Sending email to %s', email) mail.send_mail(sender=sender, to=email, subject=subject, body=body)
def get_query_result(hpo_id, query_string, table_id, query_wrapper, is_subquery, app_id=None, dataset_id=None): """ :param hpo_id: the name of the hpo_id for which validation is being done :param query_string: variable name of the query string stored in the constants :param table_id: Name of the table running analysis on :param query_wrapper: wrapper over the unioned query if required :param is_subquery: binary flag(true/false) to indicate if parsing is needed or not. :param app_id: name of the big query application id :param dataset_id: name of the big query dataset id :return: returns dictionary of rows """ if app_id is None: app_id = app_identity.get_application_id() if dataset_id is None: dataset_id = bq_utils.get_dataset_id() query = None result = None if is_subquery: sub_queries = [] for table in cdm.tables_to_map(): hpo_table = '{hpo_id}_{table_name}'.format(hpo_id=hpo_id, table_name=table) if bq_utils.table_exists(hpo_table): sub_query = query_string.format(hpo_id=hpo_id, app_id=app_id, dataset_id=dataset_id, domain_table=table) sub_queries.append(sub_query) unioned_query = main_constants.UNION_ALL.join(sub_queries) if unioned_query and query_wrapper is not None: query = query_wrapper.format(union_of_subqueries=unioned_query) else: query = unioned_query else: table_name = '{hpo_name}_{results_table}'.format( hpo_name=hpo_id, results_table=table_id) if bq_utils.table_exists(table_name): query = query_string.format(application=app_id, dataset=dataset_id, table_id=table_name) if query: # Found achilles_heel_results table(s), run the query response = bq_utils.query(query) result = bq_utils.response2rows(response) if result is None: result = [] return result
def outputjson(self, log): """Convert log entry into json to be written to loggingfs gofer.""" _, level, message, _ = log message = self._truncate(message, self._MAX_LINE_SIZE) trace = logsutil.TraceID() severity = logsutil.LogLevelString(level) entry = {'severity': severity, 'message': message} project = app_identity.get_application_id() entry['logging.googleapis.com/trace'] = ('projects/%s/traces/%s' % (project, trace)) return json.dumps(entry)
def _validate_authentication(self): import tba_config # Allow all requests in debug mode if tba_config.DEBUG: return incoming_app_id = self.request_state.headers.get('X-Appengine-Inbound-Appid', None) if incoming_app_id is None: raise remote.ApplicationError('Unauthenticated') from google.appengine.api.app_identity import app_identity if not app_identity.get_application_id() == incoming_app_id: raise remote.ApplicationError('Unauthenticated')
def post(self): j = {} for k in self.request.arguments(): j[k] = self.request.get(k) CallData(data=j).put() called = self.request.get('To') if called is None: xml = '''<?xml version="1.0" encoding="UTF-8" ?> <Response> <Say voice="alice" language="ja-JP">エラーがおきました</Say> </Response> ''' self.response.headers['Content-Type'] = 'text/xml' self.response.write(xml) return status = RoombaStatus.get_by_id(called) if status is None: xml = '''<?xml version="1.0" encoding="UTF-8" ?> <Response> <Say voice="alice" language="ja-JP">ルンバが接続されていません</Say> </Response> ''' self.response.headers['Content-Type'] = 'text/xml' self.response.write(xml) return OrderData(called=self.request.get('To'), caller=self.request.get('From'), data=j, talk='', command='START', time=1, oncall=True).put() words = [u'今日もお仕事お疲れ様、', u'今日はゆっくり休んでね', u'無理しないでね'] xml = u'''<?xml version="1.0" encoding="UTF-8" ?> <Response> <Say voice="alice" language="ja-JP">{word}。{msg}</Say> <Record action="http://{appid}.appspot.com/convert" timeout="3" maxlength="5" trim="trim-silence"></Record> </Response> '''.format(appid=get_application_id(), word=words[random.randint(0, len(words) - 1)], msg=u'ただ今掃除中。とめたい時はピーの後、停止と言ってね' if status.state == 'CLEANING' else u'掃除を始めるよ') self.response.headers['Content-Type'] = 'text/xml' self.response.write(xml)
def _authenticated(self): # Allow all requests in debug mode if tba_config.DEBUG: return True # Ignore auth check during tests if self.testing: return True incoming_app_id = self.request_state.headers.get('X-Appengine-Inbound-Appid', None) if incoming_app_id is None: return False return app_identity.get_application_id() == incoming_app_id
def _send_error_report(self, message, report_location): payload = { 'serviceContext': { 'service': self.service, }, 'message': '{0}'.format(message), 'context': { 'reportLocation': report_location } } self.logging_client.projects().events().report( projectName='projects/{}'.format(app_identity.get_application_id()), body=payload).execute()
def _validate_authentication(self): # Allow all requests in debug mode if tba_config.DEBUG: return # Ignore auth check during tests if self.testing: return incoming_app_id = self.request_state.headers.get('X-Appengine-Inbound-Appid', None) if incoming_app_id is None: raise remote.ApplicationError('Unauthenticated') if not app_identity.get_application_id() == incoming_app_id: raise remote.ApplicationError('Unauthenticated')
def post(self): logging.info("Request headers:\n%s" % "\n".join("- %s: %s" % h for h in self.request.headers.iteritems())) body = self.request.body msg = mail.InboundEmailMessage(body) logging.debug("Received %.1fkB from '%s'" % (len(body) / 1024., msg.sender)) if msg.attachments: def attachment_info((filename, payload)): return "- '%s' (%.1fkB; %s)" \ % (filename, len(payload.decode()) / 1024., mail._GetMimeType(filename)) logging.debug("%d attachments:\n%s" % (len(msg.attachments), "\n".join( map(attachment_info, msg.attachments)))) ts = datetime.utcfromtimestamp( email.utils.mktime_tz(email.utils.parsedate_tz(msg.date))) to = sorted({s.strip() for s in msg.to.split(",")}) ent = Inmail( mail_timestamp=ts, sender=msg.sender, recipients=to, # the subject field is undefined if no subject subject=getattr(msg, 'subject', None), body=body) try: ent.put() except RequestTooLargeError: logging.exception("Failed to store Inmail") mail.send_mail( sender="info@%s.appspotmail.com" % get_application_id(), to=msg.sender, subject="Auto-reply: Too large email", body="Hello dear %s,\n\n" "An email you sent couldn't be received properly due to " "its excessive size (%.1fMB). Please retry with reduced " "attachments.\n\n" "Original email recipients: %s\n" "Subject: %s" % (msg.sender, len(body) / 1024.**2, msg.to, getattr(msg, 'subject', "(empty)"))) else: logging.info("Stored email to %r" % ent.key)
def run_ppi_vocab_update(project_id, dataset_id): """ runs the query which updates the ppi vocabulary in observation table :param project_id: Name of the project :param dataset_id: Name of the dataset where the queries should be run :return: """ if project_id is None: project_id = app_identity.get_application_id() q = UPDATE_PPI_QUERY.format(project=project_id, dataset=dataset_id, vocabulary=VOCABULARY_DATASET) logging.debug('Query for PMI_Skip fix is {q}'.format(q=q)) bq_utils.query(q=q)
def post(self): self.verify_permissions() suggestion_id = int(self.request.get("suggestion_id")) verdict = self.request.get("verdict") message = self.request.get("user_message") email_body = None user = None event_key = None status = '' if verdict == "accept": status = 'accept' user, event_key, email_body = self._process_accepted(suggestion_id, message) elif verdict == "reject": suggestion = Suggestion.get_by_id(suggestion_id) event_key = suggestion.contents['event_key'] user = suggestion.author.get() event = Event.get_by_id(event_key) suggestion.review_state = Suggestion.REVIEW_REJECTED suggestion.reviewer = self.user_bundle.account.key suggestion.reviewed_at = datetime.now() suggestion.put() status = 'reject' email_body = """Hi {}, We have reviewer your request for auth tokens for {} {} and have regretfully declined with the following message: {} If you have any questions, please don't heasitate to reach out to us at [email protected] Thanks, TBA Admins """.format(user.display_name, event.year, event.name, message) # Notify the user their keys are available sender = "{}@appspot.gserviceaccount.com".format(app_identity.get_application_id()) reply_to = sender if tba_config.DEBUG else "*****@*****.**" if email_body: mail.send_mail(sender=sender, reply_to=reply_to, to=user.email, subject="The Blue Alliance Auth Tokens for {}".format(event_key), body=email_body) self.redirect("/suggest/apiwrite/review?success={}".format(status))
def start_template(filename, job_name, params={}, retries=5): """ Starts a dataflow template. Args: filename (str): The gcs filename for the template. job_name (str): The job name to use. params (dict): Job parameters. retries (int): Remaining retries. Set this to 0 to never retry. Returns: dict: The job response. """ global gcs_dataflow_path if gcs_dataflow_path is None: gcs_dataflow_path = 'gs://{}/dataflow'.format( app_identity.get_default_gcs_bucket_name()) body = { 'jobName': job_name, 'environment': { 'tempLocation': '{}/temp_jobs'.format(gcs_dataflow_path) }, 'parameters': params } request = get_client().projects().locations().templates().launch( projectId=app_identity.get_application_id(), gcsPath='{}/templates/{}'.format(gcs_dataflow_path, filename), location=os.environ.get('DATAFLOW_REGION') or 'us-central1', body=body) logging.info('Starting template...') try: return request.execute().get('job', {}) except HttpError as ex: if retries > 0: logging.warn( 'Retrying failed request ({} retries remaining)'.format( retries), exc_info=True) return start_template(filename, job_name, params, retries - 1) else: raise ex
def run_pmi_fix(project_id, dataset_id): """ runs the query which adds skipped rows in survey before 2019-04-10 as PMI_Skip :param project_id: Name of the project :param dataset_id: Name of the dataset where the queries should be run :return: """ if project_id is None: project_id = app_identity.get_application_id() q = PMI_SKIP_FIX_QUERY.format(project=project_id, dataset=dataset_id) logging.debug('Query for PMI_Skip fix is {q}'.format(q=q)) bq_utils.query(q=q, destination_table_id=OBSERVATION_TABLE_NAME, destination_dataset_id=dataset_id, write_disposition=bq_consts.WRITE_TRUNCATE)
def initialize(service_name): is_local_unittest = ('expect_tests' in sys.argv[0]) if is_local_unittest: appengine_name = 'unittest' service_name = 'unittest' hostname = 'unittest' else: # pragma: no cover appengine_name = app_identity.get_application_id() hostname = '%s, %s' % (modules.get_current_module_name(), modules.get_current_version_name()) # Only send events if we are running on the actual AppEngine. if os.environ.get('SERVER_SOFTWARE', '').startswith('Google App Engine'): run_type = 'prod' # pragma: no cover else: run_type = 'dry' config.setup_monitoring(run_type, hostname, service_name, appengine_name) logging.info( 'Initialized event_mon with run_type=%s, hostname=%s, service_name=%s, ' 'appengine_name=%s', run_type, hostname, service_name, appengine_name)
def check_routes(routes): all_statuses = [] for route in routes: if not route.profile.boxcar_send_push: continue if route.profile.silence_until and route.profile.silence_until > datetime.datetime.now(): continue statuses = call_ns_api(route.station_from, route.station_to, route.departure_time_from) if not statuses: continue boxcar_access_token = route.profile.boxcar_access_token message, message_long = create_push_message(statuses) hashed_message = base64.b64encode(message) if route.latest_push_message != hashed_message: route.latest_push_message = hashed_message route.put() url = 'https://{}.appspot.com{}'.format(app_identity.get_application_id(), uri_for('profile_edit', obj_id=route.profile.key.id())) send_push_notification(message, message_long, boxcar_access_token, url) all_statuses.extend(statuses) return all_statuses
def main(app_id, dataset_id, file_name, all_hpo=False, file_format=None): """ Retrieve most prevalent errors from achilles heel results table(s) Retrieve most prevalent errors from achilles heel results table(s) and save results to a file at specified path :param app_id: Identifies the google cloud project containing the dataset :param dataset_id: Identifies the dataset where from achilles heel results should be obtained :param file_name: Path of file to save to :param all_hpo: If `True` query <hpo_id>_achilles_heel_results, otherwise just achilles_heel_results (default) :param file_format: csv or json """ if app_id is None: app_id = app_identity.get_application_id() if dataset_id is None: dataset_id = bq_utils.get_dataset_id() if os.path.exists(file_name): # Do not overwrite existing # raise RuntimeError('File {} already exists'.format(file_name)) raise IOError('File %s already exists' % file_name) if file_format is None: # Attempt to determine format file_name_comps = file_name.lower().split('.') file_format = file_name_comps[-1] if file_format not in OUTPUT_FORMATS: raise ValueError('File format must be one of (%s)' % ', '.join(OUTPUT_FORMATS)) heel_errors = top_heel_errors(app_id, dataset_id, all_hpo) if file_format == CSV: save_csv(heel_errors, file_name) elif file_format == JSON: save_json(heel_errors, file_name) else: save_csv(heel_errors, file_name)
def post(self): payloadType = 'text' msg = self.request.get('msg').strip() if not msg: logging.debug('loading body') msg = self.request.body if not msg: self.abort(400) return msg = json.loads(msg) payloadType = 'struct' cloud_logging = self._get_service() from google.appengine.api.app_identity import app_identity projectsId = app_identity.get_application_id() deviceid = 'test-test' logsId = 'barcamp2015' severity = 'DEBUG' logs = { 'commonLabels': { 'compute.googleapis.com/resource_id': deviceid, "compute.googleapis.com/resource_type": "instance", }, 'entries': [ { "log": logsId, "metadata": { "serviceName": "compute.googleapis.com", "severity": severity, }, payloadType + "Payload": msg, }, ] } logging.debug(logs) resp = cloud_logging.projects().logs().entries().write(projectsId=projectsId, logsId=logsId, body=logs).execute() logging.info(json.dumps(resp, sort_keys=True, indent=4)) self.response.write('log saved')
def send_email_back(recipient, subject, attachments, text_content=None, html_content=None): from sendgrid.helpers import mail from sendgrid.helpers.mail import Attachment import premailer logging.info("sending mail to %s (%s/%s)", recipient, SENDGRID_API_KEY, SENDGRID_SENDER) to_email = mail.Email(recipient) from_email = mail.Email(SENDGRID_SENDER) message = mail.Mail() message.set_from(from_email) message.set_subject(subject) personalization = mail.Personalization() personalization.add_to(to_email) message.add_personalization(personalization) if not text_content and not html_content: message.add_content(mail.Content("text/plain", global_body)) if text_content: message.add_content(mail.Content("text/plain", text_content)) if html_content: message.add_content(mail.Content("text/html", html_content)) for att in attachments: data = att["data"] file_name = att["name"] if file_name.endswith(".htm") or file_name.endswith(".html"): stub_css = "https://%s.appspot.com/css/stub.css" % app_identity.get_application_id() data = re.sub( r'\"D:\\ssis\\SecureMail\\SecureMailTest\\MailItemImages\\BankB1\.gifstyle\.css
.*\"', '"%s"' % stub_css, data) logging.info("before transform(%s) %s", type(data), data) logging.info("using premailer for %s", file_name) data = data.decode("utf8") p = premailer.Premailer(data) data = p.transform().encode("utf8") logging.info("after transform(%s) %s", type(data), data) attachment = Attachment() attachment.set_content(base64.b64encode(data)) attachment.set_type(att["type"]) attachment.set_filename(att["name"]) attachment.set_disposition("attachment") attachment.set_content_id(att["name"]) message.add_attachment(attachment) data = json.dumps(message.get()) logging.debug("sending %s", data) headers = { "Authorization": 'Bearer {0}'.format(SENDGRID_API_KEY), "Content-Type": "application/json", "Accept": 'application/json' } response = urlfetch.fetch( url="https://api.sendgrid.com/v3/mail/send", payload=data, method=urlfetch.POST, headers=headers) if response.status_code > 299: logging.error("response %s(%s)", response.content, response.status_code) else: logging.info("response %s(%s)", response.content, response.status_code) if response.status_code > 299: raise Exception("Failed to call sendgrid API")
from mcfw.cache import set_cache_key from mcfw.consts import MISSING from mcfw.rpc import arguments, returns, check_function_metadata, get_parameter_types, run, get_parameters, \ get_type_details, serialize_value, parse_parameter _CALL_ACTION_RESEND = 1 _CALL_ACTION_MUST_PROCESS = 2 _CALL_ACTION_DO_NOT_PROCESS = 3 BACKLOG_CONCURRENCY_PROTECTION_INTERVAL = 120 MESSAGE_LINGER_INTERVAL = 3600 * 24 * 20 # 20 days MESSAGE_ALLOWED_FUTURE_TIME_INTERVAL = 3600 * 24 BACKLOG_MESSAGE_RETENTION_INTERVAL = 3600 * 24 + MESSAGE_LINGER_INTERVAL # 21 days BACKLOG_DUPLICATE_AVOIDANCE_RETENTION_INTERVAL = 3600 * 24 # 1 day APPENGINE_APP_ID = get_application_id() DO_NOT_SAVE_RPCCALL_OBJECTS = "DO_NOT_SAVE_RPCCALL_OBJECTS" PERFORM_CALLBACK_SYNCHRONOUS = "PERFORM_CALLBACK_SYNCHRONOUS" SKIP_ACCOUNTS = "SKIP_ACCOUNTS" MOBILE_ACCOUNT = "MOBILE_ACCOUNT" DEFER_KICK = "DEFER_KICK" TARGET_MFR = "TARGET_MFR" API_VERSION = u"av" API_DIRECT_PATH_KEY = u"ap" CALL_ID = u"ci" FUNCTION = u"f" PARAMETERS = u"a" STATUS = u"s" STATUS_SUCCESS = u"success" STATUS_FAIL = u"fail" RESULT = u"r"
import os import jinja2 import webapp2 import logging from db_reg import UserRegistration, PostRegistration, CommentRegistration from db_reg import LikeRegistration from google.appengine.ext import ndb import blog_db import encrypt from google.appengine.api.app_identity import app_identity template_dir = os.path.join(os.path.dirname(__file__), "templates") jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir), autoescape=True) logging.info("ABC " + app_identity.get_application_id()) class Handler(webapp2.RequestHandler): def write(self, *a, **kw): self.response.write(*a, **kw) def render_str(self, template, **params): t = jinja_env.get_template(template) return t.render(params) def render(self, template, **kw): self.write(self.render_str(template, **kw)) class SignupPage(Handler):
def initialize(app=None, is_enabled_fn=None, cron_module='default', is_local_unittest=None): """Instruments webapp2 `app` with gae_ts_mon metrics. Instruments all the endpoints in `app` with basic metrics. Args: app (webapp2 app): the app to instrument. is_enabled_fn (function or None): a function returning bool if ts_mon should send the actual metrics. None (default) is equivalent to lambda: True. This allows apps to turn monitoring on or off dynamically, per app. cron_module (str): the name of the module handling the /internal/cron/ts_mon/send endpoint. This allows moving the cron job to any module the user wants. is_local_unittest (bool or None): whether we are running in a unittest. """ if is_local_unittest is None: # pragma: no cover # Since gae_ts_mon.initialize is called at module-scope by appengine apps, # AppengineTestCase.setUp() won't have run yet and none of the appengine # stubs will be initialized, so accessing Datastore or even getting the # application ID will fail. is_local_unittest = ('expect_tests' in sys.argv[0]) if is_enabled_fn is not None: interface.state.flush_enabled_fn = is_enabled_fn if app is not None: instrument_wsgi_application(app) if is_local_unittest or modules.get_current_module_name() == cron_module: instrument_wsgi_application(handlers.app) # Use the application ID as the service name and the module name as the job # name. if is_local_unittest: # pragma: no cover service_name = 'unittest' job_name = 'unittest' hostname = 'unittest' else: service_name = app_identity.get_application_id() job_name = modules.get_current_module_name() hostname = modules.get_current_version_name() runtime.set_shutdown_hook(_shutdown_hook) interface.state.target = targets.TaskTarget( service_name, job_name, shared.REGION, hostname, task_num=-1) interface.state.flush_mode = 'manual' interface.state.last_flushed = datetime.datetime.utcnow() # Don't send metrics when running on the dev appserver. if (is_local_unittest or os.environ.get('SERVER_SOFTWARE', '').startswith('Development')): logging.info('Using debug monitor') interface.state.global_monitor = monitors.DebugMonitor() else: logging.info('Using pubsub monitor %s/%s', shared.PUBSUB_PROJECT, shared.PUBSUB_TOPIC) interface.state.global_monitor = monitors.PubSubMonitor( monitors.APPENGINE_CREDENTIALS, shared.PUBSUB_PROJECT, shared.PUBSUB_TOPIC) shared.register_global_metrics([shared.appengine_default_version]) shared.register_global_metrics_callback( shared.INTERNAL_CALLBACK_NAME, _internal_callback) logging.info('Initialized ts_mon with service_name=%s, job_name=%s, ' 'hostname=%s', service_name, job_name, hostname)
def _fcm_url(self): app_id = app_identity.get_application_id() return 'https://fcm.googleapis.com/v1/projects/{}/messages:send'.format(app_id)
from google.appengine.api.app_identity import app_identity from google.appengine.api.urlfetch import DeadlineExceededError import sys from methods.email.admin import send_error __author__ = 'dvpermyakov' _APP_ID = app_identity.get_application_id() def handle_500(request, response, exception): body = """URL: %s User-Agent: %s Exception: %s Logs: https://appengine.google.com/logs?app_id=s~%s&severity_level_override=0&severity_level=3""" \ % (request.url, request.headers['User-Agent'], exception, _APP_ID) if isinstance(exception, DeadlineExceededError) and ":9900/" in exception.message: send_error("iiko", "iiko deadline", body) else: send_error("server", "Error 500", body) exc_info = sys.exc_info() raise exc_info[0], exc_info[1], exc_info[2]
import logging import os from google.appengine.api.app_identity.app_identity import get_application_id try: appid = get_application_id() except AttributeError: appid = None IS_PROD = not os.environ.get('SERVER_SOFTWARE', "").startswith("Development/") \ and appid == 'jsa-pore' if IS_PROD: PRIMARY_HOST = "pore.savukoski.name" else: PRIMARY_HOST = os.environ.get('HTTP_HOST', 'localhost:8080') DEBUG = TEMPLATE_DEBUG = not IS_PROD # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name TIME_ZONE = 'GMT' # http://www.i18nguy.com/unicode/language-identifiers.html LANGUAGE_CODE = 'en-us' SITE_ID = 1 USE_I18N = USE_L10N = False USE_TZ = True
def _fcm_url(self): from google.appengine.api.app_identity import app_identity app_id = app_identity.get_application_id() return 'https://fcm.googleapis.com/v1/projects/{}/messages:send'.format(app_id)
def render_query(query_str, **kwargs): project_id = app_identity.get_application_id() dataset_id = bq_utils.get_dataset_id() return query_str.format(project_id=project_id, dataset_id=dataset_id, **kwargs)
def __create_gcs_output_url(gcs_folder_name): app_id = app_identity.get_application_id() output_url_prefix = "gs://staging.{}.appspot.com/{}" \ .format(app_id, gcs_folder_name) return output_url_prefix
import logging import webapp2 from google.appengine.api.app_identity import app_identity from google.appengine.api import urlfetch from google.appengine.api import mail from google.appengine.ext.webapp.mail_handlers import InboundMailHandler appid = app_identity.get_application_id() mail_sender = 'www@' + appid + '.appspotmail.com' class RMailHandler(InboundMailHandler): def receive(self, msg): logging.info(msg.sender + " " + msg.subject) if msg.subject: UrlHandler(msg.subject, msg.sender) def UrlHandler(url, receipt): try: logging.info("fetching " + url) result = urlfetch.Fetch(url, deadline=30) if result.status_code == 200: logging.info("done " + url) TMailHandler(receipt, result.content, "Re: " + url) else: logging.error(url + ":" + result.status) except Exception, e: print(e) return None
def initialize(app=None, is_enabled_fn=None, cron_module='default', is_local_unittest=None): """Instruments webapp2 `app` with gae_ts_mon metrics. Instruments all the endpoints in `app` with basic metrics. Args: app (webapp2 app): the app to instrument. is_enabled_fn (function or None): a function returning bool if ts_mon should send the actual metrics. None (default) is equivalent to lambda: True. This allows apps to turn monitoring on or off dynamically, per app. cron_module (str): the name of the module handling the /internal/cron/ts_mon/send endpoint. This allows moving the cron job to any module the user wants. is_local_unittest (bool or None): whether we are running in a unittest. """ if is_local_unittest is None: # pragma: no cover # Since gae_ts_mon.initialize is called at module-scope by appengine apps, # AppengineTestCase.setUp() won't have run yet and none of the appengine # stubs will be initialized, so accessing Datastore or even getting the # application ID will fail. is_local_unittest = ('expect_tests' in sys.argv[0]) if is_enabled_fn is not None: interface.state.flush_enabled_fn = is_enabled_fn if app is not None: instrument_wsgi_application(app) if is_local_unittest or modules.get_current_module_name( ) == cron_module: instrument_wsgi_application(handlers.app) # Use the application ID as the service name and the module name as the job # name. if is_local_unittest: # pragma: no cover service_name = 'unittest' job_name = 'unittest' hostname = 'unittest' else: service_name = app_identity.get_application_id() job_name = modules.get_current_module_name() hostname = modules.get_current_version_name() runtime.set_shutdown_hook(_shutdown_hook) interface.state.target = targets.TaskTarget(service_name, job_name, shared.REGION, hostname, task_num=-1) interface.state.flush_mode = 'manual' interface.state.last_flushed = datetime.datetime.utcnow() # Don't send metrics when running on the dev appserver. if (is_local_unittest or os.environ.get('SERVER_SOFTWARE', '').startswith('Development')): logging.info('Using debug monitor') interface.state.global_monitor = monitors.DebugMonitor() else: logging.info('Using https monitor %s with %s', shared.PRODXMON_ENDPOINT, shared.PRODXMON_SERVICE_ACCOUNT_EMAIL) interface.state.global_monitor = monitors.HttpsMonitor( shared.PRODXMON_ENDPOINT, monitors.DelegateServiceAccountCredentials( shared.PRODXMON_SERVICE_ACCOUNT_EMAIL, monitors.AppengineCredentials())) interface.state.use_new_proto = True interface.register_global_metrics([shared.appengine_default_version]) interface.register_global_metrics_callback(shared.INTERNAL_CALLBACK_NAME, _internal_callback) # We invoke global callbacks once for the whole application in the cron # handler. Leaving this set to True would invoke them once per task. interface.state.invoke_global_callbacks_on_flush = False standard_metrics.init() logging.info( 'Initialized ts_mon with service_name=%s, job_name=%s, ' 'hostname=%s', service_name, job_name, hostname)