def register_db_accessor(self, app_id): """ Gets a distributed datastore object to interact with the datastore for a certain application. Args: app_id: The application ID. Returns: A distributed_datastore.DatastoreDistributed object. """ ds_distributed = datastore_distributed.DatastoreDistributed( app_id, self.datastore_path, require_indexes=False) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', ds_distributed) os.environ['APPLICATION_ID'] = app_id os.environ['AUTH_DOMAIN'] = "appscale.com" return ds_distributed
def __init__(self): """ DistributedTaskQueue Constructor. """ file_io.set_logging_format() file_io.mkdir(self.LOG_DIR) file_io.mkdir(TaskQueueConfig.CELERY_WORKER_DIR) file_io.mkdir(TaskQueueConfig.CELERY_CONFIG_DIR) setup_env() # Cache all queue information in memory. self.__queue_info_cache = {} master_db_ip = appscale_info.get_db_master_ip() connection_str = master_db_ip + ":" + str(constants.DB_SERVER_PORT) ds_distrib = datastore_distributed.DatastoreDistributed( constants.DASHBOARD_APP_ID, connection_str, require_indexes=False) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', ds_distrib) os.environ['APPLICATION_ID'] = constants.DASHBOARD_APP_ID
def __init__(self, db_access, zk_client): """ DistributedTaskQueue Constructor. Args: db_access: A DatastoreProxy object. zk_client: A KazooClient. """ setup_env() db_proxy = appscale_info.get_db_proxy() connection_str = '{}:{}'.format(db_proxy, str(constants.DB_SERVER_PORT)) ds_distrib = datastore_distributed.DatastoreDistributed( constants.DASHBOARD_APP_ID, connection_str, require_indexes=False) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', ds_distrib) os.environ['APPLICATION_ID'] = constants.DASHBOARD_APP_ID self.db_access = db_access self.queue_manager = GlobalQueueManager(zk_client, db_access)
def post(self, session_id="session"): """ Handler a post request from a user uploading a blob. Args: session_id: Authentication token to validate the upload. """ app_id = self.request.headers.get('X-Appengine-Inbound-Appid', '') global datastore_path db = datastore_distributed.DatastoreDistributed(app_id, datastore_path) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', db) os.environ['APPLICATION_ID'] = app_id # Setup the app id in the datastore. # Get session info and upload success path. blob_session = get_session(session_id) if not blob_session: self.finish('Session has expired. Contact the owner of the ' + \ 'app for support.\n\n') return success_path = blob_session["success_path"] if success_path.startswith('/'): success_path = urlparse.urljoin(self.request.full_url(), success_path) server_host = success_path[:success_path.rfind("/", 3)] if server_host.startswith("http://"): # Strip off the beginging of the server host server_host = server_host[len("http://"):] server_host = server_host.split('/')[0] blob_storage = datastore_blob_storage.DatastoreBlobStorage(app_id) uploadhandler = dev_appserver_upload.UploadCGIHandler(blob_storage) datastore.Delete(blob_session) # This request is sent to the upload handler of the app # in the hope it returns a redirect to be forwarded to the user urlrequest = urllib2.Request(success_path) # Forward all relevant headers and create data for request content_type = self.request.headers["Content-Type"] kv = split_content_type(content_type) boundary = None if "boundary" in kv: boundary = kv["boundary"] urlrequest.add_header("Content-Type", 'application/x-www-form-urlencoded') urlrequest.add_header('X-AppEngine-BlobUpload', 'true') for name, value in self.request.headers.items(): if name.lower() not in STRIPPED_HEADERS: urlrequest.add_header(name, value) # Get correct redirect addresses, otherwise it will redirect back # to this port. urlrequest.add_header("Host", server_host) form = MultiPartForm(boundary) creation = datetime.datetime.now() # Loop on all files in the form. for filekey in self.request.files.keys(): data = {"blob_info_metadata": {filekey: []}} file = self.request.files[filekey][0] body = file["body"] size = len(body) filename = file["filename"] file_content_type = file["content_type"] gs_path = '' if 'gcs_bucket' in blob_session: gcs_config = {'scheme': 'https', 'port': 443} try: gcs_config.update(deployment_config.get_config('gcs')) except ConfigInaccessible: self.send_error('Unable to fetch GCS configuration.') return if 'host' not in gcs_config: self.send_error('GCS host is not defined.') return gcs_path = '{scheme}://{host}:{port}'.format(**gcs_config) gcs_bucket_name = blob_session['gcs_bucket'] gcs_url = '/'.join([gcs_path, gcs_bucket_name, filename]) response = requests.post(gcs_url, headers={'x-goog-resumable': 'start'}) if (response.status_code != 201 or GCS_UPLOAD_ID_HEADER not in response.headers): self.send_error( reason='Unable to start resumable GCS upload.') return upload_id = response.headers[GCS_UPLOAD_ID_HEADER] total_chunks = int(math.ceil(float(size) / GCS_CHUNK_SIZE)) for chunk_num in range(total_chunks): offset = GCS_CHUNK_SIZE * chunk_num current_chunk_size = min(GCS_CHUNK_SIZE, size - offset) end_byte = offset + current_chunk_size current_range = '{}-{}'.format(offset, end_byte - 1) content_range = 'bytes {}/{}'.format(current_range, size) response = requests.put( gcs_url, data=body[offset:end_byte], headers={'Content-Range': content_range}, params={'upload_id': upload_id}) if chunk_num == total_chunks - 1: if response.status_code != 200: self.send_error( reason='Unable to complete GCS upload.') return else: if response.status_code != 308: self.send_error( reason='Unable to continue GCS upload.') return gs_path = '/gs/{}/{}'.format(gcs_bucket_name, filename) blob_key = 'encoded_gs_key:' + base64.b64encode(gs_path) else: form_item = cgi.FieldStorage( headers={'content-type': file_content_type}) form_item.file = cStringIO.StringIO(body) form_item.filename = filename blob_entity = uploadhandler.StoreBlob(form_item, creation) blob_key = str(blob_entity.key().name()) if not blob_key: self.finish('Status: 500\n\n') return creation_formatted = blobstore._format_creation(creation) form.add_file(filekey, filename, cStringIO.StringIO(blob_key), blob_key, blobstore.BLOB_KEY_HEADER, size, creation_formatted) md5_handler = hashlib.md5(str(body)) blob_info = { "filename": filename, "creation-date": creation_formatted, "key": blob_key, "size": str(size), "content-type": file_content_type, "md5-hash": md5_handler.hexdigest() } if 'gcs_bucket' in blob_session: blob_info['gs-name'] = gs_path data["blob_info_metadata"][filekey].append(blob_info) # Loop through form fields for fieldkey in self.request.arguments.keys(): form.add_field(fieldkey, self.request.arguments[fieldkey][0]) data[fieldkey] = self.request.arguments[fieldkey][0] logger.debug("Callback data: \n{}".format(data)) data = urllib.urlencode(data) urlrequest.add_header("Content-Length", str(len(data))) urlrequest.add_data(data) # We are catching the redirect error here # and extracting the Location to post the redirect. try: response = urllib2.urlopen(urlrequest) output = response.read() if response.info().get('Content-Encoding') == 'gzip': buf = StringIO(output) f = gzip.GzipFile(fileobj=buf) data = f.read() output = data self.finish(output) except urllib2.HTTPError, e: if "Location" in e.hdrs: # Catch any errors, use the success path to # get the ip and port, use the redirect path # for the path. We split redirect_path just in case # its a full path. redirect_path = e.hdrs["Location"] self.redirect(redirect_path) return else: self.finish(UPLOAD_ERROR + "</br>" + str(e.hdrs) + "</br>" + str(e)) return
def setup_stubs( request_data, app_id, application_root, trusted, blobstore_path, datastore_consistency, datastore_path, datastore_require_indexes, datastore_auto_id_policy, images_host_prefix, logs_path, mail_smtp_host, mail_smtp_port, mail_smtp_user, mail_smtp_password, mail_enable_sendmail, mail_show_mail_body, matcher_prospective_search_path, search_index_path, taskqueue_auto_run_tasks, taskqueue_default_http_server, uaserver_path, user_login_url, user_logout_url, xmpp_path): """Configures the APIs hosted by this server. Args: request_data: An apiproxy_stub.RequestInformation instance used by the stubs to lookup information about the request associated with an API call. app_id: The str application id e.g. "guestbook". application_root: The path to the directory containing the user's application e.g. "/home/joe/myapp". trusted: A bool indicating if privileged APIs should be made available. blobstore_path: The path to the file that should be used for blobstore storage. datastore_consistency: The datastore_stub_util.BaseConsistencyPolicy to use as the datastore consistency policy. datastore_path: The path to the file that should be used for datastore storage. datastore_require_indexes: A bool indicating if the same production datastore indexes requirements should be enforced i.e. if True then a google.appengine.ext.db.NeedIndexError will be be raised if a query is executed without the required indexes. datastore_auto_id_policy: The type of sequence from which the datastore stub assigns auto IDs, either datastore_stub_util.SEQUENTIAL or datastore_stub_util.SCATTERED. images_host_prefix: The URL prefix (protocol://host:port) to prepend to image urls on calls to images.GetUrlBase. logs_path: Path to the file to store the logs data in. mail_smtp_host: The SMTP hostname that should be used when sending e-mails. If None then the mail_enable_sendmail argument is considered. mail_smtp_port: The SMTP port number that should be used when sending e-mails. If this value is None then mail_smtp_host must also be None. mail_smtp_user: The username to use when authenticating with the SMTP server. This value may be None if mail_smtp_host is also None or if the SMTP server does not require authentication. mail_smtp_password: The password to use when authenticating with the SMTP server. This value may be None if mail_smtp_host or mail_smtp_user is also None. mail_enable_sendmail: A bool indicating if sendmail should be used when sending e-mails. This argument is ignored if mail_smtp_host is not None. mail_show_mail_body: A bool indicating whether the body of sent e-mails should be written to the logs. matcher_prospective_search_path: The path to the file that should be used to save prospective search subscriptions. search_index_path: The path to the file that should be used for search index storage. taskqueue_auto_run_tasks: A bool indicating whether taskqueue tasks should be run automatically or it the must be manually triggered. taskqueue_default_http_server: A str containing the address of the http server that should be used to execute tasks. uaserver_path: (AppScale-specific) A str containing the FQDN or IP address of the machine that runs a UserAppServer. user_login_url: A str containing the url that should be used for user login. user_logout_url: A str containing the url that should be used for user logout. xmpp_path: (AppScale-specific) A str containing the FQDN or IP address of the machine that runs ejabberd, where XMPP clients should connect to. """ apiproxy_stub_map.apiproxy.RegisterStub( 'app_identity_service', app_identity_stub.AppIdentityServiceStub()) blob_storage = datastore_blob_storage.DatastoreBlobStorage(app_id) apiproxy_stub_map.apiproxy.RegisterStub( 'blobstore', blobstore_stub.BlobstoreServiceStub(blob_storage, request_data=request_data)) apiproxy_stub_map.apiproxy.RegisterStub( 'capability_service', capability_stub.CapabilityServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'channel', channel_service_stub.ChannelServiceStub(request_data=request_data)) datastore = datastore_distributed.DatastoreDistributed( app_id, datastore_path, require_indexes=datastore_require_indexes, trusted=trusted, root_path=application_root) apiproxy_stub_map.apiproxy.ReplaceStub( 'datastore_v3', datastore) apiproxy_stub_map.apiproxy.RegisterStub( 'datastore_v4', datastore_v4_stub.DatastoreV4Stub(app_id)) apiproxy_stub_map.apiproxy.RegisterStub( 'file', file_service_stub.FileServiceStub(blob_storage)) serve_address = os.environ['NGINX_HOST'] try: from google.appengine.api.images import images_stub except ImportError: logging.warning('Could not initialize images API; you are likely missing ' 'the Python "PIL" module.') # We register a stub which throws a NotImplementedError for most RPCs. from google.appengine.api.images import images_not_implemented_stub apiproxy_stub_map.apiproxy.RegisterStub( 'images', images_not_implemented_stub.ImagesNotImplementedServiceStub( host_prefix=images_host_prefix)) else: host_prefix = 'http://{}'.format(serve_address) apiproxy_stub_map.apiproxy.RegisterStub( 'images', images_stub.ImagesServiceStub(host_prefix=host_prefix)) apiproxy_stub_map.apiproxy.RegisterStub( 'logservice', logservice_stub.LogServiceStub(persist=True)) apiproxy_stub_map.apiproxy.RegisterStub( 'mail', mail_stub.MailServiceStub(mail_smtp_host, mail_smtp_port, mail_smtp_user, mail_smtp_password, enable_sendmail=mail_enable_sendmail, show_mail_body=mail_show_mail_body)) apiproxy_stub_map.apiproxy.RegisterStub( 'memcache', memcache_distributed.MemcacheService()) apiproxy_stub_map.apiproxy.RegisterStub( 'search', appscale_search_stub.SearchServiceStub(app_id=app_id)) apiproxy_stub_map.apiproxy.RegisterStub( 'modules', modules_stub.ModulesServiceStub(request_data)) apiproxy_stub_map.apiproxy.RegisterStub( 'system', system_stub.SystemServiceStub(request_data=request_data)) apiproxy_stub_map.apiproxy.RegisterStub( 'taskqueue', taskqueue_distributed.TaskQueueServiceStub(app_id, serve_address)) urlmatchers_to_fetch_functions = [] urlmatchers_to_fetch_functions.extend( gcs_dispatcher.URLMATCHERS_TO_FETCH_FUNCTIONS) apiproxy_stub_map.apiproxy.RegisterStub( 'urlfetch', urlfetch_stub.URLFetchServiceStub( urlmatchers_to_fetch_functions=urlmatchers_to_fetch_functions)) apiproxy_stub_map.apiproxy.RegisterStub( 'user', user_service_stub.UserServiceStub(login_url=user_login_url, logout_url=user_logout_url, request_data=request_data)) apiproxy_stub_map.apiproxy.RegisterStub( 'xmpp', xmpp_service_real.XmppService(domain=xmpp_path, uaserver=uaserver_path)) apiproxy_stub_map.apiproxy.RegisterStub( 'matcher', prospective_search_stub.ProspectiveSearchStub( matcher_prospective_search_path, apiproxy_stub_map.apiproxy.GetStub('taskqueue'))) apiproxy_stub_map.apiproxy.RegisterStub( 'remote_socket', _remote_socket_stub.RemoteSocketServiceStub())
def post(self, app_id="blob", session_id = "session"): global datastore_path #file = self.request.files['file'][0] db = datastore_distributed.DatastoreDistributed( app_id, datastore_path, False, False) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', db) os.environ['APPLICATION_ID'] = app_id # setup the app id in the datastore # Get session info and upload success path blob_session = get_session(session_id) if not blob_session: self.finish('Session has expired. Contact the owner of the app for support.\n\n') return success_path = blob_session["success_path"] server_host = success_path[:success_path.rfind("/",3)] if server_host.startswith("http://"): # strip off the beginging server_host = server_host[len("http://"):] server_host = server_host.split('/')[0] blob_storage = datastore_blob_storage.DatastoreBlobStorage("", app_id) uploadhandler = dev_appserver_upload.UploadCGIHandler(blob_storage) datastore.Delete(blob_session) # This request is sent to the upload handler of the app # in the hope it returns a redirect to be forwarded to the user urlrequest = urllib2.Request(success_path) # Forward all relevant headers # Create data for request reqbody = self.request.body content_type = self.request.headers["Content-Type"] main, kv = split_content_type(content_type) boundary = None if "boundary" in kv: boundary = kv["boundary"] urlrequest.add_header("Content-Type",'multipart/form-data; boundary="%s"'%boundary) for name, value in self.request.headers.items(): if name.lower() not in STRIPPED_HEADERS: urlrequest.add_header(name, value) # Get correct redirect addresses, otherwise it will redirect back # to this port urlrequest.add_header("Host",server_host) form = MultiPartForm(boundary) creation = datetime.datetime.now() # Loop on all files in the form for filekey in self.request.files.keys(): file = self.request.files[filekey][0] body = file["body"] size = len(body) filetype = file["content_type"] filename = file["filename"] blob_entity = uploadhandler.StoreBlob(file, creation) blob_key = str(blob_entity.key().name()) if not blob_key: self.finish('Status: 500\n\n') return creation_formatted = blobstore._format_creation(creation) form.add_file(filekey, filename, cStringIO.StringIO(blob_key), blob_key, blobstore.BLOB_KEY_HEADER, size, creation_formatted) # Loop through form fields for fieldkey in self.request.arguments.keys(): form.add_field(fieldkey, self.request.arguments[fieldkey][0]) request_body = str(form) urlrequest.add_header("Content-Length",str(len(request_body))) urlrequest.add_data(request_body) opener = urllib2.build_opener(SmartRedirectHandler()) f = None redirect_path = None # We are catching the redirect error here # and extracting the Location to post the redirect try: f = opener.open(urlrequest) output = f.read() self.finish(output) except urllib2.HTTPError, e: if "Location" in e.hdrs: #catch any errors, use the success path to #get the ip and port, use the redirect path #for the path. We split redirect_path just in case #its a full path redirect_path = e.hdrs["Location"] success_path_toks = success_path.split('/') redirect_toks = redirect_path.split("/") final_redirect_path = success_path_toks[0] + '//' + success_path_toks[2] + '/' + redirect_toks[len(redirect_toks)-1] self.redirect(final_redirect_path) return else: self.finish(UPLOAD_ERROR + "</br>" + str(e.hdrs) + "</br>" + str(e)) return
from google.appengine.runtime import apiproxy_errors from google.appengine.api import apiproxy_stub_map from google.appengine.api import datastore_errors from google.appengine.api import datastore_distributed from google.appengine.api import datastore from google.appengine.ext import db sys.path.append(TaskQueueConfig.CELERY_CONFIG_DIR) sys.path.append(TaskQueueConfig.CELERY_WORKER_DIR) app_id = 'APP_ID' module_name = TaskQueueConfig.get_celery_worker_module_name(app_id) celery = Celery(module_name, broker=rabbitmq.get_connection_string(), backend='amqp://') celery.config_from_object('CELERY_CONFIGURATION') logger = get_task_logger(__name__) logger.setLevel(logging.INFO) db_proxy = appscale_info.get_db_proxy() connection_str = '{}:{}'.format(db_proxy, str(constants.DB_SERVER_PORT)) ds_distrib = datastore_distributed.DatastoreDistributed( "appscaledashboard", connection_str, require_indexes=False) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', ds_distrib) os.environ['APPLICATION_ID'] = "appscaledashboard" # This template header and tasks can be found in appscale/AppTaskQueue/templates
def post(self, app_id="blob", session_id="session"): """ Handler a post request from a user uploading a blob. Args: app_id: The application triggering the upload. session_id: Authentication token to validate the upload. """ global datastore_path db = datastore_distributed.DatastoreDistributed(app_id, datastore_path, require_indexes=False) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', db) os.environ['APPLICATION_ID'] = app_id # Setup the app id in the datastore. # Get session info and upload success path. blob_session = get_session(session_id) if not blob_session: self.finish('Session has expired. Contact the owner of the ' + \ 'app for support.\n\n') return success_path = blob_session["success_path"] server_host = success_path[:success_path.rfind("/", 3)] if server_host.startswith("http://"): # Strip off the beginging of the server host server_host = server_host[len("http://"):] server_host = server_host.split('/')[0] blob_storage = datastore_blob_storage.DatastoreBlobStorage(app_id) uploadhandler = dev_appserver_upload.UploadCGIHandler(blob_storage) datastore.Delete(blob_session) # This request is sent to the upload handler of the app # in the hope it returns a redirect to be forwarded to the user urlrequest = urllib2.Request(success_path) # Forward all relevant headers and create data for request content_type = self.request.headers["Content-Type"] kv = split_content_type(content_type) boundary = None if "boundary" in kv: boundary = kv["boundary"] urlrequest.add_header("Content-Type", 'application/x-www-form-urlencoded') for name, value in self.request.headers.items(): if name.lower() not in STRIPPED_HEADERS: urlrequest.add_header(name, value) # Get correct redirect addresses, otherwise it will redirect back # to this port. urlrequest.add_header("Host", server_host) form = MultiPartForm(boundary) creation = datetime.datetime.now() # Loop on all files in the form. for filekey in self.request.files.keys(): data = {"blob_info_metadata": {filekey: []}} file = self.request.files[filekey][0] body = file["body"] size = len(body) filename = file["filename"] file_content_type = file["content_type"] blob_entity = uploadhandler.StoreBlob(file, creation) blob_key = str(blob_entity.key().name()) if not blob_key: self.finish('Status: 500\n\n') return creation_formatted = blobstore._format_creation(creation) form.add_file(filekey, filename, cStringIO.StringIO(blob_key), blob_key, blobstore.BLOB_KEY_HEADER, size, creation_formatted) md5_handler = hashlib.md5(str(body)) data["blob_info_metadata"][filekey].append({ "filename": filename, "creation-date": creation_formatted, "key": blob_key, "size": str(size), "content-type": file_content_type, "md5-hash": md5_handler.hexdigest() }) # Loop through form fields for fieldkey in self.request.arguments.keys(): form.add_field(fieldkey, self.request.arguments[fieldkey][0]) data[fieldkey] = self.request.arguments[fieldkey][0] logging.debug("Callback data: \n{}".format(data)) data = urllib.urlencode(data) urlrequest.add_header("Content-Length", str(len(data))) urlrequest.add_data(data) # We are catching the redirect error here # and extracting the Location to post the redirect. try: response = urllib2.urlopen(urlrequest) output = response.read() if response.info().get('Content-Encoding') == 'gzip': buf = StringIO(output) f = gzip.GzipFile(fileobj=buf) data = f.read() output = data self.finish(output) except urllib2.HTTPError, e: if "Location" in e.hdrs: # Catch any errors, use the success path to # get the ip and port, use the redirect path # for the path. We split redirect_path just in case # its a full path. redirect_path = e.hdrs["Location"] self.redirect(redirect_path) return else: self.finish(UPLOAD_ERROR + "</br>" + str(e.hdrs) + "</br>" + str(e)) return
MAX_TASK_DURATION = 13 * 60 app_id = os.environ['APP_ID'] remote_host = os.environ['HOST'] with open(get_celery_configuration_path(app_id)) as config_file: rates = json.load(config_file) celery = create_celery_for_app(app_id, rates) logger = get_task_logger(__name__) logger.setLevel(logging.INFO) db_proxy = appscale_info.get_db_proxy() connection_str = '{}:{}'.format(db_proxy, str(constants.DB_SERVER_PORT)) ds_distrib = datastore_distributed.DatastoreDistributed( 'appscaledashboard', connection_str) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', ds_distrib) os.environ['APPLICATION_ID'] = 'appscaledashboard' def get_wait_time(retries, args): """ Calculates how long we should wait to execute a failed task, based on how many times it's failed in the past. Args: retries: An int that indicates how many times this task has failed. args: A dict that contains information about when the user wants to retry the failed task. Returns: The amount of time, in seconds, that we should wait before executing this task again.