示例#1
0
文件: views.py 项目: Chartini/MorePi
def mock_request_batch(request, raw_path):
    """Handle a request batch."""
    base_api_url = string.replace(request.path, raw_path, "")
    batch_items = json_parse(request.raw_post_data)
    responses = []
    for item in batch_items["batch"]:
        batch_request = SimpleObject() #Mock a request object for each batch item.
        batch_request.method = item["type"]
        if "data" in item:
            batch_request.raw_post_data = batch_substitute(item["data"], responses)
        else:
            batch_request.raw_post_data = ""
            
        base_url = batch_substitute(item["url"], responses)
        print "Batch item: " + item["type"] + " " + base_url + " (" + item["url"] + ")"
        
        batch_request.user = request.user
        batch_request.path_info = force_unicode(base_url)
        batch_item_path = string.replace(base_url, base_api_url, "")
        
        batch_request.GET = {}
        
        query_delim_loc = batch_item_path.find('?')
        if query_delim_loc > -1:
            query_start = query_delim_loc + 1
            query_split = urlparse.parse_qs(batch_item_path[query_start:])
            batch_item_path = batch_item_path[:query_delim_loc]
            
            for k in query_split.keys():
                batch_request.GET[k] = ','.join(query_split[k])
            
        
        responses.append(mock_request_processor(batch_request, batch_item_path))
    return {"responses":responses, "success":True}
示例#2
0
文件: views.py 项目: Chartini/MorePi
def mock_post(request, raw_path):
    """Either update an individual item or create a new item."""
    rp = get_request_properties(request, raw_path)
    collection = get_collection(rp)
    
    if rp.item_id != None: #It's an item, overlay passed values.
        if rp.item_id in collection["items"]:
            collection["items"][rp.item_id].update(json_parse(request.raw_post_data))
        else:
            collection["items"][rp.item_id] = json_parse(request.raw_post_data)
        return status_message(True, "Item {} saved to {}.".format(rp.item_id, rp.key))
    
    else: #It's a collection, create item
        next_id = str(collection["next_id"]) 
        collection["next_id"] += 1
        item_json = request.raw_post_data.replace('___id___', next_id) #Replace the embedded id placeholder with the actual ID.
        collection["items"][next_id] = json_parse(item_json)
        return status_message(True, next_id)
示例#3
0
文件: views.py 项目: Chartini/MorePi
def mock_put(request, raw_path):
    """Replace or create an individual item at a specific key."""
    rp = get_request_properties(request, raw_path)
    collection = get_collection(rp)
    
    if rp.item_id != None: #It's an item, overwrite it
        collection["items"][rp.item_id] = json_parse(request.raw_post_data)
        collection["next_id"] += 1
        
        return status_message(True, "Item {} saved to {}.".format(rp.item_id, rp.key))
        
    return status_message(False, "Operation undefined.")
示例#4
0
def get_verified_data(jws, expected=None, session_token=None):
    headers = json_parse(get_jws_part(jws, 0))
    raw_username = headers['kikUsr']
    username = raw_username.lower()
    hostname = headers['kikCrdDm'].split('/')[0].lower()
    payload = get_jws_part(jws, 1)

    if expected is not None and payload != expected:
        logging.info('jws, payload does not match expected value')
        raise Exception('payload does not match expected value')

    try:
        data = json_parse(payload)
    except:
        data = None

    try:
        session = ndb.Key(urlsafe=session_token).get()
    except Exception:
        session = None
    if session is None or not isinstance(session, Session) \
            or session.username != username or session.hostname != hostname:
        session_token = None
        if username not in TEST_USERS:
            verify_jws(jws, raw_username, hostname, (headers.get('kikDbg') and DEBUG))
        elif not DEBUG:
            logging.info('jws, chrome user detected')
            raise Exception('chrome user detected')
        try:
            session = Session(username=username, hostname=hostname)
            session.put()
            session_token = session.key.urlsafe()
        except:
            pass

    return username, hostname, data, session_token
	def initialize(self, *args, **kwargs):
		value = super(BaseHandler, self).initialize(*args, **kwargs)
		try:
			self.body_params = json_parse(self.request.body)
		except:
			self.body_params = {}
		self.params = {}
		self.params.update(self.request.params)
		self.params.update(self.body_params)
		try:
			session = self.request.headers.get(KIK_SESSION)
			if self.request.method in ('POST', 'PUT', 'PATCH'):
				jws = self.request.body
				payload = None
			else:
				jws = self.request.headers[KIK_JWS]
				payload = self.request.path
			from lib.jws import get_verified_data
			self.username, self.hostname, self.auth_params, self.kik_session = get_verified_data(jws, expected=payload, session_token=session)
		except:
			pass
		return value
示例#6
0
def main(args):
	version = os.getenv("HIVE_VERSION")
	if not version:
		version = strftime("%d%b%Y", gmtime()) 
	home = os.getenv("HIVE_HOME")
	output = "llap-slider-%(version)s" % ({"version": version})
	parser = argparse.ArgumentParser()
	parser.add_argument("--instances", type=int, default=1)
	parser.add_argument("--output", default=output)
	parser.add_argument("--input", required=True)
	parser.add_argument("--args", default="")
	parser.add_argument("--name", default="llap0")
	parser.add_argument("--loglevel", default="INFO")
	parser.add_argument("--logger", default="RFA")
	parser.add_argument("--chaosmonkey", type=int, default=0)
	parser.add_argument("--slider-am-container-mb", type=int, default=1024)
	parser.add_argument("--slider-keytab-dir", default="")
	parser.add_argument("--slider-keytab", default="")
	parser.add_argument("--slider-principal", default="")
	parser.add_argument("--slider-default-keytab", dest='slider_default_keytab', action='store_true')
	parser.set_defaults(slider_default_keytab=False)
	parser.add_argument("--slider-placement", type=int, default=4)
	# Unneeded here for now: parser.add_argument("--hiveconf", action='append')
	#parser.add_argument("--size") parser.add_argument("--xmx") parser.add_argument("--cache") parser.add_argument("--executors")
	(args, unknown_args) = parser.parse_known_args(args)
	input = args.input
	output = args.output
	slider_am_jvm_heapsize = max(args.slider_am_container_mb * 0.8, args.slider_am_container_mb - 1024)
	slider_keytab_dir = args.slider_keytab_dir
	slider_keytab = args.slider_keytab
	slider_principal = args.slider_principal
	# set the defaults only if the defaults are enabled
	if args.slider_default_keytab:
		if not slider_keytab_dir:
			slider_keytab_dir = ".slider/keytabs/llap"
		if not slider_keytab:
			slider_keytab = "llap.keytab"
		if not slider_principal:
			slider_principal = "[email protected]"
	if not input:
		print "Cannot find input files"
		sys.exit(1)
		return
	config = json_parse(open(join(input, "config.json")).read())
	java_home = config["java.home"]
	max_direct_memory = config["max_direct_memory"]
	daemon_args = args.args
	if long(max_direct_memory) > 0:
		daemon_args = " -XX:MaxDirectMemorySize=%s %s" % (max_direct_memory, daemon_args)
	resource = LlapResource(config)
	# 5% container failure every monkey_interval seconds
	monkey_percentage = 5 # 5%
	vars = {
		"home" : home,
		"version" : version,
		"instances" : args.instances,
		"heap" : resource.heap_size,
		"container.mb" : resource.container_size,
		"container.cores" : resource.container_cores,
		"hadoop_home" : os.getenv("HADOOP_HOME"),
		"java_home" : java_home,
		"name" : resource.clusterName,
		"daemon_args" : daemon_args,
		"daemon_loglevel" : args.loglevel,
		"daemon_logger" : args.logger,
		"queue.string" : resource.queueString,
		"monkey_interval" : args.chaosmonkey,
		"monkey_percentage" : monkey_percentage,
		"monkey_enabled" : args.chaosmonkey > 0,
		"slider.am.container.mb" : args.slider_am_container_mb,
		"slider_am_jvm_heapsize" : slider_am_jvm_heapsize,
		"slider_keytab_dir" : slider_keytab_dir,
		"slider_keytab" : slider_keytab,
		"slider_principal" : slider_principal,
		"placement" : args.slider_placement
	}
	
	if not exists(output):
		os.makedirs(output)
	
	src = join(home, "scripts", "llap", "bin")
	dst = join(input, "bin")
	if exists(dst):
		shutil.rmtree(dst)
	shutil.copytree(src, dst)


	# Make the zip package
	tmp = join(output, "tmp")
	pkg = join(tmp, "package")

	src = join(home, "scripts", "llap", "slider")
	dst = join(pkg, "scripts")
	if exists(dst):
		shutil.rmtree(dst)
	shutil.copytree(src, dst)

	with open(join(tmp, "metainfo.xml"),"w") as f:
		f.write(metainfo % vars)

	os.mkdir(join(pkg, "files"))
	tarball = tarfile.open(join(pkg, "files", "llap-%s.tar.gz" %  version), "w:gz")
	# recursive add + -C chdir inside
	tarball.add(input, "")
	tarball.close()

	zipped = zipfile.ZipFile(join(output, "llap-%s.zip" % version), "w")
	zipdir(tmp, zipped)
	zipped.close()

	# cleanup after making zip pkg
	shutil.rmtree(tmp)

	with open(join(output, "appConfig.json"), "w") as f:
		f.write(appConfig % vars)
	
	with open(join(output, "resources.json"), "w") as f:
		f.write(resources % vars)

	with open(join(output, "run.sh"), "w") as f:
		f.write(runner % vars)
	os.chmod(join(output, "run.sh"), 0700)

	print "Prepared %s/run.sh for running LLAP on Slider" % (output)
示例#7
0
def main(args):
	version = os.getenv("HIVE_VERSION")
	if not version:
		version = strftime("%d%b%Y", gmtime()) 
	home = os.getenv("HIVE_HOME")
	output = "llap-yarn-%(version)s" % ({"version": version})
	parser = argparse.ArgumentParser()
	parser.add_argument("--instances", type=int, default=1)
	parser.add_argument("--output", default=output)
	parser.add_argument("--input", required=True)
	parser.add_argument("--args", default="")
	parser.add_argument("--name", default="llap0")
	parser.add_argument("--loglevel", default="INFO")
	parser.add_argument("--logger", default="query-routing")
	parser.add_argument("--service-am-container-mb", type=int, default=1024)
	parser.add_argument("--service-appconfig-global", nargs='*', type=service_appconfig_global_property, action='append')
	parser.add_argument("--service-keytab-dir", default="")
	parser.add_argument("--service-keytab", default="")
	parser.add_argument("--service-principal", default="")
	parser.add_argument("--service-default-keytab", dest='service_default_keytab', action='store_true')
	parser.add_argument("--service-placement", type=int, default=4)
	parser.add_argument("--health-percent", type=int, default=80)
	parser.add_argument("--health-time-window-secs", type=int, default=300)
	parser.add_argument("--health-init-delay-secs", type=int, default=400)
	parser.set_defaults(service_default_keytab=False)
	parser.add_argument("--startImmediately", dest='start_immediately', action='store_true')
	parser.add_argument("--javaChild", dest='java_child', action='store_true')
	parser.set_defaults(start_immediately=False)
	parser.set_defaults(java_child=False)
	# Unneeded here for now: parser.add_argument("--hiveconf", action='append')
	#parser.add_argument("--size") parser.add_argument("--xmx") parser.add_argument("--cache") parser.add_argument("--executors")
	(args, unknown_args) = parser.parse_known_args(args)
	if args.start_immediately and not args.java_child:
		sys.exit(0)
		return
	if args.java_child:
		print "%s Running as a child of LlapServiceDriver" % (strftime("%H:%M:%S", gmtime()))
	else:
		print "%s Running after LlapServiceDriver" % (strftime("%H:%M:%S", gmtime()))

	input = args.input
	output = args.output
	service_am_jvm_heapsize = max(args.service_am_container_mb * 0.8, args.service_am_container_mb - 1024)
	service_keytab_dir = args.service_keytab_dir
	service_keytab = args.service_keytab
	service_principal = args.service_principal
	# set the defaults only if the defaults are enabled
	if args.service_default_keytab:
		if not service_keytab_dir:
			service_keytab_dir = ".yarn/keytabs/llap"
		if not service_keytab:
			service_keytab = "llap.keytab"
		if not service_principal:
			service_principal = "[email protected]"
	service_keytab_path = service_keytab_dir
	if service_keytab_path:
		if service_keytab:
			service_keytab_path += "/" + service_keytab
	else:
		service_keytab_path = service_keytab
	if service_keytab_path:
		service_keytab_path = "hdfs:///user/hive/" + service_keytab_path

	if not input:
		print "Cannot find input files"
		sys.exit(1)
		return
	config = json_parse(open(join(input, "config.json")).read())
	java_home = config["java.home"]
	max_direct_memory = config["max_direct_memory"]

	resource = LlapResource(config)

	daemon_args = args.args
	if long(max_direct_memory) > 0:
		daemon_args = " -XX:MaxDirectMemorySize=%s %s" % (max_direct_memory, daemon_args)
	daemon_args = " -Dhttp.maxConnections=%s %s" % ((max(args.instances, resource.executors) + 1), daemon_args)
	vars = {
		"home" : home,
		"version" : version,
		"instances" : args.instances,
		"heap" : resource.heap_size,
		"container.mb" : resource.container_size,
		"container.cores" : resource.container_cores,
		"hadoop_home" : os.getenv("HADOOP_HOME"),
		"java_home" : java_home,
		"name" : resource.clusterName,
		"daemon_args" : daemon_args,
		"daemon_loglevel" : args.loglevel,
		"daemon_logger" : args.logger,
		"queue.string" : resource.queueString,
		"service.am.container.mb" : args.service_am_container_mb,
		"service_appconfig_global_append": construct_service_site_global_string(args.service_appconfig_global),
		"service_am_jvm_heapsize" : service_am_jvm_heapsize,
		"service_keytab_path" : service_keytab_path,
		"service_principal" : service_principal,
		"placement" : args.service_placement,
		"health_percent": args.health_percent,
		"health_time_window": args.health_time_window_secs,
		"health_init_delay": args.health_init_delay_secs
	}
	
	if not exists(output):
		os.makedirs(output)
	
	src = join(home, "scripts", "llap", "bin")
	dst = join(input, "bin")
	if exists(dst):
		shutil.rmtree(dst)
	shutil.copytree(src, dst)

	# Make the llap tarball
	print "%s Prepared the files" % (strftime("%H:%M:%S", gmtime()))

	tarball = tarfile.open(join(output, "llap-%s.tar.gz" %  version), "w:gz")
	# recursive add + -C chdir inside
	tarball.add(input, "")
	tarball.close()

	print "%s Packaged the files" % (strftime("%H:%M:%S", gmtime()))

	with open(join(output, "Yarnfile"), "w") as f:
		f.write(yarnfile % vars)

	with open(join(output, "run.sh"), "w") as f:
		f.write(runner % vars)
	os.chmod(join(output, "run.sh"), 0700)

	if not args.java_child:
		print "%s Prepared %s/run.sh for running LLAP on YARN" % (strftime("%H:%M:%S", gmtime()), output)
示例#8
0
def main(args):
	version = os.getenv("HIVE_VERSION")
	if not version:
		version = strftime("%d%b%Y", gmtime()) 
	home = os.getenv("HIVE_HOME")
	output = "llap-slider-%(version)s" % ({"version": version})
	parser = argparse.ArgumentParser()
	parser.add_argument("--instances", type=int, default=1)
	parser.add_argument("--output", default=output)
	parser.add_argument("--input", required=True)
	parser.add_argument("--args", default="")
	parser.add_argument("--name", default="llap0")
	parser.add_argument("--loglevel", default="INFO")
	parser.add_argument("--chaosmonkey", type=int, default="0")
	# Unneeded here for now: parser.add_argument("--hiveconf", action='append')
	#parser.add_argument("--size") parser.add_argument("--xmx") parser.add_argument("--cache") parser.add_argument("--executors")
	(args, unknown_args) = parser.parse_known_args(args)
	input = args.input
	output = args.output
	if not input:
		print "Cannot find input files"
		sys.exit(1)
		return
	config = json_parse(open(join(input, "config.json")).read())
	resource = LlapResource(config)
	# 5% container failure every monkey_interval seconds
	monkey_percentage = 5 # 5%
	vars = {
		"home" : home,
		"version" : version,
		"instances" : args.instances,
		"heap" : resource.heap_size,
		"container.mb" : resource.container_size,
		"container.cores" : resource.container_cores,
		"hadoop_home" : os.getenv("HADOOP_HOME"),
		"java_home" : os.getenv("JAVA_HOME"),
		"name" : args.name,
		"daemon_args" : args.args,
		"daemon_loglevel" : args.loglevel,
		"monkey_interval" : args.chaosmonkey,
		"monkey_percentage" : monkey_percentage,
		"monkey_enabled" : args.chaosmonkey > 0
	}
	
	if not exists(output):
		os.makedirs(output)
	
	src = join(home, "scripts", "llap", "bin")
	dst = join(input, "bin")
	if exists(dst):
		shutil.rmtree(dst)
	shutil.copytree(src, dst)


	# Make the zip package
	tmp = join(output, "tmp")
	pkg = join(tmp, "package")

	src = join(home, "scripts", "llap", "slider")
	dst = join(pkg, "scripts")
	if exists(dst):
		shutil.rmtree(dst)
	shutil.copytree(src, dst)

	with open(join(tmp, "metainfo.xml"),"w") as f:
		f.write(metainfo % vars)

	os.mkdir(join(pkg, "files"))
	tarball = tarfile.open(join(pkg, "files", "llap-%s.tar.gz" %  version), "w:gz")
	# recursive add + -C chdir inside
	tarball.add(input, "")
	tarball.close()

	zipped = zipfile.ZipFile(join(output, "llap-%s.zip" % version), "w")
	zipdir(tmp, zipped)
	zipped.close()

	# cleanup after making zip pkg
	shutil.rmtree(tmp)

	with open(join(output, "appConfig.json"), "w") as f:
		f.write(appConfig % vars)
	
	with open(join(output, "resources.json"), "w") as f:
		f.write(resources % vars)

	with open(join(output, "run.sh"), "w") as f:
		f.write(runner % vars)
	os.chmod(join(output, "run.sh"), 0700)

	print "Prepared %s/run.sh for running LLAP on Slider" % (output)
示例#9
0
def main(args):
    opts, args = getopt(
        args,
        "",
        [
            "instances=",
            "output=",
            "input=",
            "args=",
            "name=",
            "loglevel=",
            "chaosmonkey=",
            "size=",
            "xmx=",
            "cache=",
            "executors=",
            "hiveconf=",
        ],
    )
    version = os.getenv("HIVE_VERSION")
    if not version:
        version = strftime("%d%b%Y", gmtime())
    home = os.getenv("HIVE_HOME")
    output = "llap-slider-%(version)s" % ({"version": version})
    instances = 1
    name = "llap0"
    d_args = ""
    d_loglevel = "INFO"
    input = None
    monkey = "0"
    for k, v in opts:
        if k in ("--input"):
            input = v
        elif k in ("--output"):
            output = v
        elif k in ("--instances"):
            instances = int(v)
        elif k in ("--name"):
            name = v
        elif k in ("--args"):
            d_args = v
        elif k in ("--loglevel"):
            d_loglevel = v
        elif k in ("--chaosmonkey"):
            monkey = v
    if not input:
        print "Cannot find input files"
        sys.exit(1)
        return
    config = json_parse(open(join(input, "config.json")).read())
    resource = LlapResource(config)
    monkey_interval = int(monkey)
    # 5% container failure every monkey_interval seconds
    monkey_percentage = 5  # 5%
    vars = {
        "home": home,
        "version": version,
        "instances": instances,
        "heap": resource.heap_size,
        "container.mb": resource.container_size,
        "container.cores": resource.container_cores,
        "hadoop_home": os.getenv("HADOOP_HOME"),
        "java_home": os.getenv("JAVA_HOME"),
        "name": name,
        "daemon_args": d_args,
        "daemon_loglevel": d_loglevel,
        "monkey_interval": monkey_interval,
        "monkey_percentage": monkey_percentage,
        "monkey_enabled": monkey_interval > 0,
    }

    if not exists(output):
        os.makedirs(output)

    src = join(home, "scripts", "llap", "bin")
    dst = join(input, "bin")
    if exists(dst):
        shutil.rmtree(dst)
    shutil.copytree(src, dst)

    # Make the zip package
    tmp = join(output, "tmp")
    pkg = join(tmp, "package")

    src = join(home, "scripts", "llap", "slider")
    dst = join(pkg, "scripts")
    if exists(dst):
        shutil.rmtree(dst)
    shutil.copytree(src, dst)

    with open(join(tmp, "metainfo.xml"), "w") as f:
        f.write(metainfo % vars)

    os.mkdir(join(pkg, "files"))
    tarball = tarfile.open(join(pkg, "files", "llap-%s.tar.gz" % version), "w:gz")
    # recursive add + -C chdir inside
    tarball.add(input, "")
    tarball.close()

    zipped = zipfile.ZipFile(join(output, "llap-%s.zip" % version), "w")
    zipdir(tmp, zipped)
    zipped.close()

    # cleanup after making zip pkg
    shutil.rmtree(tmp)

    with open(join(output, "appConfig.json"), "w") as f:
        f.write(appConfig % vars)

    with open(join(output, "resources.json"), "w") as f:
        f.write(resources % vars)

    with open(join(output, "run.sh"), "w") as f:
        f.write(runner % vars)
    os.chmod(join(output, "run.sh"), 0700)

    print "Prepared %s/run.sh for running LLAP on Slider" % (output)