def __call__(self, env, start_response):
domain_id = (env.get('HTTP_X_DOMAIN_ID') or
env.get('HTTP_X_PROJECT_DOMAIN_ID') or
env.get('HTTP_X_USER_DOMAIN_ID'))
# if there is not domain ID (aka Keystone v2) or if it equals to
# 'default', fallback to the Contrail default-domain ID and name
if not domain_id or domain_id == self.auth_svc.default_domain_id:
env['HTTP_X_DOMAIN_ID'] =\
self.server_mgr.default_domain['uuid'].replace('-', '')
env['HTTP_X_DOMAIN_NAME'] =\
self.server_mgr.default_domain['fq_name'][-1]
else:
# ensure to set HTTP_X_DOMAIN_ID as it is the header used in the
# Contrail RBAC code and in certain setup, Keystone auth middleware
# just sets HTTP_X_PROJECT_DOMAIN_ID and/or HTTP_X_USER_DOMAIN_ID
# headers
env['HTTP_X_DOMAIN_ID'] = domain_id
get_context().set_proc_time('POST_KEYSTONE_REQ')
set_auth_context(env)
# if rbac is set, skip old admin based MT
if self.auth_svc.mt_rbac:
return self.app(env, start_response)
# only allow admin access when MT is on
roles = []
if 'HTTP_X_ROLE' in env:
roles = env['HTTP_X_ROLE'].split(',')
if not 'admin' in [x.lower() for x in roles]:
start_response('403 Permission Denied',
[('Content-type', 'text/plain')])
return '403 Permission Denied'.encode("latin-1")
return self.app(env, start_response)
def __call__(self, env, start_response):
domain_id = (env.get('HTTP_X_DOMAIN_ID')
or env.get('HTTP_X_PROJECT_DOMAIN_ID')
or env.get('HTTP_X_USER_DOMAIN_ID'))
# if there is not domain ID (aka Keystone v2) or if it equals to
# 'default', fallback to the Contrail default-domain ID and name
if not domain_id or domain_id == self.conf['default_domain_id']:
env['HTTP_X_DOMAIN_ID'] =\
self.server_mgr.default_domain['uuid'].replace('-', '')
env['HTTP_X_DOMAIN_NAME'] =\
self.server_mgr.default_domain['fq_name'][-1]
get_context().set_proc_time('POST_KEYSTONE_REQ')
set_auth_context(env)
# if rbac is set, skip old admin based MT
if self.conf['auth_svc']._mt_rbac:
return self.app(env, start_response)
# only allow admin access when MT is on
roles = []
if 'HTTP_X_ROLE' in env:
roles = env['HTTP_X_ROLE'].split(',')
if not 'admin' in [x.lower() for x in roles]:
start_response('403 Permission Denied',
[('Content-type', 'text/plain')])
return '403 Permission Denied'.encode("latin-1")
return self.app(env, start_response)
def __call__(self, env, start_response):
get_context().set_proc_time('PRE_KEYSTONE_REQ')
if self.server_mgr.path_in_white_list(env['PATH_INFO']):
# permit access to white list without requiring a token
env['HTTP_X_DOMAIN_ID'] =\
self.server_mgr.default_domain['uuid'].replace('-', '')
env['HTTP_X_DOMAIN_NAME'] =\
self.server_mgr.default_domain['fq_name'][-1]
env['HTTP_X_PROJECT_ID'] =\
self.server_mgr.default_project['uuid'].replace('-', '')
env['HTTP_X_PROJECT_NAME'] =\
self.server_mgr.default_project['fq_name'][-1]
env['HTTP_X_ROLE'] = ''
return self.server_mgr.api_bottle(env, start_response)
elif self.server_mgr.is_auth_needed():
try:
return self.app(env, start_response)
except k_exc.EmptyCatalog as e:
# https://contrail-jws.atlassian.net/browse/CEM-7641
# If API server started after Keystone was started and before
# Keystone Endpoints were provisioned, the auth_token
# middleware needs to be reloaded.
msg = "Keystone auth_token middleware failed: %s" % str(e)
self.server_mgr.sigterm_handler(msg)
else:
return self.server_mgr.api_bottle(env, start_response)
def __call__(self, env, start_response):
if self.path_in_white_list(env['PATH_INFO']):
# permit access to white list without requiring a token
env['HTTP_X_ROLE'] = ''
app = self.server_mgr.api_bottle
else:
app = self.app if self.mt else self.server_mgr.api_bottle
get_context().set_proc_time('PRE_KEYSTONE_REQ')
return app(env, start_response)
def __call__(self, env, start_response):
if self.path_in_white_list(env['PATH_INFO']):
# permit access to white list without requiring a token
env['HTTP_X_ROLE'] = ''
app = self.server_mgr.api_bottle
elif self.server_mgr.is_auth_needed():
app = self.app
else:
app = self.server_mgr.api_bottle
get_context().set_proc_time('PRE_KEYSTONE_REQ')
return app(env, start_response)
def call(self, *commands):
'''
Helper function that implements a (subset of) the RESP
protocol used by Redis >= 1.2
'''
cm = context.get_context().connection_mgr
sock = buffered_socket.BufferedSocket(cm.get_connection(self.address))
# ARRAY: first byte *, decimal length, \r\n, contents
out = ['*' + str(len(commands))] + \
["${0}\r\n{1}".format(len(e), e) for e in commands]
out = "\r\n".join(out) + "\r\n"
sock.send(out)
fbyte = sock.peek(1)
if fbyte == "-": # error string
raise RedisError(sock.recv_until('\r\n'))
elif fbyte == '+': # simple string
resp = sock.recv_until('\r\n')[1:]
elif fbyte == '$': # bulk string
length = int(sock.recv_until('\r\n')[1:])
if length == -1:
resp = None
else:
resp = sock.recv_all(length)
cm.release_connection(sock)
return resp
def _subscribe(self):
self._consumer.subscribe()
for i in range(self._processes):
if not iswin:
if fork() > 0:
continue
try:
with LogicContext():
while True:
cmd, data = self._consumer.consume()
srv_funcs = self._services.get(cmd, ())
if cmd and data:
cmd_key = '{}_{}'.format(cmd, data.get('sendtime',''))
count = get_context().get_redis().lrem(__conf__.SERVICE_LISTKEY, cmd_key)
for fun in srv_funcs:
try:
if fun.__sub_mode__ == -1 and count==0:
continue
p = Process(target=fun, args=(data,))
p.start()
p.join()
except Exception as e:
print cmd,e
except ConnectionError as e:
print 'Expception:'+e.message
exit(0)
def _queue_consumer(self):
for i in range(self._processes):
if not iswin:
if fork() > 0:
continue
with LogicContext():
log_queue_consumer = log("log/service-queue-consumer", level = 'info' if not __conf__.DEBUG else 'debug')
channel = ['REDIS_QUEUE_1', self._channel]
r = get_context().get_redis()
while True:
try:
channel, data = r.brpop(channel)
cmd, data = Pack.unpack(data)
srv_funcs = self._services.get(cmd, ())
ps = []
for func in srv_funcs:
try:
func(data)
except Exception as e:
log_queue_consumer.error("{}".format(traceback.format_exc()))
except (SystemExit, KeyboardInterrupt) as e:
log_queue_consumer.error("{}".format(traceback.format_exc()))
break;
except Exception as e:
log_queue_consumer.error("{}".format(traceback.format_exc()))
time.sleep(1)
continue
exit(0)
def _queue_consumer(self):
for i in range(self._processes):
if not iswin:
if fork() > 0:
continue
with LogicContext():
log_queue_consumer = log("log/service-queue-consumer", level = 'info' if not __conf__.DEBUG else 'debug')
channel = ['REDIS_QUEUE_1', self._channel]
r = get_context().get_redis()
while True:
try:
channel, data = r.brpop(channel)
cmd, data = Pack.unpack(data)
srv_funcs = self._services.get(cmd, ())
ps = []
for func in srv_funcs:
try:
func(data)
except Exception as e:
log_queue_consumer.error("{}".format(traceback.format_exc()))
except (SystemExit, KeyboardInterrupt) as e:
log_queue_consumer.error("{}".format(traceback.format_exc()))
break;
except Exception as e:
log_queue_consumer.error("{}".format(traceback.format_exc()))
time.sleep(1)
continue
exit(0)
def get_path(sha_1):
ctx = context.get_context()
assert len(sha_1) == 40
pathname = os.path.join(ctx.objects_dir, sha_1[:2])
pathname = os.path.join(pathname, sha_1[2:])
if os.path.exists(pathname):
return pathname
else:
return None
def __call__(self, env, start_response):
get_context().set_proc_time('POST_KEYSTONE_REQ')
# if rbac is set, skip old admin based MT
if self.conf['auth_svc']._mt_rbac:
return self.app(env, start_response)
# only allow admin access when MT is on
roles = []
if 'HTTP_X_ROLE' in env:
roles = env['HTTP_X_ROLE'].split(',')
if not 'admin' in [x.lower() for x in roles]:
start_response('403 Permission Denied',
[('Content-type', 'text/plain')])
return ['403 Permission Denied']
return self.app(env, start_response)
def __call__(self, env, start_response):
get_context().set_proc_time('POST_KEYSTONE_REQ')
# if rbac is set, skip old admin based MT
if self.conf['auth_svc']._mt_rbac:
return self.app(env, start_response)
# only allow admin access when MT is on
roles = []
if 'HTTP_X_ROLE' in env:
roles = env['HTTP_X_ROLE'].split(',')
if not 'admin' in [x.lower() for x in roles]:
start_response('403 Permission Denied',
[('Content-type', 'text/plain')])
return ['403 Permission Denied']
return self.app(env, start_response)
def __call__(self, env, start_response):
if self.server_mgr.path_in_white_list(env['PATH_INFO']):
# permit access to white list without requiring a token
env['HTTP_X_DOMAIN_ID'] =\
self.server_mgr.default_domain['uuid'].replace('-', '')
env['HTTP_X_DOMAIN_NAME'] =\
self.server_mgr.default_domain['fq_name'][-1]
env['HTTP_X_PROJECT_ID'] =\
self.server_mgr.default_project['uuid'].replace('-', '')
env['HTTP_X_PROJECT_NAME'] =\
self.server_mgr.default_project['fq_name'][-1]
env['HTTP_X_ROLE'] = ''
app = self.server_mgr.api_bottle
elif self.server_mgr.is_auth_needed():
app = self.app
else:
app = self.server_mgr.api_bottle
get_context().set_proc_time('PRE_KEYSTONE_REQ')
return app(env, start_response)
def __call__(self, env, start_response):
if self.server_mgr.path_in_white_list(env['PATH_INFO']):
# permit access to white list without requiring a token
env['HTTP_X_DOMAIN_ID'] =\
self.server_mgr.default_domain['uuid'].replace('-', '')
env['HTTP_X_DOMAIN_NAME'] =\
self.server_mgr.default_domain['fq_name'][-1]
env['HTTP_X_PROJECT_ID'] =\
self.server_mgr.default_project['uuid'].replace('-', '')
env['HTTP_X_PROJECT_NAME'] =\
self.server_mgr.default_project['fq_name'][-1]
env['HTTP_X_ROLE'] = ''
app = self.server_mgr.api_bottle
elif self.server_mgr.is_auth_needed():
app = self.app
else:
app = self.server_mgr.api_bottle
get_context().set_proc_time('PRE_KEYSTONE_REQ')
return app(env, start_response)
def __call__(self, env, start_response):
domain_id = (env.get('HTTP_X_DOMAIN_ID') or
env.get('HTTP_X_PROJECT_DOMAIN_ID') or
env.get('HTTP_X_USER_DOMAIN_ID'))
domain_name = (env.get('HTTP_X_DOMAIN_NAME') or
env.get('HTTP_X_PROJECT_DOMAIN_NAME') or
env.get('HTTP_X_USER_DOMAIN_NAME') or
'default-domain')
if domain_name and (not domain_id or not
_UUID_WITHOUT_DASH_REGEX.match(domain_id)):
if domain_name in ['default', 'Default']:
domain_name = 'default-domain'
try:
domain_id = self.server_mgr._db_conn.fq_name_to_uuid(
'domain', [domain_name])
domain_id = domain_id.replace('-', '')
except NoIdError:
# TODO(ethuleau): We allow the request even if the domain is
# not synced to Contrail. This can lead some issue for
# RBAC/perms validation
pass
env['HTTP_X_DOMAIN_ID'] = domain_id
env['HTTP_X_DOMAIN_NAME'] = domain_name
get_context().set_proc_time('POST_KEYSTONE_REQ')
set_auth_context(env)
# if rbac is set, skip old admin based MT
if self.conf['auth_svc']._mt_rbac:
return self.app(env, start_response)
# only allow admin access when MT is on
roles = []
if 'HTTP_X_ROLE' in env:
roles = env['HTTP_X_ROLE'].split(',')
if not 'admin' in [x.lower() for x in roles]:
start_response('403 Permission Denied',
[('Content-type', 'text/plain')])
return '403 Permission Denied'.encode("latin-1")
return self.app(env, start_response)
def write_sha1_object(hexdigest, data):
ctx = context.get_context()
hash_prefix = hexdigest[:2]
object_prefix_dir = os.path.join(ctx.objects_dir, hash_prefix)
hash_filename = os.path.join(object_prefix_dir, hexdigest[2:])
if not os.path.exists(object_prefix_dir):
os.mkdir(object_prefix_dir)
if not os.path.exists(hash_filename):
fd = open(hash_filename, 'wb')
fd.write(zlib.compress(data))
def add(argv):
"""
'pit add' expects a list of files to add and optional arguments
"""
ctx = context.get_context()
files_to_add = argv[2:]
for f in files_to_add:
if not os.path.exists(os.path.join(ctx.working_dir, f)):
print('fatal: pathspec "%s" did not match any files' % f)
sys.exit(2)
index.update_index(files_to_add)
def commit_tree(**kwargs):
"""
take the tree contained in the index and
write a commit objects whose layout is following:
====================================================================
commit 358<NUL>tree d306b9e74803fe248c420d731274a02e80c6619e
parent 6ceb2b9655a2c92c183c2bb7e8e2861f49edff0c
author John Placeholder <*****@*****.**> 1431980072 +0200
committer John Placeholder <*****@*****.**> 1431980072 +0200
<commit description>
====================================================================
"""
author_name = kwargs['author_name']
author_email = kwargs['author_email']
author_date = kwargs['author_date']
committer_name = kwargs['committer_name'] if 'commiter_name' in kwargs.keys() else None
committer_email = kwargs['committer_email'] if 'committer_email' in kwargs.keys() else None
committer_date = kwargs['committer_date'] if 'committer_date' in kwargs.keys() else None
description = kwargs['description']
tree_sha_1 = write_tree()
parent_commit = kwargs['parent_commit'] if 'parent_commit' in kwargs.keys() else None
content = 'tree %s\n' % tree_sha_1
if parent_commit is not None:
content += 'parent %s\n' % parent_commit
content += 'author {0:s} {1:s} {2:s}\n'.format(author_name, author_email, author_date)
if committer_name is not None and committer_email is not None and committer_date is not None:
content += 'committer {0:s} {1:s} {2:s}\n'.format(committer_name, committer_email, committer_date)
content += '\n'
content += description
content += '\n'
sha_1 = hash_commit(content.encode(), write_on_disk=True)
# update the current branch to the newly created commit
branch_file = context.get_context().get_current_branch_file()
# if not os.path.exists(branch_file):
# os.mknod(branch_file)
fd = open(branch_file, 'w')
fd.write(sha_1)
fd.close()
return sha_1
def __call__(self, env, start_response):
if ('HTTP_X_DOMAIN_ID' not in env or not env['HTTP_X_DOMAIN_ID']):
domain_id = (env.get('HTTP_X_PROJECT_DOMAIN_ID')
or env.get('HTTP_X_USER_DOMAIN_ID'))
domain_name = (env.get('HTTP_X_DOMAIN_NAME')
or env.get('HTTP_X_PROJECT_DOMAIN_NAME')
or env.get('HTTP_X_USER_DOMAIN_NAME')
or 'default-domain')
if not domain_id:
if domain_name in ['default', 'Default']:
domain_name = 'default-domain'
try:
domain_id = self.server_mgr._db_conn.fq_name_to_uuid(
'domain', [domain_name])
except NoIdError:
start_response('404 Not Found',
[('Content-type', 'text/plain')])
msg = "Cannot identifying Domain '%s'" % domain_name
return msg.encode("latin-1")
env['HTTP_X_DOMAIN_ID'] = domain_id.replace('-', '')
env['HTTP_X_DOMAIN_NAME'] = domain_name
get_context().set_proc_time('POST_KEYSTONE_REQ')
set_auth_context(env)
# if rbac is set, skip old admin based MT
if self.conf['auth_svc']._mt_rbac:
return self.app(env, start_response)
# only allow admin access when MT is on
roles = []
if 'HTTP_X_ROLE' in env:
roles = env['HTTP_X_ROLE'].split(',')
if not 'admin' in [x.lower() for x in roles]:
start_response('403 Permission Denied',
[('Content-type', 'text/plain')])
return '403 Permission Denied'.encode("latin-1")
return self.app(env, start_response)
def __call__(self, env, start_response):
if ('HTTP_X_DOMAIN_ID' not in env or not env['HTTP_X_DOMAIN_ID'] or
not UUID_REGEX.match(env['HTTP_X_DOMAIN_ID'])):
domain_id = (env.get('HTTP_X_PROJECT_DOMAIN_ID') or
env.get('HTTP_X_USER_DOMAIN_ID'))
domain_name = (env.get('HTTP_X_DOMAIN_NAME') or
env.get('HTTP_X_PROJECT_DOMAIN_NAME') or
env.get('HTTP_X_USER_DOMAIN_NAME') or
'default-domain')
if not domain_id or not UUID_REGEX.match(domain_id):
if domain_name in ['default', 'Default']:
domain_name = 'default-domain'
try:
domain_id = self.server_mgr._db_conn.fq_name_to_uuid(
'domain', [domain_name])
except NoIdError:
start_response('404 Not Found',
[('Content-type', 'text/plain')])
return "Cannot identifying Domain '%s'" % domain_name
env['HTTP_X_DOMAIN_ID'] = domain_id.replace('-', '')
env['HTTP_X_DOMAIN_NAME'] = domain_name
get_context().set_proc_time('POST_KEYSTONE_REQ')
set_auth_context(env)
# if rbac is set, skip old admin based MT
if self.conf['auth_svc']._mt_rbac:
return self.app(env, start_response)
# only allow admin access when MT is on
roles = []
if 'HTTP_X_ROLE' in env:
roles = env['HTTP_X_ROLE'].split(',')
if not 'admin' in [x.lower() for x in roles]:
start_response('403 Permission Denied',
[('Content-type', 'text/plain')])
return ['403 Permission Denied']
return self.app(env, start_response)
def hash_commit(data, write_on_disk=False):
ctx = context.get_context()
header = ('commit {0:d}\x00'.format(len(data))).encode()
sha1_object = sha1()
sha1_object.update(header)
sha1_object.update(data)
hexdigest = sha1_object.hexdigest()
if write_on_disk:
write_sha1_object(hexdigest, header + data)
return hexdigest
def write_tree():
"""
write the index trees and return the SHA-1 of the root tree
"""
ctx = context.get_context()
trees = get_trees()
root = trees.pop('root')
sha_1 = hash_tree(root, write_on_disk=True)
for tree in trees:
hash_tree(tree, write_on_disk=True)
return sha_1
def log_failure(bad_str):
"""Stats on failed logs"""
try:
import context
context.get_context().stats["log.failure"].add(1)
context.get_context().stats["log.failure." + bad_str].add(1)
if context.get_context().log_failure_print:
if context.get_context().stats["log.failure"].n < 10:
print "log failure - " + bad_str
except:
pass
def create_entry(pathname):
assert not os.path.isabs(pathname)
ctx = context.get_context()
stat = os.stat(os.path.join(ctx.working_dir, pathname))
stat_info = StatInfo()
stat_info.ctime = stat.st_ctime
stat_info.ctime_ns = stat.st_ctime_ns
stat_info.mtime = stat.st_mtime
stat_info.mtime_ns = stat.st_mtime_ns
stat_info.dev = stat.st_dev
stat_info.ino = stat.st_ino
stat_info.mode = 33188
stat_info.uid = stat.st_uid
stat_info.gid = stat.st_gid
stat_info.size = stat.st_size
sha_1 = objects.hash_file(pathname, write_on_disk=True)
return IndexEntry(pathname, sha_1, stat_info)
def get_completions(word, code, subcontext, cursor_indentation):
"""gets the completions for word after evaluating code"""
global subprogram_globals
parsed_subcontext = context.calculate_subcontext(subcontext,
cursor_indentation)
context.exec_code(code)
keys = context.get_context()
dots = word.split('.')
# If it is a not completable python expression completions = an empty list
pattern = "^[A-Za-z_][A-Za-z_0-9]*([.][A-Za-z_][A-Za-z_0-9]*)*\.?$"
if not re.match(pattern, word):
completions = []
elif word.rfind('.') == -1:
# If the word is a simple statement not containing "." completions =
# the global keys starting with the word
completions = [i for i in keys if i.startswith(word)]
else:
if word.startswith('self'):
dot_index = parsed_subcontext.rfind('.')
parsed_subcontext = parsed_subcontext[:dot_index]
word = word.replace('self', parsed_subcontext)
# If word ends with a "." strip it and execute get_dir
if word.endswith('.'):
module = context.eval_code(word[:-1])
if module:
completions = get_dir(module)
else:
completions = []
else:
# If word does not ends with "." but it contains a dot
# then eval word up to ".", split the remaining part, get
# the attributes of the module when the attribute starts
# with the remaining
dot_index = word.rfind('.')
module = context.eval_code(word[:dot_index])
if module:
mword = word[dot_index + 1:]
completions = [
i for i in get_dir(module) if i.startswith(mword)
]
else:
completions = []
return sorted(list(set(completions)))
def update_index(pathnames):
"""rewrite the index file with the given objects"""
ctx = context.get_context()
entries = get_entries()
# compare the entries in index with what we want to add from the work tree
# if the SHA-1s match, skip this entry: the file has not been modified
for p in pathnames:
add_entry = True
for i in range(len(entries)):
if entries[i].pathname == p and entries[i].sha_1 == objects.hash_file(p, write_on_disk=False):
add_entry = False
if add_entry:
entries.append(create_entry(p))
# index file header ('dir cache')
data = b'DIRC'
# version number (for now, 2)
# on 4 bytes
data += b'\x00\x00\x00\x02'
# number of entries on 4 bytes
# for now, limit it to 255 entries
data += ('\x00\x00\x00' + chr(len(entries))).encode()
# write the actual entries (sorted by bytes)
for o in sorted(entries, key=lambda ent: bytes(ent.pathname, encoding='utf-8')):
data += o.to_bytes()
# compute the SHA-1 of the data so far and append it as
# the last value in the index file
index_sha = sha1()
index_sha.update(data)
index_sha_value = index_sha.digest()
data += index_sha_value
fd = open(ctx.index, 'wb')
fd.write(data)
fd.close()
def get_entries(pathnames_only=False):
ctx = context.get_context()
if not os.path.exists(ctx.index):
return []
fd = open(ctx.index, 'rb')
content = fd.read()
number_of_entries = int.from_bytes(content[8:12], byteorder='big')
fd.close()
pos = 12
entries = []
while number_of_entries > 0:
stat_info = extract_stat_info(content[pos:pos+40])
pos += 40
# read the SHA-1 of the current entry
sha_1 = hexlify(content[pos:pos + 20])
pos += 20
# skip 2 non implemented bytes
pathname_size = int.from_bytes(content[pos:pos+2], byteorder='big')
pos += 2
pathname = (content[pos:pos + pathname_size]).decode()
pos += pathname_size
current_entry = IndexEntry(pathname, sha_1, stat_info)
while content[pos] is 0:
pos += 1
number_of_entries -= 1
if pathnames_only:
entries.append(pathname)
else:
entries.append(current_entry)
return entries
def get_completions(word, code, subcontext, cursor_indentation):
"""gets the completions for word after evaluating code"""
global subprogram_globals
parsed_subcontext = context.calculate_subcontext(subcontext, cursor_indentation)
context.exec_code(code)
keys = context.get_context()
dots = word.split('.')
# If it is a not completable python expression completions = an empty list
pattern = "^[A-Za-z_][A-Za-z_0-9]*([.][A-Za-z_][A-Za-z_0-9]*)*\.?$"
if not re.match(pattern, word):
completions = []
elif word.rfind('.') == -1:
# If the word is a simple statement not containing "." completions =
# the global keys starting with the word
completions = [i for i in keys if i.startswith(word)]
else:
if word.startswith('self'):
dot_index = parsed_subcontext.rfind('.')
parsed_subcontext = parsed_subcontext[:dot_index]
word = word.replace('self', parsed_subcontext)
# If word ends with a "." strip it and execute get_dir
if word.endswith('.'):
module = context.eval_code(word[:-1])
if module:
completions = get_dir(module)
else:
completions = []
else:
# If word does not ends with "." but it contains a dot
# then eval word up to ".", split the remaining part, get
# the attributes of the module when the attribute starts
# with the remaining
dot_index = word.rfind('.')
module = context.eval_code(word[:dot_index])
if module:
mword = word[dot_index+1:]
completions = [i for i in get_dir(module) if i.startswith(mword)]
else:
completions = []
return sorted(list(set(completions)))
def do_request(self,
method='GET',
path='/',
headers={},
body=None,
context=None):
"""Experimental method to do a request on the server"""
message = SoupMessage()
if body:
headers.setdefault('content-type',
'application/x-www-form-urlencoded')
message.set_message(method, 'http://localhost' + path, body or '')
for name, value in headers.items():
message.set_request_header(name, value)
context = context or get_context() or self.get_fake_context()
context = context.handle_request(message, path)
return {
'status': context.status,
'method': context.method,
'entity': context.entity,
'context': context
}
def get_help(obj):
"""Returns the help of the given object.
Inspired in the original pycomplete package
"""
paren = obj.rfind("(")
if paren != -1:
obj = obj[:paren]
if obj.endswith("(") or obj.endswith("."):
obj = obj[:-1]
found = False
pobj = None
context_dict = 'subprogram_globals'
if not obj in context.get_context():
context_dict = 'helper_globals'
found = context.cimport(obj, context_dict)
else:
pobj = context.eval_code(obj)
if obj not in context.subcontext_globals and found:
pobj = context.eval_code(obj, context_dict)
if not pobj:
return "no help string for " + obj
obj = context.eval_code(obj)
return pydoc.getdoc(obj)
def hash_file(filename, write_on_disk=False):
"""
appends the blob header to the file content and hashes the result
see: http://www.git-scm.com/book/en/v2/Git-Internals-Git-Objects#Object-Storage
"""
ctx = context.get_context()
content = open(os.path.join(ctx.working_dir, filename), 'r').read().encode()
size = len(content)
header = ('blob %i\x00' % size).encode()
sha1_object = sha1()
sha1_object.update(header)
sha1_object.update(content)
hexdigest = sha1_object.hexdigest()
if write_on_disk:
write_sha1_object(hexdigest, header + content)
return hexdigest
def validate_user_token(self):
if not self._user_auth_middleware:
# following config forces keystone middleware to always return the
# result back in HTTP_X_IDENTITY_STATUS env variable
conf_info = self._conf_info.copy()
conf_info['delay_auth_decision'] = True
self._user_auth_middleware = auth_token.AuthProtocol(
self.token_valid, conf_info)
if not self._user_auth_middleware:
return False, (403, " Permission denied")
request_attrs = {
'REQUEST_METHOD': get_request().route.method,
'bottle.app': get_request().environ['bottle.app'],
}
if 'HTTP_X_AUTH_TOKEN' in get_request().environ:
request_attrs['HTTP_X_AUTH_TOKEN'] =\
get_request().environ['HTTP_X_AUTH_TOKEN'].encode("ascii")
elif 'HTTP_X_USER_TOKEN' in get_request().environ:
request_attrs['HTTP_X_USER_TOKEN'] =\
get_request().environ['HTTP_X_USER_TOKEN'].encode("ascii")
else:
return False, (400, "User token needed for validation")
b_req = bottle.BaseRequest(request_attrs)
# get permissions in internal context
orig_context = get_context()
i_req = ApiInternalRequest(b_req.url, b_req.urlparts, b_req.environ,
b_req.headers, None, None)
set_context(ApiContext(internal_req=i_req))
try:
token_info = self._user_auth_middleware(
get_request().headers.environ, self.start_response)
finally:
set_context(orig_context)
return True, token_info
def validate_user_token(self):
if not self._user_auth_middleware:
# following config forces keystone middleware to always return the
# result back in HTTP_X_IDENTITY_STATUS env variable
conf_info = self._conf_info.copy()
conf_info['delay_auth_decision'] = True
self._user_auth_middleware = auth_token.AuthProtocol(
self.token_valid, conf_info)
if not self._user_auth_middleware:
return False, (403, " Permission denied")
request_attrs = {
'REQUEST_METHOD': get_request().route.method,
'bottle.app': get_request().environ['bottle.app'],
}
if 'HTTP_X_AUTH_TOKEN' in get_request().environ:
request_attrs['HTTP_X_AUTH_TOKEN'] =\
get_request().environ['HTTP_X_AUTH_TOKEN'].encode("ascii")
elif 'HTTP_X_USER_TOKEN' in get_request().environ:
request_attrs['HTTP_X_USER_TOKEN'] =\
get_request().environ['HTTP_X_USER_TOKEN'].encode("ascii")
else:
return False, (400, "User token needed for validation")
b_req = bottle.BaseRequest(request_attrs)
# get permissions in internal context
orig_context = get_context()
i_req = ApiInternalRequest(b_req.url, b_req.urlparts, b_req.environ,
b_req.headers, None, None)
set_context(ApiContext(internal_req=i_req))
try:
token_info = self._user_auth_middleware(
get_request().headers.environ, self.start_response)
finally:
set_context(orig_context)
return True, token_info
def get_signature(obj):
"""Returns the signature of the given object.
Inspired in the original pycomplete package
"""
# FIXME - make this function less ugly
paren = obj.find("(")
if paren != -1:
obj = obj[:paren]
context_dict = 'subprogram_globals'
if not obj in context.get_context():
context_dict = 'helper_globals'
if not context.cimport(obj, context_dict):
return "no signature for " + obj
try:
obj = context.eval_code(obj, context_dict)
except:
return "no signature for " + obj
print "yesy"
sig = ""
# This part is extracted from the pycomplete.py file
if type(obj) in (types.ClassType, types.TypeType):
obj = _find_constructor(obj)
elif type(obj) == types.MethodType:
obj = obj.im_func
if type(obj) in [types.FunctionType, types.LambdaType]:
(args, varargs, varkw, defaults) = inspect.getargspec(obj)
sig = ('%s: %s' % (obj.__name__,
inspect.formatargspec(args, varargs, varkw,
defaults)))
doc = getattr(obj, '__doc__', '')
if doc and not sig:
doc = doc.lstrip()
pos = doc.find('\n')
if pos < 0 or pos > 70:
pos = 70
sig = doc[:pos]
return sig
def get_random_analogy():
book_id = random.randint(0,NUM_OF_BOOKS)
path = get_path(book_id, analogy_path)
analogies = path + "book%s_analogies.csv" % book_id
#if the randomly generated path does not exist
try:
df = pd.read_csv(analogies)
except:
#print("{} does not exist".format(book_id))
return get_random_analogy()
#fixes the punctuation
df['text'] = df['text'].apply(lambda x: fix(x))
#if file selected has no analogies in it, call the function again to pick a different analogy
if len(df) == 0:
#print("no analogies here")
return get_random_analogy()
#pick a random analogy
analogy = df.sample(n=1, random_state=1)
#check if analogy is nanalogy
while like_is_verb(str(analogy['text'])):
#print("like is a verb")
return get_random_analogy()
#adds the context to the dataframe
#to change values about this, including how many sentences before and after and input directory,
#go to context.py.
name = analogy['name'].to_string(index = False)
try:
prev, after = get_context(name, book_path)
except Exception as e:
# print("get context failed because:")
#print(e)
return get_random_analogy()
analogy.insert(1,'prev_sent',prev)
analogy.insert(3, 'next_sent', after)
print("analogy returned")
return analogy
def call(self, *commands):
"""
Helper function that implements a (subset of) the RESP
protocol used by Redis >= 1.2
"""
cm = context.get_context().connection_mgr
sock = buffered_socket.BufferedSocket(cm.get_connection(self.address))
# ARRAY: first byte *, decimal length, \r\n, contents
out = ["*" + str(len(commands))] + ["${0}\r\n{1}".format(len(e), e) for e in commands]
out = "\r\n".join(out) + "\r\n"
sock.send(out)
fbyte = sock.peek(1)
if fbyte == "-": # error string
raise RedisError(sock.recv_until("\r\n"))
elif fbyte == "+": # simple string
resp = sock.recv_until("\r\n")[1:]
elif fbyte == "$": # bulk string
length = int(sock.recv_until("\r\n")[1:])
if length == -1:
resp = None
else:
resp = sock.recv_all(length)
cm.release_connection(sock)
return resp
def get_signature(obj):
"""Returns the signature of the given object.
Inspired in the original pycomplete package
"""
# FIXME - make this function less ugly
paren = obj.find("(")
if paren != -1:
obj = obj[:paren]
context_dict = 'subprogram_globals'
if not obj in context.get_context():
context_dict = 'helper_globals'
if not context.cimport(obj, context_dict):
return "no signature for " + obj
try:
obj = context.eval_code(obj, context_dict)
except:
return "no signature for " + obj
sig = ""
# This part is extracted from the pycomplete.py file
if type(obj) in (types.ClassType, types.TypeType):
obj = _find_constructor(obj)
elif type(obj) == types.MethodType:
obj = obj.im_func
if type(obj) in [types.FunctionType, types.LambdaType]:
(args, varargs, varkw, defaults) = inspect.getargspec(obj)
sig = ('%s: %s' % (obj.__name__,
inspect.formatargspec(args, varargs, varkw,
defaults)))
doc = getattr(obj, '__doc__', '')
if doc and not sig:
doc = doc.lstrip()
pos = doc.find('\n')
if pos < 0 or pos > 70:
pos = 70
sig = doc[:pos]
return sig
import os\n
import glob\n
def function(a, b=1):\n\t
bar = a\n\t
foo = 2\n
class Class(object):\n\t
def __init__(self, arg1):\n\t
self._arg1 = arg1\n
def test(self, arg):\n\t
a = self._arg1"""
print 'completions:'
print get_completions("sys.", code)[:4]
print get_completions("os.", code)[:4]
print get_completions("func", code)
print get_completions("Cla", code)
print get_completions("fo", code, [["", "def", "function"]], "\t")
print get_completions("self._", code, [[" ", "def", "test"]], "\t")
print ''
print 'signatures'
print context.get_context()
print get_signature("function(")
print get_signature("glob.glob")
print ''
print 'helpers'
print get_help("help")
print get_help("os.path")
def do(*args):
""" Shortcut for get_context().code_maker.do(...) """
get_context().code_maker.do(*args)
def _coll(self):
return get_context().get_mongoclient(self._dbname)[self._name]
t_all = 0.0
t_success = 0.0
c = 2**c
for k in range(args.instances):
success, r, t = q2(c)
t_all += t
if success:
t_success += t
success_count += 1
t_all /= args.instances
t_success = t_success / success_count if success_count != 0 else 0.0
print(c, f"{success_count}/{args.instances}", f"{t_all:.2f}",
f"{t_success:.2f}")
pass
elif args.query == 3:
context = get_context(args)
def run_(row, col):
success_count = 0.0
t_all = 0.0
t_success = 0.0
for c in range(9):
success, r, t = q3(c, row, col, context)
t_all += t
if success:
t_success += t
success_count += 1
t_all /= 9.0
t_success = t_success / success_count if success_count != 0 else 0.0
print(row, col, f"{success_count}/{args.instances}", f"{t_all:.2f}",
f"{t_success:.2f}")
def async_send_pack(cls, cmd, data, channel=None):
cls.send_pack(get_context().get_redis(), channel
or __conf__.SERVICE_CHANNEL, cmd, data)
import os\n
import glob\n
def function(a, b=1):\n\t
bar = a\n\t
foo = 2\n
class Class(object):\n\t
def __init__(self, arg1):\n\t
self._arg1 = arg1\n
def test(self, arg):\n\t
a = self._arg1"""
print 'completions:'
print get_completions("sys.", code)[:4]
print get_completions("os.", code)[:4]
print get_completions("func", code)
print get_completions("Cla", code)
print get_completions("fo", code, [["","def","function"]], "\t")
print get_completions("self._", code, [[" ","def","test"]], "\t")
print ''
print 'signatures'
print context.get_context()
print get_signature("function(")
print get_signature("glob.glob")
print ''
print 'helpers'
print get_help("help")
print get_help("os.path")
def async_send_pack(cls, cmd, data, channel=None):
r = get_context().get_redis()
pack = dumps(dict(cmd=cmd, data=data), cls=DateTimeEncoder)
r.publish(channel or __conf__.SERVICE_CHANNEL, pack)
def async_send_pack(cls, cmd, data, channel=None):
cls.send_pack(get_context(
).get_redis(), channel or __conf__.SERVICE_CHANNEL, cmd, data)
def async_send_pack(cls, cmd, data, channel=None):
"""
将数据发送到队列，以实现异步处理。
"""
cls.send_pack(get_context(
).get_redis(), channel or __conf__.SERVICE_CHANNEL, cmd, data)
from app_claims import optional_claims
from app_permissions import default_permissions, all_permissions
from context import get_context
from app_registration import AppBuilder, AppDefinitionBuilder
context = get_context()
# Working Example
app2_def = AppDefinitionBuilder.build_composite(title='SampleApp_2',
name='sample.app2',
business='personal')
app2_def.add_claims(optional_claims())
app2_def.add_permissions(default_permissions())
app2 = AppBuilder.create_composite_app(app2_def, context)
def get_status(dir): try: full_status = get_context().client.status(dir) except pysvn.ClientError, e: raise ApplicationError(e)
def async_queue(cls, cmd, data, channel=None):
r = get_context().get_redis()
pack = dumps(dict(cmd=cmd, data=data), cls=DateTimeEncoder)
r.rpush(channel or __conf__.SERVICE_CHANNEL, pack)
