def __init__(self, factspace, type_, criteria):
self.factspace = factspace
self.type_ = type_
self.criteria = criteria
subscribe(factspace, 'fact', type_)
facts = get_all_facts(factspace, type_, criteria)
self.count = 0 if facts is None else len(facts)
def get_statements(path, params, content, client_ip):
factspace, confidence, fact_type = path.split('/')[2:5]
if ('kropotkin_subscribe' in params):
id = subscribe(factspace, confidence, fact_type)
return (200, dumps(id), 'text/plain')
if (factspace == 'kropotkin'):
statements_dir = environ['KROPOTKIN_DIR']
else:
factspace_info = get_newest_fact('kropotkin',
'factspace',
{'name': factspace})
if not factspace_info:
return (404, 'Could not locate factspace %s' % factspace,
'text/plain')
statements_dir = factspace_info['directory']
statements = _fetch_statements(statements_dir, factspace, confidence,
fact_type, params)
return (200, dumps(statements), 'application/json')
subscription_facts = []
for f in get_all_facts('kropotkin', 'factspace', {}):
subscription_facts += get_all_facts(f['name'], 'subscription', {})
for f in component_facts:
if f['name'] != 'executioner':
execute_pid(f['identifier'])
for f in subscription_facts:
pid = extract_local_process(f['queue'])
if pid is not None:
execute_pid(pid)
def execute_pid(pid):
try:
kill(pid, SIGTERM)
except OSError, err:
stderr.write('Exception when killing %d: %s\n' % (pid, str(err)))
if __name__=="__main__":
subscribe('kropotkin', 'fact', 'stop_requested')
while True:
stop_fact = get_next_fact('kropotkin', 'stop_requested')
if stop_fact and stop_fact['location'] in [THIS_MACHINE, 'all']:
identifier = stop_fact['identifier']
if identifier == 'all':
execute_all()
break
else:
pid = int(identifier)
execute_pid(pid)
from kropotkin import get_newest_fact, get_next_fact, store_opinion, subscribe
from time import time
def choose(random_value, percentages, default=None):
n = 0
for name, percentage in percentages:
if random_value in range(n, n + percentage):
return name
else:
n = n + percentage
return default
def _now():
return int(round(time()))
subscribe('whooshingby', 'fact', 'completed_task')
while True:
fact = get_next_fact('whooshingby', 'completed_task')
if not fact:
continue
percentages_fact = get_newest_fact('whooshingby', 'reward_percentages', {})
reward_percentages = loads(percentages_fact['percentages'])
random_value = (hash(fact['name']) * 61 + fact['time'] * 47) % 100
name = choose(random_value, reward_percentages)
if name:
content = {'name': name, 'task_id': fact['task_id'],
'source': 'python', 'time': _now()}
if not store_opinion('whooshingby', 'reward', content):
print "Could not store reward opinion" # handle better
def get_unique_executable(files):
executables = [f for f in files if access(f, X_OK)]
return executables[0] if len(executables) == 1 else None
HASHBANG_TYPE = {'#!/usr/bin/python': 'python'}
EXTENSION_TYPE = {'py': 'python',
'js': 'javascript',
'html': 'html'}
def language_type(filename):
if access(filename, X_OK):
with open(filename, 'r') as e:
hashbang = e.readline()[:-1]
return HASHBANG_TYPE.get(hashbang)
else:
extension = filename.split('.')[-1]
return EXTENSION_TYPE[extension]
def archive(files):
with closing(StringIO()) as buffer:
with taropen(mode='w', fileobj=buffer) as tar:
for f in files:
tar.add(f, arcname=basename(f))
return b64encode(buffer.getvalue())
if __name__=="__main__":
subscribe('kropotkin', 'fact', 'component_available')
while True:
fact = get_next_fact('kropotkin', 'component_available')
publish(fact['location'])
else:
copy(original, dest)
def get_unique_executable(directory):
nodes = listdir(directory)
executables = [f for f in nodes if is_executable_file(join(directory, f))]
return executables[0] if len(executables) == 1 else None
def is_executable_file(f):
return (not isdir(f)) and access(f, X_OK)
MODULE_TYPES = {'python': 'python-module',
'javascript': 'javascript-library',
'ruby': 'ruby-gem'}
subscribe('kropotkin', 'fact', 'library_available')
while True:
fact = get_next_statement('kropotkin', 'fact', 'library_available')
original_dir = fact['directory']
language = fact['language']
build_dir = mkdtemp(prefix='library-build-%s' % basename(original_dir))
output_dir = mkdtemp(prefix='library-%s' % basename(original_dir))
copy_all(original_dir, build_dir)
executable = get_unique_executable(build_dir)
if not executable:
stderr.write("Cannot locate unique executable in %s\n" % build_dir)
continue
#!/usr/bin/python
from base64 import b64encode
from hashlib import sha512
from json import loads
from kropotkin import get_next_fact, store_fact, subscribe
from os import urandom
def generate_salt():
return b64encode(urandom(24))
def create_hash(salt, password):
return sha512(salt+password).hexdigest()
subscribe('whooshingby', 'fact', 'registration_request')
while True:
request = get_next_fact('whooshingby', 'registration_request')
if not request:
continue
name = request['name']
password = request['password']
salt = generate_salt()
hash = create_hash(salt, password)
content = {'name': name, 'salt': salt, 'hash': hash}
if not store_fact('whooshingby', 'user', content):
stderr.write('Could not store user fact, name %s\n' % name)
#!/usr/bin/python
from kropotkin import get_next_statement, store_fact, subscribe
from os.path import join
from tempfile import mkdtemp
def create_factspace(name, directory):
if not directory:
directory = mkdtemp()
content = {'name': name, 'directory': directory}
if not store_fact('kropotkin', 'factspace', content):
raise Exception("Cannot store factspace fact")
return directory
if __name__=="__main__":
subscribe('kropotkin', 'fact', 'factspace_wanted')
while True:
factspace_fact = get_next_statement('kropotkin',
'fact',
'factspace_wanted')
if factspace_fact:
create_factspace(factspace_fact['name'],
factspace_fact['directory'])
'identifier': process.pid,
'stdout_file': stdout_file.name,
'stderr_file': stderr_file.name}
if store_deployed_fact is True:
if not store_fact('kropotkin', 'component_deployed', content):
stderr.write('Cannot store component_deployed fact for %s\n' %name)
return content
def get_unique_executable(directory):
nodes = listdir(directory)
executables = [f for f in nodes if is_executable_file(join(directory, f))]
return join(directory, executables[0]) if len(executables) == 1 else None
def is_executable_file(f):
return (not isdir(f)) and access(f, X_OK)
def open_output_file(directory, process_name, file_name):
basename = '.'.join([process_name, file_name, 'log'])
filename = join(directory, basename)
return open(filename, 'w')
if __name__=="__main__":
subscribe('kropotkin', 'fact', 'component')
while True:
component_fact = get_next_fact('kropotkin', 'component')
if component_fact['content_type'] == 'component-tar':
name = component_fact['name']
directory = unpack(name, component_fact['bytes'])
deploy(name, directory)
bytes_to_read = min(current_size - log_file.position, MAX_FACT_SIZE)
if bytes_to_read == 0:
return
with open(log_file.path, 'r') as f:
f.seek(-bytes_to_read, SEEK_END)
log_data = f.read(MAX_FACT_SIZE)
content = {'component': log_file.component,
'type': log_file.type,
'file': log_file.path,
'data': log_data}
if not store_fact('kropotkin', 'log_data', content):
stderr.write("Could not store log_data\n")
log_file.position += len(log_data)
subscribe('kropotkin', 'fact', 'component_deployed')
component_facts = get_all_facts('kropotkin', 'component_deployed', {})
map(store_component_log_files, filter(should_check_component, component_facts))
while True:
target_end = time() + POLL_INTERVAL
component_fact = get_next_fact_noblock('kropotkin', 'component_deployed')
if component_fact and should_check_component(component_fact):
store_component_log_files(component_fact)
map(check_log_file, LOG_FILES)
time_left = target_end - time()
if time_left > 0:
sleep(time_left)
