def __init__(self, factspace, type_, criteria): self.factspace = factspace self.type_ = type_ self.criteria = criteria subscribe(factspace, 'fact', type_) facts = get_all_facts(factspace, type_, criteria) self.count = 0 if facts is None else len(facts)
def get_statements(path, params, content, client_ip): factspace, confidence, fact_type = path.split('/')[2:5] if ('kropotkin_subscribe' in params): id = subscribe(factspace, confidence, fact_type) return (200, dumps(id), 'text/plain') if (factspace == 'kropotkin'): statements_dir = environ['KROPOTKIN_DIR'] else: factspace_info = get_newest_fact('kropotkin', 'factspace', {'name': factspace}) if not factspace_info: return (404, 'Could not locate factspace %s' % factspace, 'text/plain') statements_dir = factspace_info['directory'] statements = _fetch_statements(statements_dir, factspace, confidence, fact_type, params) return (200, dumps(statements), 'application/json')
subscription_facts = [] for f in get_all_facts('kropotkin', 'factspace', {}): subscription_facts += get_all_facts(f['name'], 'subscription', {}) for f in component_facts: if f['name'] != 'executioner': execute_pid(f['identifier']) for f in subscription_facts: pid = extract_local_process(f['queue']) if pid is not None: execute_pid(pid) def execute_pid(pid): try: kill(pid, SIGTERM) except OSError, err: stderr.write('Exception when killing %d: %s\n' % (pid, str(err))) if __name__=="__main__": subscribe('kropotkin', 'fact', 'stop_requested') while True: stop_fact = get_next_fact('kropotkin', 'stop_requested') if stop_fact and stop_fact['location'] in [THIS_MACHINE, 'all']: identifier = stop_fact['identifier'] if identifier == 'all': execute_all() break else: pid = int(identifier) execute_pid(pid)
from kropotkin import get_newest_fact, get_next_fact, store_opinion, subscribe from time import time def choose(random_value, percentages, default=None): n = 0 for name, percentage in percentages: if random_value in range(n, n + percentage): return name else: n = n + percentage return default def _now(): return int(round(time())) subscribe('whooshingby', 'fact', 'completed_task') while True: fact = get_next_fact('whooshingby', 'completed_task') if not fact: continue percentages_fact = get_newest_fact('whooshingby', 'reward_percentages', {}) reward_percentages = loads(percentages_fact['percentages']) random_value = (hash(fact['name']) * 61 + fact['time'] * 47) % 100 name = choose(random_value, reward_percentages) if name: content = {'name': name, 'task_id': fact['task_id'], 'source': 'python', 'time': _now()} if not store_opinion('whooshingby', 'reward', content): print "Could not store reward opinion" # handle better
def get_unique_executable(files): executables = [f for f in files if access(f, X_OK)] return executables[0] if len(executables) == 1 else None HASHBANG_TYPE = {'#!/usr/bin/python': 'python'} EXTENSION_TYPE = {'py': 'python', 'js': 'javascript', 'html': 'html'} def language_type(filename): if access(filename, X_OK): with open(filename, 'r') as e: hashbang = e.readline()[:-1] return HASHBANG_TYPE.get(hashbang) else: extension = filename.split('.')[-1] return EXTENSION_TYPE[extension] def archive(files): with closing(StringIO()) as buffer: with taropen(mode='w', fileobj=buffer) as tar: for f in files: tar.add(f, arcname=basename(f)) return b64encode(buffer.getvalue()) if __name__=="__main__": subscribe('kropotkin', 'fact', 'component_available') while True: fact = get_next_fact('kropotkin', 'component_available') publish(fact['location'])
else: copy(original, dest) def get_unique_executable(directory): nodes = listdir(directory) executables = [f for f in nodes if is_executable_file(join(directory, f))] return executables[0] if len(executables) == 1 else None def is_executable_file(f): return (not isdir(f)) and access(f, X_OK) MODULE_TYPES = {'python': 'python-module', 'javascript': 'javascript-library', 'ruby': 'ruby-gem'} subscribe('kropotkin', 'fact', 'library_available') while True: fact = get_next_statement('kropotkin', 'fact', 'library_available') original_dir = fact['directory'] language = fact['language'] build_dir = mkdtemp(prefix='library-build-%s' % basename(original_dir)) output_dir = mkdtemp(prefix='library-%s' % basename(original_dir)) copy_all(original_dir, build_dir) executable = get_unique_executable(build_dir) if not executable: stderr.write("Cannot locate unique executable in %s\n" % build_dir) continue
#!/usr/bin/python from base64 import b64encode from hashlib import sha512 from json import loads from kropotkin import get_next_fact, store_fact, subscribe from os import urandom def generate_salt(): return b64encode(urandom(24)) def create_hash(salt, password): return sha512(salt+password).hexdigest() subscribe('whooshingby', 'fact', 'registration_request') while True: request = get_next_fact('whooshingby', 'registration_request') if not request: continue name = request['name'] password = request['password'] salt = generate_salt() hash = create_hash(salt, password) content = {'name': name, 'salt': salt, 'hash': hash} if not store_fact('whooshingby', 'user', content): stderr.write('Could not store user fact, name %s\n' % name)
#!/usr/bin/python from kropotkin import get_next_statement, store_fact, subscribe from os.path import join from tempfile import mkdtemp def create_factspace(name, directory): if not directory: directory = mkdtemp() content = {'name': name, 'directory': directory} if not store_fact('kropotkin', 'factspace', content): raise Exception("Cannot store factspace fact") return directory if __name__=="__main__": subscribe('kropotkin', 'fact', 'factspace_wanted') while True: factspace_fact = get_next_statement('kropotkin', 'fact', 'factspace_wanted') if factspace_fact: create_factspace(factspace_fact['name'], factspace_fact['directory'])
'identifier': process.pid, 'stdout_file': stdout_file.name, 'stderr_file': stderr_file.name} if store_deployed_fact is True: if not store_fact('kropotkin', 'component_deployed', content): stderr.write('Cannot store component_deployed fact for %s\n' %name) return content def get_unique_executable(directory): nodes = listdir(directory) executables = [f for f in nodes if is_executable_file(join(directory, f))] return join(directory, executables[0]) if len(executables) == 1 else None def is_executable_file(f): return (not isdir(f)) and access(f, X_OK) def open_output_file(directory, process_name, file_name): basename = '.'.join([process_name, file_name, 'log']) filename = join(directory, basename) return open(filename, 'w') if __name__=="__main__": subscribe('kropotkin', 'fact', 'component') while True: component_fact = get_next_fact('kropotkin', 'component') if component_fact['content_type'] == 'component-tar': name = component_fact['name'] directory = unpack(name, component_fact['bytes']) deploy(name, directory)
bytes_to_read = min(current_size - log_file.position, MAX_FACT_SIZE) if bytes_to_read == 0: return with open(log_file.path, 'r') as f: f.seek(-bytes_to_read, SEEK_END) log_data = f.read(MAX_FACT_SIZE) content = {'component': log_file.component, 'type': log_file.type, 'file': log_file.path, 'data': log_data} if not store_fact('kropotkin', 'log_data', content): stderr.write("Could not store log_data\n") log_file.position += len(log_data) subscribe('kropotkin', 'fact', 'component_deployed') component_facts = get_all_facts('kropotkin', 'component_deployed', {}) map(store_component_log_files, filter(should_check_component, component_facts)) while True: target_end = time() + POLL_INTERVAL component_fact = get_next_fact_noblock('kropotkin', 'component_deployed') if component_fact and should_check_component(component_fact): store_component_log_files(component_fact) map(check_log_file, LOG_FILES) time_left = target_end - time() if time_left > 0: sleep(time_left)