def hello_world(task): ''' complex tree of subtasks are executed: hello_world (current local task :5001, sequential) | |-- subtask (local virtual task :5001, parallel) | | | |-- subsubtask1/goodbye_cruel_world (local task :5001, simple) | | | |-- subsubtask2(goodbye_cruel_world (remote task :5000, simple) | |-- subtask2 (local virtual task :5001, simple) when all the subtasks are executed, the sender (:5000 via post_hello) is notified that the initial task is finished. ''' username = task.get_data()['input_data']['username'] from time import sleep print("hello %s! sleeping..\n" % username) sleep(5) print("woke up! time to finish =)\n") subtask = ParallelTask() task.add(subtask) subsubtask1 = SimpleTask( receiver_url='http://127.0.0.1:5001/api/queues', action="testing.goodbye_cruel_world", queue="hello_world", data={ 'username': username*2 } ) subtask.add(subsubtask1) subsubtask2 = SimpleTask( receiver_url='http://127.0.0.1:5000/api/queues', action="testing.goodbye_cruel_world", queue="hello_world", data={ 'username': username*3 } ) subtask.add(subsubtask2) subtask2 = SimpleTask( receiver_url='http://127.0.0.1:5001/api/queues', action="testing.all_goodbyes_together", queue="hello_world", ) task.add(subtask2) return dict( output_data = "hello %s!" % username )
def post_hello(username): task = SimpleTask(receiver_url='https://127.0.0.1:5001/api/queues', action="hello_world", queue="say_queue", data={'username': username}) task.create_and_send() return make_response("", 200)
def tally_task(data): if not data: print("invalid json") return False requirements = [ { 'name': u'election_id', 'isinstance': int }, { 'name': u'callback_url', 'isinstance': str }, { 'name': u'votes_url', 'isinstance': str }, { 'name': u'votes_hash', 'isinstance': str }, ] for req in requirements: if req['name'] not in data or not isinstance(data[req['name']], req['isinstance']): print(req['name'], data.get(req['name'], None), type(data[req['name']])) print("invalid %s parameter" % req['name']) return False if data['election_id'] <= 0: print("election id must be >= 1") return False if not data['votes_hash'].startswith("ni:///sha-256;"): print("invalid votes_hash, must be sha256") return False election_id = data['election_id'] election = db.session.query(Election)\ .filter(Election.id == election_id).first() if election is None: print("unknown election with election_id = %s" % election_id) return False task = SimpleTask(receiver_url=app.config.get('ROOT_URL', ''), action="tally_election", queue="launch_task", data={ 'election_id': data['election_id'], 'callback_url': data['callback_url'], 'votes_url': data['votes_url'], 'votes_hash': data['votes_hash'], }) task.create_and_send() return task
def execute(self): # this task will trigger a failure self.task.add( SimpleTask(receiver_url='http://127.0.0.1:5000/api/queues', action="hello_recoverable_fail", queue="fail_queue", data={'hello': 'world'})) # this task will be executed after the failure is handled self.task.add( SimpleTask(receiver_url='http://127.0.0.1:5000/api/queues', action="hello_propagated_failure", queue="fail_queue", data={'hello': 'world'})) # this task will never be executed because previous task will fail and # won't be recovered self.task.add( SimpleTask(receiver_url='http://127.0.0.1:5000/api/queues', action="never_land", queue="fail_queue", data={'hello': 'world'}))
def election_task(data): if not data: print("invalid json") return False try: check_election_data(data, True) except Exception as e: print("ERROR", e) return False e = Election(id=data['id'], title=data['title'][:255], description=data['description'], questions=dumps(data['questions']), start_date=data['start_date'], end_date=data['end_date'], callback_url=data['callback_url'], num_parties=len(data['authorities']), threshold_parties=len(data['authorities']), status='creating') db.session.add(e) for auth_data in data['authorities']: authority = Authority(name=auth_data['name'], ssl_cert=auth_data['ssl_cert'], orchestra_url=auth_data['orchestra_url'], election_id=data['id']) db.session.add(authority) db.session.commit() task = SimpleTask(receiver_url=app.config.get('ROOT_URL', ''), action="create_election", queue="launch_task", data={'election_id': data['id']}) task.create_and_send() return task
def review_tally(task): ''' Generates the local private info for a new election ''' sender_ssl_cert = task.get_data()['sender_ssl_cert'] data = task.get_data()['input_data'] # check input data requirements = [ { 'name': u'election_id', 'isinstance': int }, { 'name': u'callback_url', 'isinstance': basestring }, { 'name': u'votes_url', 'isinstance': basestring }, { 'name': u'votes_hash', 'isinstance': basestring }, ] for req in requirements: if req['name'] not in data or not isinstance(data[req['name']], req['isinstance']): print req['name'], data.get(req['name'], None), type(data[req['name']]) raise TaskError(dict(reason="invalid %s parameter" % req['name'])) if data['election_id'] <= 0: raise TaskError(dict(reason="election_id must be a positive int")) if not data['votes_hash'].startswith("ni:///sha-256;"): raise TaskError(dict(reason="invalid votes_hash, must be sha256")) # check election has been created successfully election_id = data['election_id'] election = db.session.query(Election)\ .filter(Election.id == election_id).first() if not election: raise TaskError(dict(reason="election not created")) # check sender is legitimate found_director = False for auth in election.authorities.all(): if not certs_differ(auth.ssl_cert, sender_ssl_cert): found_director = True if not found_director: raise TaskError( dict(reason="review tally sent by an invalid authority")) private_data_path = app.config.get('PRIVATE_DATA_PATH', '') election_privpath = os.path.join(private_data_path, str(election_id)) pubdata_path = app.config.get('PUBLIC_DATA_PATH', '') election_pubpath = os.path.join(pubdata_path, str(election_id)) tally_path = os.path.join(election_pubpath, 'tally.tar.gz') if os.path.exists(tally_path): raise TaskError(dict(reason="election already tallied")) pubkeys = [] for session in election.sessions.all(): session_privpath = os.path.join(election_privpath, session.id) protinfo_path = os.path.join(session_privpath, 'protInfo.xml') pubkey_path = os.path.join(session_privpath, 'publicKey_raw') if not os.path.exists(protinfo_path) or not os.path.exists( pubkey_path): raise TaskError(dict(reason="election not created")) # once we have checked that we have permissions to start doing the tally, # we can remove the "temporal" files of any previous tally ciphertexts_path = os.path.join(session_privpath, 'ciphertexts_json') cipherraw_path = os.path.join(session_privpath, 'ciphertexts_raw') if os.path.exists(ciphertexts_path): os.unlink(ciphertexts_path) if os.path.exists(cipherraw_path): os.unlink(cipherraw_path) pubkey_json_path = os.path.join(session_privpath, 'publicKey_json') with open(pubkey_json_path, 'r') as pubkey_file: pubkeys.append(json.loads(pubkey_file.read())) # reset securely #subprocess.check_call(["vmn", "-reset", "privInfo.xml", "protInfo.xml", # "-f"], cwd=session_privpath) v_reset(session_privpath) # if there were previous tallies, remove the tally approved flag file approve_path = os.path.join(private_data_path, str(election_id), 'tally_approved') if os.path.exists(approve_path): os.unlink(approve_path) # retrieve votes/ciphertexts callback_url = data['votes_url'] r = requests.get(data['votes_url'], verify=False, stream=True) if r.status_code != 200: raise TaskError(dict(reason="error downloading the votes")) # write ciphertexts to disk ciphertexts_path = os.path.join(election_privpath, 'ciphertexts_json') ciphertexts_file = open(ciphertexts_path, 'w') for chunk in r.iter_content(10 * 1024): ciphertexts_file.write(chunk) ciphertexts_file.close() # check votes hash input_hash = data['votes_hash'].replace('ni:///sha-256;', '') if not constant_time_compare(input_hash, hash_file(ciphertexts_path)): raise TaskError(dict(reason="invalid votes_hash")) # transform input votes into something readable by vfork. Basically # we read each line of the votes file, which corresponds with a ballot, # and split each choice to each session # So basically each input line looks like: # {"choices": [vote_for_session1, vote_for_session2, [...]], "proofs": []} # # And we generate N ciphertexts_json files, each of which, for each of # those lines input lines, will contain a line with vote_for_session<i>. # NOTE: This is the inverse of what the demociphs.py script does invotes_file = None outvotes_files = [] # pubkeys needed to verify votes. we also save it to a file pubkeys_path = os.path.join(election_privpath, 'pubkeys_json') pubkeys_s = json.dumps(pubkeys, ensure_ascii=False, sort_keys=True, indent=4, separators=(',', ': ')) with open(pubkeys_path, mode='w') as pubkeys_f: pubkeys_f.write(pubkeys_s) num_questions = len(election.sessions.all()) invalid_votes = 0 for qnum in range(num_questions): pubkeys[qnum]['g'] = int(pubkeys[qnum]['g']) pubkeys[qnum]['p'] = int(pubkeys[qnum]['p']) try: invotes_file = open(ciphertexts_path, 'r') for session in election.sessions.all(): outvotes_path = os.path.join(election_privpath, session.id, 'ciphertexts_json') outvotes_files.append(open(outvotes_path, 'w')) print( "\n------ Reading and verifying POK of plaintext for the votes..\n" ) lnum = 0 for line in invotes_file: lnum += 1 line_data = json.loads(line) assert len(line_data['choices']) == len(outvotes_files) i = 0 for choice in line_data['choices']: # NOTE: we use specific separators with no spaces, because # otherwise vfork won't read it well outvotes_files[i].write( json.dumps(choice, ensure_ascii=False, sort_keys=True, separators=(',', ':'))) outvotes_files[i].write("\n") i += 1 finally: print("\n------ Verified %d votes in total (%d invalid)\n" % (lnum, invalid_votes)) # save invalid votes invalid_votes_path = os.path.join(election_privpath, 'invalid_votes') with open(invalid_votes_path, 'w') as f: f.write("%d" % invalid_votes) if invotes_file is not None: invotes_file.close() for f in outvotes_files: f.close() # Convert each ciphertexts_json of each session into ciphertexts_raw for session in election.sessions.all(): session_privpath = os.path.join(election_privpath, session.id) #subprocess.check_call(["vmnc", "-ciphs", "-ini", "json", # "ciphertexts_json", "ciphertexts_raw"], cwd=session_privpath) v_convert_ctexts_json(session_privpath) autoaccept = app.config.get('AUTOACCEPT_REQUESTS', False) if not autoaccept: def str_date(date): if date: return date.isoformat() else: return "" # request user to decide label = "approve_election_tally" info_text = { 'Title': election.title, 'Description': election.description, 'Voting period': "%s - %s" % (str_date(election.start_date), str_date(election.end_date)), 'Question data': loads(election.questions), 'Authorities': [auth.to_dict() for auth in election.authorities] } approve_task = ExternalTask(label=label, data=info_text) task.add(approve_task) check_approval_task = SimpleTask(receiver_url=app.config.get( 'ROOT_URL', ''), action="check_tally_approval", queue="orchestra_performer", data=dict(election_id=election_id)) task.add(check_approval_task)
def execute(self): data = self.task.get_data()['input_data'] election_id = data['election_id'] election = db.session.query(Election)\ .filter(Election.id == election_id).first() session_ids = [s.id for s in db.session.query(Session).\ with_parent(election,"sessions").\ order_by(Session.question_number)] # 1. let all authorities download the votes and review the requested # tally parallel_task = ParallelTask() for authority in election.authorities: review_task = SimpleTask(receiver_url=authority.orchestra_url, action="review_tally", queue="orchestra_performer", data={ 'election_id': data['election_id'], 'callback_url': data['callback_url'], 'votes_url': data['votes_url'], 'votes_hash': data['votes_hash'], }, receiver_ssl_cert=authority.ssl_cert) parallel_task.add(review_task) self.task.add(parallel_task) # 2. once all the authorities have reviewed and accepted the tallies # (one per question/session), launch vfork to perform it seq_task = SequentialTask() self.task.add(seq_task) for session_id in session_ids: sync_task = SynchronizedTask() seq_task.add(sync_task) for authority in election.authorities: auth_task = SimpleTask(receiver_url=authority.orchestra_url, action="perform_tally", queue="vfork_queue", data={ 'election_id': data['election_id'], 'session_id': session_id }, receiver_ssl_cert=authority.ssl_cert) sync_task.add(auth_task) # once the mixing phase has been done, let all the authorities verify # the results and publish them parallel_task = ParallelTask() for authority in election.authorities: review_task = SimpleTask(receiver_url=authority.orchestra_url, action="verify_and_publish_tally", queue="orchestra_performer", data={ 'election_id': data['election_id'], 'session_ids': session_ids, }, receiver_ssl_cert=authority.ssl_cert) parallel_task.add(review_task) self.task.add(parallel_task) # finally, send the tally to the callback_url ret_task = SimpleTask(receiver_url=app.config.get('ROOT_URL', ''), action="return_tally", queue="orchestra_director", data={"empty": "empty"}) self.task.add(ret_task)
def execute(self): task = self.task input_data = task.get_data()['input_data'] election_id = input_data['election_id'] election = db.session.query(Election)\ .filter(Election.id == election_id).first() # 1. generate a session per question private_data_path = app.config.get('PRIVATE_DATA_PATH', '') election_private_path = os.path.join(private_data_path, str(election_id)) sessions = [] questions = json.loads(election.questions) i = 0 for question in questions: session_id = "%d-%s" % (i, str(uuid.uuid4())) # create stub.xml session_privpath = os.path.join(election_private_path, session_id) mkdir_recursive(session_privpath) # l = ["vmni", "-prot", "-sid", session_id, "-name", # election.title, "-nopart", str(election.num_parties), "-thres", # str(election.threshold_parties)] #subprocess.check_call(l, cwd=session_privpath) v_gen_protocol_info(session_id, str(election.id), election.num_parties, election.threshold_parties, session_privpath) # read stub file to be sent to all the authorities stub_path = os.path.join(session_privpath, 'stub.xml') stub_file = codecs.open(stub_path, 'r', encoding='utf-8') stub_content = stub_file.read() stub_file.close() sessions.append(dict(id=session_id, stub=stub_content)) session = Session(id=session_id, election_id=election_id, status='default', public_key='', question_number=i) db.session.add(session) i += 1 db.session.commit() # 2. generate private info and protocol info files on each authority # (and for each question/session). Also, each authority might require # the approval of the task by its operator. priv_info_task = SequentialTask() for authority in election.authorities: subtask = SimpleTask( receiver_url=authority.orchestra_url, receiver_ssl_cert=authority.ssl_cert, action="generate_private_info", queue="orchestra_performer", data=dict( id=election_id, title=election.title, description=election.description, sessions=sessions, questions=election.questions, start_date=election.start_date, end_date=election.end_date, num_parties=election.num_parties, threshold_parties=election.threshold_parties, authorities=[a.to_dict() for a in election.authorities])) priv_info_task.add(subtask) task.add(priv_info_task) # 3. merge the outputs into protInfo.xml files, send them to the # authorities, and generate pubkeys sequentially one session after the # other merge_protinfo_task = SimpleTask( receiver_url=app.config.get('ROOT_URL', ''), action="merge_protinfo", queue="orchestra_director", data=dict(election_id=election_id, session_ids=[s['id'] for s in sessions])) task.add(merge_protinfo_task) # 4. send protInfo.xml to the original sender (we have finished!) return_election_task = SimpleTask( receiver_url=app.config.get('ROOT_URL', ''), action="return_election", queue="orchestra_director", data=dict(election_id=election_id, session_ids=[s['id'] for s in sessions])) task.add(return_election_task)
def merge_protinfo_task(task): ''' Merge the protinfos for each of the sessions (one session per question), and then create a pubkey for each protinfo ''' input_data = task.get_data()['input_data'] election_id = input_data['election_id'] session_ids = input_data['session_ids'] priv_info_task = task.get_prev() election = db.session.query(Election)\ .filter(Election.id == election_id).first() questions = json.loads(election.questions) private_data_path = app.config.get('PRIVATE_DATA_PATH', '') election_privpath = os.path.join(private_data_path, str(election_id)) i = 0 # this task will contain one subtask of type SynchronizedTask to create # the pubkey for each session seq_task = SequentialTask() task.add(seq_task) for question in questions: j = 0 # l = ["vmni", "-merge"] l = [] session_id = session_ids[i] session_privpath = os.path.join(election_privpath, session_ids[i]) # create protInfo<j>.xml files, extracting data from subtasks for subtask in priv_info_task.get_children(): protinfo_content = subtask.get_data()['output_data'][i] protinfo_path = os.path.join(session_privpath, 'protInfo%d.xml' % j) l.append('protInfo%d.xml' % j) protinfo_file = codecs.open(protinfo_path, 'w', encoding='utf-8') protinfo_file.write(protinfo_content) protinfo_file.close() j += 1 # merge the files # subprocess.check_call(l, cwd=session_privpath) v_merge(l, session_privpath) # read protinfo protinfo_path = os.path.join(session_privpath, 'protInfo.xml') protinfo_file = codecs.open(protinfo_path, 'r', encoding='utf-8') protinfo_content = protinfo_file.read() protinfo_file.close() # send protInfo.xml to the authorities and command them to cooperate in # the generation of the publicKey send_merged_protinfo = SynchronizedTask() seq_task.add(send_merged_protinfo) for authority in election.authorities: subtask = SimpleTask(receiver_url=authority.orchestra_url, action="generate_public_key", queue="vfork_queue", data=dict(session_id=session_id, election_id=election_id, protInfo_content=protinfo_content), receiver_ssl_cert=authority.ssl_cert) send_merged_protinfo.add(subtask) i += 1 return dict(output_data=protinfo_content)
def hello_world(task): ''' complex tree of subtasks are executed: task/hello_world (current local task :5001, sequential) | |-- task1 (local virtual task :5001, parallel) | | | |-- task11 (local task :5001, synchronized) | | | | | |-- task111/GoodbyeCruelWorldHandler (local task : 5001, simple) | | | | | |-- task112/GoodbyeCruelWorldHandler (local task : 5001, simple) | | | | | |-- task113/GoodbyeCruelWorldHandler (remote task : 5000, simple) | | | |-- task12 (local task :5001, synchronized) | | | | | |-- task121/GoodbyeCruelWorldHandler (local task : 5001, simple) | | | | | |-- task122/GoodbyeCruelWorldHandler (local task : 5001, simple) | | | | | |-- task123/GoodbyeCruelWorldHandler (remote task : 5000, simple) | |-- task2/all_goodbyes_together (local virtual task :5000, simple) when all the subtasks are executed, the sender (:5000 via post_hello) is notified that the initial task is finished. ''' username = task.get_data()['input_data']['username'] task1 = ParallelTask() task.add(task1) task11 = SynchronizedTask(handler=SynchronizedGoodbyeHandler) task1.add(task11) goodbye_kwargs = dict(receiver_url='http://127.0.0.1:5001/api/queues', action="testing.goodbye_cruel_world", queue="goodbye_world", data={'username': username * 2}) goodbye_remote_kwargs = copy.deepcopy(goodbye_kwargs) goodbye_remote_kwargs.update( {'receiver_url': 'http://127.0.0.1:5000/api/queues'}) task111 = SimpleTask(**goodbye_kwargs) task11.add(task111) task112 = SimpleTask(**goodbye_kwargs) task11.add(task112) task113 = SimpleTask(**goodbye_remote_kwargs) task11.add(task113) task12 = SynchronizedTask(handler=SynchronizedGoodbyeHandler) task1.add(task12) task121 = SimpleTask(**goodbye_kwargs) task12.add(task121) task122 = SimpleTask(**goodbye_kwargs) task12.add(task122) task123 = SimpleTask(**goodbye_remote_kwargs) task12.add(task123) all_goodbyes_together_kwargs = dict( receiver_url='http://127.0.0.1:5001/api/queues', action="testing.all_goodbyes_together", queue="end_of_the_world") task2 = SimpleTask(**all_goodbyes_together_kwargs) task.add(task2) # this will get overridden by last task, all_goodbyes_together return dict(output_data="hello %s!" % username)
def generate_private_info(task): ''' Generates the local private info for a new election ''' input_data = task.get_data()['input_data'] election_id = input_data['id'] # 1. check this is a new election and check input data private_data_path = app.config.get('PRIVATE_DATA_PATH', '') election_privpath = os.path.join(private_data_path, str(election_id)) # check generic input data, similar to the data for public_api check_election_data(input_data, False) # check the sessions data if not isinstance(input_data.get('sessions', None), list) or\ not len(input_data['sessions']): raise TaskError(dict(reason="No sessions provided")) for session in input_data['sessions']: if not isinstance(session, dict) or 'id' not in session or\ 'stub' not in session or\ not isinstance(session['stub'], str) or\ not re.match("^[a-zA-Z0-9_-]+$", session['id']): raise TaskError(dict(reason="Invalid session data provided")) # check that we are indeed one of the listed authorities auth_name = None for auth_data in input_data['authorities']: if auth_data['orchestra_url'] == app.config.get('ROOT_URL', ''): auth_name = auth_data['name'] if not auth_name: raise TaskError( dict( reason="trying to process what SEEMS to be an external election" )) # localProtInfo.xml should not exist for any of the sessions, as our task is # precisely to create it. note that we only check that localProtInfo.xml # files don't exist, because if we are the director, then the stub and # parent directory will already exist for session in input_data['sessions']: session_privpath = os.path.join(election_privpath, session['id']) protinfo_path = os.path.join(session_privpath, 'localProtInfo.xml') if os.path.exists(protinfo_path): raise TaskError( dict(reason="session_id %s already created" % session['id'])) # 2. create base local data from received input in case it's needed: # create election models, dirs and stubs if we are not the director if certs_differ(task.get_data()['sender_ssl_cert'], app.config.get('SSL_CERT_STRING', '')): if os.path.exists(election_privpath): raise TaskError( dict(reason="Already existing election id %d" % input_data['id'])) election = Election( id=input_data['id'], title=input_data['title'], description=input_data['description'], questions=input_data['questions'], start_date=input_data['start_date'], end_date=input_data['end_date'], num_parties=input_data['num_parties'], threshold_parties=input_data['threshold_parties'], ) db.session.add(election) for auth_data in input_data['authorities']: authority = Authority(name=auth_data['name'], ssl_cert=auth_data['ssl_cert'], orchestra_url=auth_data['orchestra_url'], election_id=input_data['id']) db.session.add(authority) # create dirs and stubs, and session model i = 0 for session in input_data['sessions']: session_model = Session(id=session['id'], election_id=election_id, status='default', public_key='', question_number=i) db.session.add(session_model) session_privpath = os.path.join(election_privpath, session['id']) mkdir_recursive(session_privpath) stub_path = os.path.join(session_privpath, 'stub.xml') stub_file = codecs.open(stub_path, 'w', encoding='utf-8') stub_content = stub_file.write(session['stub']) stub_file.close() i += 1 db.session.commit() else: # if we are the director, models, dirs and stubs have been created # already, so we just get the election from the database election = db.session.query(Election)\ .filter(Election.id == election_id).first() # only create external task if we have configured autoaccept to false in # settings: autoaccept = app.config.get('AUTOACCEPT_REQUESTS', '') if not autoaccept: def str_date(date): if date: return date.isoformat() else: return "" label = "approve_election" info_text = { 'Title': election.title, 'Description': election.description, 'Voting period': "%s - %s" % (str_date(election.start_date), str_date(election.end_date)), 'Question data': loads(election.questions), 'Authorities': [auth.to_dict() for auth in election.authorities] } approve_task = ExternalTask(label=label, data=info_text) task.add(approve_task) vfork_task = SimpleTask(receiver_url=app.config.get('ROOT_URL', ''), action="generate_private_info_vfork", queue="orchestra_performer", data=dict()) task.add(vfork_task)
callback_url=data['callback_url'], num_parties=len(data['authorities']), threshold_parties=len(data['authorities']), status='creating') db.session.add(e) for auth_data in data['authorities']: authority = Authority(name=auth_data['name'], ssl_cert=auth_data['ssl_cert'], orchestra_url=auth_data['orchestra_url'], election_id=data['id']) db.session.add(authority) db.session.commit() task = SimpleTask(receiver_url=app.config.get('ROOT_URL', ''), action="create_election", queue="launch_task", data={'election_id': data['id']}) task.create_and_send() return task def tally_task(data): if not data: print("invalid json") return False requirements = [ { 'name': u'election_id', 'isinstance': int },