def setup(): """ Reads fi/job/remote/ for jobs Asks ModuleNameInput class for a generator to get input Stores Job, then Jobs w/Input (Instance), then Jobs w/input and future output (Assignment) rows in database """ prefix = os.path.join('fi', 'job', 'remote') files = ( f for f in os.listdir(prefix) if f.endswith('.py') and f != '__init__.py' and os.path.isfile( os.path.join(prefix, f) ) ) # For each RemoteJob file for i, filename in enumerate(files): name = filename.replace('.py', '') # Get input generator input_class = __import__('fi.job.remote.' + name).job.remote.__getattribute__(name).__getattribute__(name + "Input") # Get module code with open(os.path.join(prefix, filename), 'r') as module: module_input = module.read().split('class %sInput' % name)[0] # Add job job = Job( id=i, name=name, description=input_class.DESCRIPTION, input_desc=input_class.INPUT, output_desc=input_class.OUTPUT, module=module_input, credit=input_class.CREDIT ) # Add instances for j, instance_input in enumerate(input_class.input(fi.job.MAX_INSTANCES)): instance = Instance( id=j, job=job, input=instance_input, ) # Add assignments for k in range(fi.job.MAX_ASSIGNMENTS): Assignment( id=k, job=job, instance=instance, ) model.commit()
def allocate(cls, allocations): """ allocations:[(str, int)] -> None """ ipsToBandwidth = dict(allocations) for throttle in cls.query.all(): throttle.bandwidth = ipsToBandwidth[throttle.vpn_ip] model.commit()
def setup(): """ Reads fi/job/remote/ for jobs Asks ModuleNameInput class for a generator to get input Stores Job, then Jobs w/Input (Instance), then Jobs w/input and future output (Assignment) rows in database """ prefix = os.path.join('fi', 'job', 'remote') files = (f for f in os.listdir(prefix) if f.endswith('.py') and f != '__init__.py' and os.path.isfile(os.path.join(prefix, f))) # For each RemoteJob file for i, filename in enumerate(files): name = filename.replace('.py', '') # Get input generator input_class = __import__('fi.job.remote.' + name).job.remote.__getattribute__( name).__getattribute__(name + "Input") # Get module code with open(os.path.join(prefix, filename), 'r') as module: module_input = module.read().split('class %sInput' % name)[0] # Add job job = Job(id=i, name=name, description=input_class.DESCRIPTION, input_desc=input_class.INPUT, output_desc=input_class.OUTPUT, module=module_input, credit=input_class.CREDIT) # Add instances for j, instance_input in enumerate( input_class.input(fi.job.MAX_INSTANCES)): instance = Instance( id=j, job=job, input=instance_input, ) # Add assignments for k in range(fi.job.MAX_ASSIGNMENTS): Assignment( id=k, job=job, instance=instance, ) model.commit()
def _verifyResults(cls, key): """ id:int -> None Called when all job instances have been completed. Is a hash is favored over the others? Yes -> "Correct" to majority, "Incorrect for rest" No -> "Inconclusive" for all """ fi.logmsg(cls, "Verifying %d-%d" % tuple(map(lambda x: x.id, key))) assignments = cls.digests[key] votes = {} for assignment in assignments: o = hash(assignment.output) votes.setdefault(o, []).append(assignment) majority_digest = max(votes, key=lambda o: len(votes[o])) # False majority? other >= majority false_majority = len( filter( lambda x: x != majority_digest and len(votes[x]) >= len(votes[majority_digest]), votes ) ) # Set verified status if false_majority: for ass in assignments: ass.reset() else: # Cache correct result hash cls.done[key] = majority_digest key[1].digest = majority_digest for ass in assignments: correct = hash(ass.output) == majority_digest ass.verified = cls.messages[correct] if correct: ass.client.credit += ass.job.credit del cls.digests[key] model.commit()
def complete(cls, ip, output): """ ip:str | output:serializable -> None Mark ip's current job complete """ assignment = cls.lookup(ip) if not assignment: raise fi.exception.EmptyQueryError("No assignment for completed job") assignment.time_returned = datetime.datetime.now() assignment.output = output model.commit()
def complete(cls, ip, output): """ ip:str | output:serializable -> None Mark ip's current job complete """ assignment = cls.lookup(ip) if not assignment: raise fi.exception.EmptyQueryError( "No assignment for completed job") assignment.time_returned = datetime.datetime.now() assignment.output = output model.commit()
def setup(): Client( ip="128.164.160.197", vpn_ip="10.8.0.6", credit=1, bandwidth=10 ) Client( ip="128.164.160.199", vpn_ip="10.8.0.10", credit=1, bandwidth=10 ) model.commit()
def setup(): Client(ip="128.164.160.197", vpn_ip="10.8.0.6", credit=1, bandwidth=10) Client(ip="128.164.160.199", vpn_ip="10.8.0.10", credit=1, bandwidth=10) model.commit()
def record(cls, ip, assignment): assignment.reset() assignment.time_issued = datetime.datetime.now() assignment.client = Client.byIP(ip) model.commit()