def setup():
"""
Reads fi/job/remote/ for jobs
Asks ModuleNameInput class for a generator to get input
Stores Job, then Jobs w/Input (Instance), then Jobs w/input and future output (Assignment) rows in database
"""
prefix = os.path.join('fi', 'job', 'remote')
files = (
f
for f in os.listdir(prefix)
if f.endswith('.py') and
f != '__init__.py' and
os.path.isfile(
os.path.join(prefix, f)
)
)
# For each RemoteJob file
for i, filename in enumerate(files):
name = filename.replace('.py', '')
# Get input generator
input_class = __import__('fi.job.remote.' + name).job.remote.__getattribute__(name).__getattribute__(name + "Input")
# Get module code
with open(os.path.join(prefix, filename), 'r') as module:
module_input = module.read().split('class %sInput' % name)[0]
# Add job
job = Job(
id=i,
name=name,
description=input_class.DESCRIPTION,
input_desc=input_class.INPUT,
output_desc=input_class.OUTPUT,
module=module_input,
credit=input_class.CREDIT
)
# Add instances
for j, instance_input in enumerate(input_class.input(fi.job.MAX_INSTANCES)):
instance = Instance(
id=j,
job=job,
input=instance_input,
)
# Add assignments
for k in range(fi.job.MAX_ASSIGNMENTS):
Assignment(
id=k,
job=job,
instance=instance,
)
model.commit()
def allocate(cls, allocations):
"""
allocations:[(str, int)] -> None
"""
ipsToBandwidth = dict(allocations)
for throttle in cls.query.all():
throttle.bandwidth = ipsToBandwidth[throttle.vpn_ip]
model.commit()
def allocate(cls, allocations):
"""
allocations:[(str, int)] -> None
"""
ipsToBandwidth = dict(allocations)
for throttle in cls.query.all():
throttle.bandwidth = ipsToBandwidth[throttle.vpn_ip]
model.commit()
def setup():
"""
Reads fi/job/remote/ for jobs
Asks ModuleNameInput class for a generator to get input
Stores Job, then Jobs w/Input (Instance), then Jobs w/input and future output (Assignment) rows in database
"""
prefix = os.path.join('fi', 'job', 'remote')
files = (f for f in os.listdir(prefix)
if f.endswith('.py') and f != '__init__.py'
and os.path.isfile(os.path.join(prefix, f)))
# For each RemoteJob file
for i, filename in enumerate(files):
name = filename.replace('.py', '')
# Get input generator
input_class = __import__('fi.job.remote.' +
name).job.remote.__getattribute__(
name).__getattribute__(name + "Input")
# Get module code
with open(os.path.join(prefix, filename), 'r') as module:
module_input = module.read().split('class %sInput' % name)[0]
# Add job
job = Job(id=i,
name=name,
description=input_class.DESCRIPTION,
input_desc=input_class.INPUT,
output_desc=input_class.OUTPUT,
module=module_input,
credit=input_class.CREDIT)
# Add instances
for j, instance_input in enumerate(
input_class.input(fi.job.MAX_INSTANCES)):
instance = Instance(
id=j,
job=job,
input=instance_input,
)
# Add assignments
for k in range(fi.job.MAX_ASSIGNMENTS):
Assignment(
id=k,
job=job,
instance=instance,
)
model.commit()
def _verifyResults(cls, key):
"""
id:int -> None
Called when all job instances have been completed.
Is a hash is favored over the others?
Yes -> "Correct" to majority, "Incorrect for rest"
No -> "Inconclusive" for all
"""
fi.logmsg(cls, "Verifying %d-%d" % tuple(map(lambda x: x.id, key)))
assignments = cls.digests[key]
votes = {}
for assignment in assignments:
o = hash(assignment.output)
votes.setdefault(o, []).append(assignment)
majority_digest = max(votes, key=lambda o: len(votes[o]))
# False majority? other >= majority
false_majority = len(
filter(
lambda x: x != majority_digest and len(votes[x]) >= len(votes[majority_digest]),
votes
)
)
# Set verified status
if false_majority:
for ass in assignments:
ass.reset()
else:
# Cache correct result hash
cls.done[key] = majority_digest
key[1].digest = majority_digest
for ass in assignments:
correct = hash(ass.output) == majority_digest
ass.verified = cls.messages[correct]
if correct:
ass.client.credit += ass.job.credit
del cls.digests[key]
model.commit()
def complete(cls, ip, output):
"""
ip:str | output:serializable -> None
Mark ip's current job complete
"""
assignment = cls.lookup(ip)
if not assignment:
raise fi.exception.EmptyQueryError("No assignment for completed job")
assignment.time_returned = datetime.datetime.now()
assignment.output = output
model.commit()
def complete(cls, ip, output):
"""
ip:str | output:serializable -> None
Mark ip's current job complete
"""
assignment = cls.lookup(ip)
if not assignment:
raise fi.exception.EmptyQueryError(
"No assignment for completed job")
assignment.time_returned = datetime.datetime.now()
assignment.output = output
model.commit()
def setup():
Client(
ip="128.164.160.197",
vpn_ip="10.8.0.6",
credit=1,
bandwidth=10
)
Client(
ip="128.164.160.199",
vpn_ip="10.8.0.10",
credit=1,
bandwidth=10
)
model.commit()
def setup():
Client(ip="128.164.160.197", vpn_ip="10.8.0.6", credit=1, bandwidth=10)
Client(ip="128.164.160.199", vpn_ip="10.8.0.10", credit=1, bandwidth=10)
model.commit()
def record(cls, ip, assignment):
assignment.reset()
assignment.time_issued = datetime.datetime.now()
assignment.client = Client.byIP(ip)
model.commit()
def record(cls, ip, assignment):
assignment.reset()
assignment.time_issued = datetime.datetime.now()
assignment.client = Client.byIP(ip)
model.commit()
