def init_queue():
# 以10个任务为测试基准
for i in range(10):
_new_task = TaskQueue()
_new_task.set_download_url(
"http://localhost:5000/download_sourcefile/REDEME.md")
_new_task.set_target_node(None)
_new_task.set_upload_url(None)
_new_task.add()
def getSingleFileLinks( text ):
links = []
doLog( "getSingleFileLinks start" )
dp = xbmcgui.DialogProgress()
dp.create("Searching for single file links", "")
doLog( "getSingleFileLinks finding matches" )
link_matches = re.findall( r'(http://hotfile.com/[^"]+/([^"]+).(flv|mpg|mpeg|mov|avi|mkv).html)', text, re.M|re.S )
if not link_matches:
doLog( "getSingleFileLinks no links found" )
dp.update(100,"None found")
del dp
return links
dp.update(0, "Removing duplicate links")
link_matches[:] = dedupeList( link_matches, 0 )
num_fetch_threads = 4
url_queue = TaskQueue()
def validateUrlsThread(i, q):
"""This is the worker thread function.
It processes items in the queue one after
another. These daemon threads go into an
infinite loop, and only exit when
the main thread ends.
"""
while True:
url = q.get()
valid_link = getHFLink( hotfile_user, hotfile_pass, url )
if valid_link:
links.append( valid_link )
else:
doLog( "getSingleFileLinks NOT ADDING: " + url )
q.task_done()
# Set up some threads to validate the urls
for i in range(num_fetch_threads):
worker = Thread(target=validateUrlsThread, args=(i, url_queue,))
worker.setDaemon(True)
worker.start()
if link_matches:
doLog( "getSingleFileLinks iterate over link matches" )
for link,name,extension in link_matches:
parsed_url = urlparse(link)
dp.update(0, "Validating link:", parsed_url[1], parsed_url[2].split("/")[-1])
doLog( "getSingleFileLinks appending:\nLINK: %s\nNAME: %s\nEXT: %s" % ( link, name, extension ) )
url_queue.put(link)
# wait for queue to empty which means all urls validated
url_queue.join()
doLog( "getSingleFileLinks done" )
dp.update(100, "Done getting single file links!")
del dp
return links
def ask_task():
# 节点请求任务，用json作为传递任务参数的媒介
# 包含：任务id，任务资源下载地址，任务结果上传地址，任务目标节点
from TaskQueue import TaskQueue
print("node is asking for task...")
test = TaskQueue()
# 这里需要写test=，不明原因
test = test.get()
_response = jsonify(test.to_json_dict())
return _response
def __init__(self, id, buffer=5):
threading.Thread.__init__(self)
self.id = id
self.daemon = True
self.buffer_time = buffer
self.q = TaskQueue()
self.last_action_time = 0
self.new_task = threading.Event()
self.task_done = threading.Event()
def get_compabitible_task_for_agent(self, agent, queue):
compatible_task_list = TaskQueue()
for task in self.tasks.get_queue():
if task.is_agent_compatible(agent):
compatible_task_list.enqueue(task)
if compatible_task_list.is_empty():
print "No compatible tasks found for agent %s" %agent.name
return None
return compatible_task_list
def __init__(self, p=0.75, p1=0.8, p2=0.5, iteration_count=100000):
self.current_tick = 0
self.handled_tasks = []
self.states = []
self.iteration_count = iteration_count
self.busy_count = 0
self.source = Source(p, LemerGenerator(209715120, 3, 7))
self.queue = TaskQueue(2)
self.handlers = [
Handler(p1, LemerGenerator(209715120, 3, 7)),
Handler(p2, LemerGenerator(209715120, 3, 7))
]
def __init__(self, p1=0.4, p2=0.5, iteration_count=100000):
self.iteration_count = iteration_count
self.task_in_system_count = 0
self.current_tick = 0
self.handled_count = 0
self.refused_count = 0
self.states = []
self.p1 = p1
self.source = Source()
self.queue = TaskQueue(2)
self.handlers = [
Handler(p1, LemerGenerator(209715120, 3, 7)),
Handler(p2, LemerGenerator(209715120, 3, 7))
]
def __init__(self, size=(100, 100), agents=[], tasks=TaskQueue(), AI_ON=True):
self.size = size
self.ai_on = False;
self.agents = agents
self.tasks = tasks
self.map = np.ones(size)
self.user_priority_queue = TaskQueue()
self.world_task_queue = TaskQueue()
self.generate_agents()
self.generate_world_tasks()
if(self.ai_on):
self.assign_tasks()
self.ros_controller = ROSController()
self.restart = False
def __init__(self,
name,
pose=Pose(),
size=(1.0, 1.0, 1.0),
capabilities=[],
movement_speed=0.1):
# Tuple contain the x,y,z coordinates of the agent
self.pose = pose # Rospy pose data structure, used in Unity to determine space
self.name = name # Name of this particular agent, needs to be unique
self.size = size # the x,y,z size of the agent
self.task_queue = TaskQueue()
self.capabilities = capabilities
self.status = ""
self.current_action = "idle"
self.movement_speed = movement_speed
self.is_at_task = False
self.ros_controller = ROSController()
def __init__(self, num_workers=3):
self.monitor = Monitor()
self.google_queue = TaskQueue(-1,
"Google",
self.monitor,
num_workers=num_workers)
self.azure_queue = TaskQueue(-1,
"Azure",
self.monitor,
num_workers=num_workers)
self.amazon_queue = TaskQueue(-1,
"Amazon",
self.monitor,
num_workers=num_workers)
self.maxID = 1
self.start_reassignmet()
class Agent():
def __init__(self,
name,
pose=Pose(),
size=(1.0, 1.0, 1.0),
capabilities=[],
movement_speed=0.1):
# Tuple contain the x,y,z coordinates of the agent
self.pose = pose # Rospy pose data structure, used in Unity to determine space
self.name = name # Name of this particular agent, needs to be unique
self.size = size # the x,y,z size of the agent
self.task_queue = TaskQueue()
self.capabilities = capabilities
self.status = ""
self.current_action = "idle"
self.movement_speed = movement_speed
self.is_at_task = False
self.ros_controller = ROSController()
def get_pose(self):
return self.pose
## TODO, fill in this with a custom message.
def get_message(self):
pass
def add_task(self, task):
self.task_queue.enqueue(task)
def get_task(self):
return self.task_queue.peek()
def set_current_task(self, task):
self.is_at_task = False
self.task_queue.prioritise(task)
def update_status(self):
if self.task_queue.is_empty():
self.status = "%s is idle." % self.name
else:
task_name = self.task_queue.peek().name
self.status = "%s is %s task[ %s ] currently." % (
self.name, self.current_action, task_name)
def get_status(self):
self.update_status()
return self.status
def go_to_current_task(self):
if self.pose.position == self.task_queue.peek().pose.position:
print("Agent %s is currently at the it's current task!" %
self.name)
self.is_at_task = True
return True
self.current_action = "moving to task"
self.move_towards_target(self.task_queue.peek().pose.position)
return False
def move_towards_target(self, target_pos):
target_diff_x = target_pos.x - self.pose.position.x
target_diff_y = target_pos.y - self.pose.position.y
target_diff_z = target_pos.z - self.pose.position.z
self.pose.position.x += self.get_move_vector(target_diff_x)
self.pose.position.y += self.get_move_vector(target_diff_y)
self.pose.position.z += self.get_move_vector(target_diff_z)
direction_y = math.atan2(self.get_move_vector(target_diff_y),
self.get_move_vector(target_diff_x))
quat = quaternion_from_euler(0, direction_y, 0)
self.pose.orientation.x = quat[0]
self.pose.orientation.y = quat[1]
self.pose.orientation.z = quat[2]
self.pose.orientation.w = quat[3]
def get_move_vector(self, target_x):
if abs(target_x) > self.movement_speed:
if target_x < 0:
return -self.movement_speed
else:
return self.movement_speed
return target_x
def complete_current_task(self):
if self.task_queue.peek().is_completed():
print("Current task %s being done by %s has been completed." %
(self.task_queue.peek().name, self.name))
self.ros_controller.publish_string_update_message(
"%s has been completed by %s" %
(self.task_queue.peek().name, self.name))
self.task_queue.dequeue()
self.is_at_task = False
self.current_action = "completed task"
return True
print "Current task %s has not yet been completed by agent %s." % (
self.task_queue.peek().name, self.name)
return False
# OVERRIDE THIS FUNCTION WITH SPECIFIC AGENT FUNCTION
def do_work(self):
pass
def run(self):
while isRunning:
if not self.robot.task_queue.isVideoQueueEmpty():
url = self.robot.task_queue.getFromVideoQueue()
self.robot.getVideoInfo(url)
else:
print('等待中...')
time.sleep(random.randrange(5, 10))
if __name__ == '__main__':
v_bloom = BloomFilter(500000)
a_bloom = BloomFilter(10000)
t_bloom = BloomFilter(30000)
task_queue = TaskQueue()
robot = Robot(v_bloom, a_bloom, t_bloom, task_queue)
currentVideos = VideoDao.find_all_video_urls()
currentAuthors = AuthorDao.find_all_author_urls()
currentTags = TagDao.find_all_tag_urls()
robot.initFilter(currentVideos, currentAuthors, currentTags)
p = Producer(robot=robot)
p.start()
time.sleep(10)
c = Consumer(robot=robot)
c.start()
class Bot(threading.Thread):
'''Producer/Consumer Bot utilising TaskQueue.'''
def __init__(self, id, buffer=5):
threading.Thread.__init__(self)
self.id = id
self.daemon = True
self.buffer_time = buffer
self.q = TaskQueue()
self.last_action_time = 0
self.new_task = threading.Event()
self.task_done = threading.Event()
def schedule(self, tasks):
'''Schedule a new task by starting an independent thread.
- tasks (list): tasks to be scheduled.'''
t = threading.Thread(target=self._add_tasks,
args=(tasks, ),
daemon=True)
t.start()
def _add_tasks(self, tasks):
'''Adding tasks from a list to task queue and waiting if queue is full.'''
for task in tasks:
if self.q.full():
print('Producer: Queue full - sleeping')
self.task_done.wait()
print('Producer: Queue not full now, adding task..')
self.task_done.clear()
self.q.put(task)
print(f'Producer: Scheduled {task}')
self.new_task.set()
def run(self):
'''Runs the thread which executes tasks.
- stay idle until scheduled task time'''
while True:
if self.q.empty():
self.new_task.wait()
print('Bot: Woke up')
self.new_task.clear()
task = self._idle()
print(f'Bot executing {task}')
task.execute()
self.last_action_time = time()
self.q.task_done()
self.task_done.set()
def _idle(self):
'''Wait for next scheduled task in queue.
- If tasks are added to queue, check next task again'''
next_task = self.q.peek()
time_until = self._time_until(next_task)
print(f'Bot: Time until next task {time_until}')
flag = self.new_task.wait(timeout=time_until)
if flag:
print('Bot: New task added to queue - checking next task')
self.new_task.clear()
return self._idle()
else:
return self.q.get()
def _time_until(self, task):
'''Calculate time in seconds until scheduled task.
- Convert task.time datetime to epoch time
- Add buffer delay if scheduled task is too close to last action time'''
now = time()
time_since = now - self.last_action_time
scheduled = int(task.time.strftime('%s'))
time_until = scheduled - now
if time_until < 0:
if time_since < self.buffer_time:
time_until = time_until + self.buffer_time
else:
time_until = 0
return time_until
def getSingleFileLinks(text):
links = []
doLog("getSingleFileLinks start")
dp = xbmcgui.DialogProgress()
dp.create("Searching for single file links", "")
doLog("getSingleFileLinks finding matches")
link_matches = re.findall(
r'(http://hotfile.com/[^"]+/([^"]+).(flv|mpg|mpeg|mov|avi|mkv).html)',
text, re.M | re.S)
if not link_matches:
doLog("getSingleFileLinks no links found")
dp.update(100, "None found")
del dp
return links
dp.update(0, "Removing duplicate links")
link_matches[:] = dedupeList(link_matches, 0)
num_fetch_threads = 4
url_queue = TaskQueue()
def validateUrlsThread(i, q):
"""This is the worker thread function.
It processes items in the queue one after
another. These daemon threads go into an
infinite loop, and only exit when
the main thread ends.
"""
while True:
url = q.get()
valid_link = getHFLink(hotfile_user, hotfile_pass, url)
if valid_link:
links.append(valid_link)
else:
doLog("getSingleFileLinks NOT ADDING: " + url)
q.task_done()
# Set up some threads to validate the urls
for i in range(num_fetch_threads):
worker = Thread(target=validateUrlsThread, args=(
i,
url_queue,
))
worker.setDaemon(True)
worker.start()
if link_matches:
doLog("getSingleFileLinks iterate over link matches")
for link, name, extension in link_matches:
parsed_url = urlparse(link)
dp.update(0, "Validating link:", parsed_url[1],
parsed_url[2].split("/")[-1])
doLog(
"getSingleFileLinks appending:\nLINK: %s\nNAME: %s\nEXT: %s" %
(link, name, extension))
url_queue.put(link)
# wait for queue to empty which means all urls validated
url_queue.join()
doLog("getSingleFileLinks done")
dp.update(100, "Done getting single file links!")
del dp
return links
def still_alive():
juche.info("still alive")
import unittest
class TestSequence(unittest.TestCase):
def setUp(self):
pass
def test_create_tests(self):
create_tests()
#import cProfile
#cProfile.runctx('create_tests()',globals(),locals()) #http://stackoverflow.com/questions/1819448/cannot-make-cprofile-work-in-ipython/3305654#3305654
if __name__=="__main__":
q = TaskQueue()
q.insert(priority_fix,every=HOURLY*6,now=False)
q.insert(atlas,every=MINUTELY,now=True)
q.insert(fixup,every=HOURLY*4,now=False)
q.insert(still_alive,every=60)
q.insert(autoboss,every=HOURLY*4)
q.insert(create_tests,every=MINUTELY)
while True:
try:
q.execTop()
except Exception as e:
juche.error("That's funny, I don't feel corrupt. In fact, I feel pretty good.")
juche.exception(e)
sleep(2)
def main():
test = TaskQueue()
with app.app_context():
init_queue()
class GameWorld():
def __init__(self, size=(100, 100), agents=[], tasks=TaskQueue(), AI_ON=True):
self.size = size
self.ai_on = False;
self.agents = agents
self.tasks = tasks
self.map = np.ones(size)
self.user_priority_queue = TaskQueue()
self.world_task_queue = TaskQueue()
self.generate_agents()
self.generate_world_tasks()
if(self.ai_on):
self.assign_tasks()
self.ros_controller = ROSController()
self.restart = False
#self.agents[0].task_queue.prioritise(self.world_task_queue.get_queue()[0])
def update(self):
# Prints the tasks in each of the agent queues for debugging reasons.
#for agent in self.agents:
# print ("Agent: %s has the tasks: %s" % (agent.name, ",".join([t.name for t in agent.task_queue.get_queue()])))
if not self.world_task_queue.is_empty():
#print("Publishing tasks")
self.ros_controller.publish_tasks(self.world_task_queue.get_queue())
#print("Publishing Agents")
self.ros_controller.publish_agents(self.agents)
## This checks if the game is meant to run yet
if self.ros_controller.game_start:
if(self.ai_on):
timer = Timer()
self.check_scenario_complete()
for agent in self.agents:
agent.do_work()
if self.ros_controller.recieved_assign_message:
#print("I SHOULDN'T HAVE A ASSIGN MESSAGE YET: %s" % self.ros_controller.recieved_assign_message)
self.ros_controller.recieved_assign_message = False
assigned_agent = self.find_agent_in_list(self.ros_controller.user_assign_task[0])
assigned_task = self.find_task_in_queue(self.ros_controller.user_assign_task[1])
assigned_agent.add_task(assigned_task)
print("Assigned agent %s to do task %s" % (assigned_agent.name, assigned_task.name))
if self.ros_controller.recieved_move_message:
self.ros_controller.recieved_move_message = False
assigned_agent = self.find_agent_in_list(self.ros_controller.user_assign_move[0])
pose = self.pose_constructor(float(self.ros_controller.user_assign_move[1][0]),
float(self.ros_controller.user_assign_move[1][1]),0,0,0,0,1)
assigned_agent.set_current_task(MoveTask("TempMoveTask", "Agent needs to move to this spot", end_pose=pose, agent=assigned_agent))
print("Assigned agent %s to move to %s" %(assigned_agent.name, pose))
# DEBUG STUFF!
# print("!!!!!!CURRENT LOCATION OF AGENTS AND THEIR TARGETS!!!!!!")
# for agent in self.agents:
#
# print("AGENT: %s - Position:\n %s \n TASK: %s - Position:\n %s" % (agent.name,
# agent.pose.position,
# agent.task_queue.peek().name,
# agent.task_queue.peek().pose.position))
# print("---------------------------------------------------------")
def find_task_in_queue(self, task_name):
for task in self.world_task_queue.get_queue():
if task.name == task_name:
return task
print("Could not find the task..")
return None
def find_agent_in_list(self, agent_name):
for agent in self.agents:
print("ListAgent: %s == InputAgent: %s = %s" % (agent.name, agent_name, agent.name == agent_name) )
if agent.name == agent_name:
return agent
print [ agent.name for agent in self.agents]
print "Could not find the agent!"
return None
def check_scenario_complete(self):
for t in self.world_task_queue.get_queue():
if not t.is_complete:
return False
timer = Timer()
while not self.ros_controller.got_end_confirmation:
message = String()
message.data = "finished"
self.ros_controller.publish_update_message(message)
message.data = "end time|%s" %timer.get_elasped_time()
self.ros_controller.publish_update_message(message)
time.sleep(0.05)
print("All assigned tasks have been completed!")
self.restart = True
def add_agent(self, agent):
self.agents.append(agent)
def add_task(self, task):
self.tasks.enqueue(task)
self.world_task_queue.enqueue(task)
def get_nearest_tasks(self, agent):
nearest_task = None
# Finds the furthest possible distance between two points of a rectangluar grid.
nearest_dist = math.sqrt(math.pow(self.size[0], 2) + math.pow(self.size[1], 2))
for task in agent.task_queue.get_queue():
if not task.assigned_agent:
dist = math.sqrt(math.pow(task.pose.position.x - agent.pose.position.x, 2)
+ math.pow(task.pose.position.y - agent.pose.position.y, 2)
+ math.pow(task.pose.position.z - agent.pose.position.z, 2))
if dist < nearest_dist:
nearest_task = task
# Ensures no "current task" is assigned to multiple agents,
#self.tasks.remove(nearest_task)
task.assigned_agent = agent
return nearest_task
def sort_nearest_tasks(self, agent, queue):
compatible_list = [task for task in queue.queue if task.is_agent_compatible(agent)]
dist_list = []
for task in compatible_list:
dist = math.sqrt(math.pow(task.pose.position.x - agent.pose.position.x, 2)
+ math.pow(task.pose.position.y - agent.pose.position.y, 2)
+ math.pow(task.pose.position.z - agent.pose.position.z, 2))
dist_list.append(dist)
zipped = zip(dist_list, compatible_list)
zipped.sort()
dist_list, sorted_list = zip(*zipped)
queue.queue = list(sorted_list)
return queue
def get_compabitible_task_for_agent(self, agent, queue):
compatible_task_list = TaskQueue()
for task in self.tasks.get_queue():
if task.is_agent_compatible(agent):
compatible_task_list.enqueue(task)
if compatible_task_list.is_empty():
print "No compatible tasks found for agent %s" %agent.name
return None
return compatible_task_list
def assign_tasks(self):
for agent in self.agents:
# Unneccessary but whatever.
task_queue = copy.copy(self.world_task_queue)
agent.task_queue = self.sort_nearest_tasks(agent, task_queue)
def generate_agents(self, num_of_agents=0):
pose1 = Pose()
pose1.position.x = 0
self.add_agent(ConstructionAgent('Construction Agent 1', self.pose_constructor(60, 50, 0, 0, 0, 0, 1)))
self.add_agent(ConstructionAgent('Construction Agent 2', self.pose_constructor(40, 50, 0, 0, 0, 0, 1)))
self.add_agent(ConstructionAgent('Construction Agent 3', self.pose_constructor(50, 40, 0, 0, 0, 0, 1)))
self.add_agent(TransportAgent('Transport Agent 1', pose=self.pose_constructor(50, 60, 0, 0, 0, 0, 1)))
def pose_constructor(self, posX, posY, posZ, quadX, quadY, quadZ, quadW):
tempPose = Pose()
tempPose.position.x = posX
tempPose.position.y = posY
tempPose.position.z = posZ
tempPose.orientation.x = quadX
tempPose.orientation.y = quadY
tempPose.orientation.z = quadZ
tempPose.orientation.w = quadW
return tempPose
def generate_world_tasks(self, no_of_task=0):
self.add_task(ConstructionTask('Construction Task 1',"Construction required in this area",
pose=self.pose_constructor(50, 10, 0, 0, 0, 0, 1)))
self.add_task(ConstructionTask('Construction Task 2', "Construction required in this area",
pose=self.pose_constructor(60, 20, 0, 0, 0, 0, 1)))
self.add_task(ConstructionTask('Construction Task 3', "Construction required in this area",
pose=self.pose_constructor(40, 27, 0, 0, 0, 0, 1)))
self.add_task(ConstructionTask('Construction Task 4', "Construction required in this area",
pose=self.pose_constructor(10, 65, 0, 0, 0, 0, 1)))
self.add_task(ConstructionTask('Construction Task 5', "Construction required in this area",
pose=self.pose_constructor(65, 40, 0, 0, 0, 0, 1)))
self.add_task(ConstructionTask('Construction Task 6', "Construction required in this area",
pose=self.pose_constructor(15, 50, 0, 0, 0, 0, 1)))
self.add_task(TransportTask('Transport Task 1', "Transport required",
pose=self.pose_constructor(45, 65, 0, 0, 0, 0, 1), end_pose=self.pose_constructor(10, 80, 0, 0, 0, 0, 1)))
self.add_task(TransportTask('Transport Task 2', "Transport required",
pose=self.pose_constructor(15, 75, 0, 0, 0, 0, 1),
end_pose=self.pose_constructor(30, 10, 0, 0, 0, 0, 1)))
self.add_task(TransportTask('Transport Task 3', "Transport required",
pose=self.pose_constructor(55, 70, 0, 0, 0, 0, 1),
end_pose=self.pose_constructor(35, 80, 0, 0, 0, 0, 1)))
class Mediator:
def __init__(self, p1=0.4, p2=0.5, iteration_count=100000):
self.iteration_count = iteration_count
self.task_in_system_count = 0
self.current_tick = 0
self.handled_count = 0
self.refused_count = 0
self.states = []
self.p1 = p1
self.source = Source()
self.queue = TaskQueue(2)
self.handlers = [
Handler(p1, LemerGenerator(209715120, 3, 7)),
Handler(p2, LemerGenerator(209715120, 3, 7))
]
def run(self):
self.queue.tick()
for i in range(self.iteration_count):
self.tick()
counter = Counter(self.states)
for key in counter.keys():
counter[key] = counter[key] / self.iteration_count
print('P{0} = {1}'.format(key, counter[key]))
Loch = self.queue.sum_of_sizes / self.iteration_count
Lc = self.task_in_system_count / self.iteration_count
A = self.handled_count / self.iteration_count
Кkan2 = self.handlers[1].total_task_count / self.iteration_count
print()
print('Potk = {0}'.format(self.refused_count /
self.source.total_task_count))
print('Pбл = 0')
print('Loch = {0}'.format(Loch))
print('Lc = {0}'.format(Lc))
print('Q = {0}'.format(self.handled_count /
self.source.total_task_count))
print('A = {0}'.format(A))
print('Woch = {0}'.format(Loch / A))
# print('Wc = {0}'.format(Lc/A))
print('Wc = {0}'.format(1 / (1 - self.p1) + (Loch + Кkan2) / A))
print('Ккан1 = {0}'.format(self.handlers[0].total_task_count /
self.iteration_count))
print('Ккан2 = {0}'.format(Кkan2))
def tick(self):
self.current_tick += 1
handler_result = self.handlers[1].tick()
if handler_result is not None:
self.handled_count += 1
if len(self.queue) > 0:
task = self.queue.dequeue()
self.handlers[1].set_task(task)
handler_result = self.handlers[0].tick()
if handler_result is not None:
if not self.handlers[1].is_busy():
self.handlers[1].set_task(handler_result)
else:
if self.queue.has_place():
self.queue.enqueue(handler_result)
else:
self.refused_count += 1
source_result = self.source.tick()
if source_result is not None:
if not self.handlers[0].is_busy():
self.handlers[0].set_task(source_result)
else:
self.refused_count += 1
self.queue.tick()
state = '{0}{1}{2}{3}'.format(str(self.source), str(self.handlers[0]),
str(self.queue), str(self.handlers[1]))
self.states.append(state)
self.task_in_system_count += len(self.queue) + len(
self.handlers[0]) + len(self.handlers[1])
class Mediator:
def __init__(self, p=0.75, p1=0.8, p2=0.5, iteration_count=100000):
self.current_tick = 0
self.handled_tasks = []
self.states = []
self.iteration_count = iteration_count
self.busy_count = 0
self.source = Source(p, LemerGenerator(209715120, 3, 7))
self.queue = TaskQueue(2)
self.handlers = [
Handler(p1, LemerGenerator(209715120, 3, 7)),
Handler(p2, LemerGenerator(209715120, 3, 7))
]
def run(self):
for i in range(self.iteration_count):
self.tick()
state_count = len(self.states)
counter = Counter(self.states)
for key in counter.keys():
counter[key] = counter[key] / state_count
print('P {0} = {1}'.format(key, counter[key]))
print('Loch = {0}'.format(self.queue.sum_of_sizes / len(self.states)))
print('A = {0}'.format(len(self.handled_tasks) / len(self.states)))
print('Kkan = {0}'.format(self.busy_count / len(self.states)))
def tick(self):
self.current_tick += 1
for handler in self.handlers:
handler_result = handler.tick()
if handler_result is not None:
self.handled_tasks.append(handler_result)
for handler in self.handlers:
if len(self.queue) == 0:
break
if not handler.is_busy():
task = self.queue.dequeue()
handler.set_task(task)
if not self.source.blocked:
source_result = self.source.tick()
if source_result is not None:
if self.queue.has_place():
self.queue.enqueue(source_result)
else:
self.source.block(source_result)
else:
if self.queue.has_place():
task = self.source.unblock()
self.queue.enqueue(task)
for handler in self.handlers:
if len(self.queue) == 0:
break
if not handler.is_busy():
task = self.queue.dequeue()
handler.set_task(task)
self.queue.tick()
if self.handlers[0].is_busy():
self.busy_count += 1
elif self.handlers[1].is_busy():
self.busy_count += 1
state = '{0}{1}{2}'.format(
str(self.source), str(self.queue),
''.join(map(lambda h: str(h), self.handlers)))
self.states.append(state)
#!/usr/bin/python3
from ActionDatabase import ActionDatabase
import time
import subprocess as sp
import os
from TaskQueue import TaskQueue
print("Starting Friday...")
action_db = ActionDatabase("./actions")
print("Action DB ready")
print("Starting API server")
api = sp.Popen(["python3", "api.py"], cwd='.')
print("API ready")
def execute(task):
print('Executing command ', task['command'])
process = action_db.findAndExec(task['command'], str(task['id']))
if not process:
return False
else:
return True
queue = TaskQueue()
queue.watch(execute)
while True:
pass
class scheduler():
def __init__(self, num_workers=3):
self.monitor = Monitor()
self.google_queue = TaskQueue(-1,
"Google",
self.monitor,
num_workers=num_workers)
self.azure_queue = TaskQueue(-1,
"Azure",
self.monitor,
num_workers=num_workers)
self.amazon_queue = TaskQueue(-1,
"Amazon",
self.monitor,
num_workers=num_workers)
self.maxID = 1
self.start_reassignmet()
def push_task(self, task):
# when certain platform is down, its queue size will be set as bif value to deny enqueue
if self.monitor.googleOn:
g_size = self.google_queue.qsize()
else:
g_size = 100000
if self.monitor.azureOn:
m_size = self.azure_queue.qsize()
else:
m_size = 100000
if self.monitor.awsOn:
a_size = self.amazon_queue.qsize()
else:
a_size = 100000
if g_size < 100000 and g_size == m_size:
if m_size == a_size:
ran = random.randint(0, 2)
else:
ran = random.randint(0, 1)
if ran == 0:
task["platform"] = "Google"
self.google_queue.put(task)
print("choose google")
elif ran == 1:
task["platform"] = "Azure"
self.azure_queue.put(task)
print("choose azure")
elif ran == 2:
task["platform"] = "AWS"
self.amazon_queue.put(task)
print "choose aws"
elif g_size < m_size and g_size < a_size:
task["platform"] = "Google"
self.google_queue.put(task)
print("choose google")
elif m_size < a_size:
task["platform"] = "Azure"
self.azure_queue.put(task)
print("choose azure")
else:
task["platform"] = "AWS"
self.amazon_queue.put(task)
print "choose aws"
def task2Json(self, upload_time, mission, path, filename, bucket):
self.maxID += 1
task = {
"id": self.maxID,
"mission": mission,
"path": path,
"file_name": filename,
"time_stamp": time.time(),
"platform": " ",
"bucket": bucket,
"uploadtime": upload_time
}
return task
def input(self, mission, upload_time, path, filename, bucket_name):
if self.maxID == 1:
t1 = time.time()
self.monitor.connection_test("google", 1, 1)
self.monitor.connection_test("azure", 1, 1)
self.monitor.connection_test("AWS", 1, 1)
self.monitor.start_regular_monitor()
t2 = time.time()
upload_time = upload_time + (t2 - t1)
self.push_task(
self.task2Json(upload_time, mission, path, filename, bucket_name))
def reassignment(self):
start_time = time.time()
count = 0
while count < 60:
count += 1
try:
task = custom_api.wait_list.pop(0)
task["platform"] = "AWS"
print("Reassign task %s because of the failure of %s" %
(task["id"], task["platform"]))
self.push_task(task)
time.sleep(10 - ((time.time() - start_time) % 10))
except:
time.sleep(10 - ((time.time() - start_time) % 10))
continue
def start_reassignmet(self):
t_reassign = threading.Thread(target=self.reassignment,
name="reassignment")
t_reassign.daemon = True
t_reassign.start()
def test_reassignment(self, mission, upload_time, path, filename,
bucket_name):
task = self.task2Json(upload_time, mission, path, filename,
bucket_name)
self.amazon_queue.put(task)
