def getSingleFileLinks( text ): links = [] doLog( "getSingleFileLinks start" ) dp = xbmcgui.DialogProgress() dp.create("Searching for single file links", "") doLog( "getSingleFileLinks finding matches" ) link_matches = re.findall( r'(http://hotfile.com/[^"]+/([^"]+).(flv|mpg|mpeg|mov|avi|mkv).html)', text, re.M|re.S ) if not link_matches: doLog( "getSingleFileLinks no links found" ) dp.update(100,"None found") del dp return links dp.update(0, "Removing duplicate links") link_matches[:] = dedupeList( link_matches, 0 ) num_fetch_threads = 4 url_queue = TaskQueue() def validateUrlsThread(i, q): """This is the worker thread function. It processes items in the queue one after another. These daemon threads go into an infinite loop, and only exit when the main thread ends. """ while True: url = q.get() valid_link = getHFLink( hotfile_user, hotfile_pass, url ) if valid_link: links.append( valid_link ) else: doLog( "getSingleFileLinks NOT ADDING: " + url ) q.task_done() # Set up some threads to validate the urls for i in range(num_fetch_threads): worker = Thread(target=validateUrlsThread, args=(i, url_queue,)) worker.setDaemon(True) worker.start() if link_matches: doLog( "getSingleFileLinks iterate over link matches" ) for link,name,extension in link_matches: parsed_url = urlparse(link) dp.update(0, "Validating link:", parsed_url[1], parsed_url[2].split("/")[-1]) doLog( "getSingleFileLinks appending:\nLINK: %s\nNAME: %s\nEXT: %s" % ( link, name, extension ) ) url_queue.put(link) # wait for queue to empty which means all urls validated url_queue.join() doLog( "getSingleFileLinks done" ) dp.update(100, "Done getting single file links!") del dp return links
def getSingleFileLinks(text): links = [] doLog("getSingleFileLinks start") dp = xbmcgui.DialogProgress() dp.create("Searching for single file links", "") doLog("getSingleFileLinks finding matches") link_matches = re.findall( r'(http://hotfile.com/[^"]+/([^"]+).(flv|mpg|mpeg|mov|avi|mkv).html)', text, re.M | re.S) if not link_matches: doLog("getSingleFileLinks no links found") dp.update(100, "None found") del dp return links dp.update(0, "Removing duplicate links") link_matches[:] = dedupeList(link_matches, 0) num_fetch_threads = 4 url_queue = TaskQueue() def validateUrlsThread(i, q): """This is the worker thread function. It processes items in the queue one after another. These daemon threads go into an infinite loop, and only exit when the main thread ends. """ while True: url = q.get() valid_link = getHFLink(hotfile_user, hotfile_pass, url) if valid_link: links.append(valid_link) else: doLog("getSingleFileLinks NOT ADDING: " + url) q.task_done() # Set up some threads to validate the urls for i in range(num_fetch_threads): worker = Thread(target=validateUrlsThread, args=( i, url_queue, )) worker.setDaemon(True) worker.start() if link_matches: doLog("getSingleFileLinks iterate over link matches") for link, name, extension in link_matches: parsed_url = urlparse(link) dp.update(0, "Validating link:", parsed_url[1], parsed_url[2].split("/")[-1]) doLog( "getSingleFileLinks appending:\nLINK: %s\nNAME: %s\nEXT: %s" % (link, name, extension)) url_queue.put(link) # wait for queue to empty which means all urls validated url_queue.join() doLog("getSingleFileLinks done") dp.update(100, "Done getting single file links!") del dp return links
class Bot(threading.Thread): '''Producer/Consumer Bot utilising TaskQueue.''' def __init__(self, id, buffer=5): threading.Thread.__init__(self) self.id = id self.daemon = True self.buffer_time = buffer self.q = TaskQueue() self.last_action_time = 0 self.new_task = threading.Event() self.task_done = threading.Event() def schedule(self, tasks): '''Schedule a new task by starting an independent thread. - tasks (list): tasks to be scheduled.''' t = threading.Thread(target=self._add_tasks, args=(tasks, ), daemon=True) t.start() def _add_tasks(self, tasks): '''Adding tasks from a list to task queue and waiting if queue is full.''' for task in tasks: if self.q.full(): print('Producer: Queue full - sleeping') self.task_done.wait() print('Producer: Queue not full now, adding task..') self.task_done.clear() self.q.put(task) print(f'Producer: Scheduled {task}') self.new_task.set() def run(self): '''Runs the thread which executes tasks. - stay idle until scheduled task time''' while True: if self.q.empty(): self.new_task.wait() print('Bot: Woke up') self.new_task.clear() task = self._idle() print(f'Bot executing {task}') task.execute() self.last_action_time = time() self.q.task_done() self.task_done.set() def _idle(self): '''Wait for next scheduled task in queue. - If tasks are added to queue, check next task again''' next_task = self.q.peek() time_until = self._time_until(next_task) print(f'Bot: Time until next task {time_until}') flag = self.new_task.wait(timeout=time_until) if flag: print('Bot: New task added to queue - checking next task') self.new_task.clear() return self._idle() else: return self.q.get() def _time_until(self, task): '''Calculate time in seconds until scheduled task. - Convert task.time datetime to epoch time - Add buffer delay if scheduled task is too close to last action time''' now = time() time_since = now - self.last_action_time scheduled = int(task.time.strftime('%s')) time_until = scheduled - now if time_until < 0: if time_since < self.buffer_time: time_until = time_until + self.buffer_time else: time_until = 0 return time_until
class scheduler(): def __init__(self, num_workers=3): self.monitor = Monitor() self.google_queue = TaskQueue(-1, "Google", self.monitor, num_workers=num_workers) self.azure_queue = TaskQueue(-1, "Azure", self.monitor, num_workers=num_workers) self.amazon_queue = TaskQueue(-1, "Amazon", self.monitor, num_workers=num_workers) self.maxID = 1 self.start_reassignmet() def push_task(self, task): # when certain platform is down, its queue size will be set as bif value to deny enqueue if self.monitor.googleOn: g_size = self.google_queue.qsize() else: g_size = 100000 if self.monitor.azureOn: m_size = self.azure_queue.qsize() else: m_size = 100000 if self.monitor.awsOn: a_size = self.amazon_queue.qsize() else: a_size = 100000 if g_size < 100000 and g_size == m_size: if m_size == a_size: ran = random.randint(0, 2) else: ran = random.randint(0, 1) if ran == 0: task["platform"] = "Google" self.google_queue.put(task) print("choose google") elif ran == 1: task["platform"] = "Azure" self.azure_queue.put(task) print("choose azure") elif ran == 2: task["platform"] = "AWS" self.amazon_queue.put(task) print "choose aws" elif g_size < m_size and g_size < a_size: task["platform"] = "Google" self.google_queue.put(task) print("choose google") elif m_size < a_size: task["platform"] = "Azure" self.azure_queue.put(task) print("choose azure") else: task["platform"] = "AWS" self.amazon_queue.put(task) print "choose aws" def task2Json(self, upload_time, mission, path, filename, bucket): self.maxID += 1 task = { "id": self.maxID, "mission": mission, "path": path, "file_name": filename, "time_stamp": time.time(), "platform": " ", "bucket": bucket, "uploadtime": upload_time } return task def input(self, mission, upload_time, path, filename, bucket_name): if self.maxID == 1: t1 = time.time() self.monitor.connection_test("google", 1, 1) self.monitor.connection_test("azure", 1, 1) self.monitor.connection_test("AWS", 1, 1) self.monitor.start_regular_monitor() t2 = time.time() upload_time = upload_time + (t2 - t1) self.push_task( self.task2Json(upload_time, mission, path, filename, bucket_name)) def reassignment(self): start_time = time.time() count = 0 while count < 60: count += 1 try: task = custom_api.wait_list.pop(0) task["platform"] = "AWS" print("Reassign task %s because of the failure of %s" % (task["id"], task["platform"])) self.push_task(task) time.sleep(10 - ((time.time() - start_time) % 10)) except: time.sleep(10 - ((time.time() - start_time) % 10)) continue def start_reassignmet(self): t_reassign = threading.Thread(target=self.reassignment, name="reassignment") t_reassign.daemon = True t_reassign.start() def test_reassignment(self, mission, upload_time, path, filename, bucket_name): task = self.task2Json(upload_time, mission, path, filename, bucket_name) self.amazon_queue.put(task)