def getSingleFileLinks( text ):
links = []
doLog( "getSingleFileLinks start" )
dp = xbmcgui.DialogProgress()
dp.create("Searching for single file links", "")
doLog( "getSingleFileLinks finding matches" )
link_matches = re.findall( r'(http://hotfile.com/[^"]+/([^"]+).(flv|mpg|mpeg|mov|avi|mkv).html)', text, re.M|re.S )
if not link_matches:
doLog( "getSingleFileLinks no links found" )
dp.update(100,"None found")
del dp
return links
dp.update(0, "Removing duplicate links")
link_matches[:] = dedupeList( link_matches, 0 )
num_fetch_threads = 4
url_queue = TaskQueue()
def validateUrlsThread(i, q):
"""This is the worker thread function.
It processes items in the queue one after
another. These daemon threads go into an
infinite loop, and only exit when
the main thread ends.
"""
while True:
url = q.get()
valid_link = getHFLink( hotfile_user, hotfile_pass, url )
if valid_link:
links.append( valid_link )
else:
doLog( "getSingleFileLinks NOT ADDING: " + url )
q.task_done()
# Set up some threads to validate the urls
for i in range(num_fetch_threads):
worker = Thread(target=validateUrlsThread, args=(i, url_queue,))
worker.setDaemon(True)
worker.start()
if link_matches:
doLog( "getSingleFileLinks iterate over link matches" )
for link,name,extension in link_matches:
parsed_url = urlparse(link)
dp.update(0, "Validating link:", parsed_url[1], parsed_url[2].split("/")[-1])
doLog( "getSingleFileLinks appending:\nLINK: %s\nNAME: %s\nEXT: %s" % ( link, name, extension ) )
url_queue.put(link)
# wait for queue to empty which means all urls validated
url_queue.join()
doLog( "getSingleFileLinks done" )
dp.update(100, "Done getting single file links!")
del dp
return links
def getSingleFileLinks(text):
links = []
doLog("getSingleFileLinks start")
dp = xbmcgui.DialogProgress()
dp.create("Searching for single file links", "")
doLog("getSingleFileLinks finding matches")
link_matches = re.findall(
r'(http://hotfile.com/[^"]+/([^"]+).(flv|mpg|mpeg|mov|avi|mkv).html)',
text, re.M | re.S)
if not link_matches:
doLog("getSingleFileLinks no links found")
dp.update(100, "None found")
del dp
return links
dp.update(0, "Removing duplicate links")
link_matches[:] = dedupeList(link_matches, 0)
num_fetch_threads = 4
url_queue = TaskQueue()
def validateUrlsThread(i, q):
"""This is the worker thread function.
It processes items in the queue one after
another. These daemon threads go into an
infinite loop, and only exit when
the main thread ends.
"""
while True:
url = q.get()
valid_link = getHFLink(hotfile_user, hotfile_pass, url)
if valid_link:
links.append(valid_link)
else:
doLog("getSingleFileLinks NOT ADDING: " + url)
q.task_done()
# Set up some threads to validate the urls
for i in range(num_fetch_threads):
worker = Thread(target=validateUrlsThread, args=(
i,
url_queue,
))
worker.setDaemon(True)
worker.start()
if link_matches:
doLog("getSingleFileLinks iterate over link matches")
for link, name, extension in link_matches:
parsed_url = urlparse(link)
dp.update(0, "Validating link:", parsed_url[1],
parsed_url[2].split("/")[-1])
doLog(
"getSingleFileLinks appending:\nLINK: %s\nNAME: %s\nEXT: %s" %
(link, name, extension))
url_queue.put(link)
# wait for queue to empty which means all urls validated
url_queue.join()
doLog("getSingleFileLinks done")
dp.update(100, "Done getting single file links!")
del dp
return links
class Bot(threading.Thread):
'''Producer/Consumer Bot utilising TaskQueue.'''
def __init__(self, id, buffer=5):
threading.Thread.__init__(self)
self.id = id
self.daemon = True
self.buffer_time = buffer
self.q = TaskQueue()
self.last_action_time = 0
self.new_task = threading.Event()
self.task_done = threading.Event()
def schedule(self, tasks):
'''Schedule a new task by starting an independent thread.
- tasks (list): tasks to be scheduled.'''
t = threading.Thread(target=self._add_tasks,
args=(tasks, ),
daemon=True)
t.start()
def _add_tasks(self, tasks):
'''Adding tasks from a list to task queue and waiting if queue is full.'''
for task in tasks:
if self.q.full():
print('Producer: Queue full - sleeping')
self.task_done.wait()
print('Producer: Queue not full now, adding task..')
self.task_done.clear()
self.q.put(task)
print(f'Producer: Scheduled {task}')
self.new_task.set()
def run(self):
'''Runs the thread which executes tasks.
- stay idle until scheduled task time'''
while True:
if self.q.empty():
self.new_task.wait()
print('Bot: Woke up')
self.new_task.clear()
task = self._idle()
print(f'Bot executing {task}')
task.execute()
self.last_action_time = time()
self.q.task_done()
self.task_done.set()
def _idle(self):
'''Wait for next scheduled task in queue.
- If tasks are added to queue, check next task again'''
next_task = self.q.peek()
time_until = self._time_until(next_task)
print(f'Bot: Time until next task {time_until}')
flag = self.new_task.wait(timeout=time_until)
if flag:
print('Bot: New task added to queue - checking next task')
self.new_task.clear()
return self._idle()
else:
return self.q.get()
def _time_until(self, task):
'''Calculate time in seconds until scheduled task.
- Convert task.time datetime to epoch time
- Add buffer delay if scheduled task is too close to last action time'''
now = time()
time_since = now - self.last_action_time
scheduled = int(task.time.strftime('%s'))
time_until = scheduled - now
if time_until < 0:
if time_since < self.buffer_time:
time_until = time_until + self.buffer_time
else:
time_until = 0
return time_until
class scheduler():
def __init__(self, num_workers=3):
self.monitor = Monitor()
self.google_queue = TaskQueue(-1,
"Google",
self.monitor,
num_workers=num_workers)
self.azure_queue = TaskQueue(-1,
"Azure",
self.monitor,
num_workers=num_workers)
self.amazon_queue = TaskQueue(-1,
"Amazon",
self.monitor,
num_workers=num_workers)
self.maxID = 1
self.start_reassignmet()
def push_task(self, task):
# when certain platform is down, its queue size will be set as bif value to deny enqueue
if self.monitor.googleOn:
g_size = self.google_queue.qsize()
else:
g_size = 100000
if self.monitor.azureOn:
m_size = self.azure_queue.qsize()
else:
m_size = 100000
if self.monitor.awsOn:
a_size = self.amazon_queue.qsize()
else:
a_size = 100000
if g_size < 100000 and g_size == m_size:
if m_size == a_size:
ran = random.randint(0, 2)
else:
ran = random.randint(0, 1)
if ran == 0:
task["platform"] = "Google"
self.google_queue.put(task)
print("choose google")
elif ran == 1:
task["platform"] = "Azure"
self.azure_queue.put(task)
print("choose azure")
elif ran == 2:
task["platform"] = "AWS"
self.amazon_queue.put(task)
print "choose aws"
elif g_size < m_size and g_size < a_size:
task["platform"] = "Google"
self.google_queue.put(task)
print("choose google")
elif m_size < a_size:
task["platform"] = "Azure"
self.azure_queue.put(task)
print("choose azure")
else:
task["platform"] = "AWS"
self.amazon_queue.put(task)
print "choose aws"
def task2Json(self, upload_time, mission, path, filename, bucket):
self.maxID += 1
task = {
"id": self.maxID,
"mission": mission,
"path": path,
"file_name": filename,
"time_stamp": time.time(),
"platform": " ",
"bucket": bucket,
"uploadtime": upload_time
}
return task
def input(self, mission, upload_time, path, filename, bucket_name):
if self.maxID == 1:
t1 = time.time()
self.monitor.connection_test("google", 1, 1)
self.monitor.connection_test("azure", 1, 1)
self.monitor.connection_test("AWS", 1, 1)
self.monitor.start_regular_monitor()
t2 = time.time()
upload_time = upload_time + (t2 - t1)
self.push_task(
self.task2Json(upload_time, mission, path, filename, bucket_name))
def reassignment(self):
start_time = time.time()
count = 0
while count < 60:
count += 1
try:
task = custom_api.wait_list.pop(0)
task["platform"] = "AWS"
print("Reassign task %s because of the failure of %s" %
(task["id"], task["platform"]))
self.push_task(task)
time.sleep(10 - ((time.time() - start_time) % 10))
except:
time.sleep(10 - ((time.time() - start_time) % 10))
continue
def start_reassignmet(self):
t_reassign = threading.Thread(target=self.reassignment,
name="reassignment")
t_reassign.daemon = True
t_reassign.start()
def test_reassignment(self, mission, upload_time, path, filename,
bucket_name):
task = self.task2Json(upload_time, mission, path, filename,
bucket_name)
self.amazon_queue.put(task)
