def iterate_minibatches(self,
generate_jobs,
batch_size,
buffer_size,
postprocess_batch=None):
assert buffer_size >= batch_size
buffer_semaphore = BoundedSemaphore(
buffer_size) # jobs currently in buffer
generate_jobs = iter(generate_jobs)
postprocess_batch = postprocess_batch or nop
@background(max_prefetch=-1)
def _load_jobs_and_iterate_batch_sizes():
"""
Loads jobs into queue, yields batch_size every time a generator
can order this batch_size from the database
"""
current_batch = 0
for task in generate_jobs:
buffer_semaphore.acquire()
self.add_jobs(task)
current_batch += 1
if current_batch == batch_size:
yield current_batch # you can now load additional batch_size elements
current_batch = 0
if current_batch != 0:
yield current_batch # load the remaining elements
for allowed_batch_size in _load_jobs_and_iterate_batch_sizes():
batch = self.get_results_batch(allowed_batch_size)
for _ in batch:
buffer_semaphore.release()
yield postprocess_batch(batch)
class BoundedExecutor:
"""BoundedExecutor behaves as a ThreadPoolExecutor which will block on
calls to submit() once the limit given as "bound" work items are queued for
execution.
:param bound: Integer - the maximum number of items in the work queue
:param max_workers: Integer - the size of the thread pool
"""
def __init__(self, bound, max_workers):
self.executor = ThreadPoolExecutor(max_workers=max_workers)
self.semaphore = BoundedSemaphore(bound + max_workers)
"""See concurrent.futures.Executor#submit"""
def submit(self, fn, *args, **kwargs):
self.semaphore.acquire()
try:
future = self.executor.submit(fn, *args, **kwargs)
except:
self.semaphore.release()
raise
else:
future.add_done_callback(lambda x: self.semaphore.release())
return future
"""See concurrent.futures.Executor#shutdown"""
def shutdown(self, wait=True):
self.executor.shutdown(wait)
def _add_token_loop(self, time_delta):
"""Add token every time_delta seconds."""
while True:
try:
BoundedSemaphore.release(self)
except ValueError: # ignore if already max possible value
pass
sleep(time_delta) # ignore EINTR
class RequestManager:
def __init__(self, max_workers):
self.max_workers = max_workers
self.lock = Lock()
self.sem = BoundedSemaphore(max_workers)
self.last_request = Value('d', 0.0)
self.last_restricted_request = Value('d', 0.0)
@contextmanager
def normal_request(self):
with self.lock:
self.sem.acquire()
time.sleep(
max(
0.0, self.last_restricted_request.value + 0.6 +
(random.random() * 0.15) - time.time()))
try:
yield
except Exception as e:
raise e
finally:
self.last_request.value = time.time()
self.sem.release()
@contextmanager
def restricted_request(self):
with self.lock:
for i in range(self.max_workers):
self.sem.acquire()
time.sleep(
max(
0.0, self.last_request.value + 0.6 + (random.random() * 0.15) -
time.time()))
try:
yield
except Exception as e:
raise e
finally:
self.last_request.value = time.time()
self.last_restricted_request.value = time.time()
for i in range(self.max_workers):
self.sem.release()
def process(self, input_filepath, msi_loci, config):
self.__reset()
# Generate dictionary read counts for loci
counts = {}
loci = []
for locus in msi_loci:
counts[locus.locus()] = {}
loci.append(locus)
# Generate input and output queues and (mutex) semaphores
# for each.
queue_out = Queue()
queue_in = Queue()
queue_full = BoundedSemaphore(100)
full = Semaphore(0)
empty = BoundedSemaphore(40)
mutex_out = Semaphore(1)
mutex_in = Semaphore(1)
# Set amount of consumer threads; minimum one.
consumer_threads = config['threads'] - 1
if consumer_threads < 1:
consumer_threads = 1
# Create producer thread; currently only using single thread
# since I/O is more of the limiter than CPU bound processes.
self.__producer = Process(target=self.extract_reads,
args=(input_filepath, msi_loci, full, empty,
mutex_out, queue_in, consumer_threads))
self.__producer.start()
# Spawn the set amount of threads/processes
if self.debug_output:
tprint('Main> Generating {0} analyzer process(es).'.format(
consumer_threads))
for i in range(0, consumer_threads):
p = Process(target=self.read_analyzer,
args=(queue_in, queue_out, full, empty, mutex_in,
mutex_out, queue_full))
self.__consumers.append(p)
self.__consumers[-1].start()
# Iterate through the loci, fetching any reads and pushing them to
# the pool of threads, collecting the output as they process it.
query_delay = 0.050 # In seconds
loop_counter = 0
proc_check_interval = 100
while (not queue_out.empty() or self.has_live_threads()):
# Sleep for the set amount of time so the queue isn't constantly
# getting hammered with queries
time.sleep(query_delay)
loop_counter += 1
if loop_counter % proc_check_interval is 0:
# Time to check that the consumers
# didn't die while the producer is still producing
mutex_out.acquire()
self.status_check(queue_out.qsize())
mutex_out.release()
while not queue_out.empty():
# There is data on the queue to be processed;
# the return from the queue should be a tuple
# with (locus, repeat_count)
mutex_out.acquire()
result = queue_out.get()
locus = result[0]
repeat_count = result[1]
if repeat_count >= 0:
if locus not in counts:
counts[locus] = {}
if repeat_count not in counts[locus]:
counts[locus][repeat_count] = 0
counts[locus][repeat_count] += 1
mutex_out.release()
queue_full.release()
if not self.has_live_threads():
# All processes should have terminated.
if self.debug_output:
tprint('Main> All processes complete.')
break
# end while loop
return counts
from multiprocessing import BoundedSemaphore
if __name__ == '__main__':
bsem = BoundedSemaphore(3)
bsem.acquire()
bsem.release()
bsem.release() # 在 Mac OS 平台上不会抛出异常
"""
Traceback (most recent call last):
File "D:/code/picpython/pp_046_多进程同步之BoundedSemaphore.py", line 8, in <module>
bsem.release()
ValueError: semaphore or lock released too many times
"""
def process(self, input_filepath, msi_loci, config):
self.__reset()
# Generate dictionary read counts for loci
counts = {}
loci = []
for locus in msi_loci:
counts[locus.locus()] = {}
loci.append(locus)
# Generate input and output queues and (mutex) semaphores
# for each.
queue_out = Queue()
queue_in = Queue()
queue_full = BoundedSemaphore(100)
full = Semaphore(0)
empty = BoundedSemaphore(40)
mutex_out = Semaphore(1)
mutex_in = Semaphore(1)
# Set amount of consumer threads; minimum one.
consumer_threads = config['threads'] - 1
if consumer_threads < 1:
consumer_threads = 1
# Create producer thread; currently only using single thread
# since I/O is more of the limiter than CPU bound processes.
self.__producer = Process(target=self.extract_reads, args=(
input_filepath,
msi_loci,
full,
empty,
mutex_out,
queue_in,
consumer_threads))
self.__producer.start()
# Spawn the set amount of threads/processes
if self.debug_output:
tprint('Main> Generating {0} analyzer process(es).'.format(consumer_threads))
for i in range(0, consumer_threads):
p = Process(target=self.read_analyzer, args=(
queue_in,
queue_out,
full,
empty,
mutex_in,
mutex_out,
queue_full))
self.__consumers.append(p)
self.__consumers[-1].start()
# Iterate through the loci, fetching any reads and pushing them to
# the pool of threads, collecting the output as they process it.
query_delay = 0.050 # In seconds
loop_counter = 0
proc_check_interval = 100
while (not queue_out.empty() or self.has_live_threads()):
# Sleep for the set amount of time so the queue isn't constantly
# getting hammered with queries
time.sleep(query_delay)
loop_counter += 1
if loop_counter % proc_check_interval is 0:
# Time to check that the consumers
# didn't die while the producer is still producing
mutex_out.acquire()
self.status_check(queue_out.qsize())
mutex_out.release()
while not queue_out.empty():
# There is data on the queue to be processed;
# the return from the queue should be a tuple
# with (locus, repeat_count)
mutex_out.acquire()
result = queue_out.get()
locus = result[0]
repeat_count = result[1]
if repeat_count >= 0:
if locus not in counts:
counts[locus] = {}
if repeat_count not in counts[locus]:
counts[locus][repeat_count] = 0
counts[locus][repeat_count] += 1
mutex_out.release()
queue_full.release()
if not self.has_live_threads():
# All processes should have terminated.
if self.debug_output:
tprint('Main> All processes complete.')
break
# end while loop
return counts
class Zydra():
def __init__(self):
self.start_time = time.monotonic()
self.process_lock = BoundedSemaphore(value=cpu_count())
self.counter_lock = threading.BoundedSemaphore(value=1)
self.banner()
self.stop = Queue(maxsize=1)
self.stop.put(False)
self.count = Queue(maxsize=1)
self.threads = []
self.name = Queue(maxsize=1)
self.name.put(str("a"))
self.process_count = 0
self.limit_process = 500
self.shot = 5000
def fun(self, string):
list = []
fer = ['-', "\\", "|", '/']
for char in string:
list.append(char)
timer = 0
pointer = 0
fer_pointer = 0
while timer < 20:
list[pointer] = list[pointer].upper()
print("\r" +
self.blue("".join(str(x)
for x in list) + " " + fer[fer_pointer]),
end="")
list[pointer] = list[pointer].lower()
max_fer = len(fer) - 1
if fer_pointer == max_fer:
fer_pointer = -1
max = len(list) - 1
if pointer == max:
pointer = -1
pointer += 1
fer_pointer += 1
timer += 1
time.sleep(0.1)
if timer == 20:
print("\r" + self.blue(string) + "\n", end="")
return
def blue(self, string):
return colored(string, "blue", attrs=['bold'])
def green(self, string):
return colored(string, "green", attrs=['bold'])
def yellow(self, string):
return colored(string, "yellow", attrs=['bold'])
def red(self, string):
return colored(string, "red", attrs=['bold'])
def bwhite(self, string):
return colored(string, "white", attrs=['bold'])
def white(self, string):
return colored(string, "white")
def detect_file_type(self, file):
if str(file).split(".")[-1] == "rar":
return "rar"
elif str(file).split(".")[-1] == "zip":
return "zip"
elif str(file).split(".")[-1] == "pdf":
return "pdf"
else:
return "text"
def count_word(self, dict_file):
count = 0
with open(dict_file, "r") as wordlist:
for line in wordlist:
count += 1
return count
def count_possible_com(self, chars, min, max):
x = min
possible_com = 0
while x <= max:
possible_com += len(chars)**x
x += 1
return possible_com
def counter(self, max_words):
self.counter_lock.acquire()
num = self.count.get()
# print(self.count)
# print(num)
if num != 0:
self.count.put(num - 1)
current_word = max_words - int(num) + 1
percent = (100 * current_word) / max_words
width = (current_word + 1) / (max_words / 42) # 100 / 25
bar = "\t" + self.white("Progress : [") + "#" * int(width) + " " * (42 - int(width)) \
+ "] " + self.yellow(str("%.3f" % percent) + " %")
# time.sleep(1)
sys.stdout.write(u"\t\u001b[1000D" + bar)
sys.stdout.flush()
self.counter_lock.release()
def handling_too_many_open_files_error(self):
if self.process_count == self.limit_process:
for x in self.threads:
x.join()
self.threads = []
self.limit_process += 500
def search_zip_pass(self, passwords_list, compress_file, max_words):
try:
temp_file = self.create_temporary_copy(compress_file,
passwords_list[1])
for word in passwords_list:
password = word.strip('\r').strip('\n')
stop = self.stop.get()
self.stop.put(stop)
if stop is False: # if find password dont doing more
self.counter(max_words)
try:
with zipfile.ZipFile(temp_file, "r") as zfile:
zfile.extractall(
pwd=bytes(password, encoding='utf-8'))
self.stop.get()
self.stop.put(True)
time.sleep(3)
print("\n\t" + self.green("[+] Password Found: " +
password) + "\n")
break
except Exception as e:
# print(e)
pass
else:
break
if os.path.isfile(temp_file):
os.remove(os.path.abspath(temp_file))
# last_process_number = int(max_words / self.shot) + (max_words % self.shot > 0)
if str(self.last_process_number) in str(current_process().name):
time.sleep(20)
stop = self.stop.get()
self.stop.put(stop)
if stop is False:
print("\n\t" + self.red("[-] password not found") + "\n")
else:
pass
self.process_lock.release()
except KeyboardInterrupt:
self.process_lock.release()
def create_temporary_copy(self, file, word):
name = self.name.get()
name2 = str(word)
self.name.put(name2)
directory_path = "temp_directory"
try:
os.mkdir(directory_path)
except FileExistsError:
pass
temp_file_name = "temp" + name + "." + self.file_type
temp_file_path = directory_path + '/' + temp_file_name # linux path
shutil.copy2(file, temp_file_path)
return temp_file_path
def delete_temporary_directory(self):
if os.path.exists("temp_directory"):
shutil.rmtree("temp_directory")
def search_rar_pass(self, passwords_list, compress_file, max_words):
try:
temp_file = self.create_temporary_copy(compress_file,
passwords_list[1])
for word in passwords_list:
password = word.strip('\r').strip('\n')
stop = self.stop.get()
self.stop.put(stop)
if stop is False: # if find password dont doing more
self.counter(max_words)
try:
with rarfile.RarFile(temp_file, "r") as rfile:
# print(password) very useful for trouble shooting
rfile.extractall(pwd=password)
self.stop.get()
self.stop.put(True)
time.sleep(3)
print("\n\t" + self.green("[+] Password Found: " +
password + '\n'))
break
except Exception as e:
# print(e)
pass
else:
break
if os.path.isfile(temp_file):
os.remove(os.path.abspath(temp_file))
# last_process_number = int(max_words / 500) + (max_words % 500 > 0)
if str(self.last_process_number) in str(current_process().name):
time.sleep(20)
stop = self.stop.get()
self.stop.put(stop)
if stop is False:
print("\n\t" + self.red("[-] password not found") + "\n")
else:
pass
self.process_lock.release()
except KeyboardInterrupt:
self.process_lock.release()
def search_pdf_pass(self, passwords_list, file, max_words):
try:
temp_file = self.create_temporary_copy(file, passwords_list[1])
for word in passwords_list:
password = word.strip('\r').strip('\n')
stop = self.stop.get()
self.stop.put(stop)
if stop is False: # if find password dont doing more
self.counter(max_words)
proc = subprocess.Popen([
'qpdf', "--password="******"\n\t" +
self.green("[+] Password Found: " + password))
print("\t" + self.blue("[*]") +
self.white(" Your decrypted file is ") +
self.bwhite(self.decrypted_file_name) + "\n")
# self.end_time()
break
elif status == 2:
pass
else:
break
# for thread in self.threads:
# print(thread)
# last_process_number = int(max_words / 500) + (max_words % 500 > 0)
if str(self.last_process_number) in str(current_process().name):
time.sleep(20)
stop = self.stop.get()
self.stop.put(stop)
if stop is False:
print("\n\t" + self.red("[-] password not found") + "\n")
else:
pass
self.process_lock.release()
except KeyboardInterrupt:
self.process_lock.release()
def search_shadow_pass(self, passwords_list, salt_for_crypt, crypt_pass,
max_words, user):
try:
for word in passwords_list:
password = word.strip('\r').strip('\n')
stop = self.stop.get()
self.stop.put(stop)
if stop is False: # if find password dont doing more
self.counter(max_words)
cryptword = crypt.crypt(password, salt_for_crypt)
if cryptword == crypt_pass:
self.stop.get()
self.stop.put(True)
time.sleep(4)
print("\n\t" +
self.green("[+] Password Found: " + password) +
"\n")
break
else:
pass
else:
break
# print(last_process_number)
# print(str(current_process().name))
if str(self.last_process_number) in str(current_process().name):
time.sleep(20)
stop = self.stop.get()
self.stop.put(stop)
if stop is False:
print("\n\t" + self.red("[-] password not found") + "\n")
else:
pass
self.process_lock.release()
except KeyboardInterrupt:
self.process_lock.release()
def last_words_check(self, max_words, passwords_list, file):
while True:
if self.stop is True:
exit(0)
elif self.count == len(passwords_list): # self_cont kam mishe
if self.file_type is "rar":
self.search_rar_pass(passwords_list, file, max_words)
if self.stop is False:
print("\n\t" + self.red("[-] Password not found") + "\n")
self.delete_temporary_directory()
self.end_time()
return
else:
pass
def dict_guess_password(self, dict_file, file):
last_check = 0
passwords_group = []
possible_words = self.count_word(dict_file)
self.last_process_number = int(
possible_words / self.shot) + (possible_words % self.shot > 0)
self.count.put(possible_words)
self.file_type = self.detect_file_type(file)
self.fun("Starting password cracking for " + file)
print("\n " + self.blue("[*]") +
self.white(" Count of possible passwords: ") +
self.bwhite(str(possible_words)))
if self.file_type == "text":
file = open(file)
for line in file.readlines():
self.count.get()
self.count.put(possible_words)
crypt_pass = line.split(':')[1].strip(' ')
if crypt_pass not in ['*', '!', '!!']:
user = line.split(':')[0]
print(" " + self.blue("[**]") +
self.white(" cracking Password for: ") +
self.bwhite(user))
algorythm = crypt_pass.split('$')[1].strip(' ')
salt = crypt_pass.split('$')[2].strip(' ')
salt_for_crypt = '$' + algorythm + '$' + salt + '$'
with open(dict_file, "r") as wordlist:
for word in wordlist:
passwords_group.append(word)
last_check += 1
self.handling_too_many_open_files_error()
if (len(passwords_group)
== self.shot) or (possible_words -
last_check == 0):
passwords = passwords_group
passwords_group = []
self.process_lock.acquire()
stop = self.stop.get()
self.stop.put(stop)
if stop is False:
t = Process(target=self.search_shadow_pass,
args=(passwords,
salt_for_crypt,
crypt_pass,
possible_words, user))
self.threads.append(t)
self.process_count += 1
t.start()
else:
self.process_lock.release()
else:
continue
for x in self.threads:
x.join()
self.last_process_number *= 2
self.end_time()
elif self.file_type == "zip":
with open(dict_file, "r") as wordlist:
for word in wordlist:
passwords_group.append(word)
last_check += 1
self.handling_too_many_open_files_error()
if (len(passwords_group)
== self.shot) or (possible_words - last_check
== 0):
passwords = passwords_group
passwords_group = []
self.process_lock.acquire()
stop = self.stop.get()
self.stop.put(stop)
if stop is False:
t = Process(target=self.search_zip_pass,
args=(passwords, file, possible_words))
self.threads.append(t)
self.process_count += 1
t.start()
else:
self.process_lock.release()
else:
continue
for x in self.threads:
x.join()
self.delete_temporary_directory()
self.end_time()
elif self.file_type == "pdf":
self.decrypted_file_name = "decrypted_" + file.split('/')[-1]
with open(dict_file, "r") as wordlist:
for word in wordlist:
passwords_group.append(word)
last_check += 1
self.handling_too_many_open_files_error()
if (len(passwords_group)
== self.shot) or (possible_words - last_check
== 0):
passwords = passwords_group
passwords_group = []
self.process_lock.acquire()
stop = self.stop.get()
self.stop.put(stop)
if stop is False:
t = Process(target=self.search_pdf_pass,
args=(passwords, file, possible_words))
self.threads.append(t)
self.process_count += 1
t.start()
else:
self.process_lock.release()
else:
continue
for x in self.threads:
x.join()
self.delete_temporary_directory()
self.end_time()
elif self.file_type == "rar":
with open(dict_file, "r") as wordlist:
for word in wordlist:
passwords_group.append(word)
last_check += 1
self.handling_too_many_open_files_error()
if (len(passwords_group)
== self.shot) or (possible_words - last_check
== 0):
passwords = passwords_group
passwords_group = []
self.process_lock.acquire()
stop = self.stop.get()
self.stop.put(stop)
if stop is False: # ok finishing all process after finding password
t = Process(target=self.search_rar_pass,
args=(passwords, file, possible_words))
self.threads.append(t)
self.process_count += 1
t.start()
else:
self.process_lock.release()
else:
continue
for x in self.threads:
x.join()
self.delete_temporary_directory()
self.end_time()
def bruteforce_guess_password(self, chars, min, max, file):
last_check = 0
passwords_group = []
possible_com = self.count_possible_com(chars, int(min), int(max))
self.last_process_number = int(
possible_com / self.shot) + (possible_com % self.shot > 0)
self.count.put(possible_com)
self.file_type = self.detect_file_type(file)
self.fun("Starting password cracking for " + file)
print("\n " + self.blue("[*]") +
self.white(" Count of possible passwords: ") +
self.bwhite(str(possible_com)))
if self.file_type == "text":
file = open(file)
for line in file.readlines():
self.count.get()
self.count.put(possible_com)
crypt_pass = line.split(':')[1].strip(' ')
if crypt_pass not in ['*', '!', '!!']:
user = line.split(':')[0]
print(" " + self.blue("[**]") +
self.white(" cracking Password for: ") +
self.bwhite(user))
algorythm = crypt_pass.split('$')[1].strip(' ')
salt = crypt_pass.split('$')[2].strip(' ')
salt_for_crypt = '$' + algorythm + '$' + salt + '$'
for password_length in range(int(min), int(max) + 1):
for guess in itertools.product(chars,
repeat=password_length):
guess = ''.join(guess)
passwords_group.append(guess)
last_check += 1
self.handling_too_many_open_files_error()
if (len(passwords_group)
== self.shot) or (possible_com - last_check
== 0):
passwords = passwords_group
passwords_group = []
self.process_lock.acquire()
stop = self.stop.get()
self.stop.put(stop)
if stop is False:
t = Process(
target=self.search_shadow_pass,
args=(passwords, salt_for_crypt,
crypt_pass, possible_com, user))
self.threads.append(t)
self.process_count += 1
t.start()
else:
self.process_lock.release()
else:
continue
for x in self.threads:
x.join()
self.last_process_number *= 2
self.end_time()
elif self.file_type == "zip":
for password_length in range(int(min), int(max) + 1):
for guess in itertools.product(chars, repeat=password_length):
guess = ''.join(guess)
passwords_group.append(guess)
last_check += 1
self.handling_too_many_open_files_error()
if (len(passwords_group)
== self.shot) or (possible_com - last_check == 0):
passwords = passwords_group
passwords_group = []
self.process_lock.acquire()
stop = self.stop.get()
self.stop.put(stop)
if stop is False:
t = Process(target=self.search_zip_pass,
args=(passwords, file, possible_com))
self.threads.append(t)
self.process_count += 1
t.start()
else:
self.process_lock.release()
else:
continue
for x in self.threads:
x.join()
self.delete_temporary_directory()
self.end_time()
elif self.file_type == "pdf":
self.decrypted_file_name = "decrypted_" + file.split('/')[-1]
for password_length in range(int(min), int(max) + 1):
for guess in itertools.product(chars, repeat=password_length):
guess = ''.join(guess)
passwords_group.append(guess)
last_check += 1
self.handling_too_many_open_files_error()
if (len(passwords_group)
== self.shot) or (possible_com - last_check == 0):
passwords = passwords_group
passwords_group = []
self.process_lock.acquire()
stop = self.stop.get()
self.stop.put(stop)
if stop is False:
t = Process(target=self.search_pdf_pass,
args=(passwords, file, possible_com))
self.threads.append(t)
self.process_count += 1
t.start()
else:
self.process_lock.release()
else:
continue
for x in self.threads:
x.join()
self.delete_temporary_directory()
self.end_time()
elif self.file_type == "rar":
for password_length in range(int(min), int(max) + 1):
for guess in itertools.product(chars, repeat=password_length):
guess = ''.join(guess)
passwords_group.append(guess)
last_check += 1
self.handling_too_many_open_files_error()
if (len(passwords_group)
== self.shot) or (possible_com - last_check == 0):
passwords = passwords_group
passwords_group = []
self.process_lock.acquire()
stop = self.stop.get()
self.stop.put(stop)
if stop is False: # ok finishing all process after finding password
t = Process(target=self.search_rar_pass,
args=(passwords, file, possible_com))
self.threads.append(t)
self.process_count += 1
t.start()
else:
self.process_lock.release()
else:
continue
for x in self.threads:
x.join()
self.delete_temporary_directory()
self.end_time()
def make_chars(self, char_type):
chartype_list = char_type.split(",")
chars = ""
for chartype in chartype_list:
if chartype == "lowercase":
chars += string.ascii_lowercase
elif chartype == "uppercase":
chars += string.ascii_uppercase
elif chartype == "letters":
chars += string.ascii_letters
elif chartype == "digits":
chars += string.digits
elif chartype == "symbols":
chars += string.punctuation
elif chartype == "space":
chars += " "
else:
return False
return chars
def banner(self):
term.clear()
term.pos(1, 1)
# check if font "epic" exists on this system
# sudo wget http://www.figlet.org/fonts/epic.flf -O /usr/share/figlet/epic.flf
bannerfont = "epic" if os.path.exists(
'/usr/share/figlet/epic.flf') else "banner"
banner = pyfiglet.figlet_format("ZYDRA", font=bannerfont).replace(
"\n", "\n\t\t", 7)
cprint("\r\n\t" + "@" * 61, "blue", end="")
cprint("\n\t\t" + banner + "\t\tAuthor : Hamed Hosseini",
"blue",
attrs=['bold'])
cprint("\t" + "@" * 61 + "\n", "blue")
def end_time(self):
self.stop = True
end_time_show = time.asctime()
end_time = time.monotonic()
execution_time = (timedelta(seconds=end_time - self.start_time))
print(self.blue("End time ==> ") + self.white(end_time_show))
print(
self.blue("Execution time ==> ") +
self.white(str(execution_time)) + "\n")
term.saveCursor()
term.pos(7, 15)
term.writeLine("ok", term.green, term.blink)
term.restoreCursor()
exit(0)
def main(self):
start_time_show = time.asctime()
usage = "%prog [options] [args]" \
"\n\nDictionary Mode:" \
"\n %prog -f <file> -d <wordlist>" \
"\n\nBrute force Mode:" \
"\n %prog -f <file> -b <char_type> -m <min_length> -x <max_length>" \
"\n\n Available char_type:" \
"\n\t<lowercase> The lowercase letters abcdefghijklmnopqrstuvwxyz" \
"\n\t<uppercase> The uppercase letters ABCDEFGHIJKLMNOPQRSTUVWXYZ" \
"\n\t<letters> The concatenation of the lowercase and uppercase"\
"\n\t<digits> numbers 0123456789" \
"\n\t<symbols> punctuation characters !#$%&'()*+,-./:;<=>?@[\]^_`{|}~'" + '"' \
"\n\t<space> space character" \
"\n You can select multiple character types." \
"\n\tExample: %prog -f <file> -b <space,digits> -m 1 -x 8"
parser = optparse.OptionParser(usage)
parser.add_option("-d",
dest="dictfile",
type='string',
help="Specifies dictionary file")
parser.add_option("-f",
dest="file",
type='string',
help="Specifies the file")
parser.add_option("-b",
dest="chartype",
type='string',
help="Specifies the character type")
parser.add_option("-m",
dest="minlength",
type='string',
help="Specifies minimum length of password")
parser.add_option("-x",
dest="maxlength",
type='string',
help="Specifies maximum length of password")
(options, args) = parser.parse_args()
try:
if options.file:
if os.path.isfile(options.file):
file = os.path.abspath(options.file)
if options.dictfile:
if os.path.isfile(options.dictfile):
dictfile = os.path.abspath(options.dictfile)
print(
self.blue("Start time ==> ") +
self.white(start_time_show) + "\n")
self.dict_guess_password(dictfile, file)
else:
parser.error(" " + options.dictfile +
" dictionary file does not exist")
exit(0)
elif options.chartype:
chars = self.make_chars(options.chartype)
if chars is False:
parser.error(
" " + options.chartype +
" character type is not valid, Use --help for more info"
)
if options.minlength is None:
parser.error(" Enter minimum length of password")
exit(0)
if options.maxlength is None:
parser.error(" Enter maximum length of password")
exit(0)
if options.minlength > options.maxlength:
parser.error(
" Min and Max must be numbers and Min must be \nless than Max or be the same"
", Use --help for more info")
exit(0)
else:
print(
self.blue("Start time ==> ") +
self.white(start_time_show) + "\n")
self.bruteforce_guess_password(
chars, options.minlength, options.maxlength,
file)
else:
parser.error(
" Choose a wordlist or bruteforce method, Use --help for more info"
)
exit(0)
else:
parser.error("" + options.file + " file does not exist")
exit(0)
else:
parser.error(" Choose a file, Use --help for more info")
exit(0)
except KeyboardInterrupt:
time.sleep(1)
self.delete_temporary_directory()
print(
self.red("\n\n [-] Detected CTRL+C") +
self.white("\n closing app...\n Finish\n"))
# self.end_time()
exit(0)
class Pipeline(ABC, Process, Friendable):
"""
A superclass for OpenCV lane detection
It handles opening and closing a video feed and subclasses simply implement the lane detection algorithm in their
_run() method. Additionally, provides tools to visualize the steps of a lane detection pipeline and for the user to
manually apply a mask to an image.
Use:
start() to open video feed and calls _run()
_run() is where the lane detection algorithm is implemented (MUST be overriden by subclass)
stop() to close video feed and windows and stop calling run()
#TODO: update this list
take_screenshot()
_add_knot()
# TODO: update this list
:ivar _pipeline: a list of frames showing the different steps of the pipeline. It should only ever store the pipeline
for a single iteration at a given instant (i.e. its length should never exceed the number of steps in
the pipeline) - not guaranteed to be filled
:ivar _show_pipeline: a flag indicating whether or not each step in the pipeline should be shown
:ivar _debug: a flag indicating whether or not the use is debugging the pipeline. In debug, the pipeline is shown and
debug statements are enabled
:ivar _capture: the OpenCV capture object (CvCapture) that the lane detection algorithm should run on
:ivar _fps: the fps of the capture object that the lane detection algorithmn is run on
:ivar _name: the name of the pipeline (derived from the class name)
:ivar _screen: the image where the pipeline steps are drawn
:ivar _image_mask_enabled: indicates whether or not the current instance of the pipeline supports image masks
:ivar _region_of_interest_mask: stores the region of interest mask (will be empty if image mask is disabled)
:ivar __stop: indicates whether or not the pipeline is stopped
:ivar __source: the source that the pipeline is being run on
:ivar __show_pipeline_steps: indicates whether or not the intermediate steps in the pipeline should be shown or just
the final step
:ivar __cached_pipeline: stores the last snapshot of the pipeline since it was cleared (since the current one is not
guaranteed to be filled)
:ivar __paused: indicates whether or not the pipeline is currently paused
:ivar __while_paused: stores the function to be executed while the pipeline is paused
:friend pipeline.utils.Visualizer
"""
def __init__(self,
source: str,
*,
n_consumers: int = 0,
should_start: bool = True,
show_pipeline: bool = True,
image_mask_enabled: bool = True,
debug: bool = False):
"""
Declares instance variables (_show_pipeline, _debug, _capture) and starts the pipeline according to should_start
:param source: the filename or device that the pipeline should be run on
:param should_start: a flag indicating whether or not the pipeline should start as soon as it is instantiated
:param show_pipeline: a flag indicating whether or not each step in the pipeline should be shown
:param debug: a flag indicating whether or not the use is debugging the pipeline. In debug, the pipeline is
shown and debug statements are enabled
"""
# call superclass constructor
super().__init__()
# initialize instance variables
# private - use property accessor
self._source = source
self._frame = None
self._name = self.__class__.__name__
self._fps = None
self._image_mask_enabled = image_mask_enabled
self._debug = debug
self._show_pipeline = show_pipeline or self._debug
self._show_pipeline_steps = settings.display.show_pipeline_steps
self._knots = []
screen_dimensions = (settings.window.height, settings.window.width,
NUM_IMAGE_CHANNELS)
# protected
self._screen = numpy.zeros(screen_dimensions, numpy.uint8)
self._visualizer = None
self._region_of_interest = None
self._capture = None
self._n_consumers = n_consumers # the number of objects consuming the result of this pipeline
if self._n_consumers > 0:
# stores the currently detected lane - must be a queue as the detected result is accessed from a seperate process
self._lanes_queue = Queue()
self.__n_consuming = 0 # tracks the number of consumers currently 'consuming' the detected result
self.__n_consuming_mutex = Lock(
) # blocks access to self.__n_consuming
# used to block pipeline from running while consumers are
self.__consumer_semaphore = BoundedSemaphore(1)
# used to prevent multiple iterations of pipeline from running before being consumed
self.__producer_sempahore = BoundedSemaphore(1)
# private - only accessible by class
self.__current_knots = [
] # maybe not be filled (most likely, will be partially filled)
self.__stop = False
self.__paused = False
self.__while_paused = None
# check if the pipeline should start immediately
if should_start and not self.is_alive():
self.start()
##### Property Accessors for Read-Only Instance Variables #####
@property
def source(self) -> str:
return self._source
@property
def frame(self) -> numpy.array:
return self._frame
@property
def name(self) -> str:
return self._name
@property
def fps(self) -> int:
return self._fps
@property
def image_mask_enabled(self) -> bool:
return self._image_mask_enabled
@property
def knots(self) -> list[numpy.array]:
return self.__current_knots
@property
def region_of_interest(self) -> list[tuple[int, int]]:
return self._region_of_interest.get()
##### Property Accessors and Mutators for Instance Variables #####
@property
def show_pipeline(self) -> bool:
return self._show_pipeline
@show_pipeline.setter
def show_pipeline(self, value: bool) -> None:
self._show_pipeline = value
@property
def show_pipeline_steps(self) -> bool:
return self._show_pipeline_steps
@show_pipeline_steps.setter
def show_pipeline(self, value: bool) -> None:
self._show_pipeline_steps = value
@property
def debug(self) -> bool:
return self._debug
@debug.setter
def debug(self, value: bool) -> None:
self._debug = value
##### Method Definitions #####
def start_consumption(self):
if self._n_consumers == 0:
raise RuntimeError(
'Cannot read lanes as the number of configured consumers is',
self._n_consumers)
# read the lanes from the queue
# this also blocks until the pipeline finishes executing on the current frame
lanes = self._lanes_queue.get(block=True)
# prevent pipeline from running while consumer is running
with self.__n_consuming_mutex:
if self.__n_consuming == 0:
self.__consumer_semaphore.acquire(
) # first consumer acquires semaphore
self.__n_consuming += 1
assert self._lanes_queue.empty(
) # ensure that the queue only ever contains the most recent detection
return lanes
def end_consumption(self):
if self._n_consumers == 0:
raise RuntimeError(
'Cannot read lanes as the number of configured consumers is',
self._n_consumers)
# prevent pipeline from running while consumer is running
with self.__n_consuming_mutex:
if self.__n_consuming == self._n_consumers:
self.__n_consuming = 0 # reset the number of consumers 'consuming'
self.__consumer_semaphore.release(
) # last consumer releases semaphore
self.__producer_sempahore.release(
) # allow pipeline to start on next iteration
def _add_lanes(self, lanes):
# only enqueue if we have a consumer configured, otherwise, fail silently
# allows this method can be called by subclasses without having to worry about consumers
if self._n_consumers > 0:
self._lanes_queue.put((self._frame, lanes), block=False)
def start(self):
"""
Starts running the process which then subsequently opens the video and runs the pipeline
:return: void
"""
super().start() # call Process::start() to start the Process execution
def __open_source(self, src: str) -> None:
"""
Opens a cv2.capture object
This method is susceptible to raise any errors caused by cv2.VideoCapture(src)
:param input: the filename or device id to be opened
:raises: RuntimeError is raised if this method is called when _capture is already open
:return: void
"""
# check that capture is not already open
if not self._capture:
self._capture = cv2.VideoCapture(
src) # open capture from provided input
self._fps = self._capture.get(cv2.CAP_PROP_FPS) # get fps of video
else:
# throw error if capture is already open
raise RuntimeError(
'Cannot open {input} as a capture is already open'.format(
input=input))
def run(self):
"""
:return: void
"""
# check if function was called by Process superclass and raise an error if it was not
if inspect.stack()[1].function != '_bootstrap':
raise RuntimeError(
'pipeline::run can only be invoked by multiprocessing::Process'
)
self.__open_source(self._source) # open input
first_frame = True # initialize a flag used to indicate if init_pipeline should be run
# loop run() while the capture is open and we we have not stopped running
while not self.__stop and self._capture.isOpened():
# if the pipeline is not paused, read a frame from the capture and call the pipeline
if not self.is_paused():
start_time = time.time() # store start time of loop
return_value, frame = self._capture.read(
) # read a frame of the capture
# check that the next frame was read successfully
# i.e. that we have not hit the end of the video or encountered an error
if return_value:
self._frame = frame
# if it is the first frame of the pipeline, run init_pipeline, then set flag to false
if first_frame:
self._init_pipeline(self._frame)
first_frame = False
# prevent pipeline from running until consumers are done
if self._n_consumers > 0:
# wait until consumer is finished consuming
self.__producer_sempahore.acquire()
self.__producer_sempahore.release()
self.__consumer_semaphore.acquire(
) # prevent consumers from running until after this iteration
self._run(self._frame) # run the pipeline
if self._n_consumers > 0:
self.__consumer_semaphore.release(
) # allow consumers to run on this iteration
# prevent pipeline from continuing until the current iteration is consumed
self.__producer_sempahore.acquire()
if self._show_pipeline: # display the pipeline
# self.__display_pipeline()
self._visualizer.get()
else:
self.stop(
) # stop the pipeline if we hit the end of the video or encountered an error
# only sleep if stop was not called (i.e. we will read the next frame)
if not self.__stop:
# 1 second / fps = time to sleep for each frame subtract elapsed time
time_to_sleep = max(
1 / self._fps - (time.time() - start_time), 0)
time.sleep(time_to_sleep)
# if the pipeline is paused and the whilepaused handler is defined, call it
elif self.__while_paused is not None:
self.__while_paused(
) # NOTE: the pipeline will block until the function returns
keypress = cv2.waitKey(1) & 0xFF # get the keypress
# if a key was pressed, call the handler with the pressed key
if keypress:
self.__handle_keypress(keypress)
# reset the pipeline now that the current iteration has finished
self.__clear_pipeline()
self._add_lanes(constants.SENTINEL)
def __handle_keypress(self, keypress):
"""
Handles actions based on a keypress. Works as a delegator to delegate actions based on specific keypress. If the
keypress maps to a default action, that actions is invoked, otherwise the keypress is passed to the subclass.
:param keypress: the code of the keypress
:param frame: the current frame of the pipeline when the keypress occurred
:return: void
"""
# q - stop the pipeline
if keypress == ord('q'):
self.stop()
# p - toggle displaying the pipeline steps
elif keypress == ord('p'):
self._show_pipeline_steps = not self._show_pipeline_steps
# esc - reset the debug image (top right)
elif keypress == ord('\x1b'):
self._visualizer.reset_debug_image()
# s - take a screenshot of the pipeline (saves each knot in the pipeline)
elif keypress == ord('s'):
self.take_screenshot()
# m - allow user to edit mask of source image (if image mask is enabled)
elif keypress == ord('m') and self._image_mask_enabled:
self._region_of_interest.editor(self.frame)
# other non-default case (let the subclass handle these cases if desired)
else:
self._handle_keypress(keypress)
def _handle_keypress(self, keypress):
"""
@Override - subclass CAN override this function (it is optional)
Where subclass can add custom keypress events. Cannot override keypress events in pipeline.py. This inhibits the use
of the 'q', 'p', and 's' keys and possibly the 'm' key, depending on the state of _image_mask_enabled.
:param keypress: the code of the keypress (will never correspond to any of the keys used for default keypress events)
:param frame: the current frame of the pipeline when the keypress occurred
:return: void
"""
pass
def _init_pipeline(self, first_frame):
"""
@Override - subclass CAN override this function (it is optional)
Where subclass can do any required initialization prior to the pipeline beginning to run. Default action is to get
the user to apply a mask to the image (if image mask is enabled, otherwise there is no default action).
:param first_frame: the first frame of the pipeline
:return: void
"""
# check if image mask is enabled, if so check if a mask was already defined or get the user to define one
if self._image_mask_enabled:
self._region_of_interest = utils.RegionOfInterest(self)
assert self._region_of_interest.load(
) # assert that we loaded a region of interest mask
self._visualizer = utils.Visualizer(self)
def is_paused(self):
"""
Gets whether or not the pipeline is paused
:return: boolean
"""
return self.__paused
def pause(self, whilepaused=None):
"""
Pauses the pipeline and will call the whilepaused handler until the pipeline is unpaused
:raises: RuntimeError is raised if this method is called when the pipeline is already paused
:return: void
"""
if self.is_paused():
raise RuntimeError(
'Cannot pause a pipeline that is already paused')
self.__paused = True
self.__while_paused = whilepaused
def unpause(self):
"""
Unpauses the pipeline
:raises: RuntimeError is raised if this method is called when the pipeline is not paused
:return: void
"""
if not self.is_paused():
raise RuntimeError(
'Cannot unpause a pipeline that is not currently paused')
self.__paused = False
self.__while_paused = None
def stop(self):
"""
Closes _capture and all windows and stops looping run()
:return: void
"""
self._capture.release() # close the capture
cv2.destroyAllWindows() # remove all windows
self.__stop = True # set flag to stop process execution
def take_screenshot(self, extension='jpg'):
"""
Takes a screenshot of the pipeline (saves each knot in the pipeline)
:param extension (optional) (default=jpg): the file extension that images should be saved as
:return: void
"""
def do_screenshot() -> None:
output_dir = '{base_output_dir}/{pipeline_name}/{timestamp}' \
.format(base_output_dir='output',
pipeline_name=self._name,
timestamp=datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))
os.makedirs(output_dir, exist_ok=True)
# iterate through each step of the pipeline and save the corresponding image (which is done in a new thread)
i = 1
for name, image in self._knots:
# format the file name
file_name = '{index} - {name}.{ext}'.format(index=i,
name=name,
ext=extension)
cv2.imwrite(os.path.join(output_dir, file_name), image)
i += 1
screenshot = Thread(target=do_screenshot)
screenshot.start()
# do not join the thread here since we want the screenshot to occur in the background
def _add_knot(self, name, image):
"""
Adds a knot to the lane detection pipeline
:param name: the name of the image to be added
:param image: the image to be added to the end of the pipeline
:return: void
"""
self.__current_knots.append((name, image))
def __clear_pipeline(self):
"""
Empties the stored steps of the lane detection pipeline
:return: void
"""
self._knots = self.__current_knots
self.__current_knots = []
def __display_pipeline(self):
"""
Displays the pipeline to the user. Depending on the state of _show_pipeline_steps, the steps of the pipeline may
or may not be visible.
:return: void
"""
def add_knot_to_screen(index, knot, new_dimension, position):
"""
Displays a single knot in the pipeline
:param index: the index of the knot in the pipeline
:param knot: the knot in the pipeline to be displayed
:param new_dimension: the desired size of the knot to be displayed - a tuple of the form (width, height)
:param position: the position of the top left corner of the image on self._screen - a tuple of the form (y, x)
:return: void
"""
# destructure the knot
name, image = knot
# resize the image to the desired size
resized_image = cv2.resize(image, dsize=new_dimension)
# add the image to the screen at the specified location
start_y, start_x = position
width, height = new_dimension
self._screen[start_y:(start_y + height),
start_x:(start_x + width)] = resized_image
# add the title of the knot to the image
title = '{index} - {name}'.format(index=index, name=name)
title_bounding_box, title_basline = cv2.getTextSize(
title, settings.font.face, settings.font.scale,
settings.font.thickness)
text_width, text_height = title_bounding_box
position = (start_x + settings.font.edge_offset,
start_y + text_height + settings.font.edge_offset)
cv2.putText(self._screen, title, position, settings.font.face,
settings.font.scale, settings.font.color,
settings.font.thickness)
# split the pipeline into the final and intermediate steps
pipeline_steps = self.__current_knots[:-1]
final_step = self.__current_knots[-1]
num_pipeline_steps = len(pipeline_steps)
# display the steps of the pipeline only if that option is selected
if self._show_pipeline_steps and num_pipeline_steps > 0:
# initialize the aspect ratio (gets set later when the pipeline is checked for consistent aspect ratios)
aspect_ratio = None
# check that all steps of the pipeline have the same aspect ratio (if not raise and error)
# simultaneously, check if any images are single channel and convert them to the correct number of channels
for i in range(len(self.__current_knots)):
name, image = self.__current_knots[i]
# get the dimensions of the image
# note that if the image is single channel, then num_channels will be undefined -> set it to default value after
height, width, *num_channels = image.shape
num_channels = num_channels[0] if num_channels else 1
# check for aspect ratio consistency throughout the pipeline
if aspect_ratio is None:
aspect_ratio = height / width
elif height / width != aspect_ratio:
raise RuntimeError(
'aspect ratio of images is not consistent throughout pipeline'
)
# if the image is single channel (grayscale), convert it to 3 channels (still grayscale)
# this allows the images to be merged into one
if num_channels == 1:
temp_image = numpy.empty(
(height, width, Pipeline.NUM_IMAGE_CHANNELS))
for channel in range(Pipeline.NUM_IMAGE_CHANNELS):
temp_image[:, :, channel] = image
if i < num_pipeline_steps:
pipeline_steps[i] = (name, temp_image)
else:
final_step = (name, temp_image)
# return the next lowest square greater than num
next_square = lambda num: int(
round(math.pow(math.ceil(math.sqrt(abs(num))), 2)))
# the actual ratio of the final image (will be grater than or equal to settings.display.minimum_final_image_ratio)
RESULT_IMAGE_RATIO = settings.display.minimum_final_image_ratio
# initialize variables concerned with the size of pipeline step bins
# (will get set later when calculating RESULT_IMAGE_RATIO)
num_bins_top_left = None
horizontal_bins_dimension = None
vertical_bins_dimension = None
# minimic a do-while loop
while True:
def calculate_dimensions_given_ratio(ratio):
"""
Calculates pipeline step bin dimensions given a ratio for the final step of the pipeline
:param ratio: the ratio of the final step of the pipeline to the rest of the screen
:return: void
"""
# allow this function to modify specific variables in outer scope
nonlocal num_bins_top_left, horizontal_bins_dimension, vertical_bins_dimension
# do the bin calculations
num_bins_top_left = next_square(
math.ceil(num_pipeline_steps * (1 - ratio)))
horizontal_bins_dimension = int(
round(math.sqrt(num_bins_top_left)))
vertical_bins_dimension = math.pow(
1 - ratio, -1) * horizontal_bins_dimension
# calculate the bin dimensions for the current ratio
calculate_dimensions_given_ratio(RESULT_IMAGE_RATIO)
# if the number of vertical bins is an integer (divides evenly into the screen), then break the loop
# (the while condition of the do-while loop)
if vertical_bins_dimension.is_integer():
break
# store the previously calculated ratio
prev = RESULT_IMAGE_RATIO
# calculate the new ratio to use
RESULT_IMAGE_RATIO = 1 - horizontal_bins_dimension / math.ceil(
vertical_bins_dimension)
# due to floating point precision errors, sometimes repeating decimals get rounded in an undesirable manner
# essentially, the program has successfully found the desired ratio, but rounds it causing the program to fail
# if this occurs, raise an error and instruct the user to fix the rounding error and update the value in
# pipeline settings
if prev == RESULT_IMAGE_RATIO:
raise FloatingPointError(
'Failed trying to find best ratio for result image. This was caused by a floating point decimal error on repeating digits. Update the pipeline.config file and try again. The recomended ratio is {new_ratio} (simply fix the repeating decimals)'
.format(new_ratio=RESULT_IMAGE_RATIO))
# calculate the dimensions of a pipeline step
container_width = int(
round(settings.window.width * (1 - RESULT_IMAGE_RATIO)))
step_width = container_width // horizontal_bins_dimension
step_height = int(round(step_width * aspect_ratio))
# iterate through all but the final step and display those knots in the pipeline
i = 0
for name, image in pipeline_steps:
# add the knot to the screen at the correct position
start_y = step_height * (i // horizontal_bins_dimension)
start_x = step_width * (i % horizontal_bins_dimension)
add_knot_to_screen(i + 1,
knot=(name, image),
new_dimension=(step_width, step_height),
position=(start_y, start_x))
i += 1
# add the final step to the screen in the bottom left quarter
output_width = int(
round(settings.window.width * RESULT_IMAGE_RATIO))
output_height = int(
round(settings.window.height * RESULT_IMAGE_RATIO))
add_knot_to_screen(
len(self.__current_knots),
knot=final_step,
new_dimension=(output_width, output_height),
position=(settings.window.height - output_height,
settings.window.width - output_width))
cv2.imshow(self._name, self._screen)
else:
name, image = final_step
cv2.imshow(self._name, image)
@abstractmethod
def _run(self, frame):
"""
@Override - subclass MUST override this function
Where the lane detection algorithm is written, it is called on each frame of _capture.
:param frame: the frame of the capture that the pipeline should be run on
:return: void
"""
pass
class scheduler(object):
"""multi-process scheduler"""
def __init__(self):
# if any jobs marked in run state when scheduler starts
# replace their state with X to mark that they have been shutdown
db = DAL(config.uri, auto_import=True, migrate=False,
folder=config.dbdir)
myset = db(db.jobs.state == 'R')
myset.update(state='X')
db.commit()
self.sem = BoundedSemaphore(config.np)
self.mutex = Lock()
def poll(self):
"""start polling thread which checks queue status every second"""
t = threading.Thread(target = self.assignTask)
t.start()
def assignTask(self):
while(True):
#print "scheduler:", self.qstat(), "jobs in queued state",
#time.asctime()
j = self.qfront()
if j is not None and j > 0:
self.start(j)
time.sleep(1)
def qsub(self,app,cid,user,np,pry,desc=""):
"""queue job ... really just set state to 'Q'."""
db = DAL(config.uri, auto_import=True, migrate=False,
folder=config.dbdir)
jid = db.jobs.insert(user=user, app=app, cid=cid, state=STATE_QUEUED,
description=desc, time_submit=time.asctime(), np=np, priority=pry)
db.commit()
db.close()
return str(jid)
def qfront(self):
"""pop the top job off of the queue that is in a queued 'Q' state"""
db = DAL(config.uri, auto_import=True, migrate=False,
folder=config.dbdir)
myorder = db.jobs.priority
#myorder = db.jobs.priority | db.jobs.id
row = db(db.jobs.state==STATE_QUEUED).select(orderby=myorder).first()
db.close()
if row: return row.id
else: return None
def qdel(self,jid):
"""delete job jid from the queue"""
#try:
db = DAL(config.uri, auto_import=True, migrate=False,
folder=config.dbdir)
del db.jobs[jid]
db.commit()
db.close()
#return True
#except:
# return False
def qstat(self):
"""return the number of jobs in a queued 'Q' state"""
db = DAL(config.uri, auto_import=True, migrate=False,
folder=config.dbdir)
return db(db.jobs.state==STATE_QUEUED).count()
db.close()
def start(self,jid):
"""start running a job by creating a new process"""
global p, jobs
myjobs = []
db = DAL(config.uri, auto_import=True, migrate=False,
folder=config.dbdir)
user = db.jobs(jid).user
app = db.jobs(jid).app
cid = db.jobs(jid).cid
np = db.jobs(jid).np
if np > 1: # use mpi
command = db.apps(name=app).command
command = config.mpirun + " -np " + str(np) + " " + command
else: # dont use mpi
command = db.apps(name=app).command
exe = os.path.join(config.apps_dir,app,app)
outfn = app + ".out"
cmd = command + ' >& ' + outfn
run_dir = os.path.join(config.user_dir,user,app,cid)
# if number procs available fork new process with command
for i in range(np):
self.sem.acquire()
p = Process(target=self.start_job, args=(run_dir,cmd,app,jid,np,))
myjobs.append(p)
print len(myjobs), myjobs
p.start()
for i in range(np):
self.sem.release()
def start_job(self,run_dir,cmd,app,jid,np):
"""this is what the separate job process runs"""
global popen
#print '*** pid:', os.getpid()
for i in range(np):
self.sem.acquire()
# update state to 'R' for run
self._set_state(jid,STATE_RUN)
mycwd = os.getcwd()
os.chdir(run_dir) # change to case directory
#popen = subprocess.Popen(cmd, stdout=subprocess.PIPE)
#subprocess.call(cmd, shell=True)
popen = subprocess.Popen(cmd, shell=True)
#os.system(cmd)
# let user know job has ended
outfn = app + ".out"
with open(outfn,"a") as f:
f.write("FINISHED EXECUTION")
# update state to 'C' for completed
os.chdir(mycwd) # return to SciPaaS root directory
self._set_state(jid,STATE_COMPLETED)
for i in range(np):
self.sem.release()
def _set_state(self,jid,state):
"""update state of job"""
self.mutex.acquire()
db = DAL(config.uri, auto_import=True, migrate=False,
folder=config.dbdir)
db.jobs[jid] = dict(state=state)
db.commit()
db.close()
self.mutex.release()
def stop(self,app):
#p.shutdown
#popen.terminate()
popen.terminate()
time.sleep(0.1)
#print p, p.is_alive()
#p.join()
#print p, p.is_alive()
#print '%s.exitcode = %s' % (popen.name, popen.exitcode)
#os.system("killall " + app)
def test_qfront(self):
print self.qfront()
class Queue(object):
def __init__(self, maxsize=0):
if maxsize <= 0:
# Can raise ImportError (see issues #3770 and #23400)
from multiprocessing.synchronize import SEM_VALUE_MAX as maxsize
self._maxsize = maxsize
self._reader, self._writer = connection.Pipe(duplex=False)
self._rlock = Lock()
self._opid = os.getpid()
if sys.platform == 'win32':
self._wlock = None
else:
self._wlock = Lock()
self._sem = BoundedSemaphore(maxsize)
# For use by concurrent.futures
self._ignore_epipe = False
self._after_fork()
if sys.platform != 'win32':
register_after_fork(self, Queue._after_fork)
def __getstate__(self):
assert is_spawning()
return (self._ignore_epipe, self._maxsize, self._reader, self._writer,
self._rlock, self._wlock, self._sem, self._opid)
def __setstate__(self, state):
(self._ignore_epipe, self._maxsize, self._reader, self._writer,
self._rlock, self._wlock, self._sem, self._opid) = state
self._after_fork()
def _after_fork(self):
debug('Queue._after_fork()')
self._notempty = threading.Condition(threading.Lock())
self._buffer = collections.deque()
self._thread = None
self._jointhread = None
self._joincancelled = False
self._closed = False
self._close = None
self._send_bytes = self._writer.send_bytes
self._recv_bytes = self._reader.recv_bytes
self._poll = self._reader.poll
def put(self, obj, block=True, timeout=None):
assert not self._closed
if not self._sem.acquire(block, timeout):
raise Full
with self._notempty:
if self._thread is None:
self._start_thread()
self._buffer.append(obj)
self._notempty.notify()
def get(self, block=True, timeout=None):
if block and timeout is None:
with self._rlock:
res = self._recv_bytes()
self._sem.release()
else:
if block:
deadline = time.time() + timeout
if not self._rlock.acquire(block, timeout):
raise Empty
try:
if block:
timeout = deadline - time.time()
if timeout < 0 or not self._poll(timeout):
raise Empty
elif not self._poll():
raise Empty
res = self._recv_bytes()
self._sem.release()
finally:
self._rlock.release()
# unserialize the data after having released the lock
return LokyPickler.loads(res)
def qsize(self):
# Raises NotImplementedError on Mac OSX because of broken sem
# getvalue()
return self._maxsize - self._sem._semlock._get_value()
def empty(self):
return not self._poll()
def full(self):
return self._sem._semlock._is_zero()
def get_nowait(self):
return self.get(False)
def put_nowait(self, obj):
return self.put(obj, False)
def close(self):
self._closed = True
try:
self._reader.close()
finally:
close = self._close
if close:
self._close = None
close()
def join_thread(self):
debug('Queue.join_thread()')
assert self._closed
if self._jointhread:
self._jointhread()
def cancel_join_thread(self):
debug('Queue.cancel_join_thread()')
self._joincancelled = True
try:
self._jointhread.cancel()
except AttributeError:
pass
def _start_thread(self):
debug('Queue._start_thread()')
# Start thread which transfers data from buffer to pipe
self._buffer.clear()
self._thread = threading.Thread(
target=Queue._feed,
args=(self._buffer, self._notempty, self._send_bytes, self._wlock,
self._writer.close, self._ignore_epipe),
name='QueueFeederThread')
self._thread.daemon = True
debug('doing self._thread.start()')
self._thread.start()
debug('... done self._thread.start()')
# On process exit we will wait for data to be flushed to pipe.
#
# However, if this process created the queue then all
# processes which use the queue will be descendants of this
# process. Therefore waiting for the queue to be flushed
# is pointless once all the child processes have been joined.
created_by_this_process = (self._opid == os.getpid())
if not self._joincancelled and not created_by_this_process:
self._jointhread = Finalize(self._thread,
Queue._finalize_join,
[weakref.ref(self._thread)],
exitpriority=-5)
# Send sentinel to the thread queue object when garbage collected
self._close = Finalize(self,
Queue._finalize_close,
[self._buffer, self._notempty],
exitpriority=10)
@staticmethod
def _finalize_join(twr):
debug('joining queue thread')
thread = twr()
if thread is not None:
thread.join()
debug('... queue thread joined')
else:
debug('... queue thread already dead')
@staticmethod
def _finalize_close(buffer, notempty):
debug('telling queue thread to quit')
with notempty:
buffer.append(_sentinel)
notempty.notify()
@staticmethod
def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe):
debug('starting thread to feed data to pipe')
nacquire = notempty.acquire
nrelease = notempty.release
nwait = notempty.wait
bpopleft = buffer.popleft
sentinel = _sentinel
if sys.platform != 'win32':
wacquire = writelock.acquire
wrelease = writelock.release
else:
wacquire = None
try:
while 1:
nacquire()
try:
if not buffer:
nwait()
finally:
nrelease()
try:
while 1:
obj = bpopleft()
if obj is sentinel:
debug('feeder thread got sentinel -- exiting')
close()
return
# serialize the data before acquiring the lock
obj = LokyPickler.dumps(obj)
if wacquire is None:
send_bytes(obj)
else:
wacquire()
try:
send_bytes(obj)
finally:
wrelease()
except IndexError:
pass
except Exception as e:
if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE:
return
# Since this runs in a daemon thread the resources it uses
# may be become unusable while the process is cleaning up.
# We ignore errors which happen after the process has
# started to cleanup.
try:
if is_exiting():
info('error in queue thread: %s', e)
else:
import traceback
traceback.print_exc()
except Exception:
pass
class Scheduler(object):
"""multi-process scheduler"""
def __init__(self):
# if any jobs marked in run state when scheduler starts
# replace their state with X to mark that they have been shutdown
db = DAL(config.uri, auto_import=True, migrate=False, folder=config.dbdir)
myset = db(db.jobs.state == STATE_RUN)
myset.update(state=STATE_STOPPED)
db.commit()
self.sem = BoundedSemaphore(config.np)
self.mutex = Lock()
# set time zone
try:
os.environ['TZ'] = config.time_zone
time.tzset()
except: pass
def poll(self):
"""start polling thread which checks queue status every second"""
t = threading.Thread(target = self.assignTask)
t.daemon = True
t.start()
def assignTask(self):
global myjobs
manager = Manager()
myjobs = manager.dict()
while(True):
self.stop_expired_jobs()
j = self.qfront()
if j is not None and j > 0:
self.start(j)
time.sleep(1)
def qsub(self, app, cid, uid, cmd, np, pry, walltime, desc=""):
"""queue job ... really just set state to 'Q'."""
db = DAL(config.uri, auto_import=True, migrate=False,
folder=config.dbdir)
jid = db.jobs.insert(uid=uid, app=app, cid=cid, command=cmd, state=STATE_QUEUED,
description=desc, time_submit=time.asctime(),
walltime=walltime, np=np, priority=pry)
db.commit()
db.close()
return str(jid)
def qfront(self):
"""pop the top job off of the queue that is in a queued 'Q' state"""
db = DAL(config.uri, auto_import=True, migrate=False, folder=config.dbdir)
myorder = db.jobs.priority
#myorder = db.jobs.priority | db.jobs.id
row = db(db.jobs.state==STATE_QUEUED).select(orderby=myorder).first()
db.close()
if row: return row.id
else: return None
def qdel(self,jid):
"""delete job jid from the queue"""
db = DAL(config.uri, auto_import=True, migrate=False, folder=config.dbdir)
del db.jobs[jid]
db.commit()
db.close()
def qstat(self):
"""return the number of jobs in a queued 'Q' state"""
db = DAL(config.uri, auto_import=True, migrate=False,
folder=config.dbdir)
return db(db.jobs.state==STATE_QUEUED).count()
db.close()
def start(self,jid):
"""start running a job by creating a new process"""
db = DAL(config.uri, auto_import=True, migrate=False, folder=config.dbdir)
uid = db.jobs(jid).uid
user = db.users(uid).user
app = db.jobs(jid).app
cid = db.jobs(jid).cid
np = db.jobs(jid).np
if np > 1: # use mpi
command = db.jobs(jid).command
command = config.mpirun + " -np " + str(np) + " " + command
else: # dont use mpi
command = db.jobs(jid).command
# redirect output to appname.out file
outfn = app + ".out"
cmd = command + ' > ' + outfn + ' 2>&1 '
print "cmd:", cmd
run_dir = os.path.join(user_dir, user, app, cid)
# if number procs available fork new process with command
for i in range(np):
self.sem.acquire()
p = Process(target=self.start_job, args=(run_dir,cmd,app,jid,np,myjobs))
p.start()
for i in range(np):
self.sem.release()
def start_job(self,run_dir,cmd,app,jid,np,myjobs):
"""this is what the separate job process runs"""
for i in range(np): self.sem.acquire()
# update state to 'R' for run
self._set_state(jid,STATE_RUN)
mycwd = os.getcwd()
os.chdir(run_dir) # change to case directory
pro = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True, preexec_fn=os.setsid)
myjobs[jid] = pro
pro.wait() # wait for job to finish
myjobs.pop(long(jid),None) # remove job from buffer
# let user know job has ended
outfn = app + ".out"
with open(outfn,"a") as f:
f.write("FINISHED EXECUTION")
# update state to 'C' for completed
os.chdir(mycwd)
self._set_state(jid,STATE_COMPLETED)
for i in range(np):
self.sem.release()
def _set_state(self,jid,state):
"""update state of job"""
self.mutex.acquire()
db = DAL(config.uri, auto_import=True, migrate=False, folder=config.dbdir)
db.jobs[jid] = dict(state=state)
db.commit()
db.close()
self.mutex.release()
def stop_expired_jobs(self):
"""shutdown jobs that exceed their time limit"""
db = DAL(config.uri, auto_import=True, migrate=False, folder=config.dbdir)
rows = db(db.jobs.state==STATE_RUN).select()
for row in rows:
if row:
walltime = int(row.walltime)
time_submit = time.mktime(datetime.datetime.strptime(
row.time_submit, "%a %b %d %H:%M:%S %Y").timetuple())
now = time.mktime(datetime.datetime.now().timetuple())
runtime = now - time_submit
if runtime > walltime:
print "INFO: scheduler stopped job", row.id, "REASON: reached timeout"
self.stop(row.id)
db.close()
def stop(self,jid):
p = myjobs.pop(long(jid),None)
if p: os.killpg(os.getpgid(p.pid), signal.SIGTERM)
# the following doesn't work because it gets overwritten by line 128 above
# need a way to feedback to start_job method whether job has been stopped or not
# db = DAL(config.uri, auto_import=True, migrate=False, folder=config.dbdir)
# myset = db(db.jobs.id == jid)
# myset.update(state=STATE_STOPPED)
# db.commit()
# db.close()
def test_qfront(self):
print self.qfront()
