def run(self):
client = self.get_boto_client()
job_output = client.describe_job(vaultName=self.vault,
jobId=self.job_id)
completed = job_output["Completed"]
while not completed:
self.next_check = (datetime.now() +
timedelta(seconds=self.SLEEP_TIME)).time()
self.update_task()
time.sleep(self.SLEEP_TIME)
job_output = client.describe_job(vaultName=self.vault,
jobId=self.job_id)
completed = job_output["Completed"]
if self.job_output == JobOutput.INVENTORY:
TaskManager.add_task(
DownloadInventoryTask(self.region, self.vault, self.job))
else:
TaskManager.add_task(
DownloadArchiveTask(self.region, self.vault, self.job,
self.output_file))
def run(self):
client = self.get_boto_client()
logging.info("Initiating job to retrieve inventory")
job = client.initiate_job(vaultName=self.vault, jobParameters={"Type": "inventory-retrieval"})
logging.info(f"Initiated job to retrieve inventory. Job id = {job['jobId']}")
TaskManager.add_task(WaitForJobTask(self.region, self.vault, job, JobOutput.INVENTORY, ""))
def on_click(self):
logging.info("Create vault clicked!")
region: Region = widgets_map['region_tree'].get_selected_region()
vault_name, ok = QInputDialog.getText(self.button, "New vault name",
"Name for the new vault:")
if not ok or not vault_name:
return
TaskManager.add_task(CreateVaultTask(region, vault_name))
def on_click(self):
logging.info("Upload clicked!")
files_table = widgets_map['files_table']
region: Region = files_table.displayed_region
vault: str = files_table.displayed_vault
file_location = QFileDialog.getOpenFileName(self.button, "Select file to upload")[0]
if not file_location:
return
TaskManager.add_task(UploadFileTask(region, vault, file_location))
def on_click(self):
logging.info("Download clicked!")
files_table = widgets_map['files_table']
region: Region = files_table.displayed_region
vault: str = files_table.displayed_vault
archive: Archive = files_table.get_active_archive()
output_file = QFileDialog.getSaveFileName(self.button,
"Output file location",
archive.description)[0]
if not output_file:
return
TaskManager.add_task(
GetArchiveTask(region, vault, archive, output_file))
def initialize(self, window: QMainWindow):
self.view: QTreeView = cast(QTreeView,
window.findChild(QTreeView, 'treeView'))
self.view.clicked.connect(self.on_clicked)
self.view.customContextMenuRequested.connect(self.open_menu)
self.model = QStandardItemModel()
root: QStandardItem = self.model.invisibleRootItem()
self.view.setModel(self.model)
for region in REGIONS:
item = QStandardItem(region.name)
root.appendRow(item)
if keys.Keys.has_keys():
TaskManager.add_task(ListVaultsTask(region))
def run(self):
client = self.get_boto_client()
logging.info("Initiating job to retrieve archive")
job_params = {
"Type": "archive-retrieval",
"ArchiveId": self.archive.id,
"Tier": "Bulk"
}
job = client.initiate_job(vaultName=self.vault,
jobParameters=job_params)
logging.info(
f"Initiated job to retrieve archive. Job id = {job['jobId']}")
TaskManager.add_task(
WaitForJobTask(self.region, self.vault, job, JobOutput.ARCHIVE,
self.output_file))
class Master:
def msg_callback(self, ch, method, properties, body):
callback_set = {'SUCCESS': self.success,
'FAIL': self.fail,
'AWAKE': self.update_slave_response_time,
'STOP': self.stop,
'ADD_SLAVE': self.add_slave,
'KILL_SLAVE': self.kill_slave,
'RESTART_SLAVE': self.restart_slave,
'STAT': self.stat,
'START': self.start,
'RECONFIGURE': self.configure,
'REFRESH': self.refresh
}
try:
command = body[:body.find(' ')]
info = body[body.find(' ')+1:]
if command in callback_set:
callback_set[command](ujson.loads(info))
else:
logging.debug(" [x] Unknown command %r" % (str(body),))
except KeyError as e:
if str(e) == "'Queue.DeclareOk'":
logging.debug("Queue.DelcareOk at %r" % (str(body),))
else:
logging.error("Unknown KeyError at %r:" % (str(body),))
except RuntimeError as e:
if 'recursion' in str(e):
logging.error('MAXIMUM RECURSION ERROR')
def __init__(self, conf_file):
self.config = ConfigParser.ConfigParser(allow_no_value=True)
self.clean_time_gap = None
self.wait_time_for_slave = None
self.master_queue_name = None
self.task_queue_name = None
self.task_queue_size_limit = None
self.task_file_name = None
self.task_counter_file = None
self.ssh_key = None
self.s3_bucket = None
self.s3_folder = None
self.slave_num_every_packup = None
self.slave_max_sec_each_task = None
self.slave_python_version = None
self.master_ip = None
self.slaves_ip = None
self.slave_awake_frequency = None
self.configure(conf_file)
self.last_wake_time = None
self.repeated_timer = None
self.is_started = False
self.pop_forever_handler = None
logging.info('Starting task manager...')
self.task_manager = TaskManager(self.task_file_name, self.task_counter_file)
logging.info('Starting slave manager...')
self.slave_manager = SlaveManager(master_ip=self.master_ip,
slaves_ip=self.slaves_ip,
ssh_key=self.ssh_key,
s3_bucket=self.s3_bucket,
s3_folder=self.s3_folder,
slave_num_every_packup=self.slave_num_every_packup,
slave_max_sec_each_task=self.slave_max_sec_each_task,
slave_python_version=self.slave_python_version,
slave_awake_frequency=self.slave_awake_frequency,
slave_buffer_size=1)
logging.info('Starting connection manager...')
self.message_connection = ConnectionManager(queue_name=self.master_queue_name,
durable=False,
callback=self.msg_callback,
no_ack=True)
def run(self):
logging.info(' [*] Waiting for messages. To exit press CTRL+C')
try:
self.message_connection.start_accepting_message()
except KeyboardInterrupt:
logging.info('Stopping master...')
master.stop(None)
except EOFError:
logging.info('Download finishes. Shutting down master.')
master.stop(None)
# except Exception as e:
# logging.info(str(e))
# logging.info('Stopping master...')
# TODO: write all configuration in one file
def configure(self, conf_file):
self.config.read(conf_file)
self.clean_time_gap = self.config.getint('main', 'clean_time_gap')
self.wait_time_for_slave = self.config.getint('main', 'wait_time_for_slave')
self.slave_awake_frequency = self.config.get('main', 'slave_awake_frequency')
self.master_ip = self.config.get('main', 'master_private_ip')
self.slaves_ip = self.config.get('main', 'slaves_private_ip')
self.master_queue_name = self.config.get('main', 'master_queue_name')
self.task_queue_name = self.config.get('main', 'task_queue_name')
self.task_file_name = self.config.get('main', 'task_file')
self.task_queue_size_limit = int(self.config.get('main', 'task_queue_size_limit'))
self.task_counter_file = self.config.get('main', 'task_counter_file')
self.ssh_key = self.config.get('main', 'ssh_key')
self.s3_bucket = self.config.get('main', 's3_bucket')
self.s3_folder = self.config.get('main', 's3_folder')
self.slave_num_every_packup = self.config.get('main', 'slave_num_every_packup')
self.slave_max_sec_each_task = self.config.get('main', 'slave_max_sec_each_task')
self.slave_python_version = self.config.get('main', 'slave_python_version')
def add_slave(self, slave_info):
if self.slave_manager.exist_slave(slave_info):
logging.info('Slave ' + slave_info['host'] + ' already exists.')
return
logging.info('master: add slave' + str(slave_info))
new_slave_info = self.slave_manager.add_slave(slave_info)
self.slave_manager.run_slave(new_slave_info)
# TODO:
def kill_slave(self, slave_info):
if not self.slave_manager.exist_slave(slave_info):
return
logging.info('kill slave ' + str(slave_info))
self.slave_manager.kill_slave(slave_info)
def restart_slave(self, slave_info):
logging.info(slave_info['host'])
logging.info('restart_slave' + str(slave_info))
self.kill_slave(slave_info)
self.add_slave(slave_info)
def start(self, info):
logging.info('Master Starts')
self.last_wake_time = datetime.datetime.utcnow()
self.is_started = True
self.pop_forever_handler = threading.Thread(target=self.start_popping_tasks)
self.pop_forever_handler.start()
self.repeated_timer = RepeatedTimer(self.clean_time_gap, self.notice_refresh, None)
def pop_forever(self):
self.start_popping_tasks()
def get_task_queue_size(self):
pass
# TODO: There is a bottle neck here
def start_popping_tasks(self):
task_connection = ConnectionManager(queue_name=self.task_queue_name,
durable=True, no_ack=False)
eof_reached = False
while self.is_started and not eof_reached:
current_task_queue_size = task_connection.get_task_queue_size()
while self.is_started and current_task_queue_size < self.task_queue_size_limit:
task = self.task_manager.pop_task()
if task is None:
# TODO: Don't use Error. Just break and handle the case later in this function
logging.info('EOF Reached')
eof_reached = True
break
message = 'WORK ' + ujson.dumps(task)
task_connection.publish(message)
current_task_queue_size += 1
task_connection.stop()
def fail(self, slave_task_info):
self.task_manager.add_task(slave_task_info['task'])
self.slave_manager.update_last_response(slave_task_info)
def success(self, slave_task_info):
slave_info = self.slave_manager.update_last_response(slave_task_info)
def update_slave_response_time(self, slave_task_info):
slave_info = self.slave_manager.update_last_response(slave_task_info)
def stop(self, info):
self.is_started = False
self.notice_slaves_stop()
if self.pop_forever_handler is not None:
self.pop_forever_handler.join()
if self.repeated_timer is not None:
self.repeated_timer.stop()
self.slave_manager.stop()
self.task_manager.stop()
self.message_connection.stop()
def notice_slaves_stop(self):
task_connection = ConnectionManager(queue_name=self.task_queue_name,
durable=True, no_ack=False)
screen_list = [key for key in self.slave_manager.slave_dict.keys()]
for screen in screen_list:
task_connection.publish('STOP {}')
# task_connection.broadcast_task('STOP {}')
task_connection.stop()
def refresh(self, info):
cur_progress, total_task = self.task_manager.get_progress()
logging.info('downloading {}/{} files'.format(cur_progress, total_task))
if not self.is_started:
return
# if time interval met, check failed slave
if self.last_wake_time is None:
self.last_wake_time = datetime.datetime.utcnow()
if self.last_wake_time + datetime.timedelta(
seconds=self.clean_time_gap) > datetime.datetime.utcnow():
return
failed_slaves = self.slave_manager.get_failed_slaves(self.wait_time_for_slave)
if len(failed_slaves) != 0:
logging.info('Finding failed slaves... ' + str(failed_slaves))
for slave in failed_slaves:
self.restart_slave(slave)
self.last_wake_time = datetime.datetime.utcnow()
def notice_refresh(self, info):
try:
self.message_connection.publish('REFRESH {}')
except IndexError:
logging.critical('INDEX_ERROR')
def stat(self, info):
logging.info('=====================================')
logging.info('Num of slave: ', self.slave_manager.get_num_slaves())
logging.info('=====================================')
if len(info) > 0:
for slave in self.slave_manager.slave_list:
if slave['last_response'] is None:
delta = 'new slave'
else:
delta = datetime.datetime.utcnow() - slave['last_response']
logging.info(slave['host'], '|', slave['queue'], '|', delta)
logging.info('====================================')
class ListScreen:
global count
# 화면 구성
def __init__(self):
self.tm = TaskManager()
self.p = Profile()
CANVAS_SIZE = 600
self.root = Tk()
self.root.title("Try, Do it !")
self.root.configure(bg="white")
self.root.geometry(
str(CANVAS_SIZE) + "x" + str(CANVAS_SIZE - 300) + "+300+100")
self.root.resizable(False, False)
lb1 = Label(self.root, text='Add Tasks', bg="white")
lb1.grid(row=0, column=2)
lb2 = Label(self.root, text='To do List', bg="white")
lb2.grid(row=0, column=7)
lb3 = Label(self.root, text='Doing Task', bg="white")
lb3.grid(row=7, column=2)
# Add Tasks Part
self.task = Entry(self.root, width=40, relief="solid", bd=1)
self.task.grid(row=1, column=0, columnspan=5)
self.priority = IntVar(self.root, 1)
frame = Frame(self.root, relief="solid", bd=1, bg="white")
frame.grid(row=2, column=0, rowspan=4)
rd1 = Radiobutton(frame,
bg="white",
text="중요 & 긴급",
value=1,
variable=self.priority,
command=self.check_priority)
rd1.pack()
rd2 = Radiobutton(frame,
bg="white",
text="중요 & 긴급X",
value=2,
variable=self.priority,
command=self.check_priority)
rd2.pack()
rd3 = Radiobutton(frame,
bg="white",
text="중요X & 긴급",
value=3,
variable=self.priority,
command=self.check_priority)
rd3.pack()
rd4 = Radiobutton(frame,
bg="white",
text="중요X & 긴급X",
value=4,
variable=self.priority,
command=self.check_priority)
rd4.pack()
self.tag = Entry(self.root, width=20, relief="solid", bd=1)
self.tag.grid(row=2, column=2, columnspan=3)
self.year = Entry(self.root, width=6, relief="solid", bd=1)
self.year.grid(row=3, column=2)
self.month = Entry(self.root, width=6, relief="solid", bd=1)
self.month.grid(row=3, column=3)
self.day = Entry(self.root, width=6, relief="solid", bd=1)
self.day.grid(row=3, column=4)
self.bt_add_task = Button(self.root,
bg="white",
text="Add Task",
width=20,
command=self.add)
self.bt_add_task.grid(row=4, column=2, columnspan=3)
# Doing Tasks Part
self.task_selected = Entry(self.root,
width=40,
bg="white",
relief="solid",
bd=1)
self.task_selected.grid(row=8, column=0, columnspan=5)
self.bt_update = Button(self.root,
text="Update",
fg="blue",
width=15,
bg="white",
command=self.update)
self.bt_update.grid(row=9, column=0, columnspan=2)
self.percent = Entry(self.root, width=10, relief="solid", bd=1)
self.percent.grid(row=9, column=3)
lb4 = Label(self.root, text="%", bg="white")
lb4.grid(row=9, column=4)
self.bt_open = Button(self.root,
text="Open Your Closet",
width=40,
bg="white",
command=self.open)
self.bt_open.grid(row=10, column=0, columnspan=5)
# To do List Part
self.bt_show_list = Button(self.root,
text="Show List",
fg="blue",
width=40,
bg="white",
command=self.show_list)
self.bt_show_list.grid(row=1, column=5, columnspan=5)
self.bt_show_list = Button(self.root,
text="Sort",
fg="blue",
width=40,
bg="white",
command=self.sort)
self.bt_show_list.grid(row=2, column=5, columnspan=5)
self.lb_tasks = Listbox(self.root, width=40, relief="solid", bd=1)
self.lb_tasks.grid(row=3, column=5, rowspan=7, columnspan=5)
self.bt_delete = Button(self.root,
text="Delete",
fg="brown",
width=15,
bg="white",
command=self.delete)
self.bt_delete.grid(row=10, column=5, columnspan=2)
self.bt_complete = Button(self.root,
text="Doing Tasks",
fg="blue",
width=8,
bg="white",
command=self.list_select)
self.bt_complete.grid(row=10, column=7)
self.bt_complete = Button(self.root,
text="Complete",
fg="green",
width=15,
bg="white",
command=self.closet)
self.bt_complete.grid(row=10, column=8, columnspan=2)
self.root.mainloop()
# 리스트 박스에서 선택된 항목 찾기
def list_select(self):
task = self.lb_tasks.get("active")
self.task_selected.delete(0, "end")
self.task_selected.insert(0, task)
# 리스트 박스에 할 일 목록 보여줌
def show_list(self):
# 현재 리스트 지우기
self.clear_listbox()
# 리스트 띄우기
for task in self.tm.tasks:
self.lb_tasks.insert("end", task)
# print(self.tm.tasks)
# 할 일 목록에 추가하고 DB 에 추가함
def add(self):
# 작성한 거 추가하기
lb = self.task.get()
self.tm.tasks.append(lb)
# 현재 리스트 지우기
self.clear_listbox()
# 리스트 띄우기
for task in self.tm.tasks:
self.lb_tasks.insert("end", task)
# DB 에 값 보냄
self.tm.add_task(lb)
# 할 일 목록 업데이트
def update(self):
# 작성한 거 추가하기
lb = self.lb_tasks.get("active")
if lb != self.task_selected.get():
past = self.lb_tasks.get("active")
self.tm.tasks.remove(past)
self.tm.tasks.append(self.task_selected.get())
# 현재 리스트 지우기
self.clear_listbox()
# 리스트 띄우기
for task in self.tm.tasks:
self.lb_tasks.insert("end", task)
content = lb
percent = self.percent.get()
if percent == "":
percent = 0
listnum = self.tm.get_listnum(content)
print(content, " + ", percent, "+", listnum)
self.tm.update_task(listnum, content, percent)
# 리스트 박스에서 제거하고 DB 에서도 제거하기
def delete(self):
# 리스트 박스에서 선택된 항목 찾기
task = self.lb_tasks.get("active")
if task in self.tm.tasks:
# 지우기
self.tm.tasks.remove(task)
# 리스트 보여주기
self.show_list()
listnum = self.tm.get_listnum(task)
# DB 에서 task 항목을 지움
self.tm.delete_task(listnum)
# 리스트 박스에 보여지는 것을 다 지움
def clear_listbox(self):
self.lb_tasks.delete(0, "end")
# 우선순위 체크
def check_priority(self):
return str(self.priority.get())
# 정렬하는 함수
def sort(self):
print(type(self.tm.tasks))
self.tm.tasks.sort()
self.show_list()
# ClosetScreen 불러오는 함수
def open(self):
ClosetScreen(count)
# 할 일 완료했을 때 목록과 DB 에서 제거하고 count 를 세서 ClosetScreen 으로 보내는 함수
def closet(self):
select = self.lb_tasks.get("active")
result = self.tm.complete_task(select)
count = self.tm.get_count()
# 현재 리스트 지우기
self.clear_listbox()
# 리스트 띄우기
for task in self.tm.tasks:
self.lb_tasks.insert("end", task)
print(self.tm.tasks)
if result == 1:
# count 값을 넘김
c = ClosetScreen(count)
c.update_closet()
def get_inventory_action(self):
vault_name = self.view.selectedIndexes()[0].data()
region = self.get_selected_region()
TaskManager.add_task(GetInventoryTask(region, vault_name))
def run(self):
client = self.get_boto_client()
logging.info("Initiating job to create vault")
client.create_vault(vaultName=self.vault_name)
TaskManager.add_task(ListVaultsTask(self.region))
