def run(self): client = self.get_boto_client() job_output = client.describe_job(vaultName=self.vault, jobId=self.job_id) completed = job_output["Completed"] while not completed: self.next_check = (datetime.now() + timedelta(seconds=self.SLEEP_TIME)).time() self.update_task() time.sleep(self.SLEEP_TIME) job_output = client.describe_job(vaultName=self.vault, jobId=self.job_id) completed = job_output["Completed"] if self.job_output == JobOutput.INVENTORY: TaskManager.add_task( DownloadInventoryTask(self.region, self.vault, self.job)) else: TaskManager.add_task( DownloadArchiveTask(self.region, self.vault, self.job, self.output_file))
def run(self): client = self.get_boto_client() logging.info("Initiating job to retrieve inventory") job = client.initiate_job(vaultName=self.vault, jobParameters={"Type": "inventory-retrieval"}) logging.info(f"Initiated job to retrieve inventory. Job id = {job['jobId']}") TaskManager.add_task(WaitForJobTask(self.region, self.vault, job, JobOutput.INVENTORY, ""))
def on_click(self): logging.info("Create vault clicked!") region: Region = widgets_map['region_tree'].get_selected_region() vault_name, ok = QInputDialog.getText(self.button, "New vault name", "Name for the new vault:") if not ok or not vault_name: return TaskManager.add_task(CreateVaultTask(region, vault_name))
def on_click(self): logging.info("Upload clicked!") files_table = widgets_map['files_table'] region: Region = files_table.displayed_region vault: str = files_table.displayed_vault file_location = QFileDialog.getOpenFileName(self.button, "Select file to upload")[0] if not file_location: return TaskManager.add_task(UploadFileTask(region, vault, file_location))
def on_click(self): logging.info("Download clicked!") files_table = widgets_map['files_table'] region: Region = files_table.displayed_region vault: str = files_table.displayed_vault archive: Archive = files_table.get_active_archive() output_file = QFileDialog.getSaveFileName(self.button, "Output file location", archive.description)[0] if not output_file: return TaskManager.add_task( GetArchiveTask(region, vault, archive, output_file))
def initialize(self, window: QMainWindow): self.view: QTreeView = cast(QTreeView, window.findChild(QTreeView, 'treeView')) self.view.clicked.connect(self.on_clicked) self.view.customContextMenuRequested.connect(self.open_menu) self.model = QStandardItemModel() root: QStandardItem = self.model.invisibleRootItem() self.view.setModel(self.model) for region in REGIONS: item = QStandardItem(region.name) root.appendRow(item) if keys.Keys.has_keys(): TaskManager.add_task(ListVaultsTask(region))
def run(self): client = self.get_boto_client() logging.info("Initiating job to retrieve archive") job_params = { "Type": "archive-retrieval", "ArchiveId": self.archive.id, "Tier": "Bulk" } job = client.initiate_job(vaultName=self.vault, jobParameters=job_params) logging.info( f"Initiated job to retrieve archive. Job id = {job['jobId']}") TaskManager.add_task( WaitForJobTask(self.region, self.vault, job, JobOutput.ARCHIVE, self.output_file))
class Master: def msg_callback(self, ch, method, properties, body): callback_set = {'SUCCESS': self.success, 'FAIL': self.fail, 'AWAKE': self.update_slave_response_time, 'STOP': self.stop, 'ADD_SLAVE': self.add_slave, 'KILL_SLAVE': self.kill_slave, 'RESTART_SLAVE': self.restart_slave, 'STAT': self.stat, 'START': self.start, 'RECONFIGURE': self.configure, 'REFRESH': self.refresh } try: command = body[:body.find(' ')] info = body[body.find(' ')+1:] if command in callback_set: callback_set[command](ujson.loads(info)) else: logging.debug(" [x] Unknown command %r" % (str(body),)) except KeyError as e: if str(e) == "'Queue.DeclareOk'": logging.debug("Queue.DelcareOk at %r" % (str(body),)) else: logging.error("Unknown KeyError at %r:" % (str(body),)) except RuntimeError as e: if 'recursion' in str(e): logging.error('MAXIMUM RECURSION ERROR') def __init__(self, conf_file): self.config = ConfigParser.ConfigParser(allow_no_value=True) self.clean_time_gap = None self.wait_time_for_slave = None self.master_queue_name = None self.task_queue_name = None self.task_queue_size_limit = None self.task_file_name = None self.task_counter_file = None self.ssh_key = None self.s3_bucket = None self.s3_folder = None self.slave_num_every_packup = None self.slave_max_sec_each_task = None self.slave_python_version = None self.master_ip = None self.slaves_ip = None self.slave_awake_frequency = None self.configure(conf_file) self.last_wake_time = None self.repeated_timer = None self.is_started = False self.pop_forever_handler = None logging.info('Starting task manager...') self.task_manager = TaskManager(self.task_file_name, self.task_counter_file) logging.info('Starting slave manager...') self.slave_manager = SlaveManager(master_ip=self.master_ip, slaves_ip=self.slaves_ip, ssh_key=self.ssh_key, s3_bucket=self.s3_bucket, s3_folder=self.s3_folder, slave_num_every_packup=self.slave_num_every_packup, slave_max_sec_each_task=self.slave_max_sec_each_task, slave_python_version=self.slave_python_version, slave_awake_frequency=self.slave_awake_frequency, slave_buffer_size=1) logging.info('Starting connection manager...') self.message_connection = ConnectionManager(queue_name=self.master_queue_name, durable=False, callback=self.msg_callback, no_ack=True) def run(self): logging.info(' [*] Waiting for messages. To exit press CTRL+C') try: self.message_connection.start_accepting_message() except KeyboardInterrupt: logging.info('Stopping master...') master.stop(None) except EOFError: logging.info('Download finishes. Shutting down master.') master.stop(None) # except Exception as e: # logging.info(str(e)) # logging.info('Stopping master...') # TODO: write all configuration in one file def configure(self, conf_file): self.config.read(conf_file) self.clean_time_gap = self.config.getint('main', 'clean_time_gap') self.wait_time_for_slave = self.config.getint('main', 'wait_time_for_slave') self.slave_awake_frequency = self.config.get('main', 'slave_awake_frequency') self.master_ip = self.config.get('main', 'master_private_ip') self.slaves_ip = self.config.get('main', 'slaves_private_ip') self.master_queue_name = self.config.get('main', 'master_queue_name') self.task_queue_name = self.config.get('main', 'task_queue_name') self.task_file_name = self.config.get('main', 'task_file') self.task_queue_size_limit = int(self.config.get('main', 'task_queue_size_limit')) self.task_counter_file = self.config.get('main', 'task_counter_file') self.ssh_key = self.config.get('main', 'ssh_key') self.s3_bucket = self.config.get('main', 's3_bucket') self.s3_folder = self.config.get('main', 's3_folder') self.slave_num_every_packup = self.config.get('main', 'slave_num_every_packup') self.slave_max_sec_each_task = self.config.get('main', 'slave_max_sec_each_task') self.slave_python_version = self.config.get('main', 'slave_python_version') def add_slave(self, slave_info): if self.slave_manager.exist_slave(slave_info): logging.info('Slave ' + slave_info['host'] + ' already exists.') return logging.info('master: add slave' + str(slave_info)) new_slave_info = self.slave_manager.add_slave(slave_info) self.slave_manager.run_slave(new_slave_info) # TODO: def kill_slave(self, slave_info): if not self.slave_manager.exist_slave(slave_info): return logging.info('kill slave ' + str(slave_info)) self.slave_manager.kill_slave(slave_info) def restart_slave(self, slave_info): logging.info(slave_info['host']) logging.info('restart_slave' + str(slave_info)) self.kill_slave(slave_info) self.add_slave(slave_info) def start(self, info): logging.info('Master Starts') self.last_wake_time = datetime.datetime.utcnow() self.is_started = True self.pop_forever_handler = threading.Thread(target=self.start_popping_tasks) self.pop_forever_handler.start() self.repeated_timer = RepeatedTimer(self.clean_time_gap, self.notice_refresh, None) def pop_forever(self): self.start_popping_tasks() def get_task_queue_size(self): pass # TODO: There is a bottle neck here def start_popping_tasks(self): task_connection = ConnectionManager(queue_name=self.task_queue_name, durable=True, no_ack=False) eof_reached = False while self.is_started and not eof_reached: current_task_queue_size = task_connection.get_task_queue_size() while self.is_started and current_task_queue_size < self.task_queue_size_limit: task = self.task_manager.pop_task() if task is None: # TODO: Don't use Error. Just break and handle the case later in this function logging.info('EOF Reached') eof_reached = True break message = 'WORK ' + ujson.dumps(task) task_connection.publish(message) current_task_queue_size += 1 task_connection.stop() def fail(self, slave_task_info): self.task_manager.add_task(slave_task_info['task']) self.slave_manager.update_last_response(slave_task_info) def success(self, slave_task_info): slave_info = self.slave_manager.update_last_response(slave_task_info) def update_slave_response_time(self, slave_task_info): slave_info = self.slave_manager.update_last_response(slave_task_info) def stop(self, info): self.is_started = False self.notice_slaves_stop() if self.pop_forever_handler is not None: self.pop_forever_handler.join() if self.repeated_timer is not None: self.repeated_timer.stop() self.slave_manager.stop() self.task_manager.stop() self.message_connection.stop() def notice_slaves_stop(self): task_connection = ConnectionManager(queue_name=self.task_queue_name, durable=True, no_ack=False) screen_list = [key for key in self.slave_manager.slave_dict.keys()] for screen in screen_list: task_connection.publish('STOP {}') # task_connection.broadcast_task('STOP {}') task_connection.stop() def refresh(self, info): cur_progress, total_task = self.task_manager.get_progress() logging.info('downloading {}/{} files'.format(cur_progress, total_task)) if not self.is_started: return # if time interval met, check failed slave if self.last_wake_time is None: self.last_wake_time = datetime.datetime.utcnow() if self.last_wake_time + datetime.timedelta( seconds=self.clean_time_gap) > datetime.datetime.utcnow(): return failed_slaves = self.slave_manager.get_failed_slaves(self.wait_time_for_slave) if len(failed_slaves) != 0: logging.info('Finding failed slaves... ' + str(failed_slaves)) for slave in failed_slaves: self.restart_slave(slave) self.last_wake_time = datetime.datetime.utcnow() def notice_refresh(self, info): try: self.message_connection.publish('REFRESH {}') except IndexError: logging.critical('INDEX_ERROR') def stat(self, info): logging.info('=====================================') logging.info('Num of slave: ', self.slave_manager.get_num_slaves()) logging.info('=====================================') if len(info) > 0: for slave in self.slave_manager.slave_list: if slave['last_response'] is None: delta = 'new slave' else: delta = datetime.datetime.utcnow() - slave['last_response'] logging.info(slave['host'], '|', slave['queue'], '|', delta) logging.info('====================================')
class ListScreen: global count # 화면 구성 def __init__(self): self.tm = TaskManager() self.p = Profile() CANVAS_SIZE = 600 self.root = Tk() self.root.title("Try, Do it !") self.root.configure(bg="white") self.root.geometry( str(CANVAS_SIZE) + "x" + str(CANVAS_SIZE - 300) + "+300+100") self.root.resizable(False, False) lb1 = Label(self.root, text='Add Tasks', bg="white") lb1.grid(row=0, column=2) lb2 = Label(self.root, text='To do List', bg="white") lb2.grid(row=0, column=7) lb3 = Label(self.root, text='Doing Task', bg="white") lb3.grid(row=7, column=2) # Add Tasks Part self.task = Entry(self.root, width=40, relief="solid", bd=1) self.task.grid(row=1, column=0, columnspan=5) self.priority = IntVar(self.root, 1) frame = Frame(self.root, relief="solid", bd=1, bg="white") frame.grid(row=2, column=0, rowspan=4) rd1 = Radiobutton(frame, bg="white", text="중요 & 긴급", value=1, variable=self.priority, command=self.check_priority) rd1.pack() rd2 = Radiobutton(frame, bg="white", text="중요 & 긴급X", value=2, variable=self.priority, command=self.check_priority) rd2.pack() rd3 = Radiobutton(frame, bg="white", text="중요X & 긴급", value=3, variable=self.priority, command=self.check_priority) rd3.pack() rd4 = Radiobutton(frame, bg="white", text="중요X & 긴급X", value=4, variable=self.priority, command=self.check_priority) rd4.pack() self.tag = Entry(self.root, width=20, relief="solid", bd=1) self.tag.grid(row=2, column=2, columnspan=3) self.year = Entry(self.root, width=6, relief="solid", bd=1) self.year.grid(row=3, column=2) self.month = Entry(self.root, width=6, relief="solid", bd=1) self.month.grid(row=3, column=3) self.day = Entry(self.root, width=6, relief="solid", bd=1) self.day.grid(row=3, column=4) self.bt_add_task = Button(self.root, bg="white", text="Add Task", width=20, command=self.add) self.bt_add_task.grid(row=4, column=2, columnspan=3) # Doing Tasks Part self.task_selected = Entry(self.root, width=40, bg="white", relief="solid", bd=1) self.task_selected.grid(row=8, column=0, columnspan=5) self.bt_update = Button(self.root, text="Update", fg="blue", width=15, bg="white", command=self.update) self.bt_update.grid(row=9, column=0, columnspan=2) self.percent = Entry(self.root, width=10, relief="solid", bd=1) self.percent.grid(row=9, column=3) lb4 = Label(self.root, text="%", bg="white") lb4.grid(row=9, column=4) self.bt_open = Button(self.root, text="Open Your Closet", width=40, bg="white", command=self.open) self.bt_open.grid(row=10, column=0, columnspan=5) # To do List Part self.bt_show_list = Button(self.root, text="Show List", fg="blue", width=40, bg="white", command=self.show_list) self.bt_show_list.grid(row=1, column=5, columnspan=5) self.bt_show_list = Button(self.root, text="Sort", fg="blue", width=40, bg="white", command=self.sort) self.bt_show_list.grid(row=2, column=5, columnspan=5) self.lb_tasks = Listbox(self.root, width=40, relief="solid", bd=1) self.lb_tasks.grid(row=3, column=5, rowspan=7, columnspan=5) self.bt_delete = Button(self.root, text="Delete", fg="brown", width=15, bg="white", command=self.delete) self.bt_delete.grid(row=10, column=5, columnspan=2) self.bt_complete = Button(self.root, text="Doing Tasks", fg="blue", width=8, bg="white", command=self.list_select) self.bt_complete.grid(row=10, column=7) self.bt_complete = Button(self.root, text="Complete", fg="green", width=15, bg="white", command=self.closet) self.bt_complete.grid(row=10, column=8, columnspan=2) self.root.mainloop() # 리스트 박스에서 선택된 항목 찾기 def list_select(self): task = self.lb_tasks.get("active") self.task_selected.delete(0, "end") self.task_selected.insert(0, task) # 리스트 박스에 할 일 목록 보여줌 def show_list(self): # 현재 리스트 지우기 self.clear_listbox() # 리스트 띄우기 for task in self.tm.tasks: self.lb_tasks.insert("end", task) # print(self.tm.tasks) # 할 일 목록에 추가하고 DB 에 추가함 def add(self): # 작성한 거 추가하기 lb = self.task.get() self.tm.tasks.append(lb) # 현재 리스트 지우기 self.clear_listbox() # 리스트 띄우기 for task in self.tm.tasks: self.lb_tasks.insert("end", task) # DB 에 값 보냄 self.tm.add_task(lb) # 할 일 목록 업데이트 def update(self): # 작성한 거 추가하기 lb = self.lb_tasks.get("active") if lb != self.task_selected.get(): past = self.lb_tasks.get("active") self.tm.tasks.remove(past) self.tm.tasks.append(self.task_selected.get()) # 현재 리스트 지우기 self.clear_listbox() # 리스트 띄우기 for task in self.tm.tasks: self.lb_tasks.insert("end", task) content = lb percent = self.percent.get() if percent == "": percent = 0 listnum = self.tm.get_listnum(content) print(content, " + ", percent, "+", listnum) self.tm.update_task(listnum, content, percent) # 리스트 박스에서 제거하고 DB 에서도 제거하기 def delete(self): # 리스트 박스에서 선택된 항목 찾기 task = self.lb_tasks.get("active") if task in self.tm.tasks: # 지우기 self.tm.tasks.remove(task) # 리스트 보여주기 self.show_list() listnum = self.tm.get_listnum(task) # DB 에서 task 항목을 지움 self.tm.delete_task(listnum) # 리스트 박스에 보여지는 것을 다 지움 def clear_listbox(self): self.lb_tasks.delete(0, "end") # 우선순위 체크 def check_priority(self): return str(self.priority.get()) # 정렬하는 함수 def sort(self): print(type(self.tm.tasks)) self.tm.tasks.sort() self.show_list() # ClosetScreen 불러오는 함수 def open(self): ClosetScreen(count) # 할 일 완료했을 때 목록과 DB 에서 제거하고 count 를 세서 ClosetScreen 으로 보내는 함수 def closet(self): select = self.lb_tasks.get("active") result = self.tm.complete_task(select) count = self.tm.get_count() # 현재 리스트 지우기 self.clear_listbox() # 리스트 띄우기 for task in self.tm.tasks: self.lb_tasks.insert("end", task) print(self.tm.tasks) if result == 1: # count 값을 넘김 c = ClosetScreen(count) c.update_closet()
def get_inventory_action(self): vault_name = self.view.selectedIndexes()[0].data() region = self.get_selected_region() TaskManager.add_task(GetInventoryTask(region, vault_name))
def run(self): client = self.get_boto_client() logging.info("Initiating job to create vault") client.create_vault(vaultName=self.vault_name) TaskManager.add_task(ListVaultsTask(self.region))