class DataSyncer: _logger = logging.getLogger('DataSyncer') def __init__(self): self.root = Tk() self.file_list = {} self.bag_num_thres = 5 self.file_size = 0 self.finish_size = 0 self.usb_model = None self._lock = threading.Lock() self.sync_proc = None self.sync_thread = None self.stop_thread = None self.status = SYNC_NOT_READY self.sync_dst_bag = '' self.sync_dst_dataset = '' self.usb_status = '' self.net_status = '' self.sync_status = '' self.cur_time = 0 # GUI interface self.frame = Frame(self.root) self.root.title('TuSimple Data Syncer') self.root.geometry('500x700') self.root.protocol("WM_DELETE_WINDOW", self.exit) self.style = Style(self.root) self.style.layout('text.Horizontal.TProgressbar', [('Horizontal.Progessbar.trough', {'children': [('Horizontal.Progressbar.pbar', {'side':'left', 'sticky':'ns'})], 'sticky': 'nswe'}), ('Horizontal.Progressbar.label', {'side':'right', 'sticky':''})]) self.font_size = tkFont.Font(family='Times New Roman', size=15) self.create_layout() self.gui_update() self.search_usb_update() self.search_net_update() DataSyncer._logger.info('GUI layout created') self.root.mainloop() def create_layout(self): # usb label usb_lbl = Label(text='usb status:', height=4, width=20, font=self.font_size) usb_lbl.grid(column=0, row=0) # usb status label self.usb_status_lbl = Label(text=self.usb_status, height=4, width=20, font=self.font_size) self.usb_status_lbl.grid(column=1, row=0) # network label net_lbl = Label(text='network status:', height=4, width=20, font=self.font_size) net_lbl.grid(column=0, row=1) # network status label self.net_status_lbl = Label(text=self.net_status, height=4, font=self.font_size) self.net_status_lbl.grid(column=1, row=1) # start date label start_lbl = Label(text='start date:', height=4, font=self.font_size) start_lbl.grid(column=0, row=2) # start date text self.start_txt = Entry(width=25, font=self.font_size) self.start_txt.grid(column=1, row=2) # end date label end_lbl = Label(text='end date:', height=4, font=self.font_size) end_lbl.grid(column=0, row=3) # end date text self.end_txt = Entry(width=25, font=self.font_size) self.end_txt.grid(column=1, row=3) # start button self.usb_button = Button(text='USB sync', height=3, command= lambda: self.start_button_click('USB'), font=self.font_size) self.usb_button.grid(column=0, row=4, sticky=N+S+E+W) # stop button self.stop_button = Button(text='stop sync', height=3, command= lambda: self.stop_button_click(), font=self.font_size) self.stop_button.grid(column=1, row=4, sticky=N+S+E+W) # start button self.net_button = Button(text='Net sync', height=3, command= lambda: self.start_button_click('Net'), font=self.font_size) self.net_button.grid(column=0, row=5, sticky=N+S+E+W) # start button self.exit_button = Button(text='exit', height=3, command=self.exit, font=self.font_size) self.exit_button.grid(column=1, row=5, sticky=N+S+E+W) # sync status label self.sync_status_lbl = Label(text=self.sync_status, width=50, height=4, font=self.font_size) self.sync_status_lbl.grid(column=0, row=6, columnspan=2) # progress bar self.progressbar = Progressbar(orient='horizontal', length=240, mode='determinate', style='text.Horizontal.TProgressbar') # time estimator self.time_est = Label(height=3, font=self.font_size) def usb_status_set(self, text): self.usb_status = text def net_status_set(self, text): self.net_status = text def sync_status_set(self, text): self.sync_status = text def usb_status_config(self, text): self.usb_status_lbl.configure(text=text) def net_status_config(self, text): self.net_status_lbl.configure(text=text) def sync_status_config(self, text): self.sync_status_lbl.configure(text=text) def prog_status_config(self, val, maximum, text): self.progressbar.configure(value=val, maximum=maximum) self.style.configure('text.Horizontal.TProgressbar', text=text) def start_date_get(self): return self.start_txt.get() def end_date_get(self): return self.end_txt.get() def set_status(self, status): with self._lock: self.status = status def get_status(self): with self._lock: return self.status def forget_progressbar(self): self.cur_time = 0 self.finish_size = 0 if len(self.progressbar.grid_info()) != 0: self.progressbar.grid_forget() if len(self.time_est.grid_info()) != 0: self.time_est.grid_forget() def search_usb_update(self): self.root.after(USB_FREQ, self.search_usb) def search_net_update(self): self.root.after(NET_FREQ, self.search_net) def gui_update(self): self.root.after(UPDATE_FREQ, self.status_update) def progressbar_update(self): self.root.after(PROG_FREQ, self.progressbar_calculator) def progressbar_calculator(self): if self.get_status() == SYNCING: self.progressbar.grid(column=0, row=12, columnspan=2) self.time_est.grid(column=0, row=13, columnspan=2) finish_size = 0 for key, folders in self.file_list.iteritems(): if 'bag' in self.file_list[key].values(): path = os.path.join(self.sync_dst_bag, key) if os.path.exists(path): try: finish_size += self.get_size(path) except OSError: DataSyncer._logger.error('Unable to get size at {}'.format(path)) for data_folder, data_type in folders.iteritems(): if data_type == 'dataset': path = os.path.join(self.sync_dst_dataset, data_folder) if os.path.exists(path): try: finish_size += self.get_size(path) except OSError: DataSyncer._logger.error('Unable to get size at {}'.format(path)) # calculate percent val = finish_size * 1. / self.file_size * 100 maximum = 100 self.prog_status_config(val, maximum, '{}/{}'.format(int(val), maximum)) # calculate speed and estimated time if self.cur_time == 0: self.cur_time = time.time() else: t_diff = time.time() - self.cur_time self.cur_time = time.time() s_diff = finish_size - self.finish_size self.finish_size = finish_size # speed in MB/s, t_diff in KB, needs to divide by 1024, time left in min sync_speed = s_diff * 1. / t_diff if t_diff != 0 else 0 time_left = int(math.ceil((self.file_size - self.finish_size) * 1. / sync_speed / 60) if sync_speed != 0 else 0) self.time_est.configure(text='Sync speed: {:.1f}MB/s, Estimate: {}min'.format(sync_speed/1024, time_left)) self.progressbar_update() def start_button_click(self, sync_type): if self.get_status() == SYNCING or (self.sync_thread != None and self.sync_thread.isAlive()): self.sync_status_set('Unable to sync: syncing in progress') DataSyncer._logger.warn('Unable to sync: syncing in progress') return self.sync_thread = threading.Thread(target=self.start_sync, args=(sync_type,)) self.sync_thread.start() def stop_button_click(self): ''' if self.stop_thread != None and self.stop_thread.isAlive(): self.sync_status_set('Unable to stop: stopping now') DataSyncer._logger.warn('Unable to sync: stopping now') return ''' if self.get_status() != SYNCING: self.sync_status_set('Unable to stop: no syncing in progress') DataSyncer._logger.warn('Unable to sync: no syncing in progress') return self.stop_thread = threading.Thread(target=self.stop_sync) self.stop_thread.start() def status_update(self): self.usb_status_config(self.usb_status) self.net_status_config(self.net_status) self.sync_status_config(self.sync_status) self.gui_update() ''' if self.sync_thread != None and self.sync_thread.isAlive() and self.get_status() in [SYNC_NOT_READY, SYNC_STOPPING]: self.sync_thread.join() DataSyncer._logger.info('sync thread joined!') if self.stop_thread != None and self.stop_thread.isAlive() and self.get_status() in [SYNC_NOT_READY]: self.stop_thread.join() DataSyncer._logger.info('stop thread joined!') ''' # check if usb is availble def search_usb(self): dev_path = os.path.join('/media', USER) dev_name = None try: devs = os.listdir(dev_path) for dev in devs: _dev = dev.lower() if DEV_PRE.lower() in _dev: dev_name = dev if dev_name: self.usb_status = dev else: self.usb_status = 'No USB found' self.usb_model = dev_name if self.get_status() != SYNCING: self.search_usb_update() except OSError: self.usb_status = '{} USB not found'.format(DEV_PRE) self.usb_model = None # search if network is available def search_net(self): try: res = os.system("ping -c 1 " + NETWORK_IP + '> /dev/null 2>&1') if res == 0: if os.path.ismount(BAG_MOUNT_POINT): self.net_status = 'Network is ready for sync' else: self.net_status = 'Mount point unfound' else: self.net_status = 'Network is unreachable' if self.get_status() != SYNCING: self.search_net_update() except OSError: print('Unable to check the network availability') # check file list def check_file_type(self, path): items = os.listdir(path) if 'top.json' in items: return 'dataset' elif 'record.json' in items or 'log' in items: return 'bag' else: return None # count how many bags one folder has def count_bag(self, folder): try: items = os.listdir(folder) except OSError: DataSyncer._logger.error('Unable to open file: {}'.format(folder)) return False bag_num = 0 for item in items: if item.endswith('.bag'): bag_num += 1 return bag_num # add to bag list def add_file_list(self, start_date, end_date): # empty the file list self.file_list = {} self.file_size = 0 # adding into file list try: dates = os.listdir(SYNC_SRC) for date in dates: if date < start_date or date > end_date: continue self.file_list[date] = {} path = os.path.join(SYNC_SRC, date) self.file_size += self.get_size(path) bag_folders = os.listdir(path) for f in bag_folders: f_path = os.path.join(SYNC_SRC, date, f) file_type = self.check_file_type(f_path) if file_type == 'dataset': ds = Dataset(f_path) _t = (ds.meta['ts_end'] - ds.meta['ts_begin']) * 1. / 1e9 / 60 if _t > self.bag_num_thres * 5: self.file_list[date][f] = 'dataset' elif file_type == 'bag': bag_num = self.count_bag(f_path) if bag_num >= self.bag_num_thres: self.file_list[date][f] = 'bag' else: DataSyncer._logger.warn('{} is not a data folder'.format(f_path)) if len(self.file_list[date]) == 0: del(self.file_list[date]) except OSError as e: DataSyncer._logger.error('Unable to open files when adding to bag list') # check user input format @staticmethod def check_date_format(date_text): try: datetime.datetime.strptime(date_text, '%Y-%m-%d') return True except ValueError: return False # check if usb condition is met for sync def check_dst_condition(self, sync_type): condition = self.usb_model if sync_type == 'USB' else self.net_status if not condition: self.sync_status_set('Unable to sync: {} is not avaible'.format(sync_type)) DataSyncer._logger.error('Unable to sync: {} is not avaible'.format(sync_type)) return False return True # check if date is met for sync def check_date_condition(self): start_date = self.start_date_get() end_date = self.end_date_get() prompt = 'Unable to sync: ' if start_date == '' or end_date == '': self.sync_status_set(prompt + 'dates null') DataSyncer._logger.error(prompt + 'dates null') return False elif not self.check_date_format(start_date) or not self.check_date_format(end_date): self.sync_status_set(prompt + 'format should be YYYY-MM-DD') DataSyncer._logger.error(prompt + 'format should be YYYY-MM-DD') return False elif start_date > end_date: self.sync_status_set(prompt + 'start date later than end date') DataSyncer._logger.error(prompt + 'start date later than end date') return False else: self.add_file_list(start_date, end_date) if len(self.file_list) == 0: self.sync_status_set(prompt + 'no bag between these dates') DataSyncer._logger.error(prompt + 'no bag between these dates') return False else: return True # check both conditions def check_sync_condition(self, sync_type): if self.check_dst_condition(sync_type) and self.check_date_condition(): self.set_status(SYNC_READY) else: self.set_status(SYNC_NOT_READY) # start syncing def start_sync(self, sync_type): # check destination self.check_sync_condition(sync_type) if self.get_status() == SYNC_NOT_READY: return self.set_status(SYNCING) # generate dist path if sync_type == 'USB': self.sync_dst_bag = self.sync_dst_dataset = os.path.join('/media', USER, self.usb_model, 'import') else: self.sync_dst_bag = os.path.join(BAG_MOUNT_POINT, 'data_collection') self.sync_dst_dataset = DATASET_MOUNT_POINT try: folders = os.listdir(self.sync_dst_bag) except OSError: self.sync_status_set('No such path: {}'.format(self.sync_dst_bag)) DataSyncer._logger.error('Unable to open file: {}'.format(self.sync_dst_bag)) return # sanity check self.sync_status_set('Sanity check...please wait') self.sanity_check() DataSyncer._logger.info('Sanity check finished') self.progressbar_update() # start to sync bag one by one for key, f_list in self.file_list.iteritems(): if key not in folders and 'bag' in f_list.values(): try: os.mkdir(os.path.join(self.sync_dst_bag, key)) except OSError: self.set_status(SYNC_NOT_READY) DataSyncer._logger.error('Unable to create {} under {}'.format(key, self.sync_dst_bag)) return for f in f_list.keys(): if self.get_status() == SYNCING: self.sync_status_set('Syncing: ' + f) cmd = ['rsync', '--progress', '-r', '--append'] cmd.append(os.path.join(SYNC_SRC, key, f)) sync_dst = os.path.join(self.sync_dst_bag, key) if f_list[f] == 'bag' else self.sync_dst_dataset cmd.append(sync_dst) self.sync_proc = subprocess.Popen(cmd) self.sync_proc.communicate() if self.sync_proc.returncode not in [0, 20]: self.sync_status_set('Syncing progress error code: {}'.format(self.sync_proc.returncode)) DataSyncer._logger.error('rsync progress error code: {}'.format(self.sync_proc.returncode)) if self.get_status() == EXIT: DataSyncer._logger.info('exiting...') return # reset status if self.get_status() == SYNCING: self.set_status(SYNC_NOT_READY) if self.sync_proc.returncode == 0: self.sync_status_set(sync_type + ' sync completed') DataSyncer._logger.info(sync_type + ' sync completed') # post deletion self.post_delete() # restart update self.search_usb_update() self.search_net_update() # forget progressbar and its related speed/time estimator self.forget_progressbar() DataSyncer._logger.info('syncing thread finished') # stop syncing def stop_sync(self): if self.sync_proc != None and self.sync_proc.poll() == None: DataSyncer._logger.info('start stopping') self.sync_proc.terminate() if self.get_status() != EXIT: self.set_status(SYNC_NOT_READY) self.sync_proc.communicate() DataSyncer._logger.info('returncode is: {}'.format(self.sync_proc.returncode)) if self.sync_proc.returncode in [0, 20]: self.sync_status_set('stop success') DataSyncer._logger.info('stop syncing thread finished') # sanity check before syncing to avoid - matching the use of rsync --append def sanity_check(self): for key, bag_folder in self.file_list.iteritems(): for f in bag_folder.keys(): try: # assumption: all files in destination must be included by those in source sync_dst = os.path.join(self.sync_dst_bag, key, f) if bag_folder[f] == 'bag' else os.path.join(self.sync_dst_dataset, f) items = os.listdir(sync_dst) for item in items: src_path = os.path.join(SYNC_SRC, key, f, item) dst_path = os.path.join(sync_dst, item) if os.path.isfile(dst_path): s_size = self.get_file_size(src_path) d_size = self.get_file_size(dst_path) else: s_size = self.get_dir_size(src_path) d_size = self.get_dir_size(dst_path) if s_size != d_size: try: os.remove(dst_path) except OSError: shutil.rmtree(dst_path) finally: DataSyncer._logger.warn('removed {}'.format(dst_path)) except OSError: DataSyncer._logger.error('Unable to do OS operation in Sanity check') # post-delete the .active bag # TO-DO: include post check def post_delete(self): for key, bag_folder in self.file_list.iteritems(): for f in bag_folder.keys(): try: if bag_folder[f] == 'bag': path = os.path.join(self.sync_dst_bag, key, f) items = os.listdir(path) for item in items: if item.endswith('.active'): path = os.path.join(path, item) os.remove(path) except OSError: DataSyncer._logger.error('Unable to do OS operation in post delete') # wait threads finish def wait_thread(self): while True: if (self.sync_thread != None and self.sync_thread.isAlive()) or (self.stop_thread != None and self.stop_thread.isAlive()): time.sleep(0.5) continue else: break # close window exit def exit(self): self.set_status(EXIT) self.stop_sync() time.sleep(0.5) self.root.destroy() sys.exit(0) # get folder size in KB in general def get_size(self, path): return int(subprocess.check_output(['du', '-s', path]).split()[0]) # calculate file's logical size def get_file_size(self, path): return int(subprocess.check_output(['ls', '-l', path]).split()[4]) # calculate folde'r logical size def get_dir_size(self, path): size = 0 for item in os.listdir(path): item_path = os.path.join(path, item) size += self.get_file_size(item_path) if os.path.isfile(item_path) else self.get_dir_size(item_path) return int(size)
class RunningOperationsPage(BaseInputPage, object): def __init__(self, parent, controller, frame_number): BaseInputPage.__init__(self, parent, controller, frame_number) self.move_back = False self.need_subjects_file = False self.downloaded_file_path = '~/Downloads/' self.operation = Operations(self.controller) self.start = tk.Button(self, text='Start Execution', command=lambda : self.executeCommand()) self.start.grid(row=self.starting_row, column=0, sticky='W', padx=5, pady=3) self.stop = tk.Button(self, text='Stop Execution', state="disabled", command=lambda : self.terminateCommand()) self.stop.grid(row=self.starting_row, column=2, sticky='E', padx=5, pady=3) self.progressbar = Progressbar(self) self.progressbar.configure(mode='determinate', max=100) self.progressbar.grid(row=self.starting_row+1, column=0, columnspan=3, sticky='ew', padx=10, pady=10) self.controller.progressbar = self.progressbar self.output = Text(self, height=15, width=100) self.output.grid(row=self.starting_row+2, column=0, columnspan=3, sticky='ew', padx=10) self.hyperlink = HyperlinkManager(self.output) self.lf_subject_file = LabelFrame(self, text='Visual QC', padx=15, font='Helvetica 14 bold') self.lf_subject_file.grid(row=self.starting_row+3, column=0, columnspan=3, sticky='WE', padx=5, pady=5, ipadx=5, ipady=5) self.lf_subject_file.grid_rowconfigure(0, weight=1) self.lf_subject_file.grid_columnconfigure(4, weight=1) createToolTip(self.lf_subject_file, self.controller.desc.subject_file) self.lb_subject_file = Label(self.lf_subject_file, text="Select flagged subjects textfile", font='Helvetica 13 bold') self.lb_subject_file.grid(row=0, column=0, sticky="W", pady=3) self.select = tk.Button(self.lf_subject_file, text='Select flagged subjects file', command=lambda : self.chooseFile(self, controller, controller.selected_subjects, 'Selected Subjects', default_dir=self.downloaded_file_path)) self.select.grid(row=1, column=0, sticky='W', padx=5, pady=3) lb_or = Label(self.lf_subject_file, text="-OR-", font='Helvetica 13 bold') lb_or.grid(row=1, column=1, sticky="W", pady=3) self.continue_with_all_sub = tk.Button(self.lf_subject_file, text='Continue with all subjects', command=lambda : self.continueWithAllSub()) self.continue_with_all_sub.grid(row=1, column=2, sticky='W', padx=10, pady=3) self.controller.display = self.output def chooseFile(self, parent, controller, place_holder, message, default_dir=''): super(RunningOperationsPage, self).chooseFile(parent, controller, place_holder, message, default_dir=default_dir) self.executeCommand() def setFrameTitle(self): self.title.set('Press Start Execution') def onShowFrame(self, event): super(RunningOperationsPage, self).onShowFrame(event) self.resetAll() self.silentMode() def resetAll(self): self.resetUI() self.operation.resetOperations() def moveToNextPage(self): super(RunningOperationsPage, self).moveToNextPage() def resetClickCounter(self): self.move_back = False self.setRequiredInputError('') def userAgreed(self): if self.move_back: return True self.setRequiredInputError('Warning: All progress will be lost. If you wish to continue, press the button again.') self.move_back = True return False def moveToPrevPage(self): if self.userAgreed(): self.resetAll() super(RunningOperationsPage, self).moveToPrevPage() def resetUI(self): self.start.config(state="normal") self.start.config(text='Start Execution') self.btn_prev.config(state="normal") self.btn_next.config(state="disabled") self.stop.config(state="disabled") self.output.delete('1.0', END) self.title.set("Press 'Start Execution' to begin") self.progressbar.config(value=0) self.need_subjects_file = False self.disableChildren(self.lf_subject_file.winfo_children()) self.resetClickCounter() def continueWithAllSub(self): self.need_subjects_file = False self.executeCommand() def silentMode(self): if self.controller.silent: self.executeCommand() def executeCommand(self): if self.start['text'] == 'Continue Execution' and self.need_subjects_file and not self.controller.silent: selected_subjects_file_path = self.controller.selected_subjects.get() new_subjects = [] try: with open(selected_subjects_file_path, 'r') as f: new_subjects = f.readlines() new_subjects = [subj.strip() for subj in new_subjects] self.downloaded_file_path = os.path.dirname(selected_subjects_file_path) self.controller.selected_subjects.set('') self.operation.updateSubjects(new_subjects) self.setRequiredInputError('') self.need_subjects_file = False except Exception as e: self.controller.logger.error(e.message) self.controller.logger.error(traceback.format_exc()) self.setRequiredInputError('Please import correct textfile downloaded from the QC page.') return False self.disableChildren(self.lf_subject_file.winfo_children()) self.start.config(state="disabled") self.stop.config(state="normal") self.resetClickCounter() if self.start['text'] == 'Continue Execution': self.operation.incrementStage() self.title.set('Please wait') self.operation.startThreads(self) def terminateCommand(self): if self.userAgreed(): self.stop.config(state="disabled") self.operation.stopThreads() def pause(self, operation_name='', data='', need_pause=False): if need_pause and not self.controller.silent: self.start.config(state="disabled") self.start.config(text='Continue Execution') self.title.set('Please input subject file') self.btn_prev.config(state="normal") self.stop.config(state="disabled") self.resetClickCounter() self.need_subjects_file = True self.enableChildren(self.lf_subject_file.winfo_children()) if data: self.insertHyperLink(operation_name, data) def finished(self, operation_name='', data=''): self.start.config(text="Start Execution") self.start.config(state="disabled") self.btn_prev.config(state="normal") self.stop.config(state="disabled") self.title.set('Completed') self.controller.updateMessage('All operations completed. You may now close the application.') self.resetClickCounter() if data: self.insertHyperLink(operation_name, data) def insertHyperLink(self, heading, link): self.output.insert(END, '\n\n') self.output.insert(END, "QC Page for " + heading, self.hyperlink.add(partial(webbrowser.open, link))) self.output.insert(END, '\n\n') self.output.see(END) def toggleChildren(self): self.disableChildren(self.lf_subject_file.winfo_children()) def disableChildren(self, childList): for child in childList: try: self.disableChildren(child.winfo_children()) except: pass try: child.configure(state='disable') except: pass def enableChildren(self, childList): for child in childList: try: self.enableChildren(child.winfo_children()) except: pass try: child.configure(state='normal') except: pass