def _gc_create_workflow(config, do_freeze=True, **kwargs): # create workflow from config and do initial processing steps # set up signal handler for interrupts and debug session or stack dump requests signal.signal(signal.SIGURG, handle_debug_interrupt) signal.signal(signal.SIGINT, handle_abort_interrupt) start_daemon('debug watchdog', _debug_watchdog) # Configure logging settings logging_setup(config.change_view(set_sections=['logging'])) global_config = config.change_view(set_sections=['global']) _setup_work_path(global_config) for package_paths in global_config.get_dn_list('package paths', [], on_change=None): init_hpf_plugins(package_paths) # Query config settings before config is frozen help_cfg = global_config.get_state('display', detail='config') help_scfg = global_config.get_state('display', detail='minimal config') action_config = config.change_view(set_sections=['action']) action_cancel = action_config.get(['delete', 'cancel'], '', on_change=None) action_reset = action_config.get('reset', '', on_change=None) # Create workflow and freeze config settings workflow = global_config.get_plugin('workflow', 'Workflow:global', cls='Workflow', pkwargs=kwargs) gui = config.get_plugin('gui', 'BasicConsoleGUI', cls=GUI, on_change=None, pargs=(workflow, )) if do_freeze: config.factory.freeze( write_config=config.get_state('init', detail='config')) # Give config help if help_cfg or help_scfg: config.write(sys.stdout, print_default=help_cfg, print_unused=False, print_minimal=help_scfg, print_source=help_cfg) sys.exit(os.EX_OK) # Check if user requested deletion / reset of jobs if action_cancel: workflow.job_manager.cancel(workflow.task, workflow.backend, action_cancel) sys.exit(os.EX_OK) if action_reset: workflow.job_manager.reset(workflow.task, workflow.backend, action_reset) sys.exit(os.EX_OK) return (workflow, gui)
def _gc_create_workflow(config, do_freeze=True, **kwargs): # create workflow from config and do initial processing steps # set up signal handler for interrupts and debug session or stack dump requests signal.signal(signal.SIGURG, handle_debug_interrupt) signal.signal(signal.SIGINT, handle_abort_interrupt) start_daemon('debug watchdog', _debug_watchdog) # Configure logging settings logging_setup(config.change_view(set_sections=['logging'])) global_config = config.change_view(set_sections=['global']) _setup_work_path(global_config) for package_paths in global_config.get_dn_list('package paths', [], on_change=None): init_hpf_plugins(package_paths) # Query config settings before config is frozen help_cfg = global_config.get_state('display', detail='config') help_scfg = global_config.get_state('display', detail='minimal config') action_config = config.change_view(set_sections=['action']) action_cancel = action_config.get(['delete', 'cancel'], '', on_change=None) action_reset = action_config.get('reset', '', on_change=None) # Create workflow and freeze config settings workflow = global_config.get_plugin('workflow', 'Workflow:global', cls='Workflow', pkwargs=kwargs) gui = config.get_plugin('gui', 'BasicConsoleGUI', cls=GUI, on_change=None, pargs=(workflow,)) if do_freeze: config.factory.freeze(write_config=config.get_state('init', detail='config')) # Give config help if help_cfg or help_scfg: config.write(sys.stdout, print_default=help_cfg, print_unused=False, print_minimal=help_scfg, print_source=help_cfg) sys.exit(os.EX_OK) # Check if user requested deletion / reset of jobs if action_cancel: workflow.job_manager.cancel(workflow.task, workflow.backend, action_cancel) sys.exit(os.EX_OK) if action_reset: workflow.job_manager.reset(workflow.task, workflow.backend, action_reset) sys.exit(os.EX_OK) return (workflow, gui)
def download_single_file(opts, jobnum, fi_idx, fi, status_mon): (source_se_path, target_se_path, local_se_path) = get_fi_path_tuple(opts, fi) show_file_info(jobnum, fi_idx, fi) # Copy files to local folder if not accepted_se(opts, fi): return status_mon.register_file_result(jobnum, fi_idx, 'skipping file on blacklisted SE', FileDownloadStatus.FILE_SE_BLACKLIST) activity_check = Activity('Checking file existance') try: if opts.skip_existing and (se_exists(target_se_path).status(timeout=10, terminate=True) == 0): return status_mon.register_file_result(jobnum, fi_idx, 'skipping already existing file', FileDownloadStatus.FILE_EXISTS) finally: activity_check.finish() if se_exists(os.path.dirname(target_se_path)).status(timeout=10, terminate=True) != 0: activity = Activity('Creating target directory') try: mkdir_proc = se_mkdir(os.path.dirname(target_se_path)) if mkdir_proc.status(timeout=10, terminate=True) != 0: return status_mon.register_file_result(jobnum, fi_idx, 'unable to create target dir', FileDownloadStatus.FILE_MKDIR_FAILED, proc=mkdir_proc) finally: activity.finish() if 'file://' in target_se_path: local_se_path = target_se_path copy_timeout_event = GCEvent() copy_ended_event = GCEvent() monitor_thread = start_daemon('Download monitor %s' % jobnum, download_monitor, jobnum, fi_idx, fi, local_se_path, copy_ended_event, copy_timeout_event) cp_proc = se_copy(source_se_path, target_se_path, tmp=local_se_path) while (cp_proc.status(timeout=0) is None) and not copy_timeout_event.wait(timeout=0.1): pass copy_ended_event.set() monitor_thread.join() if copy_timeout_event.is_set(): cp_proc.terminate(timeout=1) return status_mon.register_file_result(jobnum, fi_idx, 'Transfer timeout', FileDownloadStatus.FILE_TIMEOUT) elif cp_proc.status(timeout=0, terminate=True) != 0: return status_mon.register_file_result(jobnum, fi_idx, 'Transfer error', FileDownloadStatus.FILE_TIMEOUT, proc=cp_proc) return hash_verify(opts, status_mon, local_se_path, jobnum, fi_idx, fi)
def _start_watcher(self, desc, daemon, pid, *args): desc += ' (%d:%r)' % (pid, [self._cmd] + self._args) if daemon: return start_daemon(desc, *args) return start_thread(desc, *args)
def _trigger_debug_signal(duration): def _signal_debug_console(): time.sleep(duration) os.kill(os.getpid(), signal.SIGURG) start_daemon('debug console trigger', _signal_debug_console)
def _trigger_debug_signal(duration): def _signal_debug_console(): time.sleep(duration) os.kill(os.getpid(), signal.SIGURG) start_daemon('debug console trigger', _signal_debug_console)
def download_single_file(opts, jobnum, fi_idx, fi, status_mon): (source_se_path, target_se_path, local_se_path) = get_fi_path_tuple(opts, fi) show_file_info(jobnum, fi_idx, fi) # Copy files to local folder if not accepted_se(opts, fi): return status_mon.register_file_result( jobnum, fi_idx, 'skipping file on blacklisted SE', FileDownloadStatus.FILE_SE_BLACKLIST) activity_check = Activity('Checking file existance') try: if opts.skip_existing and (se_exists(target_se_path).status( timeout=10, terminate=True) == 0): return status_mon.register_file_result( jobnum, fi_idx, 'skipping already existing file', FileDownloadStatus.FILE_EXISTS) finally: activity_check.finish() if se_exists(os.path.dirname(target_se_path)).status(timeout=10, terminate=True) != 0: activity = Activity('Creating target directory') try: mkdir_proc = se_mkdir(os.path.dirname(target_se_path)) if mkdir_proc.status(timeout=10, terminate=True) != 0: return status_mon.register_file_result( jobnum, fi_idx, 'unable to create target dir', FileDownloadStatus.FILE_MKDIR_FAILED, proc=mkdir_proc) finally: activity.finish() if 'file://' in target_se_path: local_se_path = target_se_path copy_timeout_event = GCEvent() copy_ended_event = GCEvent() monitor_thread = start_daemon('Download monitor %s' % jobnum, download_monitor, jobnum, fi_idx, fi, local_se_path, copy_ended_event, copy_timeout_event) cp_proc = se_copy(source_se_path, target_se_path, tmp=local_se_path) while (cp_proc.status(timeout=0) is None) and not copy_timeout_event.wait(timeout=0.1): pass copy_ended_event.set() monitor_thread.join() if copy_timeout_event.is_set(): cp_proc.terminate(timeout=1) return status_mon.register_file_result(jobnum, fi_idx, 'Transfer timeout', FileDownloadStatus.FILE_TIMEOUT) elif cp_proc.status(timeout=0, terminate=True) != 0: return status_mon.register_file_result(jobnum, fi_idx, 'Transfer error', FileDownloadStatus.FILE_TIMEOUT, proc=cp_proc) return hash_verify(opts, status_mon, local_se_path, jobnum, fi_idx, fi)
def start_interface(self): GCStreamHandler.set_global_lock(self._console_lock) with_lock(self._console_lock, self._element.draw_startup) self._redraw_shutdown = False # start redraw thread self._redraw_thread = start_daemon('GUI draw thread', self._redraw)
def _start_watcher(self, desc, daemon, pid, *args): desc += ' (%d:%r)' % (pid, [self._cmd] + self._args) if daemon: return start_daemon(desc, *args) return start_thread(desc, *args)