def create_watcher(name=None, watcher_type=None, base=None, exporter=None): '''create a watcher, meaning a folder with a configuration and initialized git repo. Parameters ========== name: the watcher to create, uses default or WATCHME_WATCHER watcher_type: the type of watcher to create. defaults to WATCHER_DEFAULT_TYPE ''' if name == None: name = WATCHME_WATCHER if base == None: base = WATCHME_BASE_DIR # Create the repository folder repo = os.path.join(base, name) if not os.path.exists(repo): bot.info('Adding watcher %s...' % repo) mkdir_p(repo) # Ensure no gpg signing happens run_command("git --git-dir=%s/.git init" % repo) run_command("git --git-dir=%s/.git config commit.gpgsign false" % repo) # Add the watcher configuration file generate_watcher_config(repo, watcher_type, exporter) run_command("git -C %s add watchme.cfg" % repo) return repo else: bot.info('%s already exists: %s' % (name, repo))
def test_mkdir(self): print('Testing utils.mkdir_p') from watchme.utils import mkdir_p tmpdir = os.path.join(self.tmpdir, "if", "I", "were", "an", "avocado") mkdir_p(tmpdir) self.assertTrue(os.path.exists(tmpdir)) print('Testing utils.get_tmpdir') from watchme.utils import get_tmpdir tmpdir = get_tmpdir() self.assertTrue(os.path.exists(tmpdir)) self.assertTrue('watchme' in tmpdir) shutil.rmtree(tmpdir)
def create_watcher_base(name=None, base=None): """create a watch base and default repo, if it doesn't already exist. Parameters ========== name: the watcher to create, uses default or WATCHME_WATCHER base: the watcher base, defaults to WATCHME_BASE_DIR """ if base is None: base = WATCHME_BASE_DIR if name is None: name = WATCHME_WATCHER if not os.path.exists(base): bot.info("Creating %s..." % base) mkdir_p(base)
def finish_runs(self, results): '''finish runs should take a dictionary of results, with keys as the folder name, and for each, depending on the result type, write the result to file (or update file) and then commit to git. Parameters ========== results: a dictionary of tasks, with keys as the task name, and values as the result. ''' if results is None: return for name, result in results.items(): task_folder = os.path.join(self.repo, name) if name.startswith('task'): task = self.get_task(name) # A decorator is run on the fly (not in config) elif name.startswith('decorator'): task = self.get_decorator(name) # We only allow tasks and decorators else: bot.warning('%s is not task or decorator, skipping.' % name) continue # Ensure that the task folder exists if not os.path.exists(task_folder): mkdir_p(task_folder) git_add(self.repo, task_folder) # Files to be added to the repo via git after files = task.write_results(result, self.repo) # Add files to git, and commit files.append(write_timestamp(repo=self.repo, task=name)) git_add(repo=self.repo, files=files) git_commit(repo=self.repo, task=self.name, message="ADD results %s" % name)
def _add_task(self, task, force=False, active='true'): '''add a new task to the watcher, meaning we: 1. Check first that the task doesn't already exist (if the task exists, we only add if force is set to true) 2. Validate the task (depends on the task) 3. write the task to the helper config file, if valid. Parameters ========== task: the Task object to add, should have a name and params and be child of watchme.tasks.TaskBase force: if task already exists, overwrite active: add the task as active (default "true") ''' self.load_config() if active not in ["true", "false"]: bot.exit('Active must be "true" or "false"') # Don't overwrite a section that already exists if task.name in self.config.sections(): if not force: bot.exit('%s exists, use --force to overwrite.' % task.name) self.remove_section(task.name, save=False) # Add the new section self.config[task.name] = task.export_params(active=active) self.print_section(task.name) self.save() # If the task folder doesn't exist, recreate it. task_folder = os.path.join(self.repo, task.name) if not os.path.exists(task_folder): mkdir_p(task_folder) git_add(self.repo, task.name) # Commit changes git_commit(repo=self.repo, task=self.name, message="ADD task %s" % task.name)
def finish_runs(self, results): '''finish runs should take a dictionary of results, with keys as the folder name, and for each, depending on the result type, write the result to file (or update file) and then commit to git. Parameters ========== results: a dictionary of tasks, with keys as the task name, and values as the result. ''' for name, result in results.items(): task_folder = os.path.join(self.repo, name) task = self.get_task(name, save=True) # Files to be added via Git after files = [] # Ensure that the task folder exists if not os.path.exists(task_folder): mkdir_p(task_folder) git_add(self.repo, task_folder) # Case 1. The result is a list if isinstance(result, list): # Get rid of Nones, if the user accidentally added result = [r for r in result if r] if len(result) == 0: bot.error('%s returned empty list of results.' % name) # json output is specified elif task.params.get('save_as') == 'json': bot.debug('Saving single list as one json...') files.append(task._save_json(result, self.repo)) elif task.params.get('save_as') == 'json': bot.debug('Saving single list as multiple json...') files += task._save_json_list(result, self.repo) # Otherwise, sniff for list of paths elif os.path.exists(result[0]): bot.debug('Found list of paths...') files += task._save_files_list(result, self.repo) # Finally, assume just writing text to file else: bot.debug('Saving content from list to file...') files += task._save_text_list(result, self.repo) # Case 2. The result is a string elif isinstance(result, str): # if it's a path to a file, just save to repository if os.path.exists(result): files.append(task._save_file(result, self.repo)) # Otherwise, it's a string that needs to be saved to file else: files.append(task._save_text(result, self.repo)) # Case 3. The result is a dictionary elif isinstance(result, dict): files.append(task._save_json(result, self.repo)) elif result == None: bot.error('Result for task %s is None' % name) else: bot.error('Unsupported result format %s' % type(result)) # Get rid of None results (don't check excessively for None above) files = [f for f in files if f] # Add files to git, and commit files.append(write_timestamp(repo=self.repo, task=name)) git_add(repo=self.repo, files=files) git_commit(repo=self.repo, task=self.name, message="ADD results %s" % name)