def load_from_dir(self, directory, class_, *args): objects = OrderedDict() ids = [id for id in os.listdir(directory) if os.path.isdir(join(directory, id))] for id in sorted(ids, key=int): path = join(directory, id) ctor_args = list(args) + [id, path] objects[id] = class_(*ctor_args) return objects
def _read_params(self): try: saved_params = json.load(open(self.params_path)) except IOError: saved_params = {} self.params = OrderedDict() # built-in params default_params = [{ 'label': i18n.state, 'id': 'state', 'default': 'open', 'no_send': True, 'uploader_visible': True, 'uploader_readonly': True, }, { 'label': i18n.description, 'id': 'description', 'default': '', 'no_send': True, 'uploader_visible': True, }] + self.template.defaults for default_param in default_params: param = Param() if isinstance(default_param, tuple): param.id = default_param[0] param.label = param.id param.default = default_param[1] elif isinstance(default_param, dict): param.__dict__.update(default_param) if param.default is None: param.default = '' param.default = str(param.default) param.value = param.default if not isinstance(param.label, unicode): param.label = unicode(param.label, 'utf-8') if param.id in saved_params: param.value = saved_params[param.id] self.params[param.id] = param
def _load_templates(self): # TODO: detect if the config file changed, and reload. if self.templates is not None: return self.templates = OrderedDict() config_path = join(self.project.path, 'manager', 'imports_config.py') config = { # Give access to the project 'project': self.project, } if os.path.isfile(config_path): execfile(config_path, { 'project': self.project, }, config) if 'import_templates' not in config: log.warn('No imports config') return for template_config in config['import_templates']: template = ImportTemplate(self, template_config) self.templates[template.id] = template
class ImportsManager(object): def __init__(self, project): self.project = project self.templates = None def _load_templates(self): # TODO: detect if the config file changed, and reload. if self.templates is not None: return self.templates = OrderedDict() config_path = join(self.project.path, 'manager', 'imports_config.py') config = { # Give access to the project 'project': self.project, } if os.path.isfile(config_path): execfile(config_path, { 'project': self.project, }, config) if 'import_templates' not in config: log.warn('No imports config') return for template_config in config['import_templates']: template = ImportTemplate(self, template_config) self.templates[template.id] = template def get_import_templates(self): self._load_templates() # TODO: sort by key return self.templates.values() def get_import_template(self, template_id): self._load_templates() return self.templates[template_id] def get_import(self, template_id, import_id): import_template = self.get_import_template(template_id) return import_template.get_import(import_id)
class Import(utils.DirObjectLoader): def __init__(self, template, id, path): self.template = template self.id = id self.path = path self._runs = None self.params_path = join(self.path, 'params.json') self.events_path = join(self.path, 'events.json') self.runs_path = join(self.path, 'runs') utils.maybe_makedirs(self.runs_path) config_path = join(self.path, 'config.py') if os.path.isfile(config_path): config = {} execfile(config_path, {}, config) self.config = config['config'] else: self.config = {} self._read_params() def _read_params(self): try: saved_params = json.load(open(self.params_path)) except IOError: saved_params = {} self.params = OrderedDict() # built-in params default_params = [{ 'label': i18n.state, 'id': 'state', 'default': 'open', 'no_send': True, 'uploader_visible': True, 'uploader_readonly': True, }, { 'label': i18n.description, 'id': 'description', 'default': '', 'no_send': True, 'uploader_visible': True, }] + self.template.defaults for default_param in default_params: param = Param() if isinstance(default_param, tuple): param.id = default_param[0] param.label = param.id param.default = default_param[1] elif isinstance(default_param, dict): param.__dict__.update(default_param) if param.default is None: param.default = '' param.default = str(param.default) param.value = param.default if not isinstance(param.label, unicode): param.label = unicode(param.label, 'utf-8') if param.id in saved_params: param.value = saved_params[param.id] self.params[param.id] = param def save_params(self, username=None, no_mail=False): saved_params = dict((id, param.value) for (id, param) in self.params.iteritems() if param.value != param.default) json.dump( saved_params, open(self.params_path, 'wb'), sort_keys=True, indent=2) event = Event() event.type = 'update' event.username = username self._add_event(event, None, no_mail) def get_events(self): try: saved_events = json.load(open(self.events_path)) except IOError: saved_events = [] events = [] for saved_event in saved_events: event = Event() event.__dict__.update(saved_event) events.append(event) return events events = property(get_events) def _add_event(self, event, extra=None, no_mail=False): events = self.get_events() events.append(event) saved_events = [event.__dict__ for event in events] json.dump( saved_events, open(self.events_path, 'wb'), sort_keys=True, indent=2) if not no_mail: self._send_event_notifications(event, extra) def add_comment(self, username, comment): event = Event() event.type = 'comment' event.username = username event.content = comment self._add_event(event) def _load_runs(self): if self._runs is not None: return self._runs = self.load_from_dir(self.runs_path, ImportRun, self) def get_runs(self): self._load_runs() return self._runs.values() runs = property(get_runs) def get_run(self, run_id): self._load_runs() return self._runs[run_id] def _create_run(self): self._load_runs() return self.create_object(self._runs, self.runs_path, ImportRun, self) def _send_event_notifications(self, event, extra): t = self.template to_send = [] if event.type == 'comment': emails = t.admin_emails | t.uploader_emails detail = i18n.new_comment body = i18n.new_comment_body.format( username=event.username, comment=event.content ) to_send = ((emails, body),) elif event.type == 'run': run = extra detail = i18n.import_result to_send = self._get_run_finish_notifications(run) elif event.type == 'update': emails = t.admin_emails | t.uploader_emails detail = i18n.import_update body = i18n.import_updated_body.format( username=event.username) to_send = ((emails, body),) else: raise Exception('Unknown event type %r' % event.type) config = self.template.manager.project.env.config subject = i18n.mail_summary.format( project_name=config.project_name, hostname=socket.gethostname(), template_label=self.template.label, detail=detail) for emails, body in to_send: if not emails: continue utils.send_mail(config, list(emails), subject, body) def _get_run_finish_notifications(self, run): level_to_mails = {} for level in ImportRun.LEVEL_NAMES: # TODO: also allow passing a username instead of an email level_to_mails[level] = set( self.template.run_results_notifications.get(level, [])) # admins always get errors. level_to_mails['err'].update(self.template.admin_emails) levels_with_messages = set() for level in ImportRun.LEVEL_NAMES: if run.messages[level]: levels_with_messages.add(level) log.debug('Levels with messages: %s', levels_with_messages) levels_to_notify = set() for level in ImportRun.LEVEL_NAMES: index = ImportRun.LEVEL_NAMES.index(level) same_or_higher_levels = set(ImportRun.LEVEL_NAMES[index:]) if same_or_higher_levels & levels_with_messages: levels_to_notify.add(level) log.debug('Levels to notify: %s', levels_to_notify) config = self.template.manager.project.env.config to_send = [] notified_emails = set() for level in ImportRun.LEVEL_NAMES: if level not in levels_to_notify: continue emails = level_to_mails[level] - notified_emails if not emails: continue notified_emails.update(emails) body = run.get_summary(level) to_send.append((emails, body)) return to_send def execute(self, username=None, dummy=False, no_mail=False): log.info('Executing import') run = self._create_run() run.dummy = dummy try: start = time.time() if self.template.do_import: self.template.do_import(self, run) else: self._do_import(run) except Exception, e: log.warn('Import exception: %s', traceback.format_exc(e)) run.add_failure(e) finally: