def load_from_dir(self, directory, class_, *args):
objects = OrderedDict()
ids = [id for id in os.listdir(directory) if
os.path.isdir(join(directory, id))]
for id in sorted(ids, key=int):
path = join(directory, id)
ctor_args = list(args) + [id, path]
objects[id] = class_(*ctor_args)
return objects
def _read_params(self):
try:
saved_params = json.load(open(self.params_path))
except IOError:
saved_params = {}
self.params = OrderedDict()
# built-in params
default_params = [{
'label': i18n.state,
'id': 'state',
'default': 'open',
'no_send': True,
'uploader_visible': True,
'uploader_readonly': True,
}, {
'label': i18n.description,
'id': 'description',
'default': '',
'no_send': True,
'uploader_visible': True,
}] + self.template.defaults
for default_param in default_params:
param = Param()
if isinstance(default_param, tuple):
param.id = default_param[0]
param.label = param.id
param.default = default_param[1]
elif isinstance(default_param, dict):
param.__dict__.update(default_param)
if param.default is None:
param.default = ''
param.default = str(param.default)
param.value = param.default
if not isinstance(param.label, unicode):
param.label = unicode(param.label, 'utf-8')
if param.id in saved_params:
param.value = saved_params[param.id]
self.params[param.id] = param
def _load_templates(self):
# TODO: detect if the config file changed, and reload.
if self.templates is not None:
return
self.templates = OrderedDict()
config_path = join(self.project.path, 'manager', 'imports_config.py')
config = {
# Give access to the project
'project': self.project,
}
if os.path.isfile(config_path):
execfile(config_path, {
'project': self.project,
}, config)
if 'import_templates' not in config:
log.warn('No imports config')
return
for template_config in config['import_templates']:
template = ImportTemplate(self, template_config)
self.templates[template.id] = template
class ImportsManager(object):
def __init__(self, project):
self.project = project
self.templates = None
def _load_templates(self):
# TODO: detect if the config file changed, and reload.
if self.templates is not None:
return
self.templates = OrderedDict()
config_path = join(self.project.path, 'manager', 'imports_config.py')
config = {
# Give access to the project
'project': self.project,
}
if os.path.isfile(config_path):
execfile(config_path, {
'project': self.project,
}, config)
if 'import_templates' not in config:
log.warn('No imports config')
return
for template_config in config['import_templates']:
template = ImportTemplate(self, template_config)
self.templates[template.id] = template
def get_import_templates(self):
self._load_templates()
# TODO: sort by key
return self.templates.values()
def get_import_template(self, template_id):
self._load_templates()
return self.templates[template_id]
def get_import(self, template_id, import_id):
import_template = self.get_import_template(template_id)
return import_template.get_import(import_id)
class Import(utils.DirObjectLoader):
def __init__(self, template, id, path):
self.template = template
self.id = id
self.path = path
self._runs = None
self.params_path = join(self.path, 'params.json')
self.events_path = join(self.path, 'events.json')
self.runs_path = join(self.path, 'runs')
utils.maybe_makedirs(self.runs_path)
config_path = join(self.path, 'config.py')
if os.path.isfile(config_path):
config = {}
execfile(config_path, {}, config)
self.config = config['config']
else:
self.config = {}
self._read_params()
def _read_params(self):
try:
saved_params = json.load(open(self.params_path))
except IOError:
saved_params = {}
self.params = OrderedDict()
# built-in params
default_params = [{
'label': i18n.state,
'id': 'state',
'default': 'open',
'no_send': True,
'uploader_visible': True,
'uploader_readonly': True,
}, {
'label': i18n.description,
'id': 'description',
'default': '',
'no_send': True,
'uploader_visible': True,
}] + self.template.defaults
for default_param in default_params:
param = Param()
if isinstance(default_param, tuple):
param.id = default_param[0]
param.label = param.id
param.default = default_param[1]
elif isinstance(default_param, dict):
param.__dict__.update(default_param)
if param.default is None:
param.default = ''
param.default = str(param.default)
param.value = param.default
if not isinstance(param.label, unicode):
param.label = unicode(param.label, 'utf-8')
if param.id in saved_params:
param.value = saved_params[param.id]
self.params[param.id] = param
def save_params(self, username=None, no_mail=False):
saved_params = dict((id, param.value) for
(id, param) in self.params.iteritems() if
param.value != param.default)
json.dump(
saved_params, open(self.params_path, 'wb'),
sort_keys=True, indent=2)
event = Event()
event.type = 'update'
event.username = username
self._add_event(event, None, no_mail)
def get_events(self):
try:
saved_events = json.load(open(self.events_path))
except IOError:
saved_events = []
events = []
for saved_event in saved_events:
event = Event()
event.__dict__.update(saved_event)
events.append(event)
return events
events = property(get_events)
def _add_event(self, event, extra=None, no_mail=False):
events = self.get_events()
events.append(event)
saved_events = [event.__dict__ for event in events]
json.dump(
saved_events, open(self.events_path, 'wb'),
sort_keys=True, indent=2)
if not no_mail:
self._send_event_notifications(event, extra)
def add_comment(self, username, comment):
event = Event()
event.type = 'comment'
event.username = username
event.content = comment
self._add_event(event)
def _load_runs(self):
if self._runs is not None:
return
self._runs = self.load_from_dir(self.runs_path, ImportRun, self)
def get_runs(self):
self._load_runs()
return self._runs.values()
runs = property(get_runs)
def get_run(self, run_id):
self._load_runs()
return self._runs[run_id]
def _create_run(self):
self._load_runs()
return self.create_object(self._runs, self.runs_path, ImportRun, self)
def _send_event_notifications(self, event, extra):
t = self.template
to_send = []
if event.type == 'comment':
emails = t.admin_emails | t.uploader_emails
detail = i18n.new_comment
body = i18n.new_comment_body.format(
username=event.username,
comment=event.content
)
to_send = ((emails, body),)
elif event.type == 'run':
run = extra
detail = i18n.import_result
to_send = self._get_run_finish_notifications(run)
elif event.type == 'update':
emails = t.admin_emails | t.uploader_emails
detail = i18n.import_update
body = i18n.import_updated_body.format(
username=event.username)
to_send = ((emails, body),)
else:
raise Exception('Unknown event type %r' % event.type)
config = self.template.manager.project.env.config
subject = i18n.mail_summary.format(
project_name=config.project_name,
hostname=socket.gethostname(),
template_label=self.template.label,
detail=detail)
for emails, body in to_send:
if not emails:
continue
utils.send_mail(config, list(emails), subject, body)
def _get_run_finish_notifications(self, run):
level_to_mails = {}
for level in ImportRun.LEVEL_NAMES:
# TODO: also allow passing a username instead of an email
level_to_mails[level] = set(
self.template.run_results_notifications.get(level, []))
# admins always get errors.
level_to_mails['err'].update(self.template.admin_emails)
levels_with_messages = set()
for level in ImportRun.LEVEL_NAMES:
if run.messages[level]:
levels_with_messages.add(level)
log.debug('Levels with messages: %s', levels_with_messages)
levels_to_notify = set()
for level in ImportRun.LEVEL_NAMES:
index = ImportRun.LEVEL_NAMES.index(level)
same_or_higher_levels = set(ImportRun.LEVEL_NAMES[index:])
if same_or_higher_levels & levels_with_messages:
levels_to_notify.add(level)
log.debug('Levels to notify: %s', levels_to_notify)
config = self.template.manager.project.env.config
to_send = []
notified_emails = set()
for level in ImportRun.LEVEL_NAMES:
if level not in levels_to_notify:
continue
emails = level_to_mails[level] - notified_emails
if not emails:
continue
notified_emails.update(emails)
body = run.get_summary(level)
to_send.append((emails, body))
return to_send
def execute(self, username=None, dummy=False, no_mail=False):
log.info('Executing import')
run = self._create_run()
run.dummy = dummy
try:
start = time.time()
if self.template.do_import:
self.template.do_import(self, run)
else:
self._do_import(run)
except Exception, e:
log.warn('Import exception: %s', traceback.format_exc(e))
run.add_failure(e)
finally:
