def save_trigger(self, trigger, commit_and_initialize=True, flush=False):
""" :type trigger: dart.model.trigger.Trigger """
trigger_type_name = trigger.data.trigger_type_name
if trigger_type_name == self._manual_trigger_processor.trigger_type(
).name:
raise DartValidationException('manual triggers cannot be saved')
trigger_processor = self._trigger_processors.get(trigger_type_name)
if not trigger_processor:
raise DartValidationException('unknown trigger_type_name: %s' %
trigger_type_name)
assert isinstance(trigger_processor, TriggerProcessor)
trigger = default_and_validate(
trigger,
trigger_schema(
trigger_processor.trigger_type().params_json_schema))
trigger_dao = TriggerDao()
trigger_dao.id = random_id()
trigger_dao.data = trigger.data.to_dict()
db.session.add(trigger_dao)
if flush:
db.session.flush()
trigger = trigger_dao.to_model()
if commit_and_initialize:
db.session.commit()
trigger = trigger_dao.to_model()
try:
trigger_processor.initialize_trigger(trigger, self)
except:
db.session.delete(trigger_dao)
db.session.commit()
raise
return trigger
def save_trigger(self,
trigger,
commit_and_initialize=True,
flush=False,
user_id=None):
wf_uuid = uuid.uuid4().hex # to avoid uuid serialization issues
trigger.data.tags = trigger.data.tags if (trigger.data.tags) else []
trigger.data.tags.append(wf_uuid)
if user_id:
trigger.data.user_id = user_id
""" :type trigger: dart.model.trigger.Trigger """
trigger_type_name = trigger.data.trigger_type_name
if trigger_type_name == self._manual_trigger_processor.trigger_type(
).name:
raise DartValidationException('manual triggers cannot be saved')
if trigger_type_name == self._retry_trigger_processor.trigger_type(
).name:
raise DartValidationException('retry triggers cannot be saved')
trigger_processor = self._trigger_processors.get(trigger_type_name)
if not trigger_processor:
raise DartValidationException('unknown trigger_type_name: %s' %
trigger_type_name)
assert isinstance(trigger_processor, TriggerProcessor)
trigger = default_and_validate(
trigger,
trigger_schema(
trigger_processor.trigger_type().params_json_schema))
trigger_dao = TriggerDao()
trigger_dao.id = random_id()
if trigger_type_name == 'subscription_batch':
sub = self._subscription_service.get_subscription(
trigger.data.args['subscription_id'])
if sub.data.nudge_id:
response = self.update_nudge_with_trigger(
sub.data.nudge_id,
trigger.data.args['unconsumed_data_size_in_bytes'],
trigger_dao.id, trigger.data.trigger_type_name)
assert (response.status_code == 200)
trigger_dao.data = trigger.data.to_dict()
db.session.add(trigger_dao)
if flush:
db.session.flush()
trigger = trigger_dao.to_model()
if commit_and_initialize:
db.session.commit()
trigger = trigger_dao.to_model()
try:
trigger_processor.initialize_trigger(trigger, self)
except:
db.session.delete(trigger_dao)
db.session.commit()
raise
return trigger
def patch_trigger(self, source_trigger, trigger):
trigger_type_name = trigger.data.trigger_type_name
if trigger_type_name == self._manual_trigger_processor.trigger_type(
).name:
raise DartValidationException('manual triggers cannot be saved')
if trigger_type_name == self._retry_trigger_processor.trigger_type(
).name:
raise DartValidationException('retry triggers cannot be saved')
trigger_processor = self._trigger_processors.get(trigger_type_name)
trigger = patch_difference(TriggerDao, source_trigger, trigger)
return trigger_processor.update_trigger(source_trigger, trigger)
def default_and_validate(model, schema):
instance = model.to_dict()
apply_defaults(instance, schema)
errors = list(Draft4Validator(schema).iter_errors(instance))
if len(errors) > 0:
raise DartValidationException(str(best_match(errors)))
return model.from_dict(instance)
def _validate_ecs_task_definition(self, ecs_task_definition):
if not ecs_task_definition:
return
for cd in ecs_task_definition['containerDefinitions']:
if 'memory' not in cd:
DartValidationException(
'every containerDefinition must specify memory requirements'
)
mx = self._engine_task_definition_max_total_memory_mb
mem = sum([
cd['memory'] for cd in ecs_task_definition['containerDefinitions']
])
if mem > mx:
raise DartValidationException(
'ecs task definition requires %s mb of memory, but the max is %s'
% mem, mx)
def save_accounting_event(accounting_event, commit=True, flush=False):
""" :type accounting_event: dart.model.accounting.Accounting """
accounting_dao = AccountingDao()
accounting_dao.id = random_id()
accounting_dao.user_id = accounting_event.user_id
accounting_dao.state = accounting_event.state
accounting_dao.entity = accounting_event.entity
accounting_dao.params = accounting_event.params
accounting_dao.return_code = accounting_event.return_code
accounting_dao.api_version = accounting_event.api_version
accounting_dao.extra = accounting_event.extra
db.session.add(accounting_dao)
try:
if flush:
db.session.flush()
if commit:
db.session.commit()
# accounting_event = accounting_dao.to_model()
return accounting_event
except SqlAlchemyIntegrityError as e:
if hasattr(e, 'orig') and isinstance(
e.orig,
PostgresIntegrityError) and e.orig.pgcode == '23505':
raise DartValidationException('name already exists: %s' %
dataset.data.name)
raise e
def _validate_aws_cron_expression(cron_expression):
# See the Note on: http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/ScheduledEvents.html
cron_pattern_split = cron_expression.split()
if '?' not in [cron_pattern_split[2], cron_pattern_split[4]]:
raise DartValidationException(
'CRON Validation Error: Support for specifying both a day-of-week and a '
'day-of-month value is not complete (you must currently use the "?"'
'character in one of these fields).')
def from_string(o_string):
pattern = re.compile(r'\s*(\S+?)\s+((ASC)|(DESC))\s*')
m = pattern.match(o_string)
try:
return OrderBy(m.group(1), m.group(2))
except:
raise DartValidationException('could not parse order_by: %s' %
o_string)
def save_trigger(self,
trigger,
commit_and_initialize=True,
flush=False,
user_id=None):
wf_uuid = uuid.uuid4().hex # to avoid uuid serialization issues
trigger.data.tags = trigger.data.tags if (trigger.data.tags) else []
trigger.data.tags.append(wf_uuid)
if user_id:
trigger.data.user_id = user_id
""" :type trigger: dart.model.trigger.Trigger """
trigger_type_name = trigger.data.trigger_type_name
if trigger_type_name == self._manual_trigger_processor.trigger_type(
).name:
raise DartValidationException('manual triggers cannot be saved')
if trigger_type_name == self._retry_trigger_processor.trigger_type(
).name:
raise DartValidationException('retry triggers cannot be saved')
trigger_processor = self._trigger_processors.get(trigger_type_name)
if not trigger_processor:
raise DartValidationException('unknown trigger_type_name: %s' %
trigger_type_name)
assert isinstance(trigger_processor, TriggerProcessor)
trigger = default_and_validate(
trigger,
trigger_schema(
trigger_processor.trigger_type().params_json_schema))
trigger_dao = TriggerDao()
trigger_dao.id = random_id()
trigger_dao.data = trigger.data.to_dict()
db.session.add(trigger_dao)
if flush:
db.session.flush()
trigger = trigger_dao.to_model()
if commit_and_initialize:
db.session.commit()
trigger = trigger_dao.to_model()
try:
trigger_processor.initialize_trigger(trigger, self)
except:
db.session.delete(trigger_dao)
db.session.commit()
raise
return trigger
def from_string(self, f_string):
pattern = re.compile(r'\s*(\S+?)\s*(' +
'|'.join(self._operator_handlers.keys()) +
')\s*(\S+)\s*')
m = pattern.match(f_string)
try:
return Filter(m.group(1), m.group(2), m.group(3))
except:
raise DartValidationException('could not parse filter: %s' %
f_string)
def default_and_validate_action(self, action, action_type=None):
if not action_type:
engine = self._engine_service.get_engine_by_name(action.data.engine_name)
action_types_by_name = {at.name: at for at in engine.data.supported_action_types}
action_type = action_types_by_name.get(action.data.action_type_name)
if not action_type:
raise DartValidationException('unknown action: "%s"' % action.data.action_type_name)
assert isinstance(action_type, ActionType)
action = default_and_validate(action, action_schema(action_type.params_json_schema))
return action
def get_subscription_element(subscription_id, s3_path):
""" :rtype: dart.model.subscription.SubscriptionElement """
try:
return SubscriptionElementDao.query\
.filter(SubscriptionElementDao.subscription_id == subscription_id)\
.filter(SubscriptionElementDao.s3_path == s3_path)\
.one()\
.to_model()
except NoResultFound:
values = (subscription_id, s3_path)
raise DartValidationException('no elements found for subscription (id=%s) key: %s' % values)
def default_and_validate(model, schema):
instance = model.to_dict()
apply_defaults(instance, schema)
errors = list(Draft4Validator(schema).iter_errors(instance))
if len(errors) > 0:
raise DartValidationException(str(best_match(errors)))
user_id = "anonymous"
if current_user and current_user.email:
user_id = current_user.email
else:
_logger.debug("No current_user found. using anonymous user id instead.")
if instance and ('data' in instance) and ("user_id" in instance["data"]):
instance['data']['user_id'] = user_id
else:
_logger.debug("instance['data']['user_id'] do not exist. Skipping setting a user_id.")
return model.from_dict(instance)
def save_engine(self, engine):
""" :type engine: dart.model.engine.Engine """
engine = default_and_validate(engine, engine_schema())
self._validate_ecs_task_definition(engine.data.ecs_task_definition)
engine_dao = EngineDao()
engine_dao.id = random_id()
engine_dao.name = engine.data.name
engine_dao.data = engine.data.to_dict()
db.session.add(engine_dao)
try:
db.session.commit()
engine = engine_dao.to_model()
engine.data.ecs_task_definition_arn = self._register_ecs_task_definition(
engine)
return self.update_engine_data(engine.id, engine.data)
except SqlAlchemyIntegrityError as e:
if hasattr(e, 'orig') and isinstance(
e.orig,
PostgresIntegrityError) and e.orig.pgcode == '23505':
raise DartValidationException('name already exists: %s' %
engine.data.name)
raise e
def save_dataset(dataset, commit=True, flush=False):
""" :type dataset: dart.model.dataset.Dataset """
dataset = default_and_validate(dataset, dataset_schema())
dataset_dao = DatasetDao()
dataset_dao.id = random_id()
dataset_dao.name = dataset.data.name
dataset.data.location = dataset.data.location.rstrip('/')
dataset_dao.data = dataset.data.to_dict()
db.session.add(dataset_dao)
try:
if flush:
db.session.flush()
if commit:
db.session.commit()
dataset = dataset_dao.to_model()
return dataset
except SqlAlchemyIntegrityError as e:
if hasattr(e, 'orig') and isinstance(
e.orig,
PostgresIntegrityError) and e.orig.pgcode == '23505':
raise DartValidationException('name already exists: %s' %
dataset.data.name)
raise e