def _inject_instances(trigger, rate_per_trigger, duration, payload=None, max_throughput=False):
    payload = payload or {}

    start = date_utils.get_datetime_utc_now()
    elapsed = 0.0
    count = 0

    dispatcher = TriggerDispatcher()
    while elapsed < duration:
        # print('Dispatching trigger %s at time %s', trigger, date_utils.get_datetime_utc_now())
        dispatcher.dispatch(trigger, payload)

        if rate_per_trigger:
            # NOTE: We decrease sleep delay for 56% to take into account overhead / delay because
            # of the call to dispatchet.dispatch method.
            delta = random.expovariate(rate_per_trigger)
            eventlet.sleep(delta * 0.56)

        elapsed = (date_utils.get_datetime_utc_now() - start).seconds
        count += 1

    actual_rate = int(count / elapsed)

    print('%s: Emitted %d triggers in %d seconds (actual rate=%s triggers / second)' %
          (trigger, count, elapsed, actual_rate))

    # NOTE: Due to the overhead of dispatcher.dispatch call, we allow for 10% of deviation from
    # requested rate before warning
    if rate_per_trigger and (actual_rate < (rate_per_trigger * 0.9)):
        print('')
        print('Warning, requested rate was %s triggers / second, but only achieved %s '
              'triggers / second' % (rate_per_trigger, actual_rate))
        print('Too increase the throuput you will likely need to run multiple instances of '
              'this script in parallel.')
Example #2
0
 def test_token_post_set_ttl(self):
     timestamp = date_utils.add_utc_tz(date_utils.get_datetime_utc_now())
     response = self.app.post_json(TOKEN_V1_PATH, {'ttl': 60}, expect_errors=False)
     expected_expiry = date_utils.get_datetime_utc_now() + datetime.timedelta(seconds=60)
     self.assertEqual(response.status_int, 201)
     actual_expiry = isotime.parse(response.json['expiry'])
     self.assertLess(timestamp, actual_expiry)
     self.assertLess(actual_expiry, expected_expiry)
Example #3
0
 def test_get_marker_from_db(self):
     marker_dt = date_utils.get_datetime_utc_now() - datetime.timedelta(minutes=5)
     marker_db = DumperMarkerDB(marker=isotime.format(marker_dt, offset=False),
                                updated_at=date_utils.get_datetime_utc_now())
     DumperMarker.add_or_update(marker_db)
     exec_exporter = ExecutionsExporter(None, None)
     export_marker = exec_exporter._get_export_marker_from_db()
     self.assertEqual(export_marker, date_utils.add_utc_tz(marker_dt))
Example #4
0
 def _test_token_post(self, path=TOKEN_V1_PATH):
     ttl = cfg.CONF.auth.token_ttl
     timestamp = date_utils.get_datetime_utc_now()
     response = self.app.post_json(path, {}, expect_errors=False)
     expected_expiry = date_utils.get_datetime_utc_now() + datetime.timedelta(seconds=ttl)
     expected_expiry = date_utils.add_utc_tz(expected_expiry)
     self.assertEqual(response.status_int, 201)
     self.assertIsNotNone(response.json['token'])
     self.assertEqual(response.json['user'], USERNAME)
     actual_expiry = isotime.parse(response.json['expiry'])
     self.assertLess(timestamp, actual_expiry)
     self.assertLess(actual_expiry, expected_expiry)
Example #5
0
    def test_update_canceled_liveaction(self):
        liveaction_db = LiveActionDB()
        liveaction_db.status = 'initializing'
        liveaction_db.start_timestamp = get_datetime_utc_now()
        liveaction_db.action = ResourceReference(
            name=ActionDBUtilsTestCase.action_db.name,
            pack=ActionDBUtilsTestCase.action_db.pack).ref
        params = {
            'actionstr': 'foo',
            'some_key_that_aint_exist_in_action_or_runner': 'bar',
            'runnerint': 555
        }
        liveaction_db.parameters = params
        liveaction_db = LiveAction.add_or_update(liveaction_db)
        origliveaction_db = copy.copy(liveaction_db)

        # Update by id.
        newliveaction_db = action_db_utils.update_liveaction_status(
            status='running', liveaction_id=liveaction_db.id)

        # Verify id didn't change.
        self.assertEqual(origliveaction_db.id, newliveaction_db.id)
        self.assertEqual(newliveaction_db.status, 'running')

        # Verify that state is published.
        self.assertTrue(LiveActionPublisher.publish_state.called)
        LiveActionPublisher.publish_state.assert_called_once_with(newliveaction_db, 'running')

        # Cancel liveaction.
        now = get_datetime_utc_now()
        status = 'canceled'
        newliveaction_db = action_db_utils.update_liveaction_status(
            status=status, end_timestamp=now, liveaction_id=liveaction_db.id)
        self.assertEqual(origliveaction_db.id, newliveaction_db.id)
        self.assertEqual(newliveaction_db.status, status)
        self.assertEqual(newliveaction_db.end_timestamp, now)

        # Since liveaction has already been canceled, check that anymore update of
        # status, result, context, and end timestamp are not processed.
        now = get_datetime_utc_now()
        status = 'succeeded'
        result = 'Work is done.'
        context = {'third_party_id': uuid.uuid4().hex}
        newliveaction_db = action_db_utils.update_liveaction_status(
            status=status, result=result, context=context, end_timestamp=now,
            liveaction_id=liveaction_db.id)

        self.assertEqual(origliveaction_db.id, newliveaction_db.id)
        self.assertEqual(newliveaction_db.status, 'canceled')
        self.assertNotEqual(newliveaction_db.result, result)
        self.assertNotEqual(newliveaction_db.context, context)
        self.assertNotEqual(newliveaction_db.end_timestamp, now)
Example #6
0
 def test_execution_update(self):
     liveaction = self.MODELS['liveactions']['liveaction1.yaml']
     executions_util.create_execution_object(liveaction)
     liveaction.status = 'running'
     pre_update_timestamp = date_utils.get_datetime_utc_now()
     executions_util.update_execution(liveaction)
     post_update_timestamp = date_utils.get_datetime_utc_now()
     execution = self._get_action_execution(liveaction__id=str(liveaction.id),
                                            raise_exception=True)
     self.assertEquals(len(execution.log), 2)
     self.assertEquals(execution.log[1]['status'], liveaction.status)
     self.assertGreater(execution.log[1]['timestamp'], pre_update_timestamp)
     self.assertLess(execution.log[1]['timestamp'], post_update_timestamp)
def _inject_instances(trigger, rate_per_trigger, duration, payload={}):
    start = date_utils.get_datetime_utc_now()
    elapsed = 0.0
    count = 0

    dispatcher = TriggerDispatcher()
    while elapsed < duration:
        # print('Dispatching trigger %s at time %s', trigger, date_utils.get_datetime_utc_now())
        dispatcher.dispatch(trigger, payload)
        delta = random.expovariate(rate_per_trigger)
        eventlet.sleep(delta)
        elapsed = (date_utils.get_datetime_utc_now() - start).seconds / 60.0
        count += 1

    print("%s: Emitted %d triggers in %d seconds" % (trigger, count, elapsed))
Example #8
0
    def _get_api_client(self):
        """
        Retrieve API client instance.
        """
        token_expire = self._token_expire <= get_datetime_utc_now()

        if not self._client or token_expire:
            self._logger.audit('Creating new Client object.')
            ttl = (24 * 60 * 60)
            self._token_expire = get_datetime_utc_now() + timedelta(seconds=ttl)
            temporary_token = create_token(username=self._api_username, ttl=ttl)
            api_url = get_full_public_api_url()
            self._client = Client(api_url=api_url, token=temporary_token.token)

        return self._client
Example #9
0
    def process(self, instance):
        trigger = instance['trigger']
        payload = instance['payload']

        trigger_instance = None
        try:
            trigger_instance = container_utils.create_trigger_instance(
                trigger,
                payload or {},
                date_utils.get_datetime_utc_now(),
                raise_on_no_trigger=True)
        except:
            # We got a trigger ref but we were unable to create a trigger instance.
            # This could be because a trigger object wasn't found in db for the ref.
            LOG.exception('Failed to create trigger_instance %s.', instance)
            return

        if trigger_instance:
            try:
                self.rules_engine.handle_trigger_instance(trigger_instance)
            except:
                # This could be a large message but at least in case of an exception
                # we get to see more context.
                # Beyond this point code cannot really handle the exception anyway so
                # eating up the exception.
                LOG.exception('Failed to handle trigger_instance %s.', instance)
                return
Example #10
0
    def _purge_action_executions(self):
        """
        Purge action executions and corresponding live action, stdout and stderr object which match
        the criteria defined in the config.
        """
        LOG.info('Performing garbage collection for action executions and related objects')

        utc_now = get_datetime_utc_now()
        timestamp = (utc_now - datetime.timedelta(days=self._action_executions_ttl))

        # Another sanity check to make sure we don't delete new executions
        if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)):
            raise ValueError('Calculated timestamp would violate the minimum TTL constraint')

        timestamp_str = isotime.format(dt=timestamp)
        LOG.info('Deleting action executions older than: %s' % (timestamp_str))

        assert timestamp < utc_now

        try:
            purge_executions(logger=LOG, timestamp=timestamp)
        except Exception as e:
            LOG.exception('Failed to delete executions: %s' % (six.text_type(e)))

        return True
Example #11
0
 def test_update_execution(self):
     """Test ActionExecutionDb update
     """
     self.assertTrue(self.executions['execution_1'].end_timestamp is None)
     self.executions['execution_1'].end_timestamp = date_utils.get_datetime_utc_now()
     updated = ActionExecution.add_or_update(self.executions['execution_1'])
     self.assertTrue(updated.end_timestamp == self.executions['execution_1'].end_timestamp)
Example #12
0
File: handler.py Project: mahak/st2
    def _handle_garbage_collection(self):
        """
        Periodically look for executions which have "handling" set to "True" and haven't been
        updated for a while (this likely indicates that an execution as picked up by a scheduler
        process which died before finishing the processing or similar) and reset handling to
        False so other scheduler can pick it up.
        """
        query = {
            'scheduled_start_timestamp__lte': date.append_milliseconds_to_time(
                date.get_datetime_utc_now(),
                -EXECUTION_SCHEDUELING_TIMEOUT_THRESHOLD_MS
            ),
            'handling': True
        }

        execution_queue_item_dbs = ActionExecutionSchedulingQueue.query(**query) or []

        for execution_queue_item_db in execution_queue_item_dbs:
            execution_queue_item_db.handling = False

            try:
                ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
                LOG.info('Removing lock for orphaned execution queue item: %s',
                         execution_queue_item_db.id)
            except db_exc.StackStormDBObjectWriteConflictError:
                LOG.info(
                    'Execution queue item updated before rescheduling: %s',
                    execution_queue_item_db.id
                )
Example #13
0
    def create_tarball(self, temp_dir_path):
        """
        Create tarball with the contents of temp_dir_path.

        Tarball will be written to self.output_path, if set. Otherwise it will
        be written to /tmp a name generated according to OUTPUT_FILENAME_TEMPLATE.

        :param temp_dir_path: Base directory to include in tarbal.
        :type temp_dir_path: ``str``

        :return: Path to the created tarball.
        :rtype: ``str``
        """
        LOG.info('Creating tarball...')
        if self.output_path:
            output_file_path = self.output_path
        else:
            date = date_utils.get_datetime_utc_now().strftime(DATE_FORMAT)
            values = {'hostname': socket.gethostname(), 'date': date}

            output_file_name = OUTPUT_FILENAME_TEMPLATE % values
            output_file_path = os.path.join('/tmp', output_file_name)

        with tarfile.open(output_file_path, 'w:gz') as tar:
            tar.add(temp_dir_path, arcname='')

        return output_file_path
Example #14
0
    def test_notify_triggers_end_timestamp_none(self):
        liveaction_db = LiveActionDB(action='core.local')
        liveaction_db.id = bson.ObjectId()
        liveaction_db.description = ''
        liveaction_db.status = 'succeeded'
        liveaction_db.parameters = {}
        on_success = NotificationSubSchema(message='Action succeeded.')
        on_failure = NotificationSubSchema(message='Action failed.')
        liveaction_db.notify = NotificationSchema(on_success=on_success,
                                                  on_failure=on_failure)
        liveaction_db.start_timestamp = date_utils.get_datetime_utc_now()

        # This tests for end_timestamp being set to None, which can happen when a policy cancels
        # a request.
        # The assertions within "MockDispatcher.dispatch" will validate that the underlying code
        # handles this properly, so all we need to do is keep the call to "notifier.process" below
        liveaction_db.end_timestamp = None
        LiveAction.add_or_update(liveaction_db)

        execution = MOCK_EXECUTION
        execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db))
        execution.status = liveaction_db.status

        dispatcher = NotifierTestCase.MockDispatcher(self)
        notifier = Notifier(connection=None, queues=[], trigger_dispatcher=dispatcher)
        notifier.process(execution)
Example #15
0
    def test_notify_triggers_jinja_patterns(self, dispatch):
        liveaction_db = LiveActionDB(action='core.local')
        liveaction_db.id = bson.ObjectId()
        liveaction_db.description = ''
        liveaction_db.status = 'succeeded'
        liveaction_db.parameters = {'cmd': 'mamma mia', 'runner_foo': 'foo'}
        on_success = NotificationSubSchema(message='Command {{action_parameters.cmd}} succeeded.',
                                           data={'stdout': '{{action_results.stdout}}'})
        liveaction_db.notify = NotificationSchema(on_success=on_success)
        liveaction_db.start_timestamp = date_utils.get_datetime_utc_now()
        liveaction_db.end_timestamp = \
            (liveaction_db.start_timestamp + datetime.timedelta(seconds=50))

        LiveAction.add_or_update(liveaction_db)

        execution = MOCK_EXECUTION
        execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db))
        execution.status = liveaction_db.status

        notifier = Notifier(connection=None, queues=[])
        notifier.process(execution)
        exp = {'status': 'succeeded',
               'start_timestamp': isotime.format(liveaction_db.start_timestamp),
               'route': 'notify.default', 'runner_ref': 'local-shell-cmd',
               'channel': 'notify.default', 'message': u'Command mamma mia succeeded.',
               'data': {'result': '{}', 'stdout': 'stuff happens'},
               'action_ref': u'core.local',
               'execution_id': str(MOCK_EXECUTION.id),
               'end_timestamp': isotime.format(liveaction_db.end_timestamp)}
        dispatch.assert_called_once_with('core.st2.generic.notifytrigger', payload=exp,
                                         trace_context={})
        notifier.process(execution)
Example #16
0
    def test_what_comes_in_goes_out(self):
        field = ComplexDateTimeField()

        date = date_utils.get_datetime_utc_now()
        us = field._datetime_to_microseconds_since_epoch(date)
        result = field._microseconds_since_epoch_to_datetime(us)
        self.assertEqual(date, result)
Example #17
0
File: handler.py Project: mahak/st2
    def _get_next_execution(self):
        """
        Sort execution requests by FIFO and priority and get the latest, highest priority item from
        the queue and pop it off.
        """
        query = {
            'scheduled_start_timestamp__lte': date.get_datetime_utc_now(),
            'handling': False,
            'limit': 1,
            'order_by': [
                '+scheduled_start_timestamp',
            ]
        }

        execution_queue_item_db = ActionExecutionSchedulingQueue.query(**query).first()

        if not execution_queue_item_db:
            return None

        # Mark that this scheduler process is currently handling (processing) that request
        # NOTE: This operation is atomic (CAS)
        execution_queue_item_db.handling = True

        try:
            ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
            return execution_queue_item_db
        except db_exc.StackStormDBObjectWriteConflictError:
            LOG.info('Execution queue item handled by another scheduler: %s',
                     execution_queue_item_db.id)

        return None
Example #18
0
 def _get_execution_db_model(self, status=action_constants.LIVEACTION_STATUS_REQUESTED):
     start_timestamp = date_utils.get_datetime_utc_now()
     action_ref = ResourceReference(name='test_action', pack='test_pack').ref
     parameters = None
     live_action_db = LiveActionDB(status=status, start_timestamp=start_timestamp,
                                   action=action_ref, parameters=parameters)
     return action.LiveAction.add_or_update(live_action_db, publish=False)
Example #19
0
File: base.py Project: nzlosh/st2
    def _update_live_action_db(self, liveaction_id, status, result, context):
        """
        Update LiveActionDB object for the provided liveaction id.
        """
        liveaction_db = get_liveaction_by_id(liveaction_id)

        state_changed = (
            liveaction_db.status != status and
            liveaction_db.status not in action_constants.LIVEACTION_COMPLETED_STATES
        )

        if status in action_constants.LIVEACTION_COMPLETED_STATES:
            end_timestamp = date_utils.get_datetime_utc_now()
        else:
            end_timestamp = None

        liveaction_db = update_liveaction_status(
            status=status if state_changed else liveaction_db.status,
            result=result,
            context=context,
            end_timestamp=end_timestamp,
            liveaction_db=liveaction_db
        )

        return (liveaction_db, state_changed)
Example #20
0
 def test_time(self, metrics_patch, datetime_patch):
     start_time = get_datetime_utc_now()
     middle_time = start_time + timedelta(seconds=1)
     end_time = middle_time + timedelta(seconds=1)
     datetime_patch.side_effect = [
         start_time,
         middle_time,
         middle_time,
         middle_time,
         end_time
     ]
     test_key = "test_key"
     with base.Timer(test_key) as timer:
         self.assertTrue(isinstance(timer._start_time, datetime))
         metrics_patch.time.assert_not_called()
         timer.send_time()
         metrics_patch.time.assert_called_with(
             test_key,
             (end_time - middle_time).total_seconds()
         )
         second_test_key = "lakshmi_has_toes"
         timer.send_time(second_test_key)
         metrics_patch.time.assert_called_with(
             second_test_key,
             (end_time - middle_time).total_seconds()
         )
         time_delta = timer.get_time_delta()
         self.assertEquals(
             time_delta.total_seconds(),
             (end_time - middle_time).total_seconds()
         )
     metrics_patch.time.assert_called_with(
         test_key,
         (end_time - start_time).total_seconds()
     )
Example #21
0
    def test_datastore_token_timeout(self):
        datastore_service = SensorDatastoreService(logger=mock.Mock(),
                                                   pack_name='core',
                                                   class_name='TestSensor',
                                                   api_username='******')

        mock_api_client = mock.Mock()
        kvp1 = KeyValuePair()
        kvp1.name = 'test1'
        kvp1.value = 'bar'
        mock_api_client.keys.get_by_id.return_value = kvp1

        token_expire_time = get_datetime_utc_now() - timedelta(seconds=5)
        datastore_service._client = mock_api_client
        datastore_service._token_expire = token_expire_time

        self._set_mock_api_client(mock_api_client)

        with mock.patch(
            'st2common.services.datastore.Client',
            return_value=mock_api_client
        ) as datastore_client:
            value = datastore_service.get_value(name='test1', local=False)
            self.assertTrue(datastore_client.called)
            self.assertEqual(value, kvp1.value)
            self.assertGreater(datastore_service._token_expire, token_expire_time)
Example #22
0
def recover_delayed_executions():
    coordinator = coordination.get_coordinator()
    dt_now = date_utils.get_datetime_utc_now()
    dt_delta = datetime.timedelta(seconds=cfg.CONF.scheduler.delayed_execution_recovery)
    dt_timeout = dt_now - dt_delta

    with coordinator.get_lock('st2-rescheduling-delayed-executions'):
        liveactions = LiveAction.query(status=action_constants.LIVEACTION_STATUS_DELAYED,
                                       start_timestamp__lte=dt_timeout,
                                       order_by=['start_timestamp'])

        if not liveactions:
            return

        LOG.info('There are %d liveactions that have been delayed for longer than %d seconds.',
                 len(liveactions), cfg.CONF.scheduler.delayed_execution_recovery)

        # Update status to requested and publish status for each liveactions.
        rescheduled = 0
        for instance in liveactions:
            try:
                action_service.update_status(instance,
                                             action_constants.LIVEACTION_STATUS_REQUESTED,
                                             publish=True)
                rescheduled += 1
            except:
                LOG.exception('Unable to reschedule liveaction. <LiveAction.id=%s>', instance.id)

        LOG.info('Rescheduled %d out of %d delayed liveactions.', len(liveactions), rescheduled)
Example #23
0
    def _do_cancel(self, runner, runnertype_db, action_db, liveaction_db):
        try:
            extra = {"runner": runner}
            LOG.debug("Performing cancel for runner: %s", (runner.runner_id), extra=extra)

            runner.cancel()

            liveaction_db = update_liveaction_status(
                status=action_constants.LIVEACTION_STATUS_CANCELED,
                end_timestamp=date_utils.get_datetime_utc_now(),
                liveaction_db=liveaction_db,
            )

            executions.update_execution(liveaction_db)

            LOG.debug("Performing post_run for runner: %s", runner.runner_id)
            result = {"error": "Execution canceled by user."}
            runner.post_run(status=liveaction_db.status, result=result)
            runner.container_service = None
        except:
            _, ex, tb = sys.exc_info()
            # include the error message and traceback to try and provide some hints.
            result = {"error": str(ex), "traceback": "".join(traceback.format_tb(tb, 20))}
            LOG.exception("Failed to cancel action %s." % (liveaction_db.id), extra=result)
        finally:
            # Always clean-up the auth_token
            status = liveaction_db.status
            self._clean_up_auth_token(runner=runner, status=status)

        return liveaction_db
Example #24
0
    def setup_action_models(cls):
        action_db = ActionDB()
        action_db.name = 'action-1'
        action_db.description = 'awesomeness'
        action_db.enabled = True
        action_db.pack = 'wolfpack'
        action_db.ref = ResourceReference(name=action_db.name, pack=action_db.pack).ref
        action_db.entry_point = ''
        action_db.runner_type = {'name': 'test-runner'}
        action_db.parameters = {
            'actionstr': {'type': 'string', 'position': 1, 'required': True},
            'actionint': {'type': 'number', 'default': 10, 'position': 0},
            'runnerdummy': {'type': 'string', 'default': 'actiondummy'}
        }
        ActionDBUtilsTestCase.action_db = Action.add_or_update(action_db)

        liveaction_db = LiveActionDB()
        liveaction_db.status = 'initializing'
        liveaction_db.start_timestamp = get_datetime_utc_now()
        liveaction_db.action = ActionDBUtilsTestCase.action_db.ref
        params = {
            'actionstr': 'foo',
            'some_key_that_aint_exist_in_action_or_runner': 'bar',
            'runnerint': 555
        }
        liveaction_db.parameters = params
        ActionDBUtilsTestCase.liveaction_db = LiveAction.add_or_update(liveaction_db)
Example #25
0
    def test_rule_enforcement_is_created_on_exception_3(self):
        # 1. Exception in payload_lookup.get_value
        rule_enforcement_dbs = list(RuleEnforcement.get_all())
        self.assertEqual(rule_enforcement_dbs, [])

        self._setup_sample_trigger('st2.test.trigger4')
        rule_4_db = self._setup_sample_rule(RULE_4)
        rules = [rule_4_db]
        trigger_instance = container_utils.create_trigger_instance(
            'dummy_pack_1.st2.test.trigger4',
            {'k1': 't2_p_v', 'k2': 'v2'},
            date_utils.get_datetime_utc_now()
        )
        trigger = get_trigger_db_by_ref(trigger_instance.trigger)

        rules_matcher = RulesMatcher(trigger_instance, trigger, rules)
        matching_rules = rules_matcher.get_matching_rules()
        self.assertEqual(matching_rules, [])
        self.assertEqual(len(matching_rules), 0)

        rule_enforcement_dbs = list(RuleEnforcement.get_all())
        self.assertEqual(len(rule_enforcement_dbs), 1)

        expected_failure = ('Failed to match rule "yoyohoneysingh.st2.test.rule4" against trigger '
                            'instance "%s": There might be a problem with the criteria in rule '
                            'yoyohoneysingh.st2.test.rule4: exception in equals' %
                            (str(trigger_instance.id)))
        self.assertEqual(rule_enforcement_dbs[0].failure_reason, expected_failure)
        self.assertEqual(rule_enforcement_dbs[0].trigger_instance_id, str(trigger_instance.id))
        self.assertEqual(rule_enforcement_dbs[0].rule['id'], str(rule_4_db.id))
        self.assertEqual(rule_enforcement_dbs[0].status, RULE_ENFORCEMENT_STATUS_FAILED)
Example #26
0
    def test_no_timestamp_doesnt_delete_things(self):
        now = date_utils.get_datetime_utc_now()
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        # Insert corresponding stdout and stderr db mock models
        self._insert_mock_stdout_and_stderr_objects_for_execution(exec_model['id'], count=3)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)

        expected_msg = 'Specify a valid timestamp'
        self.assertRaisesRegexp(ValueError, expected_msg, purge_executions,
                                logger=LOG, timestamp=None)
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        stdout_dbs = ActionExecutionOutput.query(output_type='stdout')
        self.assertEqual(len(stdout_dbs), 3)
        stderr_dbs = ActionExecutionOutput.query(output_type='stderr')
        self.assertEqual(len(stderr_dbs), 3)
Example #27
0
    def test_liveaction_gets_deleted(self):
        now = date_utils.get_datetime_utc_now()
        start_ts = now - timedelta(days=15)
        end_ts = now - timedelta(days=14)

        liveaction_model = copy.deepcopy(self.models['liveactions']['liveaction4.yaml'])
        liveaction_model['start_timestamp'] = start_ts
        liveaction_model['end_timestamp'] = end_ts
        liveaction_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        liveaction = LiveAction.add_or_update(liveaction_model)

        # Write one execution before cut-off threshold
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['end_timestamp'] = end_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        exec_model['liveaction']['id'] = str(liveaction.id)
        ActionExecution.add_or_update(exec_model)

        liveactions = LiveAction.get_all()
        executions = ActionExecution.get_all()
        self.assertEqual(len(liveactions), 1)
        self.assertEqual(len(executions), 1)

        purge_executions(logger=LOG, timestamp=now - timedelta(days=10))

        liveactions = LiveAction.get_all()
        executions = ActionExecution.get_all()
        self.assertEqual(len(executions), 0)
        self.assertEqual(len(liveactions), 0)
Example #28
0
 def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=False):
     # Include timestamp in the name.
     filename = filename.format(ts=str(date_utils.get_datetime_utc_now()).replace(' ', '_'),
                                pid=os.getpid())
     super(FormatNamedFileHandler, self).__init__(filename, mode=mode, maxBytes=maxBytes,
                                                  backupCount=backupCount, encoding=encoding,
                                                  delay=delay)
Example #29
0
    def setup_action_models(cls):
        pack = 'wolfpack'
        name = 'action-1'
        parameters = {
            'actionint': {'type': 'number', 'default': 10, 'position': 0},
            'actionfloat': {'type': 'float', 'required': False, 'position': 1},
            'actionstr': {'type': 'string', 'required': True, 'position': 2},
            'actionbool': {'type': 'boolean', 'required': False, 'position': 3},
            'actionlist': {'type': 'list', 'required': False, 'position': 4},
            'actionobject': {'type': 'object', 'required': False, 'position': 5},
            'actionnull': {'type': 'null', 'required': False, 'position': 6},

            'runnerdummy': {'type': 'string', 'default': 'actiondummy'}
        }
        action_db = ActionDB(pack=pack, name=name, description='awesomeness',
                             enabled=True,
                             ref=ResourceReference(name=name, pack=pack).ref,
                             entry_point='', runner_type={'name': 'test-runner'},
                             parameters=parameters)
        ActionDBUtilsTestCase.action_db = Action.add_or_update(action_db)

        liveaction_db = LiveActionDB()
        liveaction_db.status = 'initializing'
        liveaction_db.start_timestamp = get_datetime_utc_now()
        liveaction_db.action = ActionDBUtilsTestCase.action_db.ref
        params = {
            'actionstr': 'foo',
            'some_key_that_aint_exist_in_action_or_runner': 'bar',
            'runnerint': 555
        }
        liveaction_db.parameters = params
        ActionDBUtilsTestCase.liveaction_db = LiveAction.add_or_update(liveaction_db)
Example #30
0
    def test_update_same_liveaction_status(self):
        liveaction_db = LiveActionDB()
        liveaction_db.status = 'requested'
        liveaction_db.start_timestamp = get_datetime_utc_now()
        liveaction_db.action = ResourceReference(
            name=ActionDBUtilsTestCase.action_db.name,
            pack=ActionDBUtilsTestCase.action_db.pack).ref
        params = {
            'actionstr': 'foo',
            'some_key_that_aint_exist_in_action_or_runner': 'bar',
            'runnerint': 555
        }
        liveaction_db.parameters = params
        liveaction_db = LiveAction.add_or_update(liveaction_db)
        origliveaction_db = copy.copy(liveaction_db)

        # Update by id.
        newliveaction_db = action_db_utils.update_liveaction_status(
            status='requested', liveaction_id=liveaction_db.id)

        # Verify id didn't change.
        self.assertEqual(origliveaction_db.id, newliveaction_db.id)
        self.assertEqual(newliveaction_db.status, 'requested')

        # Verify that state is not published.
        self.assertFalse(LiveActionPublisher.publish_state.called)
Example #31
0
    def test_delayed_executions_recovery_before_timeout(self):
        # Create a live action that's delayed but has not passed the timeout.
        liveaction = LiveActionDB(action='wolfpack.action-1',
                                  parameters={'actionstr': 'foo'},
                                  start_timestamp=date_utils.get_datetime_utc_now(),
                                  status=action_constants.LIVEACTION_STATUS_DELAYED)

        liveaction = LiveAction.add_or_update(liveaction, publish=False)
        executions.create_execution_object(liveaction, publish=False)

        # Run the rescheduling routine.
        scheduler.recover_delayed_executions()

        # The live action is expected to stay "delayed".
        liveaction = LiveAction.get_by_id(str(liveaction.id))
        self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)
Example #32
0
def get_rescheduler():
    timer = BlockingScheduler()

    time_spec = {
        'seconds': cfg.CONF.scheduler.rescheduling_interval,
        'timezone': aps_utils.astimezone('UTC')
    }

    timer.add_job(recover_delayed_executions,
                  trigger=IntervalTrigger(**time_spec),
                  max_instances=1,
                  misfire_grace_time=60,
                  next_run_time=date_utils.get_datetime_utc_now(),
                  replace_existing=True)

    return timer
Example #33
0
File: base.py Project: vishnu81/st2
    def _update_live_action_db(self, liveaction_id, status, result, context):
        """
        Update LiveActionDB object for the provided liveaction id.
        """
        liveaction_db = get_liveaction_by_id(liveaction_id)
        if status in action_constants.LIVEACTION_COMPLETED_STATES:
            end_timestamp = date_utils.get_datetime_utc_now()
        else:
            end_timestamp = None

        liveaction_db = update_liveaction_status(status=status,
                                                 result=result,
                                                 context=context,
                                                 end_timestamp=end_timestamp,
                                                 liveaction_db=liveaction_db)
        return liveaction_db
Example #34
0
 def _get_output_schema_exec_db_model(self, params):
     status = action_constants.LIVEACTION_STATUS_REQUESTED
     start_timestamp = date_utils.get_datetime_utc_now()
     action_ref = ResourceReference(
         name=RunnerContainerTest.schema_output_action_db.name,
         pack=RunnerContainerTest.schema_output_action_db.pack,
     ).ref
     parameters = params
     context = {"user": cfg.CONF.system_user.user}
     liveaction_db = LiveActionDB(
         status=status,
         start_timestamp=start_timestamp,
         action=action_ref,
         parameters=parameters,
         context=context,
     )
     return liveaction_db
Example #35
0
    def test_get_matching_rules(self):
        self._setup_sample_trigger("st2.test.trigger1")
        rule_db_1 = self._setup_sample_rule(RULE_1)
        rule_db_2 = self._setup_sample_rule(RULE_2)
        rule_db_3 = self._setup_sample_rule(RULE_3)
        rules = [rule_db_1, rule_db_2, rule_db_3]
        trigger_instance = container_utils.create_trigger_instance(
            "dummy_pack_1.st2.test.trigger1",
            {"k1": "t1_p_v", "k2": "v2"},
            date_utils.get_datetime_utc_now(),
        )

        trigger = get_trigger_db_by_ref(trigger_instance.trigger)
        rules_matcher = RulesMatcher(trigger_instance, trigger, rules)
        matching_rules = rules_matcher.get_matching_rules()
        self.assertIsNotNone(matching_rules)
        self.assertEqual(len(matching_rules), 1)
Example #36
0
    def pre_ack_process(self, message):
        '''
        TriggerInstance from message is create prior to acknowledging the message. This
        gets us a way to not acknowledge messages.
        '''
        trigger = message['trigger']
        payload = message['payload']

        # Accomodate for not being able to create a TrigegrInstance if a TriggerDB
        # is not found.
        trigger_instance = container_utils.create_trigger_instance(
            trigger,
            payload or {},
            date_utils.get_datetime_utc_now(),
            raise_on_no_trigger=True)

        return self._compose_pre_ack_process_response(trigger_instance, message)
Example #37
0
 def __init__(self,
              filename,
              mode='a',
              maxBytes=0,
              backupCount=0,
              encoding=None,
              delay=False):
     # Include timestamp in the name.
     filename = filename.format(ts=str(
         date_utils.get_datetime_utc_now()).replace(' ', '_'),
                                pid=os.getpid())
     super(FormatNamedFileHandler, self).__init__(filename,
                                                  mode=mode,
                                                  maxBytes=maxBytes,
                                                  backupCount=backupCount,
                                                  encoding=encoding,
                                                  delay=delay)
Example #38
0
 def __init__(
     self, filename, mode="a", maxBytes=0, backupCount=0, encoding=None, delay=False
 ):
     # We add aditional values to the context which can be used in the log filename
     timestamp = int(time.time())
     isotime_str = str(date_utils.get_datetime_utc_now()).replace(" ", "_")
     pid = os.getpid()
     format_values = {"timestamp": timestamp, "ts": isotime_str, "pid": pid}
     filename = filename.format(**format_values)
     super(FormatNamedFileHandler, self).__init__(
         filename,
         mode=mode,
         maxBytes=maxBytes,
         backupCount=backupCount,
         encoding=encoding,
         delay=delay,
     )
Example #39
0
 def test_token_model(self):
     dt = date_utils.get_datetime_utc_now()
     tk1 = TokenAPI(user='******', token=uuid.uuid4().hex,
                    expiry=isotime.format(dt, offset=False))
     tkdb1 = TokenAPI.to_model(tk1)
     self.assertIsNotNone(tkdb1)
     self.assertIsInstance(tkdb1, TokenDB)
     self.assertEqual(tkdb1.user, tk1.user)
     self.assertEqual(tkdb1.token, tk1.token)
     self.assertEqual(tkdb1.expiry, isotime.parse(tk1.expiry))
     tkdb2 = Token.add_or_update(tkdb1)
     self.assertEqual(tkdb1, tkdb2)
     self.assertIsNotNone(tkdb2.id)
     tk2 = TokenAPI.from_model(tkdb2)
     self.assertEqual(tk2.user, tk1.user)
     self.assertEqual(tk2.token, tk1.token)
     self.assertEqual(tk2.expiry, tk1.expiry)
Example #40
0
 def test_backstop_apply(self):
     trigger_instance = container_utils.create_trigger_instance(
         self.models["triggers"]["trigger1.yaml"].ref,
         {"k1": "v1"},
         date_utils.get_datetime_utc_now(),
     )
     trigger = self.models["triggers"]["trigger1.yaml"]
     success_rule = self.models["rules"]["success.yaml"]
     rules = [
         rule
         for rule in six.itervalues(self.models["rules"])
         if rule != success_rule
     ]
     rules_matcher = RulesMatcher(trigger_instance, trigger, rules)
     matching_rules = rules_matcher.get_matching_rules()
     self.assertEqual(len(matching_rules), 1)
     self.assertEqual(matching_rules[0].id, self.models["rules"]["backstop.yaml"].id)
Example #41
0
    def test_get_matching_rules(self):
        self._setup_sample_trigger('st2.test.trigger1')
        rule_db_1 = self._setup_sample_rule(RULE_1)
        rule_db_2 = self._setup_sample_rule(RULE_2)
        rule_db_3 = self._setup_sample_rule(RULE_3)
        rules = [rule_db_1, rule_db_2, rule_db_3]
        trigger_instance = container_utils.create_trigger_instance(
            'dummy_pack_1.st2.test.trigger1', {
                'k1': 't1_p_v',
                'k2': 'v2'
            }, date_utils.get_datetime_utc_now())

        trigger = get_trigger_db_by_ref(trigger_instance.trigger)
        rules_matcher = RulesMatcher(trigger_instance, trigger, rules)
        matching_rules = rules_matcher.get_matching_rules()
        self.assertTrue(matching_rules is not None)
        self.assertEqual(len(matching_rules), 1)
Example #42
0
 def test_trigger_instance_payload_with_special_values(self):
     # Test a rule where TriggerInstance payload contains a dot (".") and $
     self._setup_sample_trigger('st2.test.trigger2')
     trigger_instance = container_utils.create_trigger_instance(
         'dummy_pack_1.st2.test.trigger2', {
             'k1': 't1_p_v',
             'k2.k2': 'v2',
             'k3.more.nested.deep': 'some.value',
             'k4.even.more.nested$': 'foo',
             'yep$aaa': 'b'
         }, date_utils.get_datetime_utc_now())
     trigger = get_trigger_db_by_ref(trigger_instance.trigger)
     rules = self._get_sample_rules()
     rules_matcher = RulesMatcher(trigger_instance, trigger, rules)
     matching_rules = rules_matcher.get_matching_rules()
     self.assertTrue(matching_rules is not None)
     self.assertEqual(len(matching_rules), 1)
Example #43
0
    def test_chain_pause_resume_status_change(self):
        # Tests context_result is updated when last task's status changes between pause and resume

        action = TEST_PACK + '.' + 'test_pause_resume_context_result'
        liveaction = LiveActionDB(action=action)
        liveaction, execution = action_service.request(liveaction)
        liveaction = LiveAction.get_by_id(str(liveaction.id))

        # Wait until the liveaction is paused.
        liveaction = self._wait_for_status(
            liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
        extra_info = str(liveaction)
        self.assertEqual(liveaction.status,
                         action_constants.LIVEACTION_STATUS_PAUSED, extra_info)

        # Wait for non-blocking threads to complete. Ensure runner is not running.
        MockLiveActionPublisherNonBlocking.wait_all()

        last_task_liveaction_id = liveaction.result['tasks'][-1][
            'liveaction_id']

        action_utils.update_liveaction_status(
            status=action_constants.LIVEACTION_STATUS_SUCCEEDED,
            end_timestamp=date_utils.get_datetime_utc_now(),
            result={'foo': 'bar'},
            liveaction_id=last_task_liveaction_id)

        # Request action chain to resume.
        liveaction, execution = action_service.request_resume(
            liveaction, USERNAME)

        # Wait until the liveaction is completed.
        liveaction = self._wait_for_status(
            liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED)

        self.assertEqual(liveaction.status,
                         action_constants.LIVEACTION_STATUS_SUCCEEDED,
                         str(liveaction))

        # Wait for non-blocking threads to complete.
        MockLiveActionPublisherNonBlocking.wait_all()

        # Check liveaction result.
        self.assertIn('tasks', liveaction.result)
        self.assertEqual(len(liveaction.result['tasks']), 2)
        self.assertEqual(liveaction.result['tasks'][0]['result']['foo'], 'bar')
Example #44
0
    def to_model(cls, kvp):
        name = getattr(kvp, 'name', None)
        description = getattr(kvp, 'description', None)
        value = kvp.value

        if getattr(kvp, 'ttl', None):
            expire_timestamp = (date_utils.get_datetime_utc_now() +
                                datetime.timedelta(seconds=kvp.ttl))
        else:
            expire_timestamp = None

        model = cls.model(name=name,
                          description=description,
                          value=value,
                          expire_timestamp=expire_timestamp)

        return model
Example #45
0
    def test_get_output_finished_execution(self):
        # Test the execution output API endpoint for execution which has finished
        for status in action_constants.LIVEACTION_COMPLETED_STATES:
            # Insert mock execution and output objects
            status = action_constants.LIVEACTION_STATUS_SUCCEEDED
            timestamp = date_utils.get_datetime_utc_now()
            action_execution_db = ActionExecutionDB(start_timestamp=timestamp,
                                                    end_timestamp=timestamp,
                                                    status=status,
                                                    action={'ref': 'core.local'},
                                                    runner={'name': 'run-local'},
                                                    liveaction={'ref': 'foo'})
            action_execution_db = ActionExecution.add_or_update(action_execution_db)

            for i in range(1, 6):
                stdout_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
                                                    action_ref='core.local',
                                                    runner_ref='dummy',
                                                    timestamp=timestamp,
                                                    output_type='stdout',
                                                    data='stdout %s\n' % (i))
                ActionExecutionOutput.add_or_update(stdout_db)

            for i in range(10, 15):
                stderr_db = ActionExecutionOutputDB(execution_id=str(action_execution_db.id),
                                                    action_ref='core.local',
                                                    runner_ref='dummy',
                                                    timestamp=timestamp,
                                                    output_type='stderr',
                                                    data='stderr %s\n' % (i))
                ActionExecutionOutput.add_or_update(stderr_db)

            resp = self.app.get('/v1/executions/%s/output' % (str(action_execution_db.id)),
                                expect_errors=False)
            self.assertEqual(resp.status_int, 200)
            lines = resp.text.strip().split('\n')
            self.assertEqual(len(lines), 10)
            self.assertEqual(lines[0], 'stdout 1')
            self.assertEqual(lines[9], 'stderr 14')

            # Verify "last" short-hand id works
            resp = self.app.get('/v1/executions/last/output', expect_errors=False)
            self.assertEqual(resp.status_int, 200)
            lines = resp.text.strip().split('\n')
            self.assertEqual(len(lines), 10)
Example #46
0
def create_token(username, ttl=None, metadata=None):
    """
    :param username: Username of the user to create the token for. If the account for this user
                     doesn't exist yet it will be created.
    :type username: ``str``

    :param ttl: Token TTL (in seconds).
    :type ttl: ``int``

    :param metadata: Optional metadata to associate with the token.
    :type metadata: ``dict``
    """

    if ttl:
        if ttl > cfg.CONF.auth.token_ttl:
            msg = 'TTL specified %s is greater than max allowed %s.' % (
                ttl, cfg.CONF.auth.token_ttl
            )
            raise TTLTooLargeException(msg)
    else:
        ttl = cfg.CONF.auth.token_ttl

    if username:
        try:
            User.get_by_name(username)
        except:
            user = UserDB(name=username)
            User.add_or_update(user)

            extra = {'username': username, 'user': user}
            LOG.audit('Registered new user "%s".' % (username), extra=extra)

    token = uuid.uuid4().hex
    expiry = date_utils.get_datetime_utc_now() + datetime.timedelta(seconds=ttl)
    token = TokenDB(user=username, token=token, expiry=expiry, metadata=metadata)
    Token.add_or_update(token)

    username_string = username if username else 'an anonymous user'
    token_expire_string = isotime.format(expiry, offset=False)
    extra = {'username': username, 'token_expiration': token_expire_string}

    LOG.audit('Access granted to "%s" with the token set to expire at "%s".' %
              (username_string, token_expire_string), extra=extra)

    return token
Example #47
0
    def _apply_pre_run(liveaction_db, execution_queue_item_db):
        # Apply policies defined for the action.
        liveaction_db = policy_service.apply_pre_run_policies(liveaction_db)

        liveaction_id = str(liveaction_db.id)
        queue_item_id = str(execution_queue_item_db.id)

        extra = {
            'liveaction_id': liveaction_id,
            'liveaction_status': liveaction_db.status,
            'queue_item_id': queue_item_id
        }

        LOG.info('Liveaction (%s) Status Pre-Run: %s (%s)',
                 liveaction_id,
                 liveaction_db.status,
                 queue_item_id,
                 extra=extra)

        if liveaction_db.status is action_constants.LIVEACTION_STATUS_POLICY_DELAYED:
            liveaction_db = action_service.update_status(
                liveaction_db,
                action_constants.LIVEACTION_STATUS_DELAYED,
                publish=False)
            execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
                date.get_datetime_utc_now(),
                POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS)
            try:
                ActionExecutionSchedulingQueue.add_or_update(
                    execution_queue_item_db, publish=False)
            except db_exc.StackStormDBObjectWriteConflictError:
                LOG.warning(
                    'Execution queue item update conflict during scheduling: %s',
                    execution_queue_item_db.id)

            return None

        if (liveaction_db.status
                in action_constants.LIVEACTION_COMPLETED_STATES
                or liveaction_db.status
                in action_constants.LIVEACTION_CANCEL_STATES):
            ActionExecutionSchedulingQueue.delete(execution_queue_item_db)
            return None

        return liveaction_db
Example #48
0
def update_status(liveaction,
                  new_status,
                  result=None,
                  publish=True,
                  set_result_size=False):
    if liveaction.status == new_status:
        return liveaction

    old_status = liveaction.status

    updates = {
        "liveaction_id": liveaction.id,
        "status": new_status,
        "result": result,
        "publish": False,
    }

    if new_status in action_constants.LIVEACTION_COMPLETED_STATES:
        updates["end_timestamp"] = date_utils.get_datetime_utc_now()

    liveaction = action_utils.update_liveaction_status(**updates)
    action_execution = executions.update_execution(
        liveaction, set_result_size=set_result_size)

    msg = ("The status of action execution is changed from %s to %s. "
           "<LiveAction.id=%s, ActionExecution.id=%s>" %
           (old_status, new_status, liveaction.id, action_execution.id))

    extra = {
        "action_execution_db": action_execution,
        "liveaction_db": liveaction
    }

    LOG.audit(msg, extra=extra)
    LOG.info(msg)

    # Invoke post run if liveaction status is completed or paused.
    if (new_status in action_constants.LIVEACTION_COMPLETED_STATES
            or new_status == action_constants.LIVEACTION_STATUS_PAUSED):
        runners_utils.invoke_post_run(liveaction)

    if publish:
        LiveAction.publish_status(liveaction)

    return liveaction
Example #49
0
    def test_purge_executions_with_action_ref(self):
        now = date_utils.get_datetime_utc_now()
        exec_model = copy.copy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = now - timedelta(days=15)
        exec_model['end_timestamp'] = now - timedelta(days=14)
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        ActionExecution.add_or_update(exec_model)

        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)
        purge_executions(action_ref='core.localzzz', timestamp=now - timedelta(days=10))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 1)

        purge_executions(action_ref='core.local', timestamp=now - timedelta(days=10))
        execs = ActionExecution.get_all()
        self.assertEqual(len(execs), 0)
Example #50
0
    def _write_marker_to_db(self, new_marker):
        LOG.info('Updating marker in db to: %s', new_marker)
        marker = DumperMarker.get_all()

        if len(marker) > 1:
            LOG.exception(
                'More than one dumper marker found. Using first found one.')

        marker_db = None
        if marker:
            marker = marker[0]
            marker_db = DumperMarkerDB(id=marker['id'])
        else:
            marker_db = DumperMarkerDB()

        marker_db.marker = isotime.format(new_marker, offset=False)
        marker_db.updated_at = date_utils.get_datetime_utc_now()
        return DumperMarker.add_or_update(marker_db)
Example #51
0
def _timediff(diff_target, period_seconds, operator):
    """
    :param diff_target: Date string.
    :type diff_target: ``str``

    :param period_seconds: Seconds.
    :type period_seconds: ``int``

    :rtype: ``bool``
    """
    # Pickup now in UTC to compare against
    utc_now = date_utils.get_datetime_utc_now()

    # assuming diff_target is UTC and specified in python iso format.
    # Note: date_utils.parse uses dateutil.parse which is way more flexible then strptime and
    # supports many date formats
    diff_target_utc = date_utils.parse(diff_target)
    return operator((utc_now - diff_target_utc).total_seconds(), period_seconds)
Example #52
0
    def test_print(self):
        instance = FakeModel(
            name="seesaw",
            boolean_field=True,
            datetime_field=date_utils.get_datetime_utc_now(),
            description="fun!",
            dict_field={"a": 1},
            integer_field=68,
            list_field=["abc"],
        )

        expected = (
            'FakeModel(boolean_field=True, datetime_field="%s", description="fun!", '
            "dict_field={'a': 1}, id=None, integer_field=68, list_field=['abc'], "
            'name="seesaw")' % str(instance.datetime_field)
        )

        self.assertEqual(str(instance), expected)
Example #53
0
    def _emit_trigger_instance(self, trigger):
        utc_now = date_utils.get_datetime_utc_now()
        # debug logging is reasonable for this one. A high resolution timer will end up
        # trashing standard logs.
        LOG.debug('Timer fired at: %s. Trigger: %s', str(utc_now), trigger)

        payload = {
            'executed_at': str(utc_now),
            'schedule': trigger['parameters'].get('time')
        }

        trace_context = TraceContext(trace_tag='%s-%s' %
                                     (self._get_trigger_type_name(trigger),
                                      trigger.get('name',
                                                  uuid.uuid4().hex)))
        self._trigger_dispatcher.dispatch(trigger,
                                          payload,
                                          trace_context=trace_context)
Example #54
0
    def test_purge_executions_whole_model_is_not_loaded_in_memory(self, mock_ActionExecution,
                                                                  mock_LiveAction):
        # Verify that whole execution objects are not loaded in memory and we just retrieve the
        # id field
        self.assertEqual(mock_ActionExecution.query.call_count, 0)
        self.assertEqual(mock_LiveAction.query.call_count, 0)
        now = date_utils.get_datetime_utc_now()
        purge_executions(logger=LOG, timestamp=now - timedelta(days=10), purge_incomplete=True)

        self.assertEqual(mock_ActionExecution.query.call_count, 2)
        self.assertEqual(mock_LiveAction.query.call_count, 1)

        self.assertEqual(mock_ActionExecution.query.call_args_list[0][1]['only_fields'], ['id'])
        self.assertTrue(mock_ActionExecution.query.call_args_list[0][1]['no_dereference'])
        self.assertEqual(mock_ActionExecution.query.call_args_list[1][1]['only_fields'], ['id'])
        self.assertTrue(mock_ActionExecution.query.call_args_list[1][1]['no_dereference'])
        self.assertEqual(mock_LiveAction.query.call_args_list[0][1]['only_fields'], ['id'])
        self.assertTrue(mock_LiveAction.query.call_args_list[0][1]['no_dereference'])
Example #55
0
    def test_purge(self):
        now = date_utils.get_datetime_utc_now()

        instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger',
                                        payload={'hola': 'hi', 'kuraci': 'chicken'},
                                        occurrence_time=now - timedelta(days=20),
                                        status=TRIGGER_INSTANCE_PROCESSED)
        TriggerInstance.add_or_update(instance_db)

        instance_db = TriggerInstanceDB(trigger='purge_tool.dummy.trigger',
                                        payload={'hola': 'hi', 'kuraci': 'chicken'},
                                        occurrence_time=now - timedelta(days=5),
                                        status=TRIGGER_INSTANCE_PROCESSED)
        TriggerInstance.add_or_update(instance_db)

        self.assertEqual(len(TriggerInstance.get_all()), 2)
        purge_trigger_instances(logger=LOG, timestamp=now - timedelta(days=10))
        self.assertEqual(len(TriggerInstance.get_all()), 1)
Example #56
0
    def poll(self):
        if self._update_counter > 60:
            self._update_sun_info()
            self._update_counter = 0

        checks = ['dawn', 'sunrise', 'sunset', 'dusk']

        currenttime = date.get_datetime_utc_now()

        self._logger.debug("Checking current time %s for sun information" %
                           str(currenttime))

        for key in checks:
            if self.is_within_minute(self.sun[key], currenttime):
                trigger = 'astral.' + key
                self.sensor_service.dispatch(trigger=trigger, payload={})

        self._update_counter += 1
Example #57
0
    def test_get_(self, mock_get):
        field = ComplexDateTimeField()

        # No value set
        mock_get.return_value = None
        self.assertEqual(field.__get__(instance=None, owner=None), None)

        # Already a datetime
        mock_get.return_value = date_utils.get_datetime_utc_now()
        self.assertEqual(field.__get__(instance=None, owner=None),
                         mock_get.return_value)

        # Microseconds
        dt = datetime.datetime(2015, 1, 1, 15, 0, 0).replace(microsecond=500)
        dt = date_utils.add_utc_tz(dt)
        us = field._datetime_to_microseconds_since_epoch(value=dt)
        mock_get.return_value = us
        self.assertEqual(field.__get__(instance=None, owner=None), dt)
 def test_dumper_marker_crud(self):
     saved = DumperMarkerModelTest._create_save_dumper_marker()
     retrieved = DumperMarker.get_by_id(saved.id)
     self.assertEqual(saved.marker, retrieved.marker,
                      'Same marker was not returned.')
     # test update
     time_now = date_utils.get_datetime_utc_now()
     retrieved.updated_at = time_now
     saved = DumperMarker.add_or_update(retrieved)
     retrieved = DumperMarker.get_by_id(saved.id)
     self.assertEqual(retrieved.updated_at, time_now, 'Update to marker failed.')
     # cleanup
     DumperMarkerModelTest._delete([retrieved])
     try:
         retrieved = DumperMarker.get_by_id(saved.id)
     except StackStormDBObjectNotFoundError:
         retrieved = None
     self.assertIsNone(retrieved, 'managed to retrieve after failure.')
Example #59
0
    def test_no_timestamp_doesnt_delete(self):
        now = date_utils.get_datetime_utc_now()

        instance_db = TaskExecutionDB(
            start_timestamp=now - timedelta(days=20),
            end_timestamp=now - timedelta(days=20),
            status="succeeded",
        )
        TaskExecution.add_or_update(instance_db)

        self.assertEqual(len(TaskExecution.get_all()), 1)
        expected_msg = "Specify a valid timestamp"
        self.assertRaisesRegexp(ValueError,
                                expected_msg,
                                purge_task_executions,
                                logger=LOG,
                                timestamp=None)
        self.assertEqual(len(TaskExecution.get_all()), 1)
Example #60
0
    def _delay(self, liveaction_db, execution_queue_item_db):
        liveaction_db = action_service.update_status(
            liveaction_db,
            action_constants.LIVEACTION_STATUS_DELAYED,
            publish=False)

        execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
            date.get_datetime_utc_now(),
            POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS)

        try:
            execution_queue_item_db.handling = False
            ActionExecutionSchedulingQueue.add_or_update(
                execution_queue_item_db, publish=False)
        except db_exc.StackStormDBObjectWriteConflictError:
            LOG.warning(
                'Execution queue item update conflict during scheduling: %s',
                execution_queue_item_db.id)