Ejemplo n.º 1
0
    def test_remove_old_simulations(self, simulations):
        timezone = pytz.timezone('Europe/Zurich')

        sim = MagicMock()
        sim.kill_datetime = datetime.now(timezone) - timedelta(days=2)
        sim.sim_id = 5
        sim.creation_datetime = datetime.now(timezone)
        sim.state = 'running'

        sim2 = MagicMock()
        sim2.kill_datetime = datetime.now(timezone) + timedelta(days=2)
        sim2.sim_id = 5
        sim2.creation_datetime = datetime.now(timezone)
        sim2.state = 'running'

        sim3 = MagicMock()
        sim3.kill_datetime = datetime.now(timezone) + timedelta(days=2)
        sim3.sim_id = 5
        sim3.creation_datetime = datetime.now(timezone) - timedelta(seconds=Settings.MAX_SIMULATION_TIMEOUT)
        sim3.state = 'running'

        simulations.__iter__.return_value = [sim, sim2, sim3]
        cleanup.remove_old_simulations()
        assert sim.state == 'stopped'
        assert sim2.state == 'running'
        assert sim3.state == 'stopped'
Ejemplo n.º 2
0
def test_output_handler_metric_names(dirname):

    wrapper = OutputHandler("tag", metric_names=["a", "b"])
    mock_logger = MagicMock(spec=TensorboardLogger)
    mock_logger.writer = MagicMock()

    mock_engine = MagicMock()
    mock_engine.state = State(metrics={"a": 12.23, "b": 23.45})
    mock_engine.state.iteration = 5

    wrapper(mock_engine, mock_logger, Events.ITERATION_STARTED)

    assert mock_logger.writer.add_scalar.call_count == 2
    mock_logger.writer.add_scalar.assert_has_calls([
        call("tag/a", 12.23, 5),
        call("tag/b", 23.45, 5),
    ],
                                                   any_order=True)

    wrapper = OutputHandler("tag", metric_names=[
        "a",
    ])

    mock_engine = MagicMock()
    mock_engine.state = State(
        metrics={"a": torch.Tensor([0.0, 1.0, 2.0, 3.0])})
    mock_engine.state.iteration = 5

    mock_logger = MagicMock(spec=TensorboardLogger)
    mock_logger.writer = MagicMock()

    wrapper(mock_engine, mock_logger, Events.ITERATION_STARTED)

    assert mock_logger.writer.add_scalar.call_count == 4
    mock_logger.writer.add_scalar.assert_has_calls([
        call("tag/a/0", 0.0, 5),
        call("tag/a/1", 1.0, 5),
        call("tag/a/2", 2.0, 5),
        call("tag/a/3", 3.0, 5),
    ],
                                                   any_order=True)

    wrapper = OutputHandler("tag", metric_names=["a", "c"])

    mock_engine = MagicMock()
    mock_engine.state = State(metrics={"a": 55.56, "c": "Some text"})
    mock_engine.state.iteration = 7

    mock_logger = MagicMock(spec=TensorboardLogger)
    mock_logger.writer = MagicMock()

    with pytest.warns(UserWarning):
        wrapper(mock_engine, mock_logger, Events.ITERATION_STARTED)

    assert mock_logger.writer.add_scalar.call_count == 1
    mock_logger.writer.add_scalar.assert_has_calls([
        call("tag/a", 55.56, 7),
    ],
                                                   any_order=True)
Ejemplo n.º 3
0
def test_output_handler_with_global_step_from_engine():

    mock_another_engine = MagicMock()
    mock_another_engine.state = State()
    mock_another_engine.state.epoch = 10
    mock_another_engine.state.output = 12.345

    wrapper = OutputHandler(
        "tag",
        output_transform=lambda x: {"loss": x},
        global_step_transform=global_step_from_engine(mock_another_engine))

    mock_logger = MagicMock(spec=VisdomLogger)
    mock_logger.vis = MagicMock()
    mock_logger.executor = _DummyExecutor()

    mock_engine = MagicMock()
    mock_engine.state = State()
    mock_engine.state.epoch = 1
    mock_engine.state.output = 0.123

    wrapper(mock_engine, mock_logger, Events.EPOCH_STARTED)
    assert mock_logger.vis.line.call_count == 1
    assert len(wrapper.windows) == 1 and "tag/loss" in wrapper.windows
    assert wrapper.windows["tag/loss"]['win'] is not None
    mock_logger.vis.line.assert_has_calls([
        call(X=[
            mock_another_engine.state.epoch,
        ],
             Y=[
                 mock_engine.state.output,
             ],
             env=mock_logger.vis.env,
             win=None,
             update=None,
             opts=wrapper.windows['tag/loss']['opts'],
             name="tag/loss")
    ])

    mock_another_engine.state.epoch = 11
    mock_engine.state.output = 1.123

    wrapper(mock_engine, mock_logger, Events.EPOCH_STARTED)
    assert mock_logger.vis.line.call_count == 2
    assert len(wrapper.windows) == 1 and "tag/loss" in wrapper.windows
    assert wrapper.windows["tag/loss"]['win'] is not None
    mock_logger.vis.line.assert_has_calls([
        call(X=[
            mock_another_engine.state.epoch,
        ],
             Y=[
                 mock_engine.state.output,
             ],
             env=mock_logger.vis.env,
             win=wrapper.windows["tag/loss"]['win'],
             update='append',
             opts=wrapper.windows['tag/loss']['opts'],
             name="tag/loss")
    ])
Ejemplo n.º 4
0
 def _mock_get(pk=None):
     result = MagicMock()
     if pk == '1111':
         result.state = 'published'
         result.pk = '1111'
     else:
         result.state = 'uploaded'
         result.pk = '2222'
     return result
Ejemplo n.º 5
0
def test_output_handler_metric_names():

    wrapper = OutputHandler("tag", metric_names=["a", "b", "c"])
    mock_logger = MagicMock(spec=PolyaxonLogger)
    mock_logger.log_metrics = MagicMock()

    mock_engine = MagicMock()
    mock_engine.state = State(metrics={"a": 12.23, "b": 23.45, "c": torch.tensor(10.0)})
    mock_engine.state.iteration = 5

    wrapper(mock_engine, mock_logger, Events.ITERATION_STARTED)

    assert mock_logger.log_metrics.call_count == 1
    mock_logger.log_metrics.assert_called_once_with(
        step=5,
        **{"tag/a": 12.23,
           "tag/b": 23.45,
           "tag/c": 10.0}
    )

    wrapper = OutputHandler("tag", metric_names=["a", ])

    mock_engine = MagicMock()
    mock_engine.state = State(metrics={"a": torch.Tensor([0.0, 1.0, 2.0, 3.0])})
    mock_engine.state.iteration = 5

    mock_logger = MagicMock(spec=PolyaxonLogger)
    mock_logger.log_metrics = MagicMock()

    wrapper(mock_engine, mock_logger, Events.ITERATION_STARTED)

    assert mock_logger.log_metrics.call_count == 1
    mock_logger.log_metrics.assert_has_calls([
        call(step=5,
             **{"tag/a/0": 0.0,
                "tag/a/1": 1.0,
                "tag/a/2": 2.0,
                "tag/a/3": 3.0}),
    ], any_order=True)

    wrapper = OutputHandler("tag", metric_names=["a", "c"])

    mock_engine = MagicMock()
    mock_engine.state = State(metrics={"a": 55.56, "c": "Some text"})
    mock_engine.state.iteration = 7

    mock_logger = MagicMock(spec=PolyaxonLogger)
    mock_logger.log_metrics = MagicMock()

    with pytest.warns(UserWarning):
        wrapper(mock_engine, mock_logger, Events.ITERATION_STARTED)

    assert mock_logger.log_metrics.call_count == 1
    mock_logger.log_metrics.assert_has_calls([
        call(step=7, **{"tag/a": 55.56})
    ], any_order=True)
Ejemplo n.º 6
0
    def setUp(self):
        # Mock resource model
        resource1 = MagicMock()
        resource1.name = 'Resource1'
        resource1.version = '1.0'
        resource1.description = 'Test resource 1'
        resource1.content_type = 'text/plain'
        resource1.state = 'created'
        resource1.open = False
        resource1.get_url.return_value = 'http://localhost/media/resources/resource1'

        resource2 = MagicMock()
        resource2.name = 'Resource2'
        resource2.version = '2.0'
        resource2.description = 'Test resource 2'
        resource2.content_type = 'text/plain'
        resource2.state = 'created'
        resource2.open = False
        resource2.get_url.return_value = 'http://localhost/media/resources/resource2'

        resource3 = MagicMock()
        resource3.name = 'Resource3'
        resource3.version = '2.0'
        resource3.description = 'Test resource 3'
        resource3.content_type = 'text/plain'
        resource3.state = 'created'
        resource3.open = True
        resource3.get_url.return_value = 'http://localhost/media/resources/resource3'

        resource4 = MagicMock()
        resource4.name = 'Resource4'
        resource4.version = '1.0'
        resource4.description = 'Test resource 4'
        resource4.content_type = 'text/plain'
        resource4.state = 'created'
        resource4.open = True
        resource4.get_url.return_value = 'http://localhost/media/resources/resource4'
        resource4.offerings = ['1111', '2222']

        resources_management.Resource = MagicMock()
        resources_management.Resource.objects.filter.return_value = [
            resource1,
            resource2,
            resource3,
            resource4
        ]
        self.user = MagicMock()
        self.org = MagicMock()
        self.user.userprofile.current_organization = self.org
    def test_ignores_terminated_stacks_that_have_been_replaced_by_a_new_instance(
            self, find_ec2_instances):
        old = MagicMock()
        old.id = 'i-123'
        old.state = 'terminated'
        old.tags = {'Name': 'dummy1--test--1'}

        new = MagicMock()
        new.id = 'i-456'
        new.state = 'running'
        new.tags = {'Name': 'dummy1--test--1'}

        find_ec2_instances.return_value = [old, new]
        self.assertEqual({'i-456': 'running'},
                         lifecycle._nodes_states('dummy1--test'))
Ejemplo n.º 8
0
    def test_50_start(self, get_driver):
        auth = Authentication([{'id': 'libcloud', 'type': 'LibCloud', 'username': '******',
                                'password': '******', 'driver': 'EC2'}])
        lib_cloud = self.get_lib_cloud()

        inf = MagicMock()
        inf.get_next_vm_id.return_value = 1
        vm = VirtualMachine(inf, "1", lib_cloud.cloud, "", "", lib_cloud)

        driver = MagicMock()
        get_driver.return_value = driver

        node = MagicMock()
        node.id = "1"
        node.state = "running"
        node.driver = driver
        driver.list_nodes.return_value = [node]

        driver.ex_stop_node.return_value = True

        success, _ = lib_cloud.start(vm, auth)

        self.assertTrue(success, msg="ERROR: stopping VM info.")
        self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
        self.clean_log()
Ejemplo n.º 9
0
    def _test(tag=None):
        wrapper = GradsScalarHandler(model, reduction=norm, tag=tag)
        mock_logger = MagicMock(spec=VisdomLogger)
        mock_logger.vis = MagicMock()
        mock_logger.executor = _DummyExecutor()

        mock_engine = MagicMock()
        mock_engine.state = State()
        mock_engine.state.epoch = 5

        wrapper(mock_engine, mock_logger, Events.EPOCH_STARTED)

        tag_prefix = "{}/".format(tag) if tag else ""

        assert mock_logger.vis.line.call_count == 4
        mock_logger.vis.line.assert_has_calls([
            call(X=[5, ], Y=ANY, env=mock_logger.vis.env,
                 win=None, update=None,
                 opts=wrapper.windows[tag_prefix + "grads_norm/fc1/weight"]['opts'],
                 name=tag_prefix + "grads_norm/fc1/weight"),
            call(X=[5, ], Y=ANY, env=mock_logger.vis.env,
                 win=None, update=None,
                 opts=wrapper.windows[tag_prefix + "grads_norm/fc1/bias"]['opts'],
                 name=tag_prefix + "grads_norm/fc1/bias"),

            call(X=[5, ], Y=ANY, env=mock_logger.vis.env,
                 win=None, update=None,
                 opts=wrapper.windows[tag_prefix + "grads_norm/fc2/weight"]['opts'],
                 name=tag_prefix + "grads_norm/fc2/weight"),
            call(X=[5, ], Y=ANY, env=mock_logger.vis.env,
                 win=None, update=None,
                 opts=wrapper.windows[tag_prefix + "grads_norm/fc2/bias"]['opts'],
                 name=tag_prefix + "grads_norm/fc2/bias"),
        ], any_order=True)
Ejemplo n.º 10
0
def test_ChuteApi_get_chute(ChuteStorage, ChuteContainer):
    update_manager = MagicMock()
    api = chute_api.ChuteApi(update_manager)

    request = MagicMock()
    request.user = User.get_internal_user()

    container = MagicMock()
    container.getStatus.return_value = "running"
    ChuteContainer.return_value = container

    chute = MagicMock()
    chute.name = "test"
    chute.state = "running"
    chute.version = 5
    chute.environment = {}
    chute.resources = {}

    ChuteStorage.chuteList = {"test": chute}

    data = api.get_chute(request, chute.name)
    assert isinstance(data, basestring)

    result = json.loads(data)
    assert result['name'] == chute.name
    assert result['version'] == chute.version
Ejemplo n.º 11
0
def test_ChuteApi_get_chutes(ChuteStorage, ChuteContainer):
    update_manager = MagicMock()
    api = chute_api.ChuteApi(update_manager)

    request = MagicMock()

    container = MagicMock()
    container.getStatus.return_value = "running"
    ChuteContainer.return_value = container

    storage = MagicMock()
    ChuteStorage.return_value = storage

    chute = MagicMock()
    chute.environment = {}
    chute.name = "test"
    chute.state = "running"
    chute.resources = {}
    chute.version = 5
    chute.get_owner.return_value = "paradrop"

    storage.getChuteList.return_value = [chute]

    data = api.get_chutes(request)
    assert isinstance(data, basestring)

    result = json.loads(data)
    assert result[0]['name'] == chute.name
    assert result[0]['version'] == chute.version
Ejemplo n.º 12
0
def test_optimizer_params():

    optimizer = torch.optim.SGD([torch.Tensor(0)], lr=0.01)
    wrapper = OptimizerParamsHandler(optimizer=optimizer, param_name="lr")
    mock_logger = MagicMock(spec=PolyaxonLogger)
    mock_logger.log_metrics = MagicMock()
    mock_engine = MagicMock()
    mock_engine.state = State()
    mock_engine.state.iteration = 123

    wrapper(mock_engine, mock_logger, Events.ITERATION_STARTED)
    mock_logger.log_metrics.assert_called_once_with(**{
        "lr/group_0": 0.01,
        "step": 123
    })

    wrapper = OptimizerParamsHandler(optimizer,
                                     param_name="lr",
                                     tag="generator")
    mock_logger = MagicMock(spec=PolyaxonLogger)
    mock_logger.log_metrics = MagicMock()

    wrapper(mock_engine, mock_logger, Events.ITERATION_STARTED)
    mock_logger.log_metrics.assert_called_once_with(**{
        "generator/lr/group_0": 0.01,
        "step": 123
    })
Ejemplo n.º 13
0
    def test_52_reboot(self, get_driver):
        auth = Authentication([{
            'id': 'ost',
            'type': 'CloudStack',
            'username': '******',
            'password': '******',
            'host': 'http://server.com'
        }])
        osc_cloud = self.get_osc_cloud()

        inf = MagicMock()
        vm = VirtualMachine(inf, "1", osc_cloud.cloud, "", "", osc_cloud, 1)

        driver = MagicMock()
        get_driver.return_value = driver

        node = MagicMock()
        node.id = "1"
        node.state = "running"
        node.driver = driver
        driver.list_nodes.return_value = [node]
        node.reboot_node.return_value = True

        success, _ = osc_cloud.reboot(vm, auth)

        self.assertTrue(success, msg="ERROR: rebooting VM info.")
        self.assertNotIn("ERROR",
                         self.log.getvalue(),
                         msg="ERROR found in log: %s" % self.log.getvalue())
Ejemplo n.º 14
0
def test_output_handler_output_transform():

    wrapper = OutputHandler("tag", output_transform=lambda x: x)
    mock_logger = MagicMock(spec=MLflowLogger)
    mock_logger.log_metrics = MagicMock()

    mock_engine = MagicMock()
    mock_engine.state = State()
    mock_engine.state.output = 12345
    mock_engine.state.iteration = 123

    wrapper(mock_engine, mock_logger, Events.ITERATION_STARTED)

    mock_logger.log_metrics.assert_called_once_with({"tag output": 12345},
                                                    step=123)

    wrapper = OutputHandler("another_tag",
                            output_transform=lambda x: {"loss": x})
    mock_logger = MagicMock(spec=MLflowLogger)
    mock_logger.log_metrics = MagicMock()

    wrapper(mock_engine, mock_logger, Events.ITERATION_STARTED)
    mock_logger.log_metrics.assert_called_once_with(
        {"another_tag loss": 12345},
        step=123,
    )
def test_grads_hist_frozen_layers(dummy_model_factory):
    model = dummy_model_factory(with_grads=True, with_frozen_layer=True)

    wrapper = GradsHistHandler(model)
    mock_logger = MagicMock(spec=TensorboardLogger)
    mock_logger.writer = MagicMock()

    mock_engine = MagicMock()
    mock_engine.state = State()
    mock_engine.state.epoch = 5

    wrapper(mock_engine, mock_logger, Events.EPOCH_STARTED)

    assert mock_logger.writer.add_histogram.call_count == 2
    mock_logger.writer.add_histogram.assert_has_calls([
        call(tag="grads/fc2/weight", values=ANY, global_step=5),
        call(tag="grads/fc2/bias", values=ANY, global_step=5),
    ],
                                                      any_order=True)

    with pytest.raises(AssertionError):
        mock_logger.writer.add_histogram.assert_has_calls([
            call(tag="grads/fc1/weight", values=ANY, global_step=5),
            call(tag="grads/fc1/bias", values=ANY, global_step=5),
        ],
                                                          any_order=True)
Ejemplo n.º 16
0
def test_output_handler_with_global_step_transform():
    def global_step_transform(*args, **kwargs):
        return 10

    wrapper = OutputHandler("tag",
                            output_transform=lambda x: {"loss": x},
                            global_step_transform=global_step_transform)
    mock_logger = MagicMock(spec=VisdomLogger)
    mock_logger.vis = MagicMock()
    mock_logger.executor = _DummyExecutor()

    mock_engine = MagicMock()
    mock_engine.state = State()
    mock_engine.state.epoch = 5
    mock_engine.state.output = 12345

    wrapper(mock_engine, mock_logger, Events.EPOCH_STARTED)
    assert mock_logger.vis.line.call_count == 1
    assert len(wrapper.windows) == 1 and "tag/loss" in wrapper.windows
    assert wrapper.windows["tag/loss"]['win'] is not None

    mock_logger.vis.line.assert_has_calls([
        call(X=[
            10,
        ],
             Y=[
                 12345,
             ],
             env=mock_logger.vis.env,
             win=None,
             update=None,
             opts=wrapper.windows['tag/loss']['opts'],
             name="tag/loss")
    ])
Ejemplo n.º 17
0
def test_optimizer_params():

    optimizer = torch.optim.SGD([torch.Tensor(0)], lr=0.01)
    wrapper = OptimizerParamsHandler(optimizer=optimizer, param_name="lr")
    mock_logger = MagicMock(spec=VisdomLogger)
    mock_logger.vis = MagicMock()
    mock_logger.executor = _DummyExecutor()
    mock_engine = MagicMock()
    mock_engine.state = State()
    mock_engine.state.iteration = 123

    wrapper(mock_engine, mock_logger, Events.ITERATION_STARTED)

    # mock_logger.vis.line.assert_called_once_with("lr/group_0", 0.01, 123)
    assert len(wrapper.windows) == 1 and "lr/group_0" in wrapper.windows
    assert wrapper.windows["lr/group_0"]['win'] is not None

    mock_logger.vis.line.assert_called_once_with(
        X=[
            123,
        ],
        Y=[
            0.01,
        ],
        env=mock_logger.vis.env,
        win=None,
        update=None,
        opts=wrapper.windows['lr/group_0']['opts'],
        name="lr/group_0")
Ejemplo n.º 18
0
def test_ChuteApi_get_chutes(ChuteStorage, ChuteContainer):
    update_manager = MagicMock()
    api = chute_api.ChuteApi(update_manager)

    request = MagicMock()

    container = MagicMock()
    container.getStatus.return_value = "running"
    ChuteContainer.return_value = container

    storage = MagicMock()
    ChuteStorage.return_value = storage

    chute = MagicMock()
    chute.environment = {}
    chute.name = "test"
    chute.state = "running"
    chute.resources = {}
    chute.version = 5
    chute.get_owner.return_value = "paradrop"

    storage.getChuteList.return_value = [chute]

    data = api.get_chutes(request)
    assert isinstance(data, basestring)

    result = json.loads(data)
    assert result[0]['name'] == chute.name
    assert result[0]['version'] == chute.version
def test_grads_scalar_handler_frozen_layers(dummy_model_factory, norm_mock):
    model = dummy_model_factory(with_grads=True, with_frozen_layer=True)

    wrapper = GradsScalarHandler(model, reduction=norm_mock)
    mock_logger = MagicMock(spec=TensorboardLogger)
    mock_logger.writer = MagicMock()

    mock_engine = MagicMock()
    mock_engine.state = State()
    mock_engine.state.epoch = 5
    norm_mock.reset_mock()

    wrapper(mock_engine, mock_logger, Events.EPOCH_STARTED)

    mock_logger.writer.add_scalar.assert_has_calls([
        call("grads_norm/fc2/weight", ANY, 5),
        call("grads_norm/fc2/bias", ANY, 5),
    ],
                                                   any_order=True)

    with pytest.raises(AssertionError):
        mock_logger.writer.add_scalar.assert_has_calls([
            call("grads_norm/fc1/weight", ANY, 5),
            call("grads_norm/fc1/bias", ANY, 5),
        ],
                                                       any_order=True)
    assert mock_logger.writer.add_scalar.call_count == 2
    assert norm_mock.call_count == 2
Ejemplo n.º 20
0
def test_builder_assigns_address_components_if_formatted_address_present():
  address_parser_mock = MagicMock(spec=address_parser)
  parsed_address_mock = MagicMock()

  address = 'fake_address'
  city = 'fake_city'
  state = 'fake_state'
  zip_code = 'fake_zip_code'
  formatted_address = 'fake_formatted_address '

  parsed_address_mock.address1 = address
  parsed_address_mock.city = city
  parsed_address_mock.state = state
  parsed_address_mock.zip_code = zip_code
  parsed_address_mock.formatted_address = formatted_address

  address_parser_mock.parse_address = MagicMock(return_value=parsed_address_mock)

  builder = ListingBuilder(address_parser_mock, formatted_address='x')

  builder.listing_attrs_output = MagicMock()

  builder._build_formatted_address()

  call_list = [
    call(listing_builder.ADDRESS, address),
    call(listing_builder.CITY, city),
    call(listing_builder.STATE, state),
    call(listing_builder.ZIP_CODE, zip_code),
    call(listing_builder.FORMATTED_ADDRESS, formatted_address),
  ]

  assert builder.listing_attrs_output.__setitem__.call_args_list == call_list
    def _test(tag=None):
        wrapper = GradsHistHandler(model, tag=tag)
        mock_logger = MagicMock(spec=TensorboardLogger)
        mock_logger.writer = MagicMock()

        mock_engine = MagicMock()
        mock_engine.state = State()
        mock_engine.state.epoch = 5

        wrapper(mock_engine, mock_logger, Events.EPOCH_STARTED)

        tag_prefix = "{}/".format(tag) if tag else ""

        assert mock_logger.writer.add_histogram.call_count == 4
        mock_logger.writer.add_histogram.assert_has_calls([
            call(tag=tag_prefix + "grads/fc1/weight",
                 values=ANY,
                 global_step=5),
            call(tag=tag_prefix + "grads/fc1/bias", values=ANY, global_step=5),
            call(tag=tag_prefix + "grads/fc2/weight",
                 values=ANY,
                 global_step=5),
            call(tag=tag_prefix + "grads/fc2/bias", values=ANY, global_step=5),
        ],
                                                          any_order=True)
Ejemplo n.º 22
0
def test_ChuteApi_get_chute(ChuteStorage, ChuteContainer):
    update_manager = MagicMock()
    api = chute_api.ChuteApi(update_manager)

    request = MagicMock()
    request.user = User.get_internal_user()

    container = MagicMock()
    container.getStatus.return_value = "running"
    ChuteContainer.return_value = container

    chute = MagicMock()
    chute.name = "test"
    chute.state = "running"
    chute.version = 5
    chute.environment = {}
    chute.resources = {}

    ChuteStorage.chuteList = {
        "test": chute
    }

    data = api.get_chute(request, chute.name)
    assert isinstance(data, basestring)

    result = json.loads(data)
    assert result['name'] == chute.name
    assert result['version'] == chute.version
def test_weights_scalar_handler_frozen_layers(dummy_model_factory):

    model = dummy_model_factory(with_grads=True, with_frozen_layer=True)

    wrapper = WeightsScalarHandler(model)
    mock_logger = MagicMock(spec=TensorboardLogger)
    mock_logger.writer = MagicMock()

    mock_engine = MagicMock()
    mock_engine.state = State()
    mock_engine.state.epoch = 5

    wrapper(mock_engine, mock_logger, Events.EPOCH_STARTED)

    mock_logger.writer.add_scalar.assert_has_calls([
        call("weights_norm/fc2/weight", 12.0, 5),
        call("weights_norm/fc2/bias", math.sqrt(12.0), 5),
    ],
                                                   any_order=True)

    with pytest.raises(AssertionError):
        mock_logger.writer.add_scalar.assert_has_calls([
            call("weights_norm/fc1/weight", 12.0, 5),
            call("weights_norm/fc1/bias", math.sqrt(12.0), 5),
        ],
                                                       any_order=True)

    assert mock_logger.writer.add_scalar.call_count == 2
Ejemplo n.º 24
0
def test_weights_hist_handler():
    class DummyModel(torch.nn.Module):
        def __init__(self):
            super(DummyModel, self).__init__()
            self.fc1 = torch.nn.Linear(10, 10)
            self.fc2 = torch.nn.Linear(12, 12)
            self.fc1.weight.data.zero_()
            self.fc1.bias.data.zero_()
            self.fc2.weight.data.fill_(1.0)
            self.fc2.bias.data.fill_(1.0)

    model = DummyModel()

    wrapper = WeightsHistHandler(model)
    mock_logger = MagicMock(spec=TensorboardLogger)
    mock_logger.writer = MagicMock()

    mock_engine = MagicMock()
    mock_engine.state = State()
    mock_engine.state.epoch = 5

    wrapper(mock_engine, mock_logger, Events.EPOCH_STARTED)

    assert mock_logger.writer.add_histogram.call_count == 4
    mock_logger.writer.add_histogram.assert_has_calls([
        call(tag="weights/fc1/weight", values=ANY, global_step=5),
        call(tag="weights/fc1/bias", values=ANY, global_step=5),
        call(tag="weights/fc2/weight", values=ANY, global_step=5),
        call(tag="weights/fc2/bias", values=ANY, global_step=5),
    ],
                                                      any_order=True)
Ejemplo n.º 25
0
    def test_50_start(self, get_driver):
        auth = Authentication([{'id': 'ost', 'type': 'OpenStack', 'username': '******',
                                'password': '******', 'tenant': 'tenant', 'host': 'https://server.com:5000'}])
        ost_cloud = self.get_ost_cloud()

        inf = MagicMock()
        inf.get_next_vm_id.return_value = 1
        vm = VirtualMachine(inf, "1", ost_cloud.cloud, "", "", ost_cloud)

        driver = MagicMock()
        get_driver.return_value = driver

        node = MagicMock()
        node.id = "1"
        node.state = "running"
        node.extra = {'flavorId': 'small'}
        node.public_ips = ['158.42.1.1']
        node.private_ips = ['10.0.0.1']
        node.driver = driver
        driver.list_nodes.return_value = [node]

        driver.ex_start_node.return_value = True

        success, _ = ost_cloud.start(vm, auth)

        self.assertTrue(success, msg="ERROR: stopping VM info.")
        self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
        self.clean_log()
Ejemplo n.º 26
0
    def _create_customer(self):

        customer = MagicMock()
        customer.id = 1
        customer.first_name = "Ellen"
        customer.last_name = "Johnson"
        customer.company = "Souveniropolis"
        customer.address = "14 Main Street"
        customer.city = "Pecan Springs"
        customer.state = "TX"
        # customer.zip_code   = "44628"
        customer.country = "USA"
        customer.email = "*****@*****.**"

        # TODO: putting this here for now. we will test silver and
        # strategize a little about the overall process. Thinking in the
        # interest of best practices, whatever frontend app we have can
        # collect this once, run it through the authorize.net backend to
        # create a customer profile + payment profile, we'll store those
        # and not need to store card numbers at all.

        customer.meta = {
            "cardNumber": "4111111111111111",
            "expirationDate": "2020-12",
            "cardCode": "123",
        }

        transaction = MagicMock()
        transaction.amount = 25.00
        transaction.currency = "USD"
        transaction.credit_card_details = customer.credit_card_details
        transaction.customer = customer

        self.transaction = transaction
        self.customer = customer
Ejemplo n.º 27
0
    def test_30_updateVMInfo(self, get_driver):
        radl_data = """
            network net (outbound = 'yes')
            system test (
            cpu.arch='x86_64' and
            cpu.count=1 and
            memory.size=512m and
            net_interface.0.connection = 'net' and
            net_interface.0.dns_name = 'test' and
            disk.0.os.name = 'linux' and
            disk.0.image.url = 'gce://us-central1-a/centos-6' and
            disk.0.os.credentials.username = '******' and
            disk.1.size=1GB and
            disk.1.device='hdb' and
            disk.1.mount_path='/mnt/path'
            )"""
        radl = radl_parse.parse_radl(radl_data)
        radl.check()

        auth = Authentication([{
            'id': 'one',
            'type': 'GCE',
            'username': '******',
            'password': '******',
            'project': 'proj'
        }])
        gce_cloud = self.get_gce_cloud()

        inf = MagicMock()
        inf.get_next_vm_id.return_value = 1
        vm = VirtualMachine(inf, "1", gce_cloud.cloud, radl, radl, gce_cloud)

        driver = MagicMock()
        get_driver.return_value = driver

        node = MagicMock()
        zone = MagicMock()
        node.id = "1"
        node.state = "running"
        node.extra = {'flavorId': 'small'}
        node.public_ips = []
        node.public_ips = ['158.42.1.1']
        node.private_ips = ['10.0.0.1']
        node.driver = driver
        zone.name = 'us-central1-a'
        node.extra = {'zone': zone}
        driver.ex_get_node.return_value = node

        volume = MagicMock()
        volume.id = "vol1"
        volume.attach.return_value = True
        volume.extra = {'status': 'READY'}
        driver.create_volume.return_value = volume

        success, vm = gce_cloud.updateVMInfo(vm, auth)

        self.assertTrue(success, msg="ERROR: updating VM info.")
        self.assertNotIn("ERROR",
                         self.log.getvalue(),
                         msg="ERROR found in log: %s" % self.log.getvalue())
Ejemplo n.º 28
0
def an_experiment_leaderboard_entry_dto():
    entry = MagicMock()
    entry.entryType = 'experiment'
    entry.id = a_uuid_string()
    entry.shortId = a_string()
    entry.projectId = a_uuid_string()
    entry.state = 'succeeded'
    entry.experimentStates = an_experiment_states(succeeded=1)
    entry.responding = True
    entry.name = a_string()
    entry.organizationName = a_string()
    entry.projectName = a_string()
    entry.description = a_string()
    entry.timeOfCreation = a_timestamp()
    entry.timeOfCompletion = a_timestamp()
    entry.runningTime = randint(1, 1000)
    entry.owner = a_string()
    entry.size = randint(1, 1000)
    entry.tags = [a_string(), a_string()]
    entry.environment = a_string()
    entry.workerType = a_string()
    entry.hostname = a_string()
    entry.sourceSize = randint(1, 1000)
    entry.sourceMd5 = a_string()
    entry.commitId = a_string()
    entry.properties = [a_property(), a_property()]
    entry.parameters = [a_parameter(), a_parameter()]
    entry.channelsLastValues = [a_channel_value(), a_channel_value()]
    entry.trashed = False
    entry.deleted = False
    entry.isBestExperiment = False
    return entry
Ejemplo n.º 29
0
def test_time_to_target_returns_None_until_initialized():
    with requests_mock.Mocker() as m:
        mqttc = MagicMock()
        boiler = MagicMock()
        zone = MagicMock()
        sensor = MagicMock()
        thermostat = MagicMock()
        weather = MagicMock()
        weather.get_weather.return_value = {'temperature': 5}

        thermostat.target = 20
        thermostat.state = 'On'
        sensor.temperature = None

        zc = scheduler.ZoneController(zone, boiler, sensor, thermostat,
                                      'https://scheduler/api', None, weather)

        # There is no gradient table or last recorded temperature:
        assert zc.get_time_to_target() is None

        sensor.temperature = MagicMock()
        sensor.temperature.reading = 15.0
        sensor.temperature.when = datetime.now()
        zc.temperature_change(sensor)

        # Still no gradient table: should return None:
        assert zc.get_time_to_target() is None

        # Now set a gradient table and check the correct value is used:
        # XXX shouldn't be setting the gradient table directly...
        gradient_table = [{'delta': 5.0, 'gradient': 1.0}]
        zc.gradient_table = gradient_table
        assert zc.get_time_to_target() == timedelta(hours=5)
Ejemplo n.º 30
0
    def test_50_start(self, get_driver):
        auth = Authentication([{
            'id': 'libcloud',
            'type': 'LibCloud',
            'username': '******',
            'password': '******',
            'driver': 'EC2'
        }])
        lib_cloud = self.get_lib_cloud()

        inf = MagicMock()
        vm = VirtualMachine(inf, "1", lib_cloud.cloud, "", "", lib_cloud, 1)

        driver = MagicMock()
        get_driver.return_value = driver

        node = MagicMock()
        node.id = "1"
        node.state = "running"
        node.driver = driver
        driver.list_nodes.return_value = [node]

        driver.ex_stop_node.return_value = True

        success, _ = lib_cloud.start(vm, auth)

        self.assertTrue(success, msg="ERROR: stopping VM info.")
        self.assertNotIn("ERROR",
                         self.log.getvalue(),
                         msg="ERROR found in log: %s" % self.log.getvalue())
Ejemplo n.º 31
0
def test_grads_scalar_handler():
    class DummyModel(torch.nn.Module):
        def __init__(self):
            super(DummyModel, self).__init__()
            self.fc1 = torch.nn.Linear(10, 10)
            self.fc2 = torch.nn.Linear(12, 12)
            self.fc1.weight.data.zero_()
            self.fc1.bias.data.zero_()
            self.fc2.weight.data.fill_(1.0)
            self.fc2.bias.data.fill_(1.0)

    model = DummyModel()

    def norm(x):
        return 0.0

    wrapper = GradsScalarHandler(model, reduction=norm)
    mock_logger = MagicMock(spec=TensorboardLogger)
    mock_logger.writer = MagicMock()

    mock_engine = MagicMock()
    mock_engine.state = State()
    mock_engine.state.epoch = 5

    wrapper(mock_engine, mock_logger, Events.EPOCH_STARTED)

    assert mock_logger.writer.add_scalar.call_count == 4

    mock_logger.writer.add_scalar.assert_has_calls([
        call("grads_norm/fc1/weight", ANY, 5),
        call("grads_norm/fc1/bias", ANY, 5),
        call("grads_norm/fc2/weight", ANY, 5),
        call("grads_norm/fc2/bias", ANY, 5),
    ],
                                                   any_order=True)
Ejemplo n.º 32
0
    def test_52_reboot(self, get_driver):
        auth = Authentication([{
            'id': 'ost',
            'type': 'OpenStack',
            'username': '******',
            'password': '******',
            'tenant': 'tenant',
            'host': 'https://server.com:5000'
        }])
        ost_cloud = self.get_ost_cloud()

        inf = MagicMock()
        vm = VirtualMachine(inf, "1", ost_cloud.cloud, "", "", ost_cloud, 1)

        driver = MagicMock()
        get_driver.return_value = driver

        node = MagicMock()
        node.id = "1"
        node.state = "running"
        node.extra = {'flavorId': 'small'}
        node.public_ips = ['158.42.1.1']
        node.private_ips = ['10.0.0.1']
        node.driver = driver
        driver.ex_get_node_details.return_value = node

        driver.ex_hard_reboot_node.return_value = True

        success, _ = ost_cloud.reboot(vm, auth)

        self.assertTrue(success, msg="ERROR: stopping VM info.")
        self.assertNotIn("ERROR",
                         self.log.getvalue(),
                         msg="ERROR found in log: %s" % self.log.getvalue())
 def _ec2_instance(self, state='running', id='i-456', launch_time=datetime(2017, 1, 1, tzinfo=utc)):
     instance = MagicMock()
     instance.id = id
     state_codes = {'running': 16}
     instance.state = {'Code': state_codes.get(state, -1), 'Name': state} # 'Code' should vary but probably isn't being used
     instance.tags = [{'Key': 'Name', 'Value': 'dummy1--test--1'}]
     instance.launch_time = launch_time
     return instance
Ejemplo n.º 34
0
 def _mock_get_up(pk=None):
     result = MagicMock()
     result.state = 'uploaded'
     if pk == '1111':
         result.pk = '1111'
     else:
         result.pk = '2222'
     return result
Ejemplo n.º 35
0
    def test_remove_old_simulations(self, simulations):
        timezone = pytz.timezone('Europe/Zurich')

        sim = MagicMock()
        sim.kill_datetime = datetime.now(timezone) - timedelta(days=2)
        sim.sim_id = 5
        sim.state = 'running'

        sim2 = MagicMock()
        sim2.kill_datetime = datetime.now(timezone) + timedelta(days=2)
        sim2.sim_id = 5
        sim2.state = 'running'

        simulations.__iter__.return_value = [sim, sim2]
        cleanup.remove_old_simulations()
        assert sim.state == 'stopped'
        assert sim2.state == 'running'
Ejemplo n.º 36
0
 def _mock_get(pk=None):
     result = MagicMock()
     if pk == '1111':
         result = self._published_offering
     else:
         result.state = 'uploaded'
         result.pk = '2222'
     return result
Ejemplo n.º 37
0
 def _mock_get_up(pk=None):
     result = MagicMock()
     result.state = 'uploaded'
     if pk == '1111':
         result.pk = '1111'
     else:
         result.pk = '2222'
     return result
Ejemplo n.º 38
0
    def test_30_updateVMInfo(self, get_driver):
        radl_data = """
            network net (outbound = 'yes')
            system test (
            cpu.arch='x86_64' and
            cpu.count=1 and
            memory.size=512m and
            net_interface.0.connection = 'net' and
            net_interface.0.ip = '158.42.1.1' and
            net_interface.0.dns_name = 'test' and
            disk.0.os.name = 'linux' and
            disk.0.image.url = 'aws://ami-id' and
            disk.0.os.credentials.username = '******' and
            disk.0.os.credentials.password = '******'
            )"""
        radl = radl_parse.parse_radl(radl_data)
        radl.check()

        auth = Authentication([{'id': 'libcloud', 'type': 'LibCloud', 'username': '******',
                                'password': '******', 'driver': 'EC2'}])
        lib_cloud = self.get_lib_cloud()

        inf = MagicMock()
        inf.get_next_vm_id.return_value = 1
        vm = VirtualMachine(inf, "1", lib_cloud.cloud, radl, radl, lib_cloud)

        driver = MagicMock()
        driver.name = "Amazon EC2"
        get_driver.return_value = driver

        node = MagicMock()
        node.id = "1"
        node.state = "running"
        node.extra = {'availability': 'use-east-1'}
        node.public_ips = []
        node.private_ips = ['10.0.0.1']
        node.driver = driver
        node.size = MagicMock()
        node.size.ram = 512
        node.size.price = 1
        node.size.disk = 1
        node.size.vcpus = 1
        node.size.name = "small"
        driver.list_nodes.return_value = [node]

        volume = MagicMock()
        volume.id = "vol1"
        volume.extra = {"state": "available"}
        volume.attach.return_value = True
        driver.create_volume.return_value = volume

        driver.ex_allocate_address.return_value = "10.0.0.1"

        success, vm = lib_cloud.updateVMInfo(vm, auth)

        self.assertTrue(success, msg="ERROR: updating VM info.")
        self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
        self.clean_log()
Ejemplo n.º 39
0
    def test_55_alter(self, get_driver):
        radl_data = """
            network net ()
            system test (
            cpu.arch='x86_64' and
            cpu.count=1 and
            memory.size=512m and
            net_interface.0.connection = 'net' and
            net_interface.0.dns_name = 'test' and
            disk.0.os.name = 'linux' and
            disk.0.image.url = 'one://server.com/1' and
            disk.0.os.credentials.username = '******' and
            disk.0.os.credentials.password = '******'
            )"""
        radl = radl_parse.parse_radl(radl_data)

        new_radl_data = """
            system test (
            cpu.count>=2 and
            memory.size>=2048m
            )"""
        new_radl = radl_parse.parse_radl(new_radl_data)

        auth = Authentication([{'id': 'ost', 'type': 'OpenStack', 'username': '******',
                                'password': '******', 'tenant': 'tenant', 'host': 'https://server.com:5000'}])
        ost_cloud = self.get_ost_cloud()

        inf = MagicMock()
        inf.get_next_vm_id.return_value = 1
        vm = VirtualMachine(inf, "1", ost_cloud.cloud, radl, radl, ost_cloud)

        driver = MagicMock()
        get_driver.return_value = driver

        node = MagicMock()
        node.id = "1"
        node.state = "running"
        node.extra = {'flavorId': 'small'}
        node.public_ips = ['158.42.1.1']
        node.private_ips = ['10.0.0.1']
        node.driver = driver
        driver.list_nodes.return_value = [node]

        node_size = MagicMock()
        node_size.ram = 2048
        node_size.price = 1
        node_size.disk = 1
        node_size.vcpus = 2
        node_size.name = "small"
        driver.list_sizes.return_value = [node_size]

        driver.ex_resize.return_value = True

        success, _ = ost_cloud.alterVM(vm, new_radl, auth)

        self.assertTrue(success, msg="ERROR: modifying VM info.")
        self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
        self.clean_log()
Ejemplo n.º 40
0
 def test_compute_q_value_should_return_reward_if_final_state(self):
     state = State(position=100, speed=0)
     previous_state = State(position=90, speed=10)
     transition = MagicMock(spec=Transition)
     transition.state = state
     transition.previous_state = previous_state
     transition.reward = 10
     q_value = compute_q_value(transition, None, 0)
     assert q_value == 10
Ejemplo n.º 41
0
    def test_60_finalize(self, sleep, get_driver):
        auth = Authentication([{
            'id': 'ost',
            'type': 'OpenStack',
            'username': '******',
            'password': '******',
            'tenant': 'tenant',
            'host': 'https://server.com:5000'
        }])
        ost_cloud = self.get_ost_cloud()

        radl_data = """
            system test (
            cpu.count>=2 and
            memory.size>=2048m
            )"""
        radl = radl_parse.parse_radl(radl_data)

        inf = MagicMock()
        inf.get_next_vm_id.return_value = 1
        vm = VirtualMachine(inf, "1", ost_cloud.cloud, radl, radl, ost_cloud)

        driver = MagicMock()
        driver.name = "OpenStack"
        get_driver.return_value = driver

        node = MagicMock()
        node.id = "1"
        node.state = "running"
        node.extra = {'flavorId': 'small'}
        node.public_ips = ['158.42.1.1']
        node.private_ips = ['10.0.0.1']
        node.driver = driver
        node.destroy.return_value = True
        driver.list_nodes.return_value = [node]

        sg = MagicMock()
        sg.id = sg.name = "sg1"
        driver.ex_get_node_security_groups.return_value = [sg]

        keypair = MagicMock()
        driver.get_key_pair.return_value = keypair
        vm.keypair = keypair

        driver.delete_key_pair.return_value = True

        driver.delete_security_group.return_value = True

        driver.ex_list_floating_ips.return_value = []

        success, _ = ost_cloud.finalize(vm, auth)

        self.assertTrue(success, msg="ERROR: finalizing VM info.")
        self.assertNotIn("ERROR",
                         self.log.getvalue(),
                         msg="ERROR found in log: %s" % self.log.getvalue())
        self.clean_log()
Ejemplo n.º 42
0
    def test_show_task_status(self):
        task = ProgressReportTask(self.func_mock)
        with patch("sys.stdout", new_callable=StringIO.StringIO) as std_mock:
            result_mock = MagicMock()
            result_mock.state = "PENDING"
            self.func_mock.AsyncResult.return_value = result_mock
            task.show_task_status("task_id")

        self.func_mock.AsyncResult.assert_called_once_with("task_id")
        self.assertEquals(std_mock.getvalue(), "Task not found or PENDING state\n")

        with patch("sys.stdout", new_callable=StringIO.StringIO) as std_mock:
            result_mock = MagicMock()
            result_mock.state = "SUCCESS"
            self.func_mock.AsyncResult.return_value = result_mock
            task.show_task_status("task_id")

        self.assertEquals(std_mock.getvalue(), "Current State: {}, {}\n".format(result_mock.state, result_mock.info))
Ejemplo n.º 43
0
def test_reloadChutes(mockStore, mockSettings, mResources, mWait, mTime, mOut, mTimeint):
    """
    Test that the reloadChutes function does it's job.
    """
    #Test that if pdconfd isn't enabled we return an empty list
    mockSettings.PDCONFD_ENABLED = False
    storage = MagicMock()
    mockStore.return_value = storage

    #Call
    ret = restart.reloadChutes()

    #Assertions
    assert ret == []
    assert not mockStore.called

    #Test that if pdconfd is enabled we do our job
    mTimeint.return_value = 'Now'
    mockSettings.PDCONFD_ENABLED = True
    mockSettings.RESERVED_CHUTE = 'PDROP'
    ch1 = MagicMock()
    ch2 = MagicMock()
    ch3 = MagicMock()
    ch1.state = 'running'
    ch1.name = 'ch1'
    ch2.state = 'stopped'
    ch3.state = 'running'
    ch3.name = 'ch3'
    storage.getChuteList.return_value = [ch1, ch2, ch3]
    mWait.side_effect = [None, None, '[{"success": false, "comment": "PDROP"},{"success": false, "comment": "ch1"},{"success": false, "comment": "ch2"},{"success": true, "comment": "ch3"},{"success": true, "comment": "error"}]']

    #Call
    ret = restart.reloadChutes()

    #Assertions
    mResources.assert_called_with(ch3)
    assert mResources.call_count == 2
    assert mTime.sleep.call_count == 2
    assert mTime.sleep.called_with(1)
    assert mOut.warn.call_count == 2
    assert 'Failed to load a system config section' in str(mOut.warn.call_args_list[0])
    assert 'Failed to load config section for unrecognized chute: ch2' in str(mOut.warn.call_args_list[1])
    assert dict(updateClass='CHUTE', updateType='stop', name=ch1.name, tok=mTimeint.return_value, func=restart.updateStatus, warning=restart.FAILURE_WARNING) in ret
    assert dict(updateClass='CHUTE', updateType='restart', name=ch3.name, tok=mTimeint.return_value, func=restart.updateStatus) in ret
Ejemplo n.º 44
0
Archivo: GCE.py Proyecto: indigo-dc/im
    def test_30_updateVMInfo(self, get_driver):
        radl_data = """
            network net (outbound = 'yes')
            system test (
            cpu.arch='x86_64' and
            cpu.count=1 and
            memory.size=512m and
            net_interface.0.connection = 'net' and
            net_interface.0.dns_name = 'test' and
            disk.0.os.name = 'linux' and
            disk.0.image.url = 'gce://us-central1-a/centos-6' and
            disk.0.os.credentials.username = '******' and
            disk.1.size=1GB and
            disk.1.device='hdb' and
            disk.1.mount_path='/mnt/path'
            )"""
        radl = radl_parse.parse_radl(radl_data)
        radl.check()

        auth = Authentication([{'id': 'one', 'type': 'GCE', 'username': '******',
                                'password': '******', 'project': 'proj'}])
        gce_cloud = self.get_gce_cloud()

        inf = MagicMock()
        inf.get_next_vm_id.return_value = 1
        vm = VirtualMachine(inf, "1", gce_cloud.cloud, radl, radl, gce_cloud)

        driver = MagicMock()
        get_driver.return_value = driver

        node = MagicMock()
        zone = MagicMock()
        node.id = "1"
        node.state = "running"
        node.extra = {'flavorId': 'small'}
        node.public_ips = []
        node.public_ips = ['158.42.1.1']
        node.private_ips = ['10.0.0.1']
        node.driver = driver
        zone.name = 'us-central1-a'
        node.extra = {'zone': zone}
        driver.ex_get_node.return_value = node

        volume = MagicMock()
        volume.id = "vol1"
        volume.attach.return_value = True
        volume.extra = {'status': 'READY'}
        driver.create_volume.return_value = volume

        success, vm = gce_cloud.updateVMInfo(vm, auth)

        self.assertTrue(success, msg="ERROR: updating VM info.")
        self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
        self.clean_log()
Ejemplo n.º 45
0
def mock_ec2_instance(name, state='stopped', **kwargs):
    instance = MagicMock(
        name='ec2-instance-' + name,
        tags=[
            {
                'Key': 'Name',
                'Value': name
            }
        ],
        **kwargs
    )
    instance.state = {'Name': state, 'Code': 0}
    return instance
Ejemplo n.º 46
0
    def test_60_finalize(self, get_driver):
        auth = Authentication([{'id': 'libcloud', 'type': 'LibCloud', 'username': '******',
                                'password': '******', 'driver': 'EC2'}])
        lib_cloud = self.get_lib_cloud()

        radl_data = """
            system test (
            cpu.count>=2 and
            memory.size>=2048m
            )"""
        radl = radl_parse.parse_radl(radl_data)

        inf = MagicMock()
        inf.get_next_vm_id.return_value = 1
        vm = VirtualMachine(inf, "1", lib_cloud.cloud, radl, radl, lib_cloud)
        vm.keypair = ""

        driver = MagicMock()
        driver.name = "Amazon EC2"
        get_driver.return_value = driver

        node = MagicMock()
        node.id = "1"
        node.state = "running"
        node.driver = driver
        node.destroy.return_value = True
        driver.list_nodes.return_value = [node]

        sg = MagicMock()
        sg.id = sg.name = "sg1"
        driver.ex_get_node_security_groups.return_value = [sg]

        keypair = MagicMock()
        driver.get_key_pair.return_value = keypair
        vm.keypair = keypair
        volume = MagicMock()
        volume.id = "id"
        vm.volumes = [volume]

        driver.delete_key_pair.return_value = True

        driver.ex_describe_addresses_for_node.return_value = ["ip"]
        driver.ex_disassociate_address.return_value = True

        success, _ = lib_cloud.finalize(vm, auth)

        self.assertTrue(success, msg="ERROR: finalizing VM info.")
        self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
        self.clean_log()
Ejemplo n.º 47
0
    def test_60_finalize(self, sleep, get_driver):
        auth = Authentication([{'id': 'ost', 'type': 'OpenStack', 'username': '******',
                                'password': '******', 'tenant': 'tenant', 'host': 'https://server.com:5000'}])
        ost_cloud = self.get_ost_cloud()

        radl_data = """
            system test (
            cpu.count>=2 and
            memory.size>=2048m
            )"""
        radl = radl_parse.parse_radl(radl_data)

        inf = MagicMock()
        inf.get_next_vm_id.return_value = 1
        vm = VirtualMachine(inf, "1", ost_cloud.cloud, radl, radl, ost_cloud)

        driver = MagicMock()
        driver.name = "OpenStack"
        get_driver.return_value = driver

        node = MagicMock()
        node.id = "1"
        node.state = "running"
        node.extra = {'flavorId': 'small'}
        node.public_ips = ['158.42.1.1']
        node.private_ips = ['10.0.0.1']
        node.driver = driver
        node.destroy.return_value = True
        driver.list_nodes.return_value = [node]

        sg = MagicMock()
        sg.id = sg.name = "sg1"
        driver.ex_get_node_security_groups.return_value = [sg]

        keypair = MagicMock()
        driver.get_key_pair.return_value = keypair
        vm.keypair = keypair

        driver.delete_key_pair.return_value = True

        driver.delete_security_group.return_value = True

        driver.ex_list_floating_ips.return_value = []

        success, _ = ost_cloud.finalize(vm, auth)

        self.assertTrue(success, msg="ERROR: finalizing VM info.")
        self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
        self.clean_log()
    def _mock_resource(self, info, provider):
        # Mock resource model
        resource = MagicMock()
        resource.pk = info['pk']
        resource.provider = provider
        resource.version = info['version']
        resource.content_type = info['content_type']
        resource.state = info['state']
        resource.download_link = info['download_link']
        resource.resource_type = info['type']
        resource.meta_info = {}

        resource.get_url.return_value = info['download_link']
        resource.get_uri.return_value = info['uri']

        return resource
Ejemplo n.º 49
0
Archivo: EC2.py Proyecto: indigo-dc/im
    def test_55_alter(self, get_connection):
        radl_data = """
            network net ()
            system test (
            cpu.arch='x86_64' and
            cpu.count=1 and
            memory.size=512m and
            net_interface.0.connection = 'net' and
            net_interface.0.dns_name = 'test' and
            disk.0.os.name = 'linux' and
            disk.0.image.url = 'one://server.com/1' and
            disk.0.os.credentials.username = '******' and
            disk.0.os.credentials.password = '******'
            )"""
        radl = radl_parse.parse_radl(radl_data)

        new_radl_data = """
            system test (
            cpu.count>=2 and
            memory.size>=2048m
            )"""
        new_radl = radl_parse.parse_radl(new_radl_data)

        auth = Authentication([{'id': 'ec2', 'type': 'EC2', 'username': '******', 'password': '******'}])
        ec2_cloud = self.get_ec2_cloud()

        inf = MagicMock()
        inf.get_next_vm_id.return_value = 1
        vm = VirtualMachine(inf, "us-east-1;sid-1", ec2_cloud.cloud, radl, radl, ec2_cloud)

        conn = MagicMock()
        get_connection.return_value = conn

        reservation = MagicMock()
        instance = MagicMock()
        instance.update.return_value = True
        instance.stop.return_value = True
        instance.state = "stopped"
        reservation.instances = [instance]
        conn.get_all_instances.return_value = [reservation]

        success, _ = ec2_cloud.alterVM(vm, new_radl, auth)

        self.assertTrue(success, msg="ERROR: modifying VM info.")
        self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
        self.clean_log()
Ejemplo n.º 50
0
    def setUp(self):
        # Mock resource model
        resource1 = MagicMock()
        resource1.name = 'Resource1'
        resource1.version = '1.0'
        resource1.description = 'Test resource 1'
        resource1.content_type = 'text/plain'
        resource1.state = 'created'
        resource1.open = False
        resource1.get_url.return_value = 'http://localhost/media/resources/resource1'
        resource1.resource_type = 'API'
        resource1.meta_info = {}

        resource2 = MagicMock()
        resource2.name = 'Resource2'
        resource2.version = '2.0'
        resource2.description = 'Test resource 2'
        resource2.content_type = 'text/plain'
        resource2.state = 'created'
        resource2.open = False
        resource2.get_url.return_value = 'http://localhost/media/resources/resource2'
        resource2.resource_type = 'API'
        resource2.meta_info = {}

        resource3 = MagicMock()
        resource3.name = 'Resource3'
        resource3.version = '2.0'
        resource3.description = 'Test resource 3'
        resource3.content_type = 'text/plain'
        resource3.state = 'created'
        resource3.open = True
        resource3.get_url.return_value = 'http://localhost/media/resources/resource3'
        resource3.resource_type = 'API'
        resource3.meta_info = {}

        resource4 = MagicMock()
        resource4.name = 'Resource4'
        resource4.version = '1.0'
        resource4.description = 'Test resource 4'
        resource4.content_type = 'text/plain'
        resource4.state = 'created'
        resource4.open = True
        resource4.get_url.return_value = 'http://localhost/media/resources/resource4'
        resource4.resource_type = 'API'
        resource4.offerings = ['1111', '2222']
        resource4.meta_info = {}

        resources_management.Resource = MagicMock()

        def resource_filter(provider=None, open=None):
            if provider != self.org:
                return []

            if open is None:
                result = [
                    resource1,
                    resource2,
                    resource3,
                    resource4
                ]
            elif not open:
                result = [
                    resource1,
                    resource2
                ]
            else:
                result = [
                    resource3,
                    resource4
                ]
            return result

        resources_management.Resource.objects.filter = resource_filter

        self.user = MagicMock()
        self.org = MagicMock()
        self.user.userprofile.current_organization = self.org
Ejemplo n.º 51
0
    def test_auth_resource_show(self, exist_pkg, creator_user_id, user_obj_id, user, private, state, owner_org,
                                owner_member, db_auth, authorized):
        #Recover the exception
        auth.tk.ObjectNotFound = self._tk.ObjectNotFound

        # Configure the mocks
        if exist_pkg:
            returned_package = MagicMock()
            returned_package.creator_user_id = creator_user_id
            returned_package.private = private
            returned_package.state = state
            returned_package.owner_org = owner_org
            returned_package.extras = {}
        else:
            returned_package = None

        returned_resource = MagicMock()
        returned_resource.package_id = 1

        # Configure the database
        db_response = []
        if db_auth is True:
            out = auth.db.AllowedUser()
            out.package_id = 'package_id'
            out.user_name = user
            db_response.append(out)

        # Prepare the context
        context = {'model': MagicMock()}
        if user is not None:
            context['user'] = user
        if user_obj_id is not None:
            context['auth_user_obj'] = MagicMock()
            context['auth_user_obj'].id = user_obj_id

        auth.db.AllowedUser.get = MagicMock(return_value=db_response)
        auth.logic_auth.get_resource_object = MagicMock(return_value=returned_resource)
        auth.logic_auth.get_package_object = MagicMock(return_value=returned_package)
        auth.authz.has_user_permission_for_group_or_org = MagicMock(return_value=owner_member)

        # Prepare the context
        context = {'model': MagicMock()}
        if user is not None:
            context['user'] = user
        if user_obj_id is not None:
            context['auth_user_obj'] = MagicMock()
            context['auth_user_obj'].id = user_obj_id

        if not exist_pkg:
            self.assertRaises(self._tk.ObjectNotFound, auth.resource_show, context, {})
        else:
            result = auth.resource_show(context, {})
            self.assertEquals(authorized, result['success'])

            if private and owner_org and state == 'active' and creator_user_id != user_obj_id:
                auth.authz.has_user_permission_for_group_or_org.assert_called_once_with(owner_org, user, 'read')
            else:
                self.assertEquals(0, auth.authz.has_user_permission_for_group_or_org.call_count)

            if private and state == 'active' and (not owner_org or not owner_member) and (creator_user_id != user_obj_id or user_obj_id is None):
                # Check that the database has been initialized properly
                auth.db.init_db.assert_called_once_with(context['model'])
            else:
                self.assertEquals(0, auth.db.init_db.call_count)
Ejemplo n.º 52
0
    def test_auth_package_show(self, creator_user_id, user_obj_id, user, private, state, owner_org,
                               owner_member, db_auth, acquire_url, request_path, authorized):

        # Configure the mocks
        returned_package = MagicMock()
        returned_package.creator_user_id = creator_user_id
        returned_package.private = private
        returned_package.state = state
        returned_package.owner_org = owner_org
        returned_package.extras = {}

        # Configure the database
        db_response = []
        if db_auth is True:
            out = auth.db.AllowedUser()
            out.package_id = 'package_id'
            out.user_name = user
            db_response.append(out)

        auth.db.AllowedUser.get = MagicMock(return_value=db_response)

        if acquire_url:
            returned_package.extras['acquire_url'] = acquire_url

        auth.logic_auth.get_package_object = MagicMock(return_value=returned_package)
        auth.new_authz.has_user_permission_for_group_or_org = MagicMock(return_value=owner_member)
        auth.request.path = MagicMock(return_value=request_path)

        # Prepare the context
        context = {'model': MagicMock()}
        if user is not None:
            context['user'] = user
        if user_obj_id is not None:
            context['auth_user_obj'] = MagicMock()
            context['auth_user_obj'].id = user_obj_id

        # Function to be tested
        result = auth.package_show(context, {})

        # Check the result
        self.assertEquals(authorized, result['success'])

        # Premissions for organization are checked when the dataset is private, it belongs to an organization
        # and when the dataset has not been created by the user who is asking for it
        if private and owner_org and state == 'active' and creator_user_id != user_obj_id:
            auth.new_authz.has_user_permission_for_group_or_org.assert_called_once_with(owner_org, user, 'read')
        else:
            self.assertEquals(0, auth.new_authz.has_user_permission_for_group_or_org.call_count)

        # The databse is only initialized when:
        # * the dataset is private AND
        # * the dataset is active AND
        # * the dataset has no organization OR the user does not belong to that organization AND
        # * the dataset has not been created by the user who is asking for it OR the user is not specified
        if private and state == 'active' and (not owner_org or not owner_member) and (creator_user_id != user_obj_id or user_obj_id is None):
            # Check that the database has been initialized properly
            auth.db.init_db.assert_called_once_with(context['model'])
        else:
            self.assertEquals(0, auth.db.init_db.call_count)

        # Conditions to buy a dataset; It should be private, active and should not belong to any organization
        if not authorized and state == 'active' and not owner_org and request_path.startswith('/dataset/'):
            auth.helpers.flash_error.assert_called_once()
        else:
            self.assertEquals(0, auth.helpers.flash_error.call_count)
Ejemplo n.º 53
0
    def test_30_updateVMInfo(self, get_driver):
        radl_data = """
            network net (outbound = 'yes')
            system test (
            cpu.arch='x86_64' and
            cpu.count=1 and
            memory.size=512m and
            net_interface.0.connection = 'net' and
            net_interface.0.dns_name = 'test' and
            disk.0.os.name = 'linux' and
            disk.0.image.url = 'one://server.com/1' and
            disk.0.os.credentials.username = '******' and
            disk.0.os.credentials.password = '******'
            )"""
        radl = radl_parse.parse_radl(radl_data)
        radl.check()

        auth = Authentication([{'id': 'ost', 'type': 'OpenStack', 'username': '******',
                                'password': '******', 'tenant': 'tenant', 'host': 'https://server.com:5000'}])
        ost_cloud = self.get_ost_cloud()

        inf = MagicMock()
        inf.get_next_vm_id.return_value = 1
        vm = VirtualMachine(inf, "1", ost_cloud.cloud, radl, radl, ost_cloud)

        driver = MagicMock()
        get_driver.return_value = driver

        node = MagicMock()
        node.id = "1"
        node.state = "running"
        node.extra = {'flavorId': 'small', 'addresses': {'os-lan': [{'addr': '10.0.0.1',
                                                                     'OS-EXT-IPS:type': 'fixed'}]}}
        node.public_ips = []
        node.private_ips = ['10.0.0.1']
        node.driver = driver
        driver.list_nodes.return_value = [node]

        node_size = MagicMock()
        node_size.ram = 512
        node_size.price = 1
        node_size.disk = 1
        node_size.vcpus = 1
        node_size.name = "small"
        driver.ex_get_size.return_value = node_size

        volume = MagicMock()
        volume.id = "vol1"
        volume.attach.return_value = True
        driver.create_volume.return_value = volume

        pool = MagicMock()
        pool.name = "pool1"
        pool.list_floating_ips.return_value = []
        pool.create_floating_ip.return_value = True
        driver.ex_list_floating_ip_pools.return_value = [pool]

        success, vm = ost_cloud.updateVMInfo(vm, auth)

        self.assertTrue(success, msg="ERROR: updating VM info.")
        self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
        self.clean_log()
Ejemplo n.º 54
0
    def test_get_inf_state(self):
        """
        Test GetInfrastructureState.
        """
        auth0 = self.getAuth([0], [], [("Dummy", 0)])

        inf = MagicMock()
        IM.infrastructure_list = {"1": inf}
        inf.id = "1"
        inf.auth = auth0
        inf.deleted = False
        vm1 = MagicMock()
        vm1.im_id = 0
        vm1.state = VirtualMachine.RUNNING
        vm2 = MagicMock()
        vm2.im_id = 1
        vm2.state = VirtualMachine.RUNNING
        vm3 = MagicMock()
        vm3.im_id = 2
        vm3.state = VirtualMachine.RUNNING
        inf.get_vm_list.return_value = [vm1, vm2, vm3]

        state = IM.GetInfrastructureState("1", auth0)
        self.assertEqual(state["state"], "running")

        vm1.state = VirtualMachine.FAILED
        vm2.state = VirtualMachine.RUNNING
        vm3.state = VirtualMachine.UNKNOWN

        state = IM.GetInfrastructureState("1", auth0)
        self.assertEqual(state["state"], "failed")

        vm1.state = VirtualMachine.PENDING
        vm2.state = VirtualMachine.RUNNING
        vm3.state = VirtualMachine.CONFIGURED

        state = IM.GetInfrastructureState("1", auth0)
        self.assertEqual(state["state"], "pending")

        vm1.state = VirtualMachine.PENDING
        vm2.state = VirtualMachine.CONFIGURED
        vm3.state = VirtualMachine.UNCONFIGURED

        state = IM.GetInfrastructureState("1", auth0)
        self.assertEqual(state["state"], "pending")

        vm1.state = VirtualMachine.RUNNING
        vm2.state = VirtualMachine.CONFIGURED
        vm3.state = VirtualMachine.UNCONFIGURED

        state = IM.GetInfrastructureState("1", auth0)
        self.assertEqual(state["state"], "running")

        vm1.state = VirtualMachine.RUNNING
        vm2.state = VirtualMachine.CONFIGURED
        vm3.state = VirtualMachine.STOPPED

        state = IM.GetInfrastructureState("1", auth0)
        self.assertEqual(state["state"], "running")
Ejemplo n.º 55
0
Archivo: EC2.py Proyecto: indigo-dc/im
    def test_30_updateVMInfo_spot(self, get_connection):
        radl_data = """
            network net (outbound = 'yes')
            system test (
            cpu.arch='x86_64' and
            cpu.count=1 and
            memory.size=512m and
            net_interface.0.connection = 'net' and
            net_interface.0.dns_name = 'test' and
            disk.0.os.name = 'linux' and
            disk.0.image.url = 'one://server.com/1' and
            disk.0.os.credentials.username = '******' and
            disk.0.os.credentials.password = '******' and
            disk.1.size=1GB and
            disk.1.device='hdb' and
            disk.1.mount_path='/mnt/path'
            )"""
        radl = radl_parse.parse_radl(radl_data)
        radl.check()

        auth = Authentication([{'id': 'ec2', 'type': 'EC2', 'username': '******', 'password': '******'}])
        ec2_cloud = self.get_ec2_cloud()

        inf = MagicMock()
        inf.get_next_vm_id.return_value = 1
        vm = VirtualMachine(inf, "us-east-1;sid-1", ec2_cloud.cloud, radl, radl, ec2_cloud)

        conn = MagicMock()
        get_connection.return_value = conn

        reservation = MagicMock()
        instance = MagicMock()
        instance.update.return_value = True
        instance.tags = []
        instance.virtualization_type = "vt"
        instance.placement = "us-east-1"
        instance.state = "running"
        instance.instance_type = "t1.micro"
        instance.launch_time = "2016-12-31T00:00:00"
        instance.ip_address = "158.42.1.1"
        instance.private_ip_address = "10.0.0.1"
        instance.connection = conn
        reservation.instances = [instance]
        conn.get_all_instances.return_value = [reservation]

        conn.get_all_addresses.return_value = []

        sir = MagicMock()
        sir.state = ""
        sir.id = "id"
        conn.get_all_spot_instance_requests.return_value = [sir]

        volume = MagicMock()
        volume.status = "available"
        volume.id = "volid"
        conn.create_volume.return_value = volume
        conn.attach_volume.return_value = True

        success, vm = ec2_cloud.updateVMInfo(vm, auth)

        self.assertTrue(success, msg="ERROR: updating VM info.")
        self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
        self.clean_log()