def test_create_progressbar(self):
sess = mock.MagicMock()
sess.post = mock.MagicMock(return_value=self._mkresp(
status_code=202,
headers={'Location': 'http://host.com/somewhere'},
payload={"value": 'http://host.com/progress/1'}
))
sess.get = mock.MagicMock(side_effect=[
self._mkresp(status_code=200, payload={'value': {'progress': i*10}}) for i in range(11)
])
c = Catalog(self='http://host.com/catalog', session=sess)
class FakeStdout(object):
writes = []
def write(self, text):
self.writes.append(text)
def flush(self):
pass
with mock.patch.object(sys, 'stdout', FakeStdout()):
c.create({'somedata': 1}, progress_tracker=SimpleTextBarProgressTracking(timeout=None,
interval=0.1))
# Check we printed the progressbar up to 100%.
self.assertEqual(''.join(FakeStdout.writes).count('-'),
SimpleTextBarProgressTracking.BAR_WIDTH)
def test_create_progressbar(self):
sess = mock.MagicMock()
sess.post = mock.MagicMock(return_value=self._mkresp(
status_code=202,
headers={'Location': 'http://host.com/somewhere'},
payload={"value": 'http://host.com/progress/1'}
))
sess.get = mock.MagicMock(side_effect=[
self._mkresp(status_code=200, payload={'value': {'progress': i*10}}) for i in range(11)
])
c = Catalog(self='http://host.com/catalog', session=sess)
class FakeStdout(object):
writes = []
def write(self, text):
self.writes.append(text)
def flush(self):
pass
with mock.patch.object(sys, 'stdout', FakeStdout()):
c.create({'somedata': 1}, progress_tracker=SimpleTextBarProgressTracking(timeout=None,
interval=0.1))
# Check we printed the progressbar up to 100%.
self.assertEqual(''.join(FakeStdout.writes).count('-'),
SimpleTextBarProgressTracking.BAR_WIDTH)
def test_create_timesout_continuation(self):
sess = mock.MagicMock()
sess.progress_tracking = DefaultProgressTracking(timeout=0.1,
interval=0.1)
sess.post = mock.MagicMock(return_value=self._mkresp(
status_code=202,
headers={'Location': 'http://host.com/somewhere'},
payload={"value": 'http://host.com/progress/1'}))
sess.get = mock.MagicMock(side_effect=[
self._mkresp(status_code=200, payload={'value': {
'progress': 30
}}),
self._mkresp(status_code=200, payload={'value': {
'progress': 60
}}),
self._mkresp(status_code=200, payload={'value': {
'progress': 100
}}),
])
c = Catalog(self='http://host.com/catalog', session=sess)
try:
c.create({'somedata': 1})
except TaskProgressTimeoutError as e:
sess.progress_tracking = DefaultProgressTracking(timeout=None,
interval=0.1)
e.entity.wait_progress(e.response)
self.assertEqual(sess.get.call_count, 3)
else:
assert False, "Should have raised TaskProgressTimeoutError"
def test_create_waits_for_progress(self):
sess = mock.MagicMock()
sess.progress_tracking = DefaultProgressTracking(timeout=1.0,
interval=0.1)
sess.post = mock.MagicMock(return_value=self._mkresp(
status_code=202,
headers={'Location': 'http://host.com/somewhere'},
payload={"value": 'http://host.com/progress/1'}))
sess.get = mock.MagicMock(side_effect=[
self._mkresp(status_code=200, payload={'value': {
'progress': 30
}}),
self._mkresp(status_code=200, payload={'value': {
'progress': 60
}}),
self._mkresp(status_code=200, payload={'value': {
'progress': 100
}}),
])
c = Catalog(self='http://host.com/catalog', session=sess)
c.create({'somedata': 1})
# Assert progress got called until it completed
self.assertEqual(sess.get.call_count, 3)
def test_unique_folders_no_secure():
session = MockSession()
dataset_url = 'http://host/api/datasets/abc/'
folders_url = 'http://host/api/datasets/abc/folders/'
hidden_url = 'http://host/api/datasets/abc/folders/hidden/'
trash_url = 'http://host/api/datasets/abc/folders/trash/'
dataset_resource = Entity(session, **{
"element": "shoji:entity",
"self": dataset_url,
"body": {
"name": "test_dataset_project"
},
"catalogs": {
"folders": folders_url,
}
})
dataset_resource.variables = MagicMock()
dataset_resource.settings = MagicMock()
folders_resource = Catalog(session, **{
"element": "shoji:catalog",
"self": folders_url,
"index": {},
"body": {
"name": "Root"
},
"catalogs": {
"hidden": hidden_url,
# Viewer users don't have the secure folder available
# "secure": secure_url,
"trash": trash_url,
}
})
hidden_resource = Catalog(session, **{
"element": "shoji:catalog",
"self": hidden_url,
"index": {},
"body": {
"name": "Hidden"
},
})
trash_resource = Catalog(session, **{
"element": "shoji:catalog",
"self": trash_url,
"index": {},
"body": {
"name": "Trash"
},
})
session.add_fixture(folders_url, folders_resource)
session.add_fixture(hidden_url, hidden_resource)
session.add_fixture(trash_url, trash_resource)
dataset = MutableDataset(dataset_resource)
assert dataset.folders.root.name == "Root"
assert dataset.folders.hidden.name == "Hidden"
assert dataset.folders.trash.name == "Trash"
assert not hasattr(dataset.folders, "secure")
def test_create_does_post_catalog(self):
sess = mock.MagicMock()
c = Catalog(self='http://host.com/catalog', session=sess)
c.create({'somedata': 1})
sess.post.assert_called_once_with(
'http://host.com/catalog',
json.dumps({"somedata": 1, "body": {}, "element": "shoji:entity"}, indent=4),
headers={'Content-Type': 'application/json'}
)
def test_create_does_post_catalog(self):
sess = mock.MagicMock()
c = Catalog(self='http://host.com/catalog', session=sess)
c.create({'somedata': 1})
sess.post.assert_called_once_with(
'http://host.com/catalog',
json.dumps({"somedata": 1, "body": {}, "element": "shoji:entity"}, indent=None, separators=(',', ':')),
headers={'Content-Type': 'application/json'}
)
def test_accepts_document_instance(self):
sess = mock.MagicMock()
location = 'http://host.com/somewhere'
sess.post = mock.MagicMock(return_value=self._mkresp(
status_code=201,
headers={'Location': location}
))
catalog = Catalog(self='http://host.com/catalog', session=sess)
body = {
"name": 'subcatalog'
}
sub_catalog = Catalog(session=sess, body=body)
retval = catalog.create(sub_catalog)
self.assertEqual(retval.self, location)
self.assertEqual(retval.body, body)
def test_accepts_document_instance(self):
sess = mock.MagicMock()
location = 'http://host.com/somewhere'
sess.post = mock.MagicMock(return_value=self._mkresp(
status_code=201,
headers={'Location': location}
))
catalog = Catalog(self='http://host.com/catalog', session=sess)
body = {
"name": 'subcatalog'
}
sub_catalog = Catalog(session=sess, body=body)
retval = catalog.create(sub_catalog)
self.assertEqual(retval.self, location)
self.assertEqual(retval.body, body)
def test_detect_correct_handler(self):
session = Mock(feature_flags={'old_projects_order': True})
dataset_order = Order(session, **{'graph': []})
datasets_catalog = Catalog(session, **{
'index': {},
'order': dataset_order
})
shoji_resource = Entity(
session, **{
'self': '/project/url/',
'body': {},
'index': {},
'datasets': datasets_catalog
})
project = Project(shoji_resource)
self.assertTrue(isinstance(project.order, ProjectDatasetsOrder))
session = Mock(feature_flags={'old_projects_order': False})
shoji_resource = Entity(
session, **{
'self': '/project/url/',
'body': {},
'index': {},
'graph': []
})
project = Project(shoji_resource)
self.assertTrue(isinstance(project.order, Project))
def test_create_waits_for_progress(self):
sess = mock.MagicMock()
sess.progress_tracking = DefaultProgressTracking(timeout=1.0, interval=0.1)
sess.post = mock.MagicMock(return_value=self._mkresp(
status_code=202,
headers={'Location': 'http://host.com/somewhere'},
payload={"value": 'http://host.com/progress/1'}
))
sess.get = mock.MagicMock(side_effect=[
self._mkresp(status_code=200, payload={'value': {'progress': 30}}),
self._mkresp(status_code=200, payload={'value': {'progress': 60}}),
self._mkresp(status_code=200, payload={'value': {'progress': 100}}),
])
c = Catalog(self='http://host.com/catalog', session=sess)
c.create({'somedata': 1})
# Assert progress got called until it completed
self.assertEqual(sess.get.call_count, 3)
def test_void_catalog_create(self):
sess = mock.MagicMock()
c = Catalog(self='http://host.com/catalog', session=sess)
things_url = "http://host.com/catalog/things"
c["catalogs"] = {"things": things_url}
# void_catalog...
things = c.void("things")
# ...MUST NOT call GET...
sess.get.assert_not_called()
# ...MUST return a dummy Catalog...
assert things.__class__ == Catalog
assert things.self == things_url
assert things.session is c.session
assert dict(things) == {"element": "shoji:catalog", "self": things_url}
# ...MUST have a working `create` method...
things.create({'somedata': 1})
sess.post.assert_called_once_with(
'http://host.com/catalog/things',
json.dumps({"somedata": 1, "body": {}, "element": "shoji:entity"}, indent=None, separators=(',', ':')),
headers={'Content-Type': 'application/json'}
)
def create_folder(self, folder_name, position=None, after=None, before=None, alias=None):
new_ent = self.folder_ent.create(Catalog(self.folder_ent.session, body={
'name': folder_name
}))
new_ent.refresh()
subfolder = Folder(new_ent, self.root, self)
if position is not None or after is not None or before is not None:
children = self._position_items([subfolder], position, before, after)
self.reorder(children)
if alias:
subfolder.move_here(alias)
self.folder_ent.refresh()
return subfolder
def test_create_raises_failures(self):
sess = mock.MagicMock()
sess.progress_tracking = DefaultProgressTracking(timeout=1.0, interval=0.1)
sess.post = mock.MagicMock(return_value=self._mkresp(
status_code=202,
headers={'Location': 'http://host.com/somewhere'},
payload={"value": 'http://host.com/progress/1'}
))
sess.get = mock.MagicMock(side_effect=[
self._mkresp(status_code=200, payload={'value': {'progress': -1,
'message': 'Some Failure'}}),
])
c = Catalog(self='http://host.com/catalog', session=sess)
self.assertRaises(TaskError, c.create, {'somedata': 1})
def test_create_timesout_continuation(self):
sess = mock.MagicMock()
sess.progress_tracking = DefaultProgressTracking(timeout=0.1, interval=0.1)
sess.post = mock.MagicMock(return_value=self._mkresp(
status_code=202,
headers={'Location': 'http://host.com/somewhere'},
payload={"value": 'http://host.com/progress/1'}
))
sess.get = mock.MagicMock(side_effect=[
self._mkresp(status_code=200, payload={'value': {'progress': 30}}),
self._mkresp(status_code=200, payload={'value': {'progress': 60}}),
self._mkresp(status_code=200, payload={'value': {'progress': 100}}),
])
c = Catalog(self='http://host.com/catalog', session=sess)
try:
c.create({'somedata': 1})
except TaskProgressTimeoutError as e:
sess.progress_tracking = DefaultProgressTracking(timeout=None, interval=0.1)
e.entity.wait_progress(e.response)
self.assertEqual(sess.get.call_count, 3)
else:
assert False, "Should have raised TaskProgressTimeoutError"
def test_unique_folders_no_hidden():
session = MockSession()
dataset_url = 'http://host/api/datasets/abc/'
folders_url = 'http://host/api/datasets/abc/folders/'
dataset_resource = Entity(session, **{
"element": "shoji:entity",
"self": dataset_url,
"body": {
"name": "test_dataset_project"
},
"catalogs": {
"folders": folders_url,
}
})
dataset_resource.variables = MagicMock()
dataset_resource.settings = MagicMock()
folders_resource = Catalog(session, **{
"element": "shoji:catalog",
"self": folders_url,
"index": {},
"body": {
"name": "Root"
},
"catalogs": {
# Standard exposed catalogs
"personal": "./personal/",
"parents": "./parents/",
# Viewer users don't have the secure folder available
# "secure": secure_url,
# Viewers also don't get the hidden folder exposed
# "hidden": hidden_url,
# Nor the trash
# "trash": trash_url,
}
})
session.add_fixture(folders_url, folders_resource)
dataset = MutableDataset(dataset_resource)
assert dataset.folders.root.name == "Root"
assert not hasattr(dataset.folders, "secure")
assert not hasattr(dataset.folders, "hidden")
assert not hasattr(dataset.folders, "trash")
def setup_folders(ds):
sess = ds.session
ds.settings.edit(variable_folders=True)
ds.variables.create({
'element': 'shoji:entity',
'body': {'name': 'testvar1', 'type': 'numeric'}
})
ds.variables.create({
'element': 'shoji:entity',
'body': {'name': 'testvar2', 'type': 'numeric'}
})
ds.variables.create({
'element': 'shoji:entity',
'body': {'name': 'testvar3', 'type': 'numeric'}
})
ds.variables.create({
'element': 'shoji:entity',
'body': {'name': 'testvar4', 'type': 'numeric'}
})
ds.variables.create({
'element': 'shoji:entity',
'body': {'name': 'testvar5', 'type': 'numeric'}
})
ds.variables.create({
'element': 'shoji:entity',
'body': {'name': 'testvar6', 'type': 'numeric'}
})
ds.refresh()
sf1 = Catalog(sess, body={
'name': 'Subfolder 1'
})
sf1 = ds.folders.create(sf1)
sfa = Catalog(sess, body={
'name': 'Subfolder A'
})
sfa = sf1.create(sfa)
sf2 = Catalog(sess, body={
'name': 'Subfolder 2'
})
sf2 = ds.folders.create(sf2)
variables = ds.variables.by('alias')
sf1.patch({'index': {
variables['age'].entity_url: {}
}})
sfa.patch({'index': {
variables['gender'].entity_url: {}
}})
sf2.patch({'index': {
variables['socialmedia'].entity_url: {}
}})
