def setUp(self):
"""Create and configure a new app instance for each test."""
# create the app with common test config
self.app = create_app('test')
self.app_context = self.app.app_context()
self.app_context.push()
self.client = self.app.test_client()
self.headers = {
"Authorization":
f"Bearer {create_access_token('00000000-0000-0000-0000-000000000001')}"
}
db.create_all()
set_initial()
self.soft = Software(id='aaaaaaaa-1234-5678-1234-56781234aaa1',
name='test',
version='1',
filename='file')
self.soft2 = Software(id='aaaaaaaa-1234-5678-1234-56781234aaa2',
name='test',
version='2',
filename='file')
self.ssa = SoftwareServerAssociation(software=self.soft,
server=Server.get_current(),
path='/root')
db.session.add_all([self.soft, self.soft2, self.ssa])
db.session.commit()
def setUp(self, mocked_now):
self.initials = dict(self.initials)
self.initials.update(action_template=False)
mocked_now.return_value = now1
super().setUp()
with self.app2_context:
mocked_now.return_value = now2
soft = Software(id='aaaaaaaa-1234-5678-1234-56781234aaa1', name='test', version='1', filename='file')
at = ActionTemplate(id='aaaaaaaa-1234-5678-1234-56781234aaa2', name='mkdir', version=1,
action_type=ActionType.SHELL, code='mkdir {dir}')
db.session.add_all([soft, at])
db.session.commit()
mocked_now.return_value = now3
ssa = SoftwareServerAssociation(software=soft, server=Server.get_current(), path='/root')
db.session.add(ssa)
db.session.commit()
self.soft_json = soft.to_json()
self.at_json = at.to_json()
self.catalog = fetch_catalog(now1)
self.mock_queue = mock.Mock()
self.mock_dm = mock.Mock()
self.mock_dm.flask_app = self.app
self.mock_dm.engine = db.engine
self.mock_dm.manager.dict.return_value = dict()
self.mock_dm.server_id = self.s1.id
self.cm = CatalogManager("Catalog", startup_event=threading.Event(), shutdown_event=threading.Event(),
publish_q=self.mock_queue, event_q=None, dimensigon=self.mock_dm)
db.session.commit()
def setUp(self) -> None:
super().setUp()
self.source_path = '/software'
self.filename = 'filename.zip'
self.content = b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
self.size = len(self.content)
self.checksum = hashlib.md5(self.content).hexdigest()
self.soft = Software(name='test_software',
version=1,
filename='software.zip',
size=self.size,
checksum=self.checksum)
self.ssa = SoftwareServerAssociation(software=self.soft,
server=self.s1,
path=self.source_path)
self.dest_path = '/dest_repo'
db.session.add(self.soft)
db.session.add(self.ssa)
db.session.commit()
self.setUpPyfakefs()
self.fs.create_dir(self.source_path)
self.fs.create_dir(self.dest_path)
self.fs.create_file(os.path.join(self.source_path, self.filename),
contents=self.content)
self.fs.create_file(os.path.join(self.source_path, self.soft.filename),
contents=self.content)
self.app.config['DEBUG'] = False
def test_execute_send_software_error(self, mock_post, mock_get):
at = ActionTemplate.query.get('00000000-0000-0000-000a-000000000001')
soft = Software(name='test', version=1, filename='test.zip')
node1 = Server('nodeA', port=5000)
ssa1 = SoftwareServerAssociation(software=soft, server=node1, path='/')
db.session.add_all([soft, node1, ssa1])
mock_post.return_value = Response(msg={'error': 'message'}, code=400)
mock_get.return_value = Response(code=400)
ro = NativeSoftwareSendOperation(code,
expected_stdout=at.expected_stdout,
expected_stderr=at.expected_stderr,
expected_rc=at.expected_rc)
cp = ro._execute(
dict(input=dict(software=str(soft.id), server=str(node1.id))),
timeout=10,
context=self.context)
mock_post.assert_called_once_with(node1,
'api_1_0.send',
json=dict(software_id=str(soft.id),
dest_server_id=str(
node1.id),
background=False,
include_transfer_data=True,
force=True),
timeout=10,
identity=ROOT)
self.assertFalse(cp.success)
self.assertEqual(flask.json.dumps(mock_post.return_value.msg),
cp.stdout)
def test_execute_send_software(self, mock_post, mock_get):
at = ActionTemplate.query.get('00000000-0000-0000-000a-000000000001')
soft = Software(name='test', version=1, filename='test.zip')
node1 = Server('nodeA', port=5000)
node2 = Server('nodeB', port=5000)
ssa1 = SoftwareServerAssociation(software=soft, server=node1, path='/')
ssa2 = SoftwareServerAssociation(software=soft, server=node2, path='/')
ssa3 = SoftwareServerAssociation(software=soft,
server=self.s1,
path='/')
db.session.add_all([soft, node1, node2, ssa1, ssa2, ssa3])
mock_post.return_value = Response(msg={'transfer_id': 1},
code=at.expected_rc)
mock_get.return_value = Response(msg={
"route_list": [{
"cost": 0,
"destination_id": f"{node1.id}",
}, {
"cost": 1,
"destination_id": f"{self.s1.id}",
}],
},
code=200,
server=node2)
ro = NativeSoftwareSendOperation(code,
expected_stdout=at.expected_stdout,
expected_stderr=at.expected_stderr,
expected_rc=at.expected_rc)
cp = ro._execute(dict(input=dict(software=soft.id,
server=node2.id,
dest_path='dest',
chunk_size=20,
max_senders=2)),
timeout=None,
context=self.context)
mock_post.assert_called_once_with(node1,
'api_1_0.send',
json=dict(software_id=str(soft.id),
dest_server_id=str(
node2.id),
background=False,
include_transfer_data=True,
force=True,
dest_path='dest',
chunk_size=20,
max_senders=2),
timeout=None,
identity=ROOT)
self.assertTrue(cp.success)
self.assertEqual(flask.json.dumps(mock_post.return_value.msg),
cp.stdout)
def test_execute_send_software_no_destination_server(self):
at = ActionTemplate.query.get('00000000-0000-0000-000a-000000000001')
soft = Software(name='test', version='1', filename='test.zip')
soft2 = Software(name='test', version='2', filename='test.zip')
node1 = Server('nodeA', port=5000)
ssa1 = SoftwareServerAssociation(software=soft2,
server=node1,
path='/')
db.session.add_all([soft, soft2, node1, ssa1])
ro = NativeSoftwareSendOperation(code,
expected_stdout=at.expected_stdout,
expected_stderr=at.expected_stderr,
expected_rc=at.expected_rc)
cp = ro._execute(dict(input=dict(software='test', server='a')),
context=self.context)
self.assertFalse(cp.success)
self.assertEqual(f"destination server 'a' not found", cp.stderr)
def fill_database(self):
soft = Software(name='test_software',
version=1,
filename=self.filename,
size=self.size,
checksum=self.checksum,
id=self.soft_id)
ssa = SoftwareServerAssociation(software=soft,
server_id=self.SERVER1,
path=self.source_path)
db.session.add_all([soft, ssa])
def fill_database(self):
soft = Software(name="python",
version="3.8.3",
family="programming",
filename=self.filename,
size=self.size,
checksum=self.checksum,
id=self.SOFTWARE,
last_modified_at=defaults.INITIAL_DATEMARK.strftime(
defaults.DATEMARK_FORMAT))
ssa = SoftwareServerAssociation(
software_id=self.SOFTWARE,
server_id=self.SERVER1,
path=self.source_folder,
last_modified_at=defaults.INITIAL_DATEMARK.strftime(
defaults.DATEMARK_FORMAT))
o = Orchestration(name="install python 3.8.3",
version=1,
description="installs python 3.8.3",
id=self.ORCH,
last_modified_at=defaults.INITIAL_DATEMARK.strftime(
defaults.DATEMARK_FORMAT))
s1 = o.add_step(id=self.STEP1,
action_template=ActionTemplate.query.get(
ActionTemplate.SEND_SOFTWARE),
undo=False,
undo_on_error=False,
schema={
"mapping": {
"software": self.SOFTWARE,
"server": {
"from": "env.server_id"
}
}
},
last_modified_at=defaults.INITIAL_DATEMARK.strftime(
defaults.DATEMARK_FORMAT))
o.add_step(id=self.STEP2,
undo=False,
undo_on_error=False,
parents=[s1],
code="{{input.file}}",
action_type="SHELL",
expected_rc=0,
last_modified_at=defaults.INITIAL_DATEMARK.strftime(
defaults.DATEMARK_FORMAT))
db.session.add_all([soft, ssa, o])
def test_execute_send_software_no_ssa(self, mock_get):
at = ActionTemplate.query.get('00000000-0000-0000-000a-000000000001')
soft = Software(name='test', version=1, filename='test.zip')
node1 = Server('nodeA', port=5000)
db.session.add_all([soft, node1])
mock_get.return_value = Response(code=400)
ro = NativeSoftwareSendOperation(code,
expected_stdout=at.expected_stdout,
expected_stderr=at.expected_stderr,
expected_rc=at.expected_rc)
cp = ro._execute(dict(input=dict(software=soft.id, server=self.s1.id)),
context=self.context)
self.assertFalse(cp.success)
self.assertEqual(f'{soft.id} has no server association', cp.stderr)
def fill_database(self):
self.soft1 = Software(id='11111111-2222-3333-4444-555555550001',
name='Dimensigon',
version='0.0.1',
filename='Dimensigon_0.0.1.tar.gz')
self.soft2 = Software(id='11111111-2222-3333-4444-555555550002',
name='Dimensigon',
version='0.0.2',
filename='Dimensigon_0.0.2.tar.gz')
self.soft3 = Software(id='11111111-2222-3333-4444-555555550003',
name='python',
version='3.6.8',
filename='python_3.6.8.x64.tar.gz')
db.session.add_all([self.soft1, self.soft2, self.soft3])
def test_to_json(self):
d = get_now()
s = Software(id='aaaaaaaa-1234-5678-1234-56781234aaa1', name='test', version='1', filename='file')
t = Transfer(id='aaaaaaaa-1234-5678-1234-56781234aaa2', software=s, dest_path='/folder', num_chunks=0,
created_on=d)
self.assertDictEqual(
dict(id='aaaaaaaa-1234-5678-1234-56781234aaa2', software_id='aaaaaaaa-1234-5678-1234-56781234aaa1',
dest_path='/folder', num_chunks=0, file='/folder/file',
status='WAITING_CHUNKS', created_on=d.strftime(defaults.DATETIME_FORMAT)),
t.to_json())
t = Transfer(id='aaaaaaaa-1234-5678-1234-56781234aaa2', software='filename', size=10, checksum='abc12',
dest_path='/folder', num_chunks=0, created_on=d)
self.assertDictEqual(
dict(id='aaaaaaaa-1234-5678-1234-56781234aaa2', size=10, checksum='abc12',
dest_path='/folder', num_chunks=0, file='/folder/filename',
status='WAITING_CHUNKS', created_on=d.strftime(defaults.DATETIME_FORMAT)),
t.to_json())
def test_wait_transfer(self):
s = Software(name='test', version='1', filename='file')
t = Transfer(software=s, dest_path='', num_chunks=0)
db.session.add_all([s, t])
db.session.commit()
status = t.wait_transfer(timeout=0.1, refresh_interval=0.01)
self.assertEqual(TransferStatus.WAITING_CHUNKS, status)
def update_transfer(app, transfer_id):
with app.app_context():
tr = Transfer.query.get(transfer_id)
tr.status = TransferStatus.COMPLETED
db.session.commit()
th = threading.Timer(0.05, update_transfer, (self.app, t.id))
th.start()
status = t.wait_transfer(timeout=0.1, refresh_interval=0.01)
self.assertEqual(TransferStatus.COMPLETED, status)
def post(self):
json = request.get_json()
file = json['file']
if not os.path.exists(file):
raise errors.FileNotFound(file)
soft = Software(name=json['name'],
version=json['version'],
filename=os.path.basename(json['file']),
size=os.path.getsize(file),
checksum=md5(file),
family=json.get('family', None))
ssa = set_software_server(soft, g.server,
os.path.dirname(json['file']))
if not isinstance(ssa, SoftwareServerAssociation):
return ssa
db.session.add_all([soft, ssa])
db.session.commit()
return {'id': str(soft.id)}, 201
def test_execute_send_software_no_software(self):
at = ActionTemplate.query.get('00000000-0000-0000-000a-000000000001')
ro = NativeSoftwareSendOperation(code,
expected_stdout=at.expected_stdout,
expected_stderr=at.expected_stderr,
expected_rc=at.expected_rc)
with self.subTest("pass an invalid id"):
soft_id = '00000000-0000-0000-0000-000000000001'
cp = ro._execute(
dict(input=dict(software=soft_id, server=self.s1.id)),
timeout=None,
context=self.context)
self.assertFalse(cp.success)
self.assertEqual(f"software id '{soft_id}' not found", cp.stderr)
with self.subTest("pass an invalid name"):
cp = ro._execute(
dict(input=dict(software='software', server=self.s1.id)),
timeout=None,
context=self.context)
self.assertFalse(cp.success)
self.assertEqual(f"No software found for 'software'", cp.stderr)
soft = Software(name='test', version="1", filename='test.zip')
db.session.add(soft)
with self.subTest("pass an invalid version"):
cp = ro._execute(dict(
input=dict(software='test', version="2.1", server=self.s1.id)),
timeout=None,
context=self.context)
self.assertFalse(cp.success)
self.assertEqual(f"No software found for 'test' and version '2.1'",
cp.stderr)
def setUp(self):
"""Create and configure a new app instance for each test."""
# create the app with common test config
super().setUp()
self.source_path = '/software'
self.filename = 'filename.zip'
self.content = b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
self.size = len(self.content)
self.checksum = hashlib.md5(self.content).hexdigest()
self.dest_path = '/dest_repo'
self.soft = Software(name='test_software',
version=1,
filename=self.filename,
size=self.size,
checksum=self.checksum)
self.ssa = SoftwareServerAssociation(software=self.soft,
server=self.s1,
path=self.source_path)
self.node2 = Server('node2', port=5000)
db.session.add_all([self.soft, self.ssa, self.node2])
db.session.commit()
class TestSoftwareList(TestResourceBase):
def fill_database(self):
self.soft1 = Software(id='11111111-2222-3333-4444-555555550001',
name='Dimensigon',
version='0.0.1',
filename='Dimensigon_0.0.1.tar.gz')
self.soft2 = Software(id='11111111-2222-3333-4444-555555550002',
name='Dimensigon',
version='0.0.2',
filename='Dimensigon_0.0.2.tar.gz')
self.soft3 = Software(id='11111111-2222-3333-4444-555555550003',
name='python',
version='3.6.8',
filename='python_3.6.8.x64.tar.gz')
db.session.add_all([self.soft1, self.soft2, self.soft3])
def test_get(self):
resp = self.client.get(url_for('api_1_0.softwarelist'),
headers=self.auth.header)
self.assertListEqual([
self.soft1.to_json(no_delete=False),
self.soft2.to_json(no_delete=False),
self.soft3.to_json(no_delete=False)
], resp.get_json())
def test_get_with_filter(self):
resp = self.client.get(url_for('api_1_0.softwarelist',
**{'filter[name]': 'Dimensigon'}),
headers=self.auth.header)
self.assertListEqual([
self.soft1.to_json(no_delete=False),
self.soft2.to_json(no_delete=False)
], resp.get_json())
def test_get_with_filter2(self):
resp = self.client.get(url_for('api_1_0.softwarelist',
**{'filter[version]': '0.0.1,3.6.8'}),
headers=self.auth.header)
self.assertListEqual([
self.soft1.to_json(no_delete=False),
self.soft3.to_json(no_delete=False)
], resp.get_json())
def test_post(self):
size = os.path.getsize(__file__)
checksum = md5(__file__)
filename = os.path.basename(__file__)
data = dict(name="Dimensigon",
version="0.0.3",
family='middleware',
file=__file__)
resp = self.client.post(url_for('api_1_0.softwarelist'),
headers=self.auth.header,
json=data)
self.assertEqual(201, resp.status_code)
soft = Software.query.filter_by(name="Dimensigon",
version="0.0.3").one()
self.assertEqual(size, soft.size)
self.assertEqual(checksum, soft.checksum)
self.assertEqual(filename, soft.filename)
self.assertEqual(1, len(soft.ssas))
ssa = soft.ssas[0]
self.assertEqual(os.path.dirname(__file__), ssa.path)