def setUp(self):
super().setUp()
self.test_subdirectory = tempfile.mkdtemp(dir=flags.FLAGS.test_tmpdir)
reaction1 = reaction_pb2.Reaction()
dummy_input = reaction1.inputs['dummy_input']
dummy_component = dummy_input.components.add()
dummy_component.identifiers.add(type='CUSTOM')
dummy_component.identifiers[0].details = 'custom_identifier'
dummy_component.identifiers[0].value = 'custom_value'
dummy_component.is_limiting = True
dummy_component.mass.value = 1
dummy_component.mass.units = reaction_pb2.Mass.GRAM
reaction1.outcomes.add().conversion.value = 75
dataset1 = dataset_pb2.Dataset(reactions=[reaction1])
self.dataset1_filename = os.path.join(self.test_subdirectory,
'dataset1.pb')
with open(self.dataset1_filename, 'wb') as f:
f.write(dataset1.SerializeToString())
# reaction2 is empty.
reaction2 = reaction_pb2.Reaction()
dataset2 = dataset_pb2.Dataset(reactions=[reaction1, reaction2])
self.dataset2_filename = os.path.join(self.test_subdirectory,
'dataset2.pb')
with open(self.dataset2_filename, 'wb') as f:
f.write(dataset2.SerializeToString())
def setUp(self):
super().setUp()
# Suppress RDKit warnings to clean up the test output.
RDLogger.logger().setLevel(RDLogger.CRITICAL)
self.test_subdirectory = tempfile.mkdtemp(dir=flags.FLAGS.test_tmpdir)
reaction1 = reaction_pb2.Reaction()
dummy_input = reaction1.inputs['dummy_input']
dummy_component = dummy_input.components.add()
dummy_component.identifiers.add(type='CUSTOM')
dummy_component.identifiers[0].details = 'custom_identifier'
dummy_component.identifiers[0].value = 'custom_value'
dummy_component.is_limiting = reaction_pb2.Boolean.TRUE
dummy_component.mass.value = 1
dummy_component.mass.units = reaction_pb2.Mass.GRAM
reaction1.outcomes.add().conversion.value = 75
dataset1 = dataset_pb2.Dataset(reactions=[reaction1])
self.dataset1_filename = os.path.join(self.test_subdirectory,
'dataset1.pbtxt')
message_helpers.write_message(dataset1, self.dataset1_filename)
# reaction2 is empty.
reaction2 = reaction_pb2.Reaction()
dataset2 = dataset_pb2.Dataset(reactions=[reaction1, reaction2])
self.dataset2_filename = os.path.join(self.test_subdirectory,
'dataset2.pbtxt')
message_helpers.write_message(dataset2, self.dataset2_filename)
def test_with_no_updates(self):
message = reaction_pb2.Reaction()
message.provenance.record_created.time.value = '2020-05-08'
message.reaction_id = 'ord-c0bbd41f095a44a78b6221135961d809'
copied = reaction_pb2.Reaction()
copied.CopyFrom(message)
updates.update_reaction(copied)
self.assertEqual(copied, message)
def test_add_dataset_with_large_data(self):
reaction = reaction_pb2.Reaction()
ethylamine = reaction.inputs['ethylamine']
component = ethylamine.components.add()
component.identifiers.add(type='SMILES', value='CCN')
component.is_limiting = reaction_pb2.Boolean.TRUE
component.moles.value = 2
component.moles.units = reaction_pb2.Moles.MILLIMOLE
reaction.outcomes.add().conversion.value = 25
image = reaction.observations.add().image
image.bytes_value = b'test data value'
image.format = 'png'
dataset = dataset_pb2.Dataset(reactions=[reaction])
dataset_filename = os.path.join(self.test_subdirectory, 'test.pbtxt')
message_helpers.write_message(dataset, dataset_filename)
filenames = self._run_main(min_size=0.0)
self.assertLen(filenames, 2)
filenames.pop(filenames.index(self.dataset_filename))
dataset = message_helpers.load_message(filenames[0],
dataset_pb2.Dataset)
relative_path = (
'data/36/ord_data-'
'36443a1839bf1160087422b7468a93c7b97dac7eea423bfac189208a15823139'
'.png')
expected = ('https://github.com/Open-Reaction-Database/'
'ord-submissions-test/tree/' + relative_path)
self.assertEqual(dataset.reactions[0].observations[0].image.url,
expected)
with open(os.path.join(self.test_subdirectory, relative_path),
'rb') as f:
self.assertEqual(b'test data value', f.read())
def test_modify_dataset(self):
dataset = message_helpers.load_message(self.dataset_filename,
dataset_pb2.Dataset)
# Modify the existing reaction...
dataset.reactions[0].inputs['methylamine'].components[
0].moles.value = 2
# ...and add a new reaction.
reaction = reaction_pb2.Reaction()
ethylamine = reaction.inputs['ethylamine']
component = ethylamine.components.add()
component.identifiers.add(type='SMILES', value='CCN')
component.is_limiting = reaction_pb2.Boolean.TRUE
component.moles.value = 2
component.moles.units = reaction_pb2.Moles.MILLIMOLE
reaction.outcomes.add().conversion.value = 25
dataset.reactions.add().CopyFrom(reaction)
message_helpers.write_message(dataset, self.dataset_filename)
filenames = self._run_main()
self.assertCountEqual([self.dataset_filename], filenames)
# Check for preservation of dataset and record IDs.
updated_dataset = message_helpers.load_message(self.dataset_filename,
dataset_pb2.Dataset)
self.assertLen(updated_dataset.reactions, 2)
self.assertEqual(dataset.dataset_id, updated_dataset.dataset_id)
self.assertEqual(dataset.reactions[0].reaction_id,
updated_dataset.reactions[0].reaction_id)
self.assertNotEmpty(updated_dataset.reactions[1].reaction_id)
def test_add_dataset_with_existing_reaction_ids(self):
reaction = reaction_pb2.Reaction()
ethylamine = reaction.inputs['ethylamine']
component = ethylamine.components.add()
component.identifiers.add(type='SMILES', value='CCN')
component.is_limiting = reaction_pb2.Boolean.TRUE
component.moles.value = 2
component.moles.units = reaction_pb2.Moles.MILLIMOLE
reaction.outcomes.add().conversion.value = 25
reaction_id = 'ord-10aed8b5dffe41fab09f5b2cc9c58ad9'
reaction.reaction_id = reaction_id
reaction.provenance.record_created.time.value = '2020-01-01 11 am'
dataset = dataset_pb2.Dataset(reactions=[reaction])
dataset_filename = os.path.join(self.test_subdirectory, 'test.pbtxt')
message_helpers.write_message(dataset, dataset_filename)
filenames = self._run_main()
self.assertLen(filenames, 2)
self.assertFalse(os.path.exists(dataset_filename))
filenames.pop(filenames.index(self.dataset_filename))
self.assertLen(filenames, 1)
dataset = message_helpers.load_message(filenames[0],
dataset_pb2.Dataset)
# Check that existing record IDs for added datasets are not overridden.
self.assertEqual(dataset.reactions[0].reaction_id, reaction_id)
self.assertLen(dataset.reactions[0].provenance.record_modified, 0)
def setUp(self):
super().setUp()
self.test_subdirectory = tempfile.mkdtemp(dir=flags.FLAGS.test_tmpdir)
os.chdir(self.test_subdirectory)
subprocess.run(['git', 'init'], check=True)
subprocess.run(
['git', 'config', '--local', 'user.email', 'test@ord-schema'],
check=True)
subprocess.run(
['git', 'config', '--local', 'user.name', 'Test Runner'],
check=True)
# Add some initial data.
reaction = reaction_pb2.Reaction()
methylamine = reaction.inputs['methylamine']
component = methylamine.components.add()
component.identifiers.add(type='SMILES', value='CN')
component.is_limiting = reaction_pb2.Boolean.TRUE
component.moles.value = 1
component.moles.units = reaction_pb2.Moles.MILLIMOLE
reaction.outcomes.add().conversion.value = 75
reaction.provenance.record_created.time.value = '2020-01-01'
reaction.reaction_id = 'ord-10aed8b5dffe41fab09f5b2cc9c58ad9'
dataset_id = 'ord_dataset-64b14868c5cd46dd8e75560fd3589a6b'
dataset = dataset_pb2.Dataset(reactions=[reaction],
dataset_id=dataset_id)
# Make sure the initial dataset is valid.
validations.validate_message(dataset)
os.makedirs(os.path.join('data', '64'))
self.dataset_filename = os.path.join(self.test_subdirectory, 'data',
'64', f'{dataset_id}.pbtxt')
message_helpers.write_message(dataset, self.dataset_filename)
subprocess.run(['git', 'add', 'data'], check=True)
subprocess.run(['git', 'commit', '-m', 'Initial commit'], check=True)
def test_modify_dataset(self):
dataset = message_helpers.load_message(self.dataset_filename,
dataset_pb2.Dataset)
# Modify the existing reaction...
reaction1 = dataset.reactions[0]
reaction1.inputs['methylamine'].components[0].amount.moles.value = 2
# ...and add a new reaction.
reaction = reaction_pb2.Reaction()
ethylamine = reaction.inputs['ethylamine']
component = ethylamine.components.add()
component.identifiers.add(type='SMILES', value='CCN')
component.is_limiting = True
component.amount.moles.value = 2
component.amount.moles.units = reaction_pb2.Moles.MILLIMOLE
reaction.outcomes.add().conversion.value = 25
reaction.provenance.record_created.time.value = '2020-01-01'
reaction.provenance.record_created.person.username = '******'
reaction.provenance.record_created.person.email = '*****@*****.**'
reaction.reaction_id = 'test'
dataset.reactions.add().CopyFrom(reaction)
message_helpers.write_message(dataset, self.dataset_filename)
added, removed, changed, filenames = self._run()
self.assertEqual(added, {'test'})
self.assertEmpty(removed)
self.assertEqual(changed, {'ord-10aed8b5dffe41fab09f5b2cc9c58ad9'})
self.assertCountEqual([self.dataset_filename], filenames)
# Check for preservation of dataset and record IDs.
updated_dataset = message_helpers.load_message(self.dataset_filename,
dataset_pb2.Dataset)
self.assertLen(updated_dataset.reactions, 2)
self.assertEqual(dataset.dataset_id, updated_dataset.dataset_id)
self.assertEqual(dataset.reactions[0].reaction_id,
updated_dataset.reactions[0].reaction_id)
self.assertNotEmpty(updated_dataset.reactions[1].reaction_id)
def test_add_sharded_dataset(self):
reaction = reaction_pb2.Reaction()
ethylamine = reaction.inputs['ethylamine']
component = ethylamine.components.add()
component.identifiers.add(type='SMILES', value='CCN')
component.is_limiting = True
component.amount.moles.value = 2
component.amount.moles.units = reaction_pb2.Moles.MILLIMOLE
reaction.outcomes.add().conversion.value = 25
reaction.provenance.record_created.time.value = '2020-01-02'
reaction.provenance.record_created.person.username = '******'
reaction.provenance.record_created.person.email = '*****@*****.**'
reaction.reaction_id = 'test1'
dataset1 = dataset_pb2.Dataset(reactions=[reaction])
dataset1_filename = os.path.join(self.test_subdirectory, 'test1.pbtxt')
message_helpers.write_message(dataset1, dataset1_filename)
reaction.provenance.record_created.time.value = '2020-01-03'
reaction.provenance.record_created.person.username = '******'
reaction.provenance.record_created.person.email = '*****@*****.**'
reaction.reaction_id = 'test2'
dataset2 = dataset_pb2.Dataset(reactions=[reaction])
dataset2_filename = os.path.join(self.test_subdirectory, 'test2.pbtxt')
message_helpers.write_message(dataset2, dataset2_filename)
added, removed, changed, filenames = self._run()
self.assertEqual(added, {'test1', 'test2'})
self.assertEmpty(removed)
self.assertEmpty(changed)
self.assertLen(filenames, 2)
filenames.pop(filenames.index(self.dataset_filename))
self.assertLen(filenames, 1)
dataset = message_helpers.load_message(filenames[0],
dataset_pb2.Dataset)
self.assertLen(dataset.reactions, 2)
self.assertFalse(os.path.exists(dataset1_filename))
self.assertFalse(os.path.exists(dataset2_filename))
def test_add_dataset(self):
reaction = reaction_pb2.Reaction()
ethylamine = reaction.inputs['ethylamine']
component = ethylamine.components.add()
component.identifiers.add(type='SMILES', value='CCN')
component.is_limiting = True
component.amount.moles.value = 2
component.amount.moles.units = reaction_pb2.Moles.MILLIMOLE
reaction.outcomes.add().conversion.value = 25
reaction.provenance.record_created.time.value = '2020-01-01'
reaction.provenance.record_created.person.username = '******'
reaction.provenance.record_created.person.email = '*****@*****.**'
reaction.reaction_id = 'test'
dataset = dataset_pb2.Dataset(reactions=[reaction])
dataset_filename = os.path.join(self.test_subdirectory, 'test.pbtxt')
message_helpers.write_message(dataset, dataset_filename)
added, removed, changed, filenames = self._run()
self.assertEqual(added, {'test'})
self.assertEmpty(removed)
self.assertEmpty(changed)
self.assertLen(filenames, 2)
self.assertFalse(os.path.exists(dataset_filename))
# Check for assignment of dataset and reaction IDs.
filenames.pop(filenames.index(self.dataset_filename))
self.assertLen(filenames, 1)
dataset = message_helpers.load_message(filenames[0],
dataset_pb2.Dataset)
self.assertNotEmpty(dataset.dataset_id)
self.assertLen(dataset.reactions, 1)
self.assertNotEmpty(dataset.reactions[0].reaction_id)
# Check for binary output.
root, ext = os.path.splitext(filenames[0])
self.assertEqual(ext, '.pbtxt')
self.assertTrue(os.path.exists(root + '.pb'))
def test_keep_existing_reaction_id(self):
message = reaction_pb2.Reaction()
message.reaction_id = 'foo'
message.provenance.record_created.time.value = '11 am'
updates.update_reaction(message)
self.assertEqual(message.reaction_id, 'foo')
self.assertLen(message.provenance.record_modified, 0)
def test_compounds(self):
message = reaction_pb2.Reaction()
message.inputs['test'].components.add().identifiers.add(
type='NAME', value='aspirin')
self.assertLen(
message_helpers.find_submessages(message, reaction_pb2.Compound),
1)
def dataset_filename(tmp_path) -> str:
# Create a test database.
connection = connect(ord_interface.client.POSTGRES_DB)
connection.set_session(autocommit=True)
with connection.cursor() as cursor:
cursor.execute("CREATE DATABASE test;")
connection.close()
# Create a test dataset.
reaction = reaction_pb2.Reaction()
reaction.reaction_id = "test"
reaction.identifiers.add(value="reaction", type="REACTION_SMILES")
input1 = reaction.inputs["input1"]
input1.components.add().identifiers.add(value="input1", type="SMILES")
input2 = reaction.inputs["input2"]
input2.components.add().identifiers.add(value="input2a", type="SMILES")
input2.components.add().identifiers.add(value="input2b", type="SMILES")
outcome = reaction.outcomes.add()
product = outcome.products.add()
product.measurements.add(type="YIELD", percentage={"value": 2.5})
product.identifiers.add(value="product", type="SMILES")
reaction.provenance.doi = "10.0000/test.foo"
dataset = dataset_pb2.Dataset(dataset_id="test_dataset", reactions=[reaction])
dataset_filename = (tmp_path / "test.pb").as_posix()
message_helpers.write_message(dataset, dataset_filename)
yield dataset_filename
# Remove the test database.
connection = connect(ord_interface.client.POSTGRES_DB)
connection.set_session(autocommit=True)
with connection.cursor() as cursor:
cursor.execute("DROP DATABASE test;")
connection.close()
def test_resolver(self):
reaction = reaction_pb2.Reaction()
ethylamine = reaction.inputs['ethylamine']
component = ethylamine.components.add()
component.identifiers.add(type='NAME', value='ethylamine')
component.is_limiting = True
component.moles.value = 2
component.moles.units = reaction_pb2.Moles.MILLIMOLE
reaction.outcomes.add().conversion.value = 25
dataset = dataset_pb2.Dataset(reactions=[reaction])
dataset_filename = os.path.join(self.test_subdirectory, 'test.pbtxt')
message_helpers.write_message(dataset, dataset_filename)
filenames = self._run_main()
self.assertLen(filenames, 2)
self.assertFalse(os.path.exists(dataset_filename))
filenames.pop(filenames.index(self.dataset_filename))
self.assertLen(filenames, 1)
dataset = message_helpers.load_message(filenames[0],
dataset_pb2.Dataset)
self.assertLen(dataset.reactions, 1)
identifiers = (dataset.reactions[0].inputs['ethylamine'].components[0].
identifiers)
self.assertLen(identifiers, 3)
self.assertEqual(
identifiers[1],
reaction_pb2.CompoundIdentifier(
type='SMILES', value='CCN',
details='NAME resolved by PubChem'))
self.assertEqual(identifiers[2].type,
reaction_pb2.CompoundIdentifier.RDKIT_BINARY)
def test_modify_repeated_submessage(self):
"""See https://git.io/JfPf9."""
message = reaction_pb2.Reaction()
message.workup.add(type='ADDITION')
frozen = self._freeze(message)
with self.assertRaises(dataclasses.FrozenInstanceError):
frozen.workup[0].type = reaction_pb2.ReactionWorkup.TEMPERATURE
def test_valid_templating(self):
template_string = self.template_string.replace('value: "CCO"',
'value: "$my_smiles$"')
template_string = template_string.replace('value: 75',
'value: $conversion$')
df = pd.DataFrame.from_dict({
'$my_smiles$': ['CCO', 'CCCO', 'CCCCO'],
'$conversion$': [75, 50, 30],
})
dataset = templating.generate_dataset(template_string, df)
expected_reactions = []
for smiles, conversion in zip(['CCO', 'CCCO', 'CCCCO'], [75, 50, 30]):
reaction = reaction_pb2.Reaction()
reaction.CopyFrom(self.valid_reaction)
reaction.inputs['in'].components[0].identifiers[0].value = smiles
reaction.outcomes[0].conversion.value = conversion
expected_reactions.append(reaction)
expected_dataset = dataset_pb2.Dataset(reactions=expected_reactions)
self.assertEqual(dataset, expected_dataset)
# Test without "$" in column names
df = pd.DataFrame.from_dict({
'my_smiles': ['CCO', 'CCCO', 'CCCCO'],
'conversion': [75, 50, 30],
})
dataset = templating.generate_dataset(template_string, df)
self.assertEqual(dataset, expected_dataset)
def test_set_map_value(self):
frozen = self._freeze(reaction_pb2.Reaction())
with self.assertRaises(KeyError):
frozen.inputs['test'].addition_order = 1
with self.assertRaises(KeyError):
frozen.inputs['test'].CopyFrom(
reaction_pb2.ReactionInput(addition_order=1))
def test_set_submessage(self):
frozen = self._freeze(reaction_pb2.Reaction())
with self.assertRaises(AttributeError):
frozen.setup.automation_platform = 'test'
with self.assertRaises(AttributeError):
frozen.setup.CopyFrom(
reaction_pb2.ReactionSetup(automation_platform='test'))
def test_render_reaction(self):
reaction = reaction_pb2.Reaction()
component = reaction.inputs['test'].components.add()
component.identifiers.add(value='c1ccccc1', type='SMILES')
response = self.client.post('/render/reaction',
data=reaction.SerializeToString(),
follow_redirects=True)
self.assertEqual(response.status_code, 200)
def test_keep_existing_reaction_id(self):
message = reaction_pb2.Reaction()
message.reaction_id = 'ord-c0bbd41f095a44a78b6221135961d809'
message.provenance.record_created.time.value = '2020-01-01'
updates.update_reaction(message)
self.assertEqual(message.reaction_id,
'ord-c0bbd41f095a44a78b6221135961d809')
self.assertLen(message.provenance.record_modified, 0)
def test_render_reaction(client):
reaction = reaction_pb2.Reaction()
component = reaction.inputs["test"].components.add()
component.identifiers.add(value="c1ccccc1", type="SMILES")
response = client.post("/render/reaction",
data=reaction.SerializeToString(),
follow_redirects=True)
assert response.status_code == 200
def test_download_reaction(self):
reaction = self._get_dataset().reactions[0]
response = self.client.post('/reaction/download',
data=reaction.SerializeToString(),
follow_redirects=True)
self.assertEqual(response.status_code, 200)
downloaded_reaction = reaction_pb2.Reaction()
text_format.Parse(response.data, downloaded_reaction)
self.assertEqual(downloaded_reaction, reaction)
def download_reaction():
"""Returns a pbtxt file parsed from POST data as an attachment."""
reaction = reaction_pb2.Reaction()
reaction.ParseFromString(flask.request.get_data())
data = io.BytesIO(text_format.MessageToBytes(reaction))
return flask.send_file(data,
mimetype='application/protobuf',
as_attachment=True,
attachment_filename='reaction.pbtxt')
def test_download_reaction(client):
reaction = _get_dataset().reactions[0]
response = client.post("/reaction/download",
data=reaction.SerializeToString(),
follow_redirects=True)
assert response.status_code == 200
downloaded_reaction = reaction_pb2.Reaction()
text_format.Parse(response.data, downloaded_reaction)
assert downloaded_reaction == reaction
def new_reaction(file_name):
"""Adds a new Reaction to the given Dataset and view the Dataset."""
dataset = get_dataset(file_name)
if dataset is None:
flask.abort(404)
reaction = reaction_pb2.Reaction()
dataset.reactions.append(reaction)
put_dataset(file_name, dataset)
return flask.redirect('/dataset/%s' % file_name)
def test_reaction_recursive_noraise_on_error(self):
message = reaction_pb2.Reaction()
message.inputs['dummy_input'].components.add()
errors = self._run_validation(message, raise_on_error=False)
expected = [
'Compounds must have at least one identifier',
"Reaction input's components require an amount",
'Reactions should have at least 1 reaction outcome',
]
self.assertEqual(errors, expected)
def test_simple(self):
reaction = reaction_pb2.Reaction()
reaction.identifiers.add(value='C(C)Cl.Br>>C(C)Br.Cl',
type='REACTION_SMILES')
self.assertTrue(reaction.IsInitialized())
self.assertLen(reaction.identifiers, 1)
self.assertFalse(reaction.HasField('setup'))
with self.assertRaisesRegex(ValueError,
'Reaction has no field not_a_field'):
reaction.HasField('not_a_field')
def test_find_data_messages(self):
message = reaction_pb2.Reaction()
self.assertEmpty(
message_helpers.find_submessages(message, reaction_pb2.Data))
message = reaction_pb2.ReactionObservation()
message.image.value = 'not an image'
self.assertLen(
message_helpers.find_submessages(message, reaction_pb2.Data), 1)
message = reaction_pb2.ReactionSetup()
message.automation_code['test1'].value = 'test data 1'
message.automation_code['test2'].bytes_value = b'test data 2'
self.assertLen(
message_helpers.find_submessages(message, reaction_pb2.Data), 2)
message = reaction_pb2.Reaction()
message.observations.add().image.value = 'not an image'
message.setup.automation_code['test1'].value = 'test data 1'
message.setup.automation_code['test2'].bytes_value = b'test data 2'
self.assertLen(
message_helpers.find_submessages(message, reaction_pb2.Data), 3)
def test_with_resolve_names(self):
reaction = reaction_pb2.Reaction()
component = reaction.inputs['ethylamine'].components.add()
component.identifiers.add(type='NAME', value='ethylamine')
updates.update_reaction(reaction)
self.assertLen(component.identifiers, 2)
self.assertEqual(component.identifiers[1].value, 'CCN')
self.assertEqual(component.identifiers[1].type,
reaction_pb2.CompoundIdentifier.IdentifierType.SMILES)
self.assertRegex(component.identifiers[1].details, 'NAME resolved')
def test_access_repeated_submessage(self):
message = reaction_pb2.Reaction()
message.observations.add(comment='test')
frozen = self._freeze(message)
self.assertLen(frozen.observations, 1)
self.assertEqual(frozen.observations[0].comment, 'test')
with self.assertRaises(IndexError):
_ = frozen.observations[1]
self.assertEmpty(frozen.outcomes)
with self.assertRaises(IndexError):
_ = frozen.outcomes[0]