def test_parent_application__streams_instead_of_files(
self, generated_rows):
yaml_file = StringIO("""- object: Foo""")
generate_cci_mapping_file = StringIO()
output_file = StringIO()
output_files = [StringIO(), StringIO()]
continuation_file = StringIO("""
id_manager:
last_used_ids:
Foo: 6
intertable_dependencies: []
nicknames_and_tables:
Foo: Foo
persistent_nicknames: {}
persistent_objects_by_table: {}
today: 2021-04-07""")
generate_continuation_file = StringIO()
decls = """[{"sf_object": Opportunity, "api": bulk}]"""
load_declarations = [StringIO(decls), StringIO(decls)]
generate_data(
yaml_file=yaml_file,
generate_cci_mapping_file=generate_cci_mapping_file,
output_file=output_file,
output_files=output_files,
output_format="txt",
continuation_file=continuation_file,
generate_continuation_file=generate_continuation_file,
load_declarations=load_declarations,
)
assert generated_rows.table_values("Foo", 0)["id"] == 7
def test_plugin_context_vars(self, write_row):
yaml = """
- plugin: tests.test_custom_plugins_and_providers.PluginThatNeedsState
- object: OBJ
fields:
name: OBJ1
path: ${{PluginThatNeedsState.object_path()}}
child:
- object: OBJ
fields:
name: OBJ2
path: ${{PluginThatNeedsState.object_path()}}
- object: OBJ
fields:
name: OBJ3
path: ${{PluginThatNeedsState.object_path()}}
child:
- object: OBJ
fields:
name: OBJ4
path: ${{PluginThatNeedsState.object_path()}}
"""
generate_data(StringIO(yaml))
assert row_values(write_row, 0, "path") == "ROOT.OBJ1.OBJ2"
assert row_values(write_row, 1, "path") == "ROOT.OBJ1"
assert row_values(write_row, 2, "path") == "ROOT.OBJ3.OBJ4"
assert row_values(write_row, 3, "path") == "ROOT.OBJ3"
def snowfakery_output_for(name, primary_example, secondary_example):
"""Generate the Snowfakery output for some YAML
Attempt to generate a two-part example, but fall back to single
or nothing if worse comes to worst."""
output = None
exception = None
for yaml_data in [primary_example, secondary_example]:
with StringIO() as s:
try:
generate_data(StringIO(yaml_data),
output_file=s,
output_format="txt")
output = s.getvalue()
exception = None
except Exception as e:
exception = e
if exception and name.lower() not in IGNORE_ERRORS:
print(f"Cannot generate sample for {name}: {str(exception)[0:80]}")
IGNORE_ERRORS.add(name.lower())
if output:
return yaml_data, output
def test_soql_dataset_in_order(self, sf, org_config, generated_rows):
filename = Path(
__file__).parent.parent / "examples/soql_dataset.recipe.yml"
generate_data(filename, plugin_options={"org_name": org_config.name})
assert len(generated_rows.mock_calls) == 10
for mock_call in generated_rows.mock_calls:
row_type, row_data = mock_call[1]
assert row_type == "Contact"
assert row_data["OwnerId"].startswith("005")
assert row_data["LastName"]
first_user_lastname = sf.query(
"select LastName from User")["records"][0]["LastName"]
assert generated_rows.mock_calls[0][1][1][
"LastName"] == first_user_lastname
# TODO: anon apex is better, so IDs don't end up in the VCR logs.
sf.restful(
"tooling/executeAnonymous",
{
"anonymousBody":
"delete [SELECT Id FROM Contact WHERE Name LIKE 'TestUser%'];"
},
)
def test_content_version(self, generated_rows):
content_version = "examples/salesforce/ContentVersion.recipe.yml"
generate_data(content_version)
b64data = generated_rows.table_values("ContentVersion",
0)["VersionData"]
rawdata = b64decode(b64data)
assert rawdata.startswith(b"%PDF-1.3")
assert b"Helvetica" in rawdata
def snowfakery(recipe, num_records, tablename, outputfile):
assert Path(recipe).exists(), recipe
output = f"sqlite:///{outputfile}"
print(output)
generate_data(
recipe,
target_number=(num_records, tablename),
dburl=output,
)
def test_constants(self, write_row_mock):
yaml = """
- plugin: snowfakery.standard_plugins.Math
- object: OBJ
fields:
pi: ${{Math.pi}}
"""
generate_data(StringIO(yaml))
assert row_values(write_row_mock, 0, "pi") == math.pi
def test_plugin_does_not_close(self):
yaml = """
- plugin: tests.test_custom_plugins_and_providers.DoesNotClosePlugin
- object: B
fields:
foo:
DoesNotClosePlugin.foo:
"""
with pytest.warns(UserWarning, match="close"):
generate_data(StringIO(yaml))
def test_math_deconstructed(self, write_row_mock):
yaml = """
- plugin: snowfakery.standard_plugins.Math
- object: OBJ
fields:
twelve:
Math.sqrt: ${{Math.min(144, 169)}}
"""
generate_data(StringIO(yaml))
assert row_values(write_row_mock, 0, "twelve") == 12
def test_missing_plugin(self):
yaml = """
- plugin: xyzzy.test_custom_plugins_and_providers.TestCustomPlugin
- object: OBJ
fields:
service_name: saascrmlightning
"""
with pytest.raises(DataGenImportError) as e:
generate_data(StringIO(yaml))
assert "xyzzy" in str(e.value)
def test_null_attributes(self):
yaml = """
- plugin: tests.test_custom_plugins_and_providers.WrongTypePlugin # 2
- object: B #3
fields: #4
foo: #5
WrongTypePlugin.junk: 5 #6
"""
with pytest.raises(DataGenError) as e:
generate_data(StringIO(yaml))
assert 6 > e.value.line_num >= 3
def test_pretend_cci_not_available(self):
filename = (Path(__file__).parent.parent /
"examples/salesforce_soql_example.recipe.yml")
with unittest.mock.patch(
"snowfakery.standard_plugins.Salesforce.SalesforceConnectionMixin._get_CliRuntime"
) as conn:
conn.side_effect = ImportError(
"CumulusCI Runtime cannot be loaded")
with pytest.raises(Exception,
match="CumulusCI Runtime cannot be loaded"):
generate_data(filename, plugin_options={"org_name": "None"})
def test_bogus_plugin(self):
yaml = """
- plugin: tests.test_custom_plugins_and_providers.TestCustomPlugin
- object: OBJ
fields:
service_name: saascrmlightning
"""
with pytest.raises(DataGenTypeError) as e:
generate_data(StringIO(yaml))
assert "TestCustomPlugin" in str(e.value)
assert ":2" in str(e.value)
def test_custom_faker_provider(self, write_row_mock):
yaml = """
- plugin: faker_microservice.Provider
- object: OBJ
fields:
service_name:
fake:
microservice
"""
generate_data(StringIO(yaml))
assert row_values(write_row_mock, 0, "service_name")
def test_simple_plugin(self, write_row_mock):
yaml = """
- plugin: tests.test_custom_plugins_and_providers.SimpleTestPlugin
- object: OBJ
fields:
four:
SimpleTestPlugin.double: 2
six: ${{SimpleTestPlugin.double(3)}}
"""
generate_data(StringIO(yaml))
assert row_values(write_row_mock, 0, "four") == 4
assert row_values(write_row_mock, 0, "six") == 6
def test_profile_id(self, generated_rows, org_config):
yaml = """
- plugin: snowfakery.standard_plugins.Salesforce
- object: foo
fields:
ProfileId:
Salesforce.ProfileId: Identity User
"""
generate_data(StringIO(yaml),
plugin_options={"org_name": org_config.name})
assert generated_rows.table_values("foo", 0,
"ProfileId").startswith("00e")
def test_find_records_returns_nothing(self, org_config):
yaml = """
- plugin: snowfakery.standard_plugins.Salesforce.SalesforceQuery
- object: Contact
fields:
FirstName: Suzy
LastName: Salesforce
AccountId:
SalesforceQuery.find_record: Contract
"""
with pytest.raises(DataGenError, match="No records returned"):
generate_data(StringIO(yaml),
plugin_options={"org_name": org_config.name})
def test_dataset_no_from(self, org_config, sf, generated_rows):
yaml = """
- plugin: snowfakery.standard_plugins.Salesforce.SOQLDataset
- object: Contact
count: 10
fields:
__users_from_salesforce:
SOQLDataset.shuffle:
fields: Junk3
"""
with pytest.raises(DataGenError, match="SOQLDataset needs a 'from'"):
generate_data(StringIO(yaml),
plugin_options={"org_name": org_config.name})
def test_parent_application__echo(self):
called = False
class MyEmbedder(SnowfakeryApplication):
def echo(self, *args, **kwargs):
nonlocal called
called = True
meth = "snowfakery.output_streams.DebugOutputStream.close"
with mock.patch(meth) as close:
close.side_effect = AssertionError
generate_data(yaml_file="examples/company.yml",
parent_application=MyEmbedder())
assert called
def test_math(self, write_row_mock):
yaml = """
- plugin: snowfakery.standard_plugins.Math
- object: OBJ
fields:
sqrt: ${{Math.sqrt(144)}}
max: ${{Math.max(144, 200, 100)}}
eleven: ${{Math.round(10.7)}}
min: ${{Math.min(144, 200, 100)}}
"""
generate_data(StringIO(yaml))
assert row_values(write_row_mock, 0, "sqrt") == 12
assert row_values(write_row_mock, 0, "max") == 200
assert row_values(write_row_mock, 0, "eleven") == 11
assert row_values(write_row_mock, 0, "min") == 100
def test_stringification(self, write_row):
yaml = """
- plugin: tests.test_custom_plugins_and_providers.EvalPlugin
- object: OBJ
fields:
some_value:
- EvalPlugin.add:
- 1
- EvalPlugin.sub:
- 5
- 3
"""
with StringIO() as s:
generate_data(StringIO(yaml), output_file=s, output_format="json")
assert eval(s.getvalue())[0]["some_value"] == 3
def test_find_records_returns_multiple(self, org_config, sf,
generated_rows):
yaml = """
- plugin: snowfakery.standard_plugins.Salesforce.SalesforceQuery
- object: Contact
fields:
FirstName: Suzy
LastName: Salesforce
AccountId:
SalesforceQuery.find_record: User
"""
generate_data(StringIO(yaml),
plugin_options={"org_name": org_config.name})
first_user_id = sf.query("select Id from User")["records"][0]["Id"]
assert generated_rows.mock_calls[0][1][1]["AccountId"] == first_user_id
def test_arguments(self):
with TemporaryDirectory() as t:
outfile = Path(t) / "foo.txt"
continuation = Path(t) / "out.yml"
generate_data(
yaml_file="examples/company.yml",
user_options={"A": "B"},
target_number=(20, "Employee"),
debug_internals=True,
output_format="json",
output_file=outfile,
generate_continuation_file=continuation,
)
assert outfile.exists()
assert continuation.exists()
with continuation.open() as f:
assert yaml.safe_load(f)
def test_lazy_with_context(self, write_row):
yaml = """
- plugin: tests.test_custom_plugins_and_providers.DoubleVisionPlugin
- plugin: tests.test_custom_plugins_and_providers.PluginThatNeedsState
- object: OBJ
fields:
some_value:
- DoubleVisionPlugin.do_it_twice:
- abc
some_value_2:
- DoubleVisionPlugin.do_it_twice:
- ${{PluginThatNeedsState.count()}}
"""
generate_data(StringIO(yaml))
assert row_values(write_row, 0, "some_value") == "abc : abc"
assert row_values(write_row, 0, "some_value_2") == "1 : 2"
def test_arguments(self):
with TemporaryDirectory() as t:
outfile = Path(t) / "foo.txt"
continuation = Path(t) / "out.yml"
generate_data(
yaml_file="tests/BDI_generator.yml",
user_options={"num_accounts": "15"},
target_number=(20, "Account"),
debug_internals=True,
output_format="json",
output_file=outfile,
generate_continuation_file=continuation,
)
assert outfile.exists()
assert continuation.exists()
with continuation.open() as f:
assert yaml.safe_load(f)
def test_parent_application__early_finish(self, generated_rows):
class MyEmbedder(SnowfakeryApplication):
count = 0
def check_if_finished(self, idmanager):
assert isinstance(idmanager, IdManager)
self.__class__.count += 1
assert self.__class__.count < 100, "Runaway recipe!"
return idmanager["Employee"] >= 10
meth = "snowfakery.output_streams.DebugOutputStream.close"
with mock.patch(meth) as close:
close.side_effect = AssertionError
generate_data(yaml_file="examples/company.yml",
parent_application=MyEmbedder())
# called 5 times, after generating 2 employees each
assert MyEmbedder.count == 5
def doit(recipe_data, *args, **kwargs):
db = tmpdir / "testdb.db"
dburl = f"sqlite:///{db}"
recipe = tmpdir / "recipe.yml"
mapping_file = tmpdir / "mapping.yml"
recipe.write_text(recipe_data)
generate_data(
recipe,
*args,
generate_cci_mapping_file=mapping_file,
dburl=dburl,
should_create_cci_record_type_tables=True,
**kwargs,
)
mapping = yaml.safe_load(mapping_file.read_text())
e = create_engine(dburl)
with e.connect() as connection:
yield mapping, connection
def test_soql_dataset_bulk(self, sf, org_config, generated_rows):
filename = (Path(__file__).parent.parent /
"examples/soql_dataset_where.recipe.yml")
# pretend there are 5000 records in org
pretend_5000 = patch(
"simple_salesforce.api.Salesforce.restful",
lambda *args, **kwargs:
{"sObjects": [{
"name": "User",
"count": 5000
}]},
)
csv_data = """"Id","FirstName","LastName"
"0051F00000nc59NQAQ","Automated","Process" """
@contextmanager
def download_file(*args, **kwargs):
yield StringIO(csv_data)
do_not_really_download = patch(
"cumulusci.tasks.bulkdata.step.download_file",
download_file,
)
with pretend_5000, do_not_really_download:
generate_data(filename,
plugin_options={"org_name": org_config.name})
assert len(generated_rows.mock_calls) == 10
for mock_call in generated_rows.mock_calls:
row_type, row_data = mock_call[1]
assert row_type == "Contact"
assert row_data["OwnerId"].startswith("005")
assert row_data["FirstName"].startswith("A")
# anon apex is better, so IDs don't end up in the VCR logs.
sf.restful(
"tooling/executeAnonymous",
{
"anonymousBody":
"delete [SELECT Id FROM Contact WHERE Name LIKE 'TestUser%'];"
},
)
def test_continuation_as_open_file(self):
with TemporaryDirectory() as t:
outfile = Path(t) / "foo.json"
continuation = Path(t) / "cont.yml"
mapping_file = Path(t) / "mapping.yml"
with open(continuation, "w") as cont, open(mapping_file, "w") as mapf:
generate_data(
yaml_file="examples/company.yml",
target_number=(20, "Employee"),
debug_internals=False,
output_file=outfile,
generate_continuation_file=cont,
generate_cci_mapping_file=mapf,
)
assert outfile.exists()
assert continuation.exists()
with continuation.open() as f:
assert yaml.safe_load(f)
assert mapping_file.exists()
with mapping_file.open() as f:
assert yaml.safe_load(f)
def test_soql_dataset_where(self, sf, org_config, generated_rows):
filename = (Path(__file__).parent.parent /
"examples/soql_dataset_where.recipe.yml")
generate_data(filename, plugin_options={"org_name": org_config.name})
assert len(generated_rows.mock_calls) == 10
for mock_call in generated_rows.mock_calls:
row_type, row_data = mock_call[1]
assert row_type == "Contact"
assert row_data["OwnerId"].startswith("005")
assert row_data["FirstName"].startswith("A")
# TODO: anon apex is better, so IDs don't end up in the VCR logs.
sf.restful(
"tooling/executeAnonymous",
{
"anonymousBody":
"delete [SELECT Id FROM Contact WHERE Name LIKE 'TestUser%'];"
},
)