def setUp(self):
# clean out any old environments
for key in os.environ.keys():
if key.startswith('HADRON'):
del os.environ[key]
os.environ['HADRON_PM_PATH'] = os.path.join('work', 'config')
os.environ['HADRON_DEFAULT_PATH'] = os.path.join('work', 'data')
try:
os.makedirs(os.environ['HADRON_PM_PATH'])
os.makedirs(os.environ['HADRON_DEFAULT_PATH'])
except:
raise IOError('Unable to create directories')
PropertyManager._remove_all()
builder = SyntheticBuilder.from_env('task1', has_contract=False)
builder.set_persist()
builder.pm_persist()
tr = Transition.from_env('task2', has_contract=False)
tr.set_source_uri(builder.get_persist_contract().raw_uri)
tr.set_persist()
tr.pm_persist()
wr = Wrangle.from_env('task3', has_contract=False)
wr.set_source_uri(tr.get_persist_contract().raw_uri)
wr.set_persist()
wr.pm_persist()
def setUp(self):
# clean out any old environments
for key in os.environ.keys():
if key.startswith('HADRON'):
del os.environ[key]
# Local Domain Contract
os.environ['HADRON_PM_PATH'] = os.path.join('working', 'contracts')
os.environ['HADRON_PM_TYPE'] = 'json'
# Local Connectivity
os.environ['HADRON_DEFAULT_PATH'] = Path('working/data').as_posix()
# Specialist Component
try:
os.makedirs(os.environ['HADRON_PM_PATH'])
except:
pass
try:
os.makedirs(os.environ['HADRON_DEFAULT_PATH'])
except:
pass
PropertyManager._remove_all()
tr = Transition.from_env('task1', has_contract=False)
tr.set_source_uri(
"https://raw.githubusercontent.com/mwaskom/seaborn-data/master/titanic.csv"
)
tr.set_persist()
wr = Wrangle.from_env('task2', has_contract=False)
wr.set_source_uri(tr.get_persist_contract().raw_uri)
wr.set_persist()
controller = Controller.from_env(has_contract=False)
controller.intent_model.transition(canonical=pd.DataFrame(),
task_name='task1',
intent_level='transition')
controller.intent_model.wrangle(canonical=pd.DataFrame(),
task_name='task2',
intent_level='wrangle')
def setUp(self):
# clean out any old environments
for key in os.environ.keys():
if key.startswith('HADRON'):
del os.environ[key]
os.environ['HADRON_PM_PATH'] = os.path.join('work', 'config')
os.environ['HADRON_DEFAULT_PATH'] = os.path.join('work', 'data')
try:
os.makedirs(os.environ['HADRON_PM_PATH'])
os.makedirs(os.environ['HADRON_DEFAULT_PATH'])
except:
pass
PropertyManager._remove_all()
tr = Transition.from_env('task1', has_contract=False)
tr.set_source_uri(
"https://raw.githubusercontent.com/mwaskom/seaborn-data/master/titanic.csv"
)
tr.set_persist()
wr = Wrangle.from_env('task2', has_contract=False)
wr.set_source_uri(tr.get_persist_contract().raw_uri)
wr.set_persist()
controller = Controller.from_env(has_contract=False)
controller.intent_model.transition(canonical=pd.DataFrame(),
task_name='task1',
intent_level='task1_tr')
controller.intent_model.wrangle(canonical=pd.DataFrame(),
task_name='task2',
intent_level='task2_wr')
def setUp(self):
os.environ['HADRON_PM_PATH'] = os.path.join('work', 'config')
os.environ['HADRON_DEFAULT_PATH'] = os.path.join('work', 'data')
try:
os.makedirs(os.environ['HADRON_PM_PATH'])
os.makedirs(os.environ['HADRON_DEFAULT_PATH'])
except:
pass
PropertyManager._remove_all()
def setUp(self):
# clean out any old environments
for key in os.environ.keys():
if key.startswith('HADRON'):
del os.environ[key]
# Local Domain Contract
os.environ['HADRON_PM_PATH'] = os.path.join('working', 'contracts')
os.environ['HADRON_PM_TYPE'] = 'json'
# Local Connectivity
os.environ['HADRON_DEFAULT_PATH'] = Path('working/data').as_posix()
# Specialist Component
try:
os.makedirs(os.environ['HADRON_PM_PATH'])
except:
pass
try:
os.makedirs(os.environ['HADRON_DEFAULT_PATH'])
except:
pass
PropertyManager._remove_all()
builder = SyntheticBuilder.from_env('builder', has_contract=False)
builder.set_persist()
sample_size = 10
df = pd.DataFrame()
df['cat'] = builder.tools.get_category(selection=['a', 'b', 'c', 'd'],
size=sample_size,
column_name='cat')
df['norm'] = builder.tools.get_dist_normal(mean=4,
std=1,
size=sample_size,
column_name='norm')
df['pois'] = builder.tools.get_dist_poisson(interval=7,
size=sample_size,
column_name='pois')
df['norm_std'] = builder.tools.correlate_numbers(
df, header='norm', standardize=True, column_name='norm_std')
df['jitter1'] = builder.tools.correlate_numbers(df,
header='pois',
jitter=0.1,
column_name='jitter1')
df['jitter2'] = builder.tools.correlate_numbers(df,
header='pois',
jitter=0.8,
column_name='jitter2')
df['jitter3'] = builder.tools.correlate_numbers(df,
header='pois',
jitter=1.5,
column_name='jitter3')
df['jitter4'] = builder.tools.correlate_numbers(df,
header='pois',
jitter=2,
column_name='jitter4')
df['jitter5'] = builder.tools.correlate_numbers(df,
header='pois',
jitter=3,
column_name='jitter5')
builder.run_component_pipeline()
def setUp(self):
# set environment variables
os.environ['HADRON_PM_PATH'] = os.path.join(os.environ['PWD'], 'work',
'config')
os.environ['HADRON_DEFAULT_PATH'] = os.path.join(
os.environ['PWD'], 'work', 'data', '0_raw')
try:
shutil.copytree('../data', os.path.join(os.environ['PWD'], 'work'))
except:
pass
PropertyManager._remove_all()
def setUp(self):
os.environ['HADRON_PM_PATH'] = os.path.join('work', 'config')
os.environ['HADRON_DEFAULT_PATH'] = os.path.join('work', 'data')
try:
os.makedirs(os.environ['HADRON_PM_PATH'])
os.makedirs(os.environ['HADRON_DEFAULT_PATH'])
except:
pass
PropertyManager._remove_all()
self.tools = SyntheticBuilder.scratch_pad()
self.clean = Transition.scratch_pad()
def setUp(self):
os.environ['HADRON_PM_PATH'] = os.path.join('work', 'config')
os.environ['HADRON_DEFAULT_PATH'] = os.path.join('work', 'data')
try:
os.makedirs(os.environ['HADRON_PM_PATH'])
os.makedirs(os.environ['HADRON_DEFAULT_PATH'])
except:
pass
PropertyManager._remove_all()
self.builder: SyntheticBuilder = SyntheticBuilder.from_env(
'sample', has_contract=False)
self.builder.setup_bootstrap()
self.tools: SyntheticIntentModel = self.builder.tools
def setUp(self):
# set environment variables
os.environ['HADRON_PM_PATH'] = os.path.join(os.environ['PWD'], 'work', 'config')
os.environ['HADRON_DEFAULT_SOURCE_PATH'] = os.path.join(os.environ['HOME'], 'code', 'projects', 'prod', 'data', 'raw')
PropertyManager._remove_all()
try:
shutil.rmtree('work')
except:
pass
try:
shutil.copytree('../data', os.path.join(os.environ['PWD'], 'work'))
except:
pass
def setUp(self):
# clean out any old environments
for key in os.environ.keys():
if key.startswith('HADRON'):
del os.environ[key]
os.environ['HADRON_PM_PATH'] = os.path.join('work', 'config')
os.environ['HADRON_DEFAULT_PATH'] = os.path.join('work', 'data')
try:
os.makedirs(os.environ['HADRON_PM_PATH'])
os.makedirs(os.environ['HADRON_DEFAULT_PATH'])
except:
pass
PropertyManager._remove_all()
def setUp(self):
# clean out any old environments
for key in os.environ.keys():
if key.startswith("HADRON"):
del os.environ[key]
os.environ["HADRON_PM_PATH"] = os.path.join("work", "config")
os.environ["HADRON_DEFAULT_PATH"] = os.path.join("work", "data")
try:
os.makedirs(os.environ["HADRON_PM_PATH"])
os.makedirs(os.environ["HADRON_DEFAULT_PATH"])
except:
pass
PropertyManager._remove_all()
def setUp(self):
# clean out any old environments
for key in os.environ.keys():
if key.startswith('HADRON'):
del os.environ[key]
os.environ['HADRON_PM_REPO'] = "https://raw.githubusercontent.com/project-hadron/hadron-asset-bank/master/contracts/healthcare/factory/members/"
os.environ['HADRON_PM_PATH'] = os.path.join('work', 'config')
os.environ['HADRON_DEFAULT_PATH'] = os.path.join('work', 'data')
try:
os.makedirs(os.environ['HADRON_PM_PATH'])
os.makedirs(os.environ['HADRON_DEFAULT_PATH'])
except:
pass
PropertyManager._remove_all()
def setUp(self):
os.environ['HADRON_PM_PATH'] = os.path.join('work', 'config')
os.environ['HADRON_DEFAULT_PATH'] = os.path.join('work', 'data')
try:
os.makedirs(os.environ['HADRON_PM_PATH'])
os.makedirs(os.environ['HADRON_DEFAULT_PATH'])
except:
pass
PropertyManager._remove_all()
self.tools = SyntheticBuilder.from_env('tester',
default_save=False,
default_save_intent=False,
has_contract=False).intent_model
self.fc: FeatureCatalog = FeatureCatalog.from_env('tester',
default_save=False,
has_contract=False)
def test_synthetic_with_no_source(self):
shutil.rmtree('work/config')
os.makedirs(os.environ['HADRON_PM_PATH'])
PropertyManager._remove_all()
builder = SyntheticBuilder.from_env('task3', has_contract=False)
tools: SyntheticIntentModel = builder.tools
builder.set_persist()
df = pd.DataFrame(index=range(10))
tools.model_noise(df, num_columns=5, column_name='noise')
controller = Controller.from_env(has_contract=False)
controller.intent_model.synthetic_builder(df, task_name='task3')
controller.run_controller()
self.assertIn(
builder.CONNECTOR_PERSIST,
builder.report_connectors(
stylise=False)['connector_name'].to_list())
self.assertNotIn(
builder.CONNECTOR_SOURCE,
builder.report_connectors(
stylise=False)['connector_name'].to_list())
def setUp(self):
# clean out any old environments
for key in os.environ.keys():
if key.startswith('HADRON'):
del os.environ[key]
# Local Domain Contract
os.environ['HADRON_PM_PATH'] = os.path.join('working', 'contracts')
os.environ['HADRON_PM_TYPE'] = 'json'
# Local Connectivity
os.environ['HADRON_DEFAULT_PATH'] = Path('working/data').as_posix()
# Specialist Component
try:
os.makedirs(os.environ['HADRON_PM_PATH'])
except:
pass
try:
os.makedirs(os.environ['HADRON_DEFAULT_PATH'])
except:
pass
PropertyManager._remove_all()
builder = SyntheticBuilder.from_env('tester', has_contract=False)
builder.set_persist()
tools: SyntheticIntentModel = builder.tools