def setUp(self): # clean out any old environments for key in os.environ.keys(): if key.startswith('HADRON'): del os.environ[key] # Local Domain Contract os.environ['HADRON_PM_PATH'] = os.path.join('working', 'contracts') os.environ['HADRON_PM_TYPE'] = 'json' # Local Connectivity os.environ['HADRON_DEFAULT_PATH'] = Path('working/data').as_posix() # Specialist Component try: os.makedirs(os.environ['HADRON_PM_PATH']) except: pass try: os.makedirs(os.environ['HADRON_DEFAULT_PATH']) except: pass PropertyManager._remove_all() tr = Transition.from_env('task1', has_contract=False) tr.set_source_uri( "https://raw.githubusercontent.com/mwaskom/seaborn-data/master/titanic.csv" ) tr.set_persist() wr = Wrangle.from_env('task2', has_contract=False) wr.set_source_uri(tr.get_persist_contract().raw_uri) wr.set_persist() controller = Controller.from_env(has_contract=False) controller.intent_model.transition(canonical=pd.DataFrame(), task_name='task1', intent_level='transition') controller.intent_model.wrangle(canonical=pd.DataFrame(), task_name='task2', intent_level='wrangle')
def setUp(self): # clean out any old environments for key in os.environ.keys(): if key.startswith('HADRON'): del os.environ[key] os.environ['HADRON_PM_PATH'] = os.path.join('work', 'config') os.environ['HADRON_DEFAULT_PATH'] = os.path.join('work', 'data') try: os.makedirs(os.environ['HADRON_PM_PATH']) os.makedirs(os.environ['HADRON_DEFAULT_PATH']) except: raise IOError('Unable to create directories') PropertyManager._remove_all() builder = SyntheticBuilder.from_env('task1', has_contract=False) builder.set_persist() builder.pm_persist() tr = Transition.from_env('task2', has_contract=False) tr.set_source_uri(builder.get_persist_contract().raw_uri) tr.set_persist() tr.pm_persist() wr = Wrangle.from_env('task3', has_contract=False) wr.set_source_uri(tr.get_persist_contract().raw_uri) wr.set_persist() wr.pm_persist()
def setUp(self): # clean out any old environments for key in os.environ.keys(): if key.startswith('HADRON'): del os.environ[key] os.environ['HADRON_PM_PATH'] = os.path.join('work', 'config') os.environ['HADRON_DEFAULT_PATH'] = os.path.join('work', 'data') try: os.makedirs(os.environ['HADRON_PM_PATH']) os.makedirs(os.environ['HADRON_DEFAULT_PATH']) except: pass PropertyManager._remove_all() tr = Transition.from_env('task1', has_contract=False) tr.set_source_uri( "https://raw.githubusercontent.com/mwaskom/seaborn-data/master/titanic.csv" ) tr.set_persist() wr = Wrangle.from_env('task2', has_contract=False) wr.set_source_uri(tr.get_persist_contract().raw_uri) wr.set_persist() controller = Controller.from_env(has_contract=False) controller.intent_model.transition(canonical=pd.DataFrame(), task_name='task1', intent_level='task1_tr') controller.intent_model.wrangle(canonical=pd.DataFrame(), task_name='task2', intent_level='task2_wr')
def tearDown(self): try: shutil.rmtree('work') except: pass props = PropertyManager().get_all() for key in props.keys(): PropertyManager().remove(key)
def setUp(self): os.environ['HADRON_PM_PATH'] = os.path.join('work', 'config') os.environ['HADRON_DEFAULT_PATH'] = os.path.join('work', 'data') try: os.makedirs(os.environ['HADRON_PM_PATH']) os.makedirs(os.environ['HADRON_DEFAULT_PATH']) except: pass PropertyManager._remove_all()
def setUp(self): # clean out any old environments for key in os.environ.keys(): if key.startswith('HADRON'): del os.environ[key] # Local Domain Contract os.environ['HADRON_PM_PATH'] = os.path.join('working', 'contracts') os.environ['HADRON_PM_TYPE'] = 'json' # Local Connectivity os.environ['HADRON_DEFAULT_PATH'] = Path('working/data').as_posix() # Specialist Component try: os.makedirs(os.environ['HADRON_PM_PATH']) except: pass try: os.makedirs(os.environ['HADRON_DEFAULT_PATH']) except: pass PropertyManager._remove_all() builder = SyntheticBuilder.from_env('builder', has_contract=False) builder.set_persist() sample_size = 10 df = pd.DataFrame() df['cat'] = builder.tools.get_category(selection=['a', 'b', 'c', 'd'], size=sample_size, column_name='cat') df['norm'] = builder.tools.get_dist_normal(mean=4, std=1, size=sample_size, column_name='norm') df['pois'] = builder.tools.get_dist_poisson(interval=7, size=sample_size, column_name='pois') df['norm_std'] = builder.tools.correlate_numbers( df, header='norm', standardize=True, column_name='norm_std') df['jitter1'] = builder.tools.correlate_numbers(df, header='pois', jitter=0.1, column_name='jitter1') df['jitter2'] = builder.tools.correlate_numbers(df, header='pois', jitter=0.8, column_name='jitter2') df['jitter3'] = builder.tools.correlate_numbers(df, header='pois', jitter=1.5, column_name='jitter3') df['jitter4'] = builder.tools.correlate_numbers(df, header='pois', jitter=2, column_name='jitter4') df['jitter5'] = builder.tools.correlate_numbers(df, header='pois', jitter=3, column_name='jitter5') builder.run_component_pipeline()
def setUp(self): # set environment variables os.environ['HADRON_PM_PATH'] = os.path.join(os.environ['PWD'], 'work', 'config') os.environ['HADRON_DEFAULT_PATH'] = os.path.join( os.environ['PWD'], 'work', 'data', '0_raw') try: shutil.copytree('../data', os.path.join(os.environ['PWD'], 'work')) except: pass PropertyManager._remove_all()
def setUp(self): os.environ['HADRON_PM_PATH'] = os.path.join('work', 'config') os.environ['HADRON_DEFAULT_PATH'] = os.path.join('work', 'data') try: os.makedirs(os.environ['HADRON_PM_PATH']) os.makedirs(os.environ['HADRON_DEFAULT_PATH']) except: pass PropertyManager._remove_all() self.tools = SyntheticBuilder.scratch_pad() self.clean = Transition.scratch_pad()
def setUp(self): os.environ['HADRON_PM_PATH'] = os.path.join('work', 'config') os.environ['HADRON_DEFAULT_PATH'] = os.path.join('work', 'data') try: os.makedirs(os.environ['HADRON_PM_PATH']) os.makedirs(os.environ['HADRON_DEFAULT_PATH']) except: pass PropertyManager._remove_all() self.builder: SyntheticBuilder = SyntheticBuilder.from_env( 'sample', has_contract=False) self.builder.setup_bootstrap() self.tools: SyntheticIntentModel = self.builder.tools
def setUp(self): # set environment variables os.environ['HADRON_PM_PATH'] = os.path.join(os.environ['PWD'], 'work', 'config') os.environ['HADRON_DEFAULT_SOURCE_PATH'] = os.path.join(os.environ['HOME'], 'code', 'projects', 'prod', 'data', 'raw') PropertyManager._remove_all() try: shutil.rmtree('work') except: pass try: shutil.copytree('../data', os.path.join(os.environ['PWD'], 'work')) except: pass
def setUp(self): # clean out any old environments for key in os.environ.keys(): if key.startswith('HADRON'): del os.environ[key] os.environ['HADRON_PM_PATH'] = os.path.join('work', 'config') os.environ['HADRON_DEFAULT_PATH'] = os.path.join('work', 'data') try: os.makedirs(os.environ['HADRON_PM_PATH']) os.makedirs(os.environ['HADRON_DEFAULT_PATH']) except: pass PropertyManager._remove_all()
def setUp(self): # clean out any old environments for key in os.environ.keys(): if key.startswith("HADRON"): del os.environ[key] os.environ["HADRON_PM_PATH"] = os.path.join("work", "config") os.environ["HADRON_DEFAULT_PATH"] = os.path.join("work", "data") try: os.makedirs(os.environ["HADRON_PM_PATH"]) os.makedirs(os.environ["HADRON_DEFAULT_PATH"]) except: pass PropertyManager._remove_all()
def setUp(self): # clean out any old environments for key in os.environ.keys(): if key.startswith('HADRON'): del os.environ[key] os.environ['HADRON_PM_REPO'] = "https://raw.githubusercontent.com/project-hadron/hadron-asset-bank/master/contracts/healthcare/factory/members/" os.environ['HADRON_PM_PATH'] = os.path.join('work', 'config') os.environ['HADRON_DEFAULT_PATH'] = os.path.join('work', 'data') try: os.makedirs(os.environ['HADRON_PM_PATH']) os.makedirs(os.environ['HADRON_DEFAULT_PATH']) except: pass PropertyManager._remove_all()
def setUp(self): os.environ['HADRON_PM_PATH'] = os.path.join('work', 'config') os.environ['HADRON_DEFAULT_PATH'] = os.path.join('work', 'data') try: os.makedirs(os.environ['HADRON_PM_PATH']) os.makedirs(os.environ['HADRON_DEFAULT_PATH']) except: pass PropertyManager._remove_all() self.tools = SyntheticBuilder.from_env('tester', default_save=False, default_save_intent=False, has_contract=False).intent_model self.fc: FeatureCatalog = FeatureCatalog.from_env('tester', default_save=False, has_contract=False)
def test_synthetic_with_no_source(self): shutil.rmtree('work/config') os.makedirs(os.environ['HADRON_PM_PATH']) PropertyManager._remove_all() builder = SyntheticBuilder.from_env('task3', has_contract=False) tools: SyntheticIntentModel = builder.tools builder.set_persist() df = pd.DataFrame(index=range(10)) tools.model_noise(df, num_columns=5, column_name='noise') controller = Controller.from_env(has_contract=False) controller.intent_model.synthetic_builder(df, task_name='task3') controller.run_controller() self.assertIn( builder.CONNECTOR_PERSIST, builder.report_connectors( stylise=False)['connector_name'].to_list()) self.assertNotIn( builder.CONNECTOR_SOURCE, builder.report_connectors( stylise=False)['connector_name'].to_list())
def setUp(self): # clean out any old environments for key in os.environ.keys(): if key.startswith('HADRON'): del os.environ[key] # Local Domain Contract os.environ['HADRON_PM_PATH'] = os.path.join('working', 'contracts') os.environ['HADRON_PM_TYPE'] = 'json' # Local Connectivity os.environ['HADRON_DEFAULT_PATH'] = Path('working/data').as_posix() # Specialist Component try: os.makedirs(os.environ['HADRON_PM_PATH']) except: pass try: os.makedirs(os.environ['HADRON_DEFAULT_PATH']) except: pass PropertyManager._remove_all() builder = SyntheticBuilder.from_env('tester', has_contract=False) builder.set_persist() tools: SyntheticIntentModel = builder.tools