def test_registry():
def test_fn():
pass
r = Registry('test')
r.register(test_fn)
assert r['test_fn'] is not None
# view_pandas.py
from utils import Registry, get_column_intersect
import pandas as pd
from assets.mapping import colmap, local_identifiers
df_registry = Registry('df_view')
## Access/View functions
def dfcolumns(data: pd.DataFrame) -> list:
return data.columns.tolist()
df_registry.register(dfcolumns, name='columns')
def dflen(data: pd.DataFrame) -> int:
return len(data)
df_registry.register(dflen, name='dlen')
def getitem(data: pd.DataFrame, idx) -> pd.Series:
return data[idx]
df_registry.register(getitem)
def head(data: pd.DataFrame, nrows:int) -> pd.DataFrame:
return data.head(nrows)
df_registry.register(head)
query = _build_iframe_selection_query(*args, **kwargs)
preplogger.debug('Preparing identity view with: \n' + query)
try:
iframe = pd.read_sql_query(query, get_db()).drop_duplicates()
except Exception as e:
iframe = pd.DataFrame()
preplogger.warn(f'Could not create iframe. Returning empty dataframe.\n{e}')
if 'index' in iframe.columns:
iframe.drop('index', axis=1, inplace=True)
iframe = iframe.dropna(axis=1, how='all')
preplogger.info(f'IView created with columns:\n{iframe.columns}\nlength:{len(iframe)}')
return View(iframe)
view_registry.register(full)
# Route Function - Looks up IDVIEW fn listed in settings.ini and returns view from mapped columns
def create_identity_view(*args, **kwargs) -> pd.DataFrame:
preplogger.info(f'Using IDVIEWTYPE: {IDVIEWTYPE}')
fn = view_registry[IDVIEWTYPE]
return fn(*args, **kwargs)
## Utility Functions
def get_intersection(i1, i2) -> set:
def _cast_set(x) -> set:
if type(x) == set:
from utils import Registry, OptimizerRegistry
from torch.optim import Adam, SGD
from torch.optim.lr_scheduler import StepLR, MultiStepLR, CosineAnnealingLR
optimizer_default_cfg = {'type': 'Adam', 'lr': 0.001}
lr_scheduler_default_cfg = None
OPTIMIZERS = OptimizerRegistry(default_cfg=optimizer_default_cfg)
OPTIMIZERS.register(Adam)
OPTIMIZERS.register(SGD)
LR_SCHEDULER = Registry(default_cfg=lr_scheduler_default_cfg)
LR_SCHEDULER.register(StepLR)
LR_SCHEDULER.register(CosineAnnealingLR)
LR_SCHEDULER.register(MultiStepLR)
try:
tx = str(x)
assert len(tx) == 9
assert tx[0:3] != '000'
assert tx[0:3] != '666'
assert int(tx[0:3]) < 900
assert int(tx[3:5]) > 0
assert int(tx[-4:]) > 0
return int(x)
except:
return None
return view[colname].apply(_test_ssn)
pandas_standards_registry.register(ssn_pool)
def guid(view: View, colname='guid'):
return view[colname]
pandas_standards_registry.register(guid)
def last_name_pool(view: View, colname='last_name_pool'):
return clean(view[colname])
pandas_standards_registry.register(last_name_pool)
