def setUp(self):
super(BaseTestController, self).setUp()
self.m_datasource = self.create_datasource(
name=u'datasource mongo',
url=u'mongodb://localhost:27017/moletest')
self.m_dataset_find = self.create_dataset(
self.m_datasource,
name=u'dataset mongo find',
query=u'''#collection=main_collection
{"data": {"$gt": 5}}''')
self.m_dataset_aggregate = self.create_dataset(
self.m_datasource,
name=u'dataset mongo aggregate',
query=u'''#collection=main_collection
[{"$match": {"data": {"$gt": 5}}}]''')
self._db = self.m_datasource.dbsession._session.get_default_database()
self._db.create_collection('main_collection')
self._db.main_collection.insert_many([{
"data":
i,
"data2":
i + 100,
"day":
datetime.utcnow() + timedelta(days=i)
} for i in range(105)])
self.ext1 = self.create_extraction(name='mongo_ext1', )
self.extds1 = model.ExtractionDataSet(
dataset_id=self.m_dataset_find.uid, extraction_id=self.ext1.uid)
DBSession.add(self.m_datasource)
DBSession.add(self.m_dataset_find)
DBSession.add(self.m_dataset_aggregate)
DBSession.add(self.ext1)
DBSession.add(self.extds1)
DBSession.flush()
transaction.commit()
def setup_class(cls):
cls.app = load_app('main', 'test_e2e.ini')
setup_app('test_e2e.ini')
# cmd = ServeCommand(Bunch(options=Bunch(debug=True, log_file=None, relative_plugins=False, verbose_level=1)), Bunch(verbose_level=2))
# cmd.run(Bunch(app_name=None, args=[], config_file='test_e2e.ini', daemon=False, monitor_restart=False, pid_file=None, reload=False, reload_interval=1, server=None, server_name=None, set_group=None, set_user=None, show_status=False, stop_daemon=False))
cat = app_model.Category(name='Default category 1')
DBSession.add(cat)
ds = model.Datasource(name='default_ds',
url=u'sqlite:///etl/tests/e2e/sales.db',
uid=randint(1, 100000))
model.DBSession.add(ds)
dataset1 = model.DataSet(name='products',
query='SELECT * FROM products',
datasource=ds,
uid=randint(1, 100000))
model.DBSession.add(dataset1)
dataset2 = model.DataSet(name='regions',
query='SELECT * FROM regions',
datasource=ds,
uid=randint(1, 100000))
model.DBSession.add(dataset2)
dataset3 = model.DataSet(name='sales',
query='SELECT * FROM sales',
datasource=ds,
uid=randint(1, 100000))
model.DBSession.add(dataset3)
dataset4 = model.DataSet(name='time',
query='SELECT * FROM time',
datasource=ds,
uid=randint(1, 100000))
model.DBSession.add(dataset4)
extraction = model.Extraction(name="Estrazione uno",
category=cat,
uid=randint(1, 100000))
model.DBSession.add(extraction)
DBSession.flush()
transaction.commit()
# create a new Chrome session
options = webdriver.ChromeOptions()
options.add_argument('--headless')
cls.driver = webdriver.Chrome(chrome_options=options)
cls.driver.implicitly_wait(30)
cls.driver.maximize_window()
def evolve(self):
log.info('DashboardCreationEvolution migration running')
engine = create_engine(config.get('sqlalchemy.url'), echo=True)
DeclarativeBase.metadata.create_all(engine)
main = Dashboard()
main.name = 'main dashboard'
DBSession.add(main)
DBSession.flush()
transaction.commit()
def evolve(self):
log.info('TgappCategories migration running')
m = DBSession.query(Group).filter(
Group.group_name == "managers").first()
a = DBSession.query(Group).filter(Group.group_name == "admin").first()
p = DBSession.query(Permission).filter(
Permission.permission_name == "tgappcategories").first()
p.groups.append(m)
p.groups.append(a)
DBSession.flush()
transaction.commit()
def test_extraction_widget_pie_multicolor(self):
entities = self.create_dashboard(visualization='pie',
graph_axis='email_address,user_id')
DBSession.query(model.ExtractionStep).delete()
DBSession.flush()
transaction.commit()
response = self.app.get(
'/dashboard/extraction_widget/' + str(entities['dashboard']),
params=dict(uid=entities['dashboard_extraction_association']),
extra_environ=self.admin_env,
status=200)
assert 'default_ext' in response.text
assert len(response.html.find_all('div', class_="bk-root")) > 0
def evolve(self):
for step in DBSession.query(ExtractionStep).filter_by(
function='slice').all():
options = json.loads(step.options)
if isinstance(options, list):
step.options = json.dumps({'fields': options})
for step in DBSession.query(ExtractionStep).filter_by(
function='concatcols').all():
options = json.loads(step.options)
if isinstance(options, list):
step.options = json.dumps({'columns': options})
for step in DBSession.query(ExtractionStep).filter_by(
function='setvalue').all():
options = json.loads(step.options)
if isinstance(options, list):
step.options = json.dumps({
'field': options[0],
'value': options[1]
})
for step in DBSession.query(ExtractionStep).filter_by(
function='striptime').all():
options = json.loads(step.options)
if isinstance(options, list):
step.options = json.dumps({'field': options[0]})
for step in DBSession.query(ExtractionStep).filter_by(
function='stripday').all():
options = json.loads(step.options)
if isinstance(options, list):
step.options = json.dumps({'field': options[0]})
for step in DBSession.query(ExtractionStep).filter_by(
function='rename').all():
options = json.loads(step.options)
if isinstance(options, list):
step.options = json.dumps({
'source': options[0],
'target': options[1]
})
for step in DBSession.query(ExtractionStep).filter_by(
function='setindex').all():
options = json.loads(step.options)
if isinstance(options, list):
step.options = json.dumps({'index': options[0]})
DBSession.flush()
transaction.commit()
def populate_for_chart_visualization(self, visualization, axis):
entities = {}
datasource = self.create_datasource(
name='fruits',
url=self.url
)
DBSession.flush()
entities['datasource_uid'] = datasource.uid
cat = DBSession.query(app_model.Category).get(self.category)
extraction = model.Extraction(
name='Fruits Extraction',
visualization=visualization,
category=cat,
graph_axis=axis
)
DBSession.add(extraction)
DBSession.flush()
entities['extraction_uid'] = extraction.uid
dataset = self.create_dataset(
datasource,
name='All fruits',
query='[{"$match": {"name": {"$ne": null}}}, {"$project": {"name": 1, "value": 1, "day": 1, "active":1, "created": 1}}]'
)
DBSession.flush()
entities['dataset_uid'] = dataset.uid
extractiondataset = model.ExtractionDataSet(
dataset=dataset,
extraction=extraction,
uid=randint(1, 1000)
)
DBSession.add(extractiondataset)
DBSession.flush()
entities['extractiondataset_uid'] = extractiondataset.uid
transaction.commit()
return entities
def post(self, filter=None, extraction=None, **kwargs):
e_filter = ExtractionFilter(extraction=extraction,
name=filter.get('name'))
if filter.get('default'):
ExtractionFilter.set_default(e_filter)
DBSession.add(
ExtractionStep(extraction_filter=e_filter,
function='query',
priority=0,
options=json.dumps(
{'expression': filter.get('query')})))
DBSession.flush()
return dict(filter=e_filter)
def setUp(self):
"""Setup test fixture for each model test method."""
try:
new_attrs = {}
new_attrs.update(self.attrs)
new_attrs.update(self.do_get_dependencies())
self.obj = self.klass(**new_attrs)
for k, v in self.after_creation().items(
): # usually klass does not have update method
setattr(self.obj, k, v)
DBSession.add(self.obj)
DBSession.flush()
return self.obj
except:
DBSession.rollback()
raise
def create_extraction_association(
self,
dashboard_uid,
index,
visualization='histogram',
graph_axis='email_address,user_id',
columns=4,
):
dashboard_extraction_association = model.DashboardExtractionAssociation(
dashboard_id=dashboard_uid,
extraction_id=self.extraction,
index=index,
visualization=visualization,
graph_axis=graph_axis,
columns=columns,
)
DBSession.add(dashboard_extraction_association)
DBSession.flush()
return dashboard_extraction_association
def add_dataset(self, query, name):
group_ds = self.create_datasource(
name=name
)
DBSession.flush()
group_ds_uid = group_ds.uid
group_dt = self.create_dataset(
group_ds,
name='Group dataset',
query=query
)
DBSession.flush()
group_dt_uid = group_dt.uid
transaction.commit()
return {
'group_dt_uid': group_dt_uid,
'group_ds_uid': group_ds_uid
}
def test_pipeline_stringfy_exc_indexerror(self):
flt = DBSession.query(model.ExtractionFilter).get(self.filter)
step = model.ExtractionStep(priority=1,
function='sort',
options='{"columns": ["not_there"]}',
extraction_filter_id=flt.uid,
extraction_id=flt.extraction.uid)
DBSession.add(step)
DBSession.flush()
transaction.commit()
response = self.app.get('/editor/' + str(self.extraction) +
'/test_pipeline',
extra_environ=self.admin_env,
status=200)
assert response.json['results'][1]['errors'] in [
"KeyError: u'not_there'", # py2
"KeyError: 'not_there'", # py3
], response.json['results'][1]['errors']
def filters_from_template(self,
template=None,
field=None,
extraction=None):
if template == 'alphabetical':
for i in range(ord('A'), ord('Z') + 1):
c = chr(i)
next_c = chr(i + 1)
e_filter = ExtractionFilter(extraction=extraction, name=c)
expr = '({} >= "{}" and {} <= "{}") or ({} >= "{}" and {} <= "{}")'.format(
field, c, field, next_c, field, c.lower(), field,
next_c.lower())
DBSession.add(
ExtractionStep(extraction_filter=e_filter,
function='query',
priority=0,
options=json.dumps({'expression': expr})))
expr = '({} < "0") or ({} >= ":" and {} <= "A") or ({} >= "[" and {} <= "a") or ({} >= "{")'.replace(
'{}', field)
e_filter = ExtractionFilter(extraction=extraction, name='Symbols')
DBSession.add(
ExtractionStep(extraction_filter=e_filter,
function='query',
priority=0,
options=json.dumps({'expression': expr})))
expr = '({} >= "0" and {} <= ":")'.replace('{}', field)
e_filter = ExtractionFilter(extraction=extraction, name='0-9')
DBSession.add(
ExtractionStep(extraction_filter=e_filter,
function='query',
priority=0,
options=json.dumps({'expression': expr})))
else:
return dict(extraction=extraction,
error=dict(
template='unknown template: {}'.format(template),
field=None,
))
DBSession.flush()
extraction = DBSession.merge(extraction)
return dict(extraction=extraction, error=None)
def create_dashboard(self,
name='Test purpose dashboard',
index=0,
visualization='histogram',
graph_axis='email_address,user_id'):
entities = dict()
dashboard = model.Dashboard(name=name)
transaction.begin()
DBSession.add(dashboard)
DBSession.flush()
entities.update(dict(dashboard=dashboard.uid))
dashboard_extraction_association = self.create_extraction_association(
dashboard.uid,
index,
visualization=visualization,
graph_axis=graph_axis,
columns=4,
)
entities.update(
dict(dashboard_extraction_association=
dashboard_extraction_association.uid))
transaction.commit()
return entities
def test_datasource_cache(self):
from etl.model.datasource import DS_CACHE
datasource = model.DBSession.query(model.Datasource).get(self.datasource)
cache_key = datasource.cache_key
original = datasource.url
assert datasource.dbsession is DS_CACHE.get_value(cache_key)
datasource.url = 'wrong_url'
DBSession.add(datasource)
DBSession.flush()
transaction.commit()
assert_raises(
KeyError,
DS_CACHE.get_value,
cache_key
)
datasource = model.DBSession.query(model.Datasource).get(self.datasource)
datasource.url = original
DBSession.add(datasource)
DBSession.flush()
transaction.commit()
datasource = model.DBSession.query(model.Datasource).get(self.datasource)
assert datasource.dbsession is DS_CACHE.get_value(cache_key)
model.DBSession.delete(datasource)
model.DBSession.flush()
transaction.commit()
assert_raises(
KeyError,
DS_CACHE.get_value,
cache_key
)
def setUp(self):
reset_cache()
super(BaseTestController, self).setUp()
cat = app_model.Category(name='Default category 1')
DBSession.add(cat)
DBSession.flush()
ds = self.create_datasource(name='default_ds')
DBSession.flush()
dt = self.create_dataset(ds, name='default_dts')
DBSession.flush()
ext = self.create_extraction(name='default_ext', category=cat)
assert list(ext.sample) == []
DBSession.flush()
extdt = model.ExtractionDataSet(dataset_id=dt.uid,
extraction_id=ext.uid)
DBSession.add(extdt)
flt = model.ExtractionFilter(extraction_id=ext.uid,
name='default_flt',
default=True)
DBSession.add(flt)
DBSession.flush()
step = model.ExtractionStep(
priority=0,
function='query',
options=json.dumps({'expression': self.filter_data['query']}),
extraction_filter_id=flt.uid,
extraction_id=ext.uid)
DBSession.add(step)
DBSession.flush()
self.datasource = ds.uid
self.dataset = dt.uid
self.extraction = ext.uid
self.filter = flt.uid
self.step = step.uid
self.category = cat._id
self.extractiondataset = extdt.uid
transaction.commit()
