def populate_for_chart_visualization(self, visualization, axis):
entities = {}
datasource = self.create_datasource(
name='fruits',
url=self.url
)
DBSession.flush()
entities['datasource_uid'] = datasource.uid
cat = DBSession.query(app_model.Category).get(self.category)
extraction = model.Extraction(
name='Fruits Extraction',
visualization=visualization,
category=cat,
graph_axis=axis
)
DBSession.add(extraction)
DBSession.flush()
entities['extraction_uid'] = extraction.uid
dataset = self.create_dataset(
datasource,
name='All fruits',
query='[{"$match": {"name": {"$ne": null}}}, {"$project": {"name": 1, "value": 1, "day": 1, "active":1, "created": 1}}]'
)
DBSession.flush()
entities['dataset_uid'] = dataset.uid
extractiondataset = model.ExtractionDataSet(
dataset=dataset,
extraction=extraction,
uid=randint(1, 1000)
)
DBSession.add(extractiondataset)
DBSession.flush()
entities['extractiondataset_uid'] = extractiondataset.uid
transaction.commit()
return entities
def save_extraction(self, dashboard_id, **kw):
axis = request.json['graph_axis']
visualization_type = request.json['visualization']
validate_axis_against_extraction_visualization(
visualization_type, axis,
DBSession.query(Extraction).get(int(
request.json['extraction_id'])))
try:
de = DBSession.query(DashboardExtractionAssociation).filter(
DashboardExtractionAssociation.uid == request.json['uid'],
).one()
except (NoResultFound, KeyError):
de = DashboardExtractionAssociation()
de.dashboard_id = int(dashboard_id)
de.extraction_id = request.json['extraction_id']
de.extraction_id = request.json['extraction_id']
de.visualization = visualization_type
de.graph_axis = axis
de.index = request.json['index']
try:
columns = int(request.json['columns'])
except ValueError:
abort(412, detail='columns must be an integer beetween 4 and 8')
if 8 < columns or 4 > columns:
abort(412, detail='columns must be between 4 and 8')
de.columns = columns
DBSession.add(de)
return dict(
de=de,
dashboard=de.dashboard,
extraction=de.extraction,
)
def setUp(self):
super(BaseTestController, self).setUp()
self.m_datasource = self.create_datasource(
name=u'datasource mongo',
url=u'mongodb://localhost:27017/moletest')
self.m_dataset_find = self.create_dataset(
self.m_datasource,
name=u'dataset mongo find',
query=u'''#collection=main_collection
{"data": {"$gt": 5}}''')
self.m_dataset_aggregate = self.create_dataset(
self.m_datasource,
name=u'dataset mongo aggregate',
query=u'''#collection=main_collection
[{"$match": {"data": {"$gt": 5}}}]''')
self._db = self.m_datasource.dbsession._session.get_default_database()
self._db.create_collection('main_collection')
self._db.main_collection.insert_many([{
"data":
i,
"data2":
i + 100,
"day":
datetime.utcnow() + timedelta(days=i)
} for i in range(105)])
self.ext1 = self.create_extraction(name='mongo_ext1', )
self.extds1 = model.ExtractionDataSet(
dataset_id=self.m_dataset_find.uid, extraction_id=self.ext1.uid)
DBSession.add(self.m_datasource)
DBSession.add(self.m_dataset_find)
DBSession.add(self.m_dataset_aggregate)
DBSession.add(self.ext1)
DBSession.add(self.extds1)
DBSession.flush()
transaction.commit()
def setup_class(cls):
cls.app = load_app('main', 'test_e2e.ini')
setup_app('test_e2e.ini')
# cmd = ServeCommand(Bunch(options=Bunch(debug=True, log_file=None, relative_plugins=False, verbose_level=1)), Bunch(verbose_level=2))
# cmd.run(Bunch(app_name=None, args=[], config_file='test_e2e.ini', daemon=False, monitor_restart=False, pid_file=None, reload=False, reload_interval=1, server=None, server_name=None, set_group=None, set_user=None, show_status=False, stop_daemon=False))
cat = app_model.Category(name='Default category 1')
DBSession.add(cat)
ds = model.Datasource(name='default_ds',
url=u'sqlite:///etl/tests/e2e/sales.db',
uid=randint(1, 100000))
model.DBSession.add(ds)
dataset1 = model.DataSet(name='products',
query='SELECT * FROM products',
datasource=ds,
uid=randint(1, 100000))
model.DBSession.add(dataset1)
dataset2 = model.DataSet(name='regions',
query='SELECT * FROM regions',
datasource=ds,
uid=randint(1, 100000))
model.DBSession.add(dataset2)
dataset3 = model.DataSet(name='sales',
query='SELECT * FROM sales',
datasource=ds,
uid=randint(1, 100000))
model.DBSession.add(dataset3)
dataset4 = model.DataSet(name='time',
query='SELECT * FROM time',
datasource=ds,
uid=randint(1, 100000))
model.DBSession.add(dataset4)
extraction = model.Extraction(name="Estrazione uno",
category=cat,
uid=randint(1, 100000))
model.DBSession.add(extraction)
DBSession.flush()
transaction.commit()
# create a new Chrome session
options = webdriver.ChromeOptions()
options.add_argument('--headless')
cls.driver = webdriver.Chrome(chrome_options=options)
cls.driver.implicitly_wait(30)
cls.driver.maximize_window()
def new(self):
dashboard = Dashboard()
dashboard.name = 'New dashboard'
DBSession.add(dashboard)
all_extractions = DBSession.query(Extraction).all()
return dict(
dashboard=dashboard,
form_dashboard_name=DashboardChangeName(),
form_dashboard_name_values=dashboard,
visualizationtypes=visualizationtypes,
all_extractions=all_extractions,
)
def evolve(self):
log.info('DashboardCreationEvolution migration running')
engine = create_engine(config.get('sqlalchemy.url'), echo=True)
DeclarativeBase.metadata.create_all(engine)
main = Dashboard()
main.name = 'main dashboard'
DBSession.add(main)
DBSession.flush()
transaction.commit()
def post(self, function=None, priority=None, options=None, **kwargs):
try:
if options is None:
options = {}
options = ExtractionStep.formfor(function).validate(options)
except Exception as e:
response.status = 412
return dict(errors=str(e))
DBSession.add(
ExtractionStep(extraction_id=tmpl_context.extraction.uid,
function=function,
priority=priority,
options=json.dumps(options)))
return dict()
def post(self, filter=None, extraction=None, **kwargs):
e_filter = ExtractionFilter(extraction=extraction,
name=filter.get('name'))
if filter.get('default'):
ExtractionFilter.set_default(e_filter)
DBSession.add(
ExtractionStep(extraction_filter=e_filter,
function='query',
priority=0,
options=json.dumps(
{'expression': filter.get('query')})))
DBSession.flush()
return dict(filter=e_filter)
def setUp(self):
"""Setup test fixture for each model test method."""
try:
new_attrs = {}
new_attrs.update(self.attrs)
new_attrs.update(self.do_get_dependencies())
self.obj = self.klass(**new_attrs)
for k, v in self.after_creation().items(
): # usually klass does not have update method
setattr(self.obj, k, v)
DBSession.add(self.obj)
DBSession.flush()
return self.obj
except:
DBSession.rollback()
raise
def create_extraction_association(
self,
dashboard_uid,
index,
visualization='histogram',
graph_axis='email_address,user_id',
columns=4,
):
dashboard_extraction_association = model.DashboardExtractionAssociation(
dashboard_id=dashboard_uid,
extraction_id=self.extraction,
index=index,
visualization=visualization,
graph_axis=graph_axis,
columns=columns,
)
DBSession.add(dashboard_extraction_association)
DBSession.flush()
return dashboard_extraction_association
def post(self,
datasetid=None,
join_type=None,
join_self_col=None,
join_other_col=None,
**kwargs):
try:
priority = tmpl_context.extraction.datasets[-1].priority + 1
except:
priority = 0
DBSession.add(
ExtractionDataSet(extraction=tmpl_context.extraction,
dataset_id=datasetid,
priority=priority,
join_type=join_type,
join_self_col=join_self_col,
join_other_col=join_other_col))
return dict()
def test_pipeline_stringfy_exc_indexerror(self):
flt = DBSession.query(model.ExtractionFilter).get(self.filter)
step = model.ExtractionStep(priority=1,
function='sort',
options='{"columns": ["not_there"]}',
extraction_filter_id=flt.uid,
extraction_id=flt.extraction.uid)
DBSession.add(step)
DBSession.flush()
transaction.commit()
response = self.app.get('/editor/' + str(self.extraction) +
'/test_pipeline',
extra_environ=self.admin_env,
status=200)
assert response.json['results'][1]['errors'] in [
"KeyError: u'not_there'", # py2
"KeyError: 'not_there'", # py3
], response.json['results'][1]['errors']
def create_dashboard(self,
name='Test purpose dashboard',
index=0,
visualization='histogram',
graph_axis='email_address,user_id'):
entities = dict()
dashboard = model.Dashboard(name=name)
transaction.begin()
DBSession.add(dashboard)
DBSession.flush()
entities.update(dict(dashboard=dashboard.uid))
dashboard_extraction_association = self.create_extraction_association(
dashboard.uid,
index,
visualization=visualization,
graph_axis=graph_axis,
columns=4,
)
entities.update(
dict(dashboard_extraction_association=
dashboard_extraction_association.uid))
transaction.commit()
return entities
def test_datasource_cache(self):
from etl.model.datasource import DS_CACHE
datasource = model.DBSession.query(model.Datasource).get(self.datasource)
cache_key = datasource.cache_key
original = datasource.url
assert datasource.dbsession is DS_CACHE.get_value(cache_key)
datasource.url = 'wrong_url'
DBSession.add(datasource)
DBSession.flush()
transaction.commit()
assert_raises(
KeyError,
DS_CACHE.get_value,
cache_key
)
datasource = model.DBSession.query(model.Datasource).get(self.datasource)
datasource.url = original
DBSession.add(datasource)
DBSession.flush()
transaction.commit()
datasource = model.DBSession.query(model.Datasource).get(self.datasource)
assert datasource.dbsession is DS_CACHE.get_value(cache_key)
model.DBSession.delete(datasource)
model.DBSession.flush()
transaction.commit()
assert_raises(
KeyError,
DS_CACHE.get_value,
cache_key
)
def setUp(self):
reset_cache()
super(BaseTestController, self).setUp()
cat = app_model.Category(name='Default category 1')
DBSession.add(cat)
DBSession.flush()
ds = self.create_datasource(name='default_ds')
DBSession.flush()
dt = self.create_dataset(ds, name='default_dts')
DBSession.flush()
ext = self.create_extraction(name='default_ext', category=cat)
assert list(ext.sample) == []
DBSession.flush()
extdt = model.ExtractionDataSet(dataset_id=dt.uid,
extraction_id=ext.uid)
DBSession.add(extdt)
flt = model.ExtractionFilter(extraction_id=ext.uid,
name='default_flt',
default=True)
DBSession.add(flt)
DBSession.flush()
step = model.ExtractionStep(
priority=0,
function='query',
options=json.dumps({'expression': self.filter_data['query']}),
extraction_filter_id=flt.uid,
extraction_id=ext.uid)
DBSession.add(step)
DBSession.flush()
self.datasource = ds.uid
self.dataset = dt.uid
self.extraction = ext.uid
self.filter = flt.uid
self.step = step.uid
self.category = cat._id
self.extractiondataset = extdt.uid
transaction.commit()
def filters_from_template(self,
template=None,
field=None,
extraction=None):
if template == 'alphabetical':
for i in range(ord('A'), ord('Z') + 1):
c = chr(i)
next_c = chr(i + 1)
e_filter = ExtractionFilter(extraction=extraction, name=c)
expr = '({} >= "{}" and {} <= "{}") or ({} >= "{}" and {} <= "{}")'.format(
field, c, field, next_c, field, c.lower(), field,
next_c.lower())
DBSession.add(
ExtractionStep(extraction_filter=e_filter,
function='query',
priority=0,
options=json.dumps({'expression': expr})))
expr = '({} < "0") or ({} >= ":" and {} <= "A") or ({} >= "[" and {} <= "a") or ({} >= "{")'.replace(
'{}', field)
e_filter = ExtractionFilter(extraction=extraction, name='Symbols')
DBSession.add(
ExtractionStep(extraction_filter=e_filter,
function='query',
priority=0,
options=json.dumps({'expression': expr})))
expr = '({} >= "0" and {} <= ":")'.replace('{}', field)
e_filter = ExtractionFilter(extraction=extraction, name='0-9')
DBSession.add(
ExtractionStep(extraction_filter=e_filter,
function='query',
priority=0,
options=json.dumps({'expression': expr})))
else:
return dict(extraction=extraction,
error=dict(
template='unknown template: {}'.format(template),
field=None,
))
DBSession.flush()
extraction = DBSession.merge(extraction)
return dict(extraction=extraction, error=None)
def create(self, name, **kw):
DBSession.add(Extraction(name=name))
flash('New Extraction successfully created', 'ok')
return redirect('./index')