def post(self):
req = request.get_json(True)
require_fields(req, ('options', 'name', 'type'))
schema = get_configuration_schema_for_destination_type(req['type'])
if schema is None:
abort(400)
config = ConfigurationContainer(req['options'], schema)
if not config.is_valid():
abort(400)
destination = models.NotificationDestination(org=self.current_org,
name=req['name'],
type=req['type'],
options=config,
user=self.current_user)
try:
models.db.session.add(destination)
models.db.session.commit()
except IntegrityError as e:
if 'name' in e.message:
abort(400, message=u"Alert Destination with the name {} already exists.".format(req['name']))
abort(500)
return destination.to_dict(all=True)
def post(self):
req = request.get_json(True)
require_fields(req, ('options', 'name', 'type'))
schema = get_configuration_schema_for_query_runner_type(req['type'])
if schema is None:
abort(400)
config = ConfigurationContainer(filter_none(req['options']), schema)
# from IPython import embed
# embed()
if not config.is_valid():
abort(400)
try:
datasource = models.DataSource.create_with_group(org=self.current_org,
name=req['name'],
type=req['type'],
options=config)
models.db.session.commit()
except IntegrityError as e:
if req['name'] in e.message:
abort(400, message="Data source with the name {} already exists.".format(req['name']))
abort(400)
self.record_event({
'action': 'create',
'object_id': datasource.id,
'object_type': 'datasource'
})
return datasource.to_dict(all=True)
def post(self):
req = request.get_json(True)
required_fields = ('options', 'name', 'type')
for f in required_fields:
if f not in req:
abort(400)
schema = get_configuration_schema_for_type(req['type'])
if schema is None:
abort(400)
config = ConfigurationContainer(req['options'], schema)
if not config.is_valid():
abort(400)
datasource = models.DataSource.create_with_group(org=self.current_org,
name=req['name'],
type=req['type'],
options=config)
self.record_event({
'action': 'create',
'object_id': datasource.id,
'object_type': 'datasource'
})
return datasource.to_dict(all=True)
def new(name=None, type=None, options=None):
"""Create new data source."""
if name is None:
name = click.prompt("Name")
if type is None:
print "Select type:"
for i, query_runner_name in enumerate(query_runners.keys()):
print "{}. {}".format(i + 1, query_runner_name)
idx = 0
while idx < 1 or idx > len(query_runners.keys()):
idx = click.prompt("[{}-{}]".format(1, len(query_runners.keys())), type=int)
type = query_runners.keys()[idx - 1]
else:
validate_data_source_type(type)
if options is None:
query_runner = query_runners[type]
schema = query_runner.configuration_schema()
types = {
'string': unicode,
'number': int,
'boolean': bool
}
options_obj = {}
for k, prop in schema['properties'].iteritems():
required = k in schema.get('required', [])
default_value = "<<DEFAULT_VALUE>>"
if required:
default_value = None
prompt = prop.get('title', k.capitalize())
if required:
prompt = "{} (required)".format(prompt)
else:
prompt = "{} (optional)".format(prompt)
value = click.prompt(prompt, default=default_value, type=types[prop['type']], show_default=False)
if value != default_value:
options_obj[k] = value
options = ConfigurationContainer(options_obj, schema)
if not options.is_valid():
print "Error: invalid configuration."
exit()
print "Creating {} data source ({}) with options:\n{}".format(type, name, options)
data_source = models.DataSource.create(name=name,
type=type,
options=options,
org=models.Organization.get_by_slug('default'))
print "Id: {}".format(data_source.id)
def test_doesnt_leave_leftovers(self):
container = ConfigurationContainer({'a': 1, 'b': 'test', 'e': 3}, configuration_schema)
new_config = container.to_dict(mask_secrets=True)
new_config.pop('e')
container.update(new_config)
self.assertEqual(container['a'], 1)
self.assertEqual('test', container['b'])
self.assertNotIn('e', container)
class TestConfigurationUpdate(TestCase):
def setUp(self):
self.config = {'a': 1, 'b': 'test'}
self.container = ConfigurationContainer(self.config, configuration_schema)
def test_rejects_invalid_new_config(self):
self.assertRaises(ValidationError, lambda: self.container.update({'c': 3}))
def test_fails_if_no_schema_set(self):
self.container.set_schema(None)
self.assertRaises(RuntimeError, lambda: self.container.update({'c': 3}))
def test_ignores_secret_placehodler(self):
self.container.update(self.container.to_dict(mask_secrets=True))
self.assertEqual(self.container['b'], self.config['b'])
def test_updates_secret(self):
new_config = {'a': 2, 'b': 'new'}
self.container.update(new_config)
self.assertDictEqual(self.container._config, new_config)
def test_doesnt_leave_leftovers(self):
container = ConfigurationContainer({'a': 1, 'b': 'test', 'e': 3}, configuration_schema)
new_config = container.to_dict(mask_secrets=True)
new_config.pop('e')
container.update(new_config)
self.assertEqual(container['a'], 1)
self.assertEqual('test', container['b'])
self.assertNotIn('e', container)
class TestConfigurationToJson(TestCase):
def setUp(self):
self.config = {'a': 1, 'b': 'test'}
self.container = ConfigurationContainer(self.config, configuration_schema)
def test_returns_plain_dict(self):
self.assertDictEqual(self.config, self.container.to_dict())
def test_raises_exception_when_no_schema_set(self):
self.container.set_schema(None)
self.assertRaises(RuntimeError, lambda: self.container.to_dict(mask_secrets=True))
def test_returns_dict_with_masked_secrets(self):
d = self.container.to_dict(mask_secrets=True)
self.assertEqual(d['a'], self.config['a'])
self.assertNotEqual(d['b'], self.config['b'])
self.assertEqual(self.config['b'], self.container['b'])
def post(self):
req = request.get_json(True)
required_fields = ("options", "name", "type")
for f in required_fields:
if f not in req:
abort(400)
schema = get_configuration_schema_for_query_runner_type(req["type"])
if schema is None:
abort(400)
config = ConfigurationContainer(req["options"], schema)
if not config.is_valid():
abort(400)
datasource = models.DataSource.create_with_group(
org=self.current_org, name=req["name"], type=req["type"], options=config
)
self.record_event({"action": "create", "object_id": datasource.id, "object_type": "datasource"})
return datasource.to_dict(all=True)
def post(self):
req = request.get_json(True)
required_fields = ('options', 'name', 'type')
for f in required_fields:
if f not in req:
abort(400)
schema = get_configuration_schema_for_destination_type(req['type'])
if schema is None:
abort(400)
config = ConfigurationContainer(req['options'], schema)
if not config.is_valid():
abort(400)
destination = models.NotificationDestination(org=self.current_org,
name=req['name'],
type=req['type'],
options=config,
user=self.current_user)
destination.save()
return destination.to_dict(all=True)
user_factory = ModelFactory(redash.models.User,
name='John Doe', email=Sequence('test{}@example.com'),
groups=[2],
org=1)
org_factory = ModelFactory(redash.models.Organization,
name=Sequence("Org {}"),
slug=Sequence("org{}.example.com"),
settings={})
data_source_factory = ModelFactory(redash.models.DataSource,
name=Sequence('Test {}'),
type='pg',
# If we don't use lambda here it will reuse the same options between tests:
options=lambda: ConfigurationContainer.from_json('{"dbname": "test"}'),
org=1)
dashboard_factory = ModelFactory(redash.models.Dashboard,
name='test', user=user_factory.create, layout='[]', org=1)
api_key_factory = ModelFactory(redash.models.ApiKey,
object=dashboard_factory.create)
query_factory = ModelFactory(redash.models.Query,
name='New Query',
description='',
query='SELECT 1',
user=user_factory.create,
is_archived=False,
schedule=None,
class DataSource(BelongsToOrgMixin, db.Model):
id = primary_key("DataSource")
org_id = Column(key_type("Organization"), db.ForeignKey("organizations.id"))
org = db.relationship(Organization, backref="data_sources")
name = Column(db.String(255))
type = Column(db.String(255))
options = Column(
"encrypted_options",
ConfigurationContainer.as_mutable(
EncryptedConfiguration(
db.Text, settings.DATASOURCE_SECRET_KEY, FernetEngine
)
),
)
queue_name = Column(db.String(255), default="queries")
scheduled_queue_name = Column(db.String(255), default="scheduled_queries")
created_at = Column(db.DateTime(True), default=db.func.now())
data_source_groups = db.relationship(
"DataSourceGroup", back_populates="data_source", cascade="all"
)
__tablename__ = "data_sources"
__table_args__ = (db.Index("data_sources_org_id_name", "org_id", "name"),)
def __eq__(self, other):
return self.id == other.id
def __hash__(self):
return hash(self.id)
def to_dict(self, all=False, with_permissions_for=None):
d = {
"id": self.id,
"name": self.name,
"type": self.type,
"syntax": self.query_runner.syntax,
"paused": self.paused,
"pause_reason": self.pause_reason,
"supports_auto_limit": self.query_runner.supports_auto_limit
}
if all:
schema = get_configuration_schema_for_query_runner_type(self.type)
self.options.set_schema(schema)
d["options"] = self.options.to_dict(mask_secrets=True)
d["queue_name"] = self.queue_name
d["scheduled_queue_name"] = self.scheduled_queue_name
d["groups"] = self.groups
if with_permissions_for is not None:
d["view_only"] = (
db.session.query(DataSourceGroup.view_only)
.filter(
DataSourceGroup.group == with_permissions_for,
DataSourceGroup.data_source == self,
)
.one()[0]
)
return d
def __str__(self):
return str(self.name)
@classmethod
def create_with_group(cls, *args, **kwargs):
data_source = cls(*args, **kwargs)
data_source_group = DataSourceGroup(
data_source=data_source, group=data_source.org.default_group
)
db.session.add_all([data_source, data_source_group])
return data_source
@classmethod
def all(cls, org, group_ids=None):
data_sources = cls.query.filter(cls.org == org).order_by(cls.id.asc())
if group_ids:
data_sources = data_sources.join(DataSourceGroup).filter(
DataSourceGroup.group_id.in_(group_ids)
)
return data_sources.distinct()
@classmethod
def get_by_id(cls, _id):
return cls.query.filter(cls.id == _id).one()
def delete(self):
Query.query.filter(Query.data_source == self).update(
dict(data_source_id=None, latest_query_data_id=None)
)
QueryResult.query.filter(QueryResult.data_source == self).delete()
res = db.session.delete(self)
db.session.commit()
redis_connection.delete(self._schema_key)
return res
def get_cached_schema(self):
cache = redis_connection.get(self._schema_key)
return json_loads(cache) if cache else None
def get_schema(self, refresh=False):
out_schema = None
if not refresh:
out_schema = self.get_cached_schema()
if out_schema is None:
query_runner = self.query_runner
schema = query_runner.get_schema(get_stats=refresh)
try:
out_schema = self._sort_schema(schema)
except Exception:
logging.exception(
"Error sorting schema columns for data_source {}".format(self.id)
)
out_schema = schema
finally:
redis_connection.set(self._schema_key, json_dumps(out_schema))
return out_schema
def _sort_schema(self, schema):
return [
{"name": i["name"], "columns": sorted(i["columns"], key=lambda x: x["name"] if isinstance(x, dict) else x)}
for i in sorted(schema, key=lambda x: x["name"])
]
@property
def _schema_key(self):
return "data_source:schema:{}".format(self.id)
@property
def _pause_key(self):
return "ds:{}:pause".format(self.id)
@property
def paused(self):
return redis_connection.exists(self._pause_key)
@property
def pause_reason(self):
return redis_connection.get(self._pause_key)
def pause(self, reason=None):
redis_connection.set(self._pause_key, reason or "")
def resume(self):
redis_connection.delete(self._pause_key)
def add_group(self, group, view_only=False):
dsg = DataSourceGroup(group=group, data_source=self, view_only=view_only)
db.session.add(dsg)
return dsg
def remove_group(self, group):
DataSourceGroup.query.filter(
DataSourceGroup.group == group, DataSourceGroup.data_source == self
).delete()
db.session.commit()
def update_group_permission(self, group, view_only):
dsg = DataSourceGroup.query.filter(
DataSourceGroup.group == group, DataSourceGroup.data_source == self
).one()
dsg.view_only = view_only
db.session.add(dsg)
return dsg
@property
def uses_ssh_tunnel(self):
return "ssh_tunnel" in self.options
@property
def query_runner(self):
query_runner = get_query_runner(self.type, self.options)
if self.uses_ssh_tunnel:
query_runner = with_ssh_tunnel(query_runner, self.options.get("ssh_tunnel"))
return query_runner
@classmethod
def get_by_name(cls, name):
return cls.query.filter(cls.name == name).one()
# XXX examine call sites to see if a regular SQLA collection would work better
@property
def groups(self):
groups = DataSourceGroup.query.filter(DataSourceGroup.data_source == self)
return dict([(group.group_id, group.view_only) for group in groups])
try:
org = Organization.get_by_slug('default')
user = User.select().where(User.org==org, peewee.SQL("%s = ANY(groups)", org.admin_group.id)).get()
except Exception:
print("!!! Warning: failed finding default organization or admin user, won't migrate Webhook/HipChat alert subscriptions.")
exit()
if WEBHOOK_ENDPOINT:
# Have all existing alerts send to webhook if already configured
schema = get_configuration_schema_for_destination_type('webhook')
conf = {'url': WEBHOOK_ENDPOINT}
if WEBHOOK_USERNAME:
conf['username'] = WEBHOOK_USERNAME
conf['password'] = WEBHOOK_PASSWORD
options = ConfigurationContainer(conf, schema)
webhook = NotificationDestination.create(
org=org,
user=user,
name="Webhook",
type="webhook",
options=options
)
for alert in Alert.select():
AlertSubscription.create(
user=user,
destination=webhook,
alert=alert
)
def process_result_value(self, value, dialect):
return ConfigurationContainer.from_json(super(EncryptedConfiguration, self).process_result_value(value, dialect))
def setUp(self):
self.config = {'a': 1, 'b': 'test'}
self.container = ConfigurationContainer(self.config, configuration_schema)
user_factory = ModelFactory(redash.models.User,
name='John Doe', email=Sequence('test{}@example.com'),
groups=[2],
org=1)
org_factory = ModelFactory(redash.models.Organization,
name=Sequence("Org {}"),
slug=Sequence("org{}.example.com"),
settings={})
data_source_factory = ModelFactory(redash.models.DataSource,
name=Sequence('Test {}'),
type='pg',
options=ConfigurationContainer.from_json('{"dbname": "test"}'),
org=1)
dashboard_factory = ModelFactory(redash.models.Dashboard,
name='test', user=user_factory.create, layout='[]', org=1)
query_factory = ModelFactory(redash.models.Query,
name='New Query',
description='',
query='SELECT 1',
user=user_factory.create,
is_archived=False,
schedule=None,
data_source=data_source_factory.create,
org=1)
def test_adds_data_source_to_default_group(self):
data_source = DataSource.create_with_group(org=self.factory.org, name='test', options=ConfigurationContainer.from_json('{"dbname": "test"}'), type='pg')
self.assertIn(self.factory.org.default_group.id, data_source.groups)
def new(name=None, type=None, options=None):
"""Create new data source."""
if name is None:
name = click.prompt("Name")
if type is None:
print "Select type:"
for i, query_runner_name in enumerate(query_runners.keys()):
print "{}. {}".format(i + 1, query_runner_name)
idx = 0
while idx < 1 or idx > len(query_runners.keys()):
idx = click.prompt("[{}-{}]".format(1, len(query_runners.keys())),
type=int)
type = query_runners.keys()[idx - 1]
else:
validate_data_source_type(type)
query_runner = query_runners[type]
schema = query_runner.configuration_schema()
if options is None:
types = {'string': unicode, 'number': int, 'boolean': bool}
options_obj = {}
for k, prop in schema['properties'].iteritems():
required = k in schema.get('required', [])
default_value = "<<DEFAULT_VALUE>>"
if required:
default_value = None
prompt = prop.get('title', k.capitalize())
if required:
prompt = "{} (required)".format(prompt)
else:
prompt = "{} (optional)".format(prompt)
value = click.prompt(prompt,
default=default_value,
type=types[prop['type']],
show_default=False)
if value != default_value:
options_obj[k] = value
options = ConfigurationContainer(options_obj, schema)
else:
options = ConfigurationContainer(json.loads(options), schema)
if not options.is_valid():
print "Error: invalid configuration."
exit()
print "Creating {} data source ({}) with options:\n{}".format(
type, name, options.to_json())
data_source = models.DataSource.create_with_group(
name=name,
type=type,
options=options,
org=models.Organization.get_by_slug('default'))
print "Id: {}".format(data_source.id)
class DataSource(BelongsToOrgMixin, db.Model):
id = Column(db.Integer, primary_key=True)
org_id = Column(db.Integer, db.ForeignKey('organizations.id'))
org = db.relationship(Organization, backref="data_sources")
name = Column(db.String(255))
type = Column(db.String(255))
options = Column(
'encrypted_options',
ConfigurationContainer.as_mutable(
EncryptedConfiguration(db.Text, settings.SECRET_KEY,
FernetEngine)))
queue_name = Column(db.String(255), default="queries")
scheduled_queue_name = Column(db.String(255), default="scheduled_queries")
created_at = Column(db.DateTime(True), default=db.func.now())
data_source_groups = db.relationship("DataSourceGroup",
back_populates="data_source",
cascade="all")
__tablename__ = 'data_sources'
__table_args__ = (db.Index('data_sources_org_id_name', 'org_id', 'name'), )
def __eq__(self, other):
return self.id == other.id
def to_dict(self, all=False, with_permissions_for=None):
d = {
'id': self.id,
'name': self.name,
'type': self.type,
'syntax': self.query_runner.syntax,
'paused': self.paused,
'pause_reason': self.pause_reason
}
if all:
schema = get_configuration_schema_for_query_runner_type(self.type)
self.options.set_schema(schema)
d['options'] = self.options.to_dict(mask_secrets=True)
d['queue_name'] = self.queue_name
d['scheduled_queue_name'] = self.scheduled_queue_name
d['groups'] = self.groups
if with_permissions_for is not None:
d['view_only'] = db.session.query(
DataSourceGroup.view_only).filter(
DataSourceGroup.group == with_permissions_for,
DataSourceGroup.data_source == self).one()[0]
return d
def __str__(self):
return text_type(self.name)
@classmethod
def create_with_group(cls, *args, **kwargs):
data_source = cls(*args, **kwargs)
data_source_group = DataSourceGroup(
data_source=data_source, group=data_source.org.default_group)
db.session.add_all([data_source, data_source_group])
return data_source
@classmethod
def all(cls, org, group_ids=None):
data_sources = cls.query.filter(cls.org == org).order_by(cls.id.asc())
if group_ids:
data_sources = data_sources.join(DataSourceGroup).filter(
DataSourceGroup.group_id.in_(group_ids))
return data_sources.distinct()
@classmethod
def get_by_id(cls, _id):
return cls.query.filter(cls.id == _id).one()
def delete(self):
Query.query.filter(Query.data_source == self).update(
dict(data_source_id=None, latest_query_data_id=None))
QueryResult.query.filter(QueryResult.data_source == self).delete()
res = db.session.delete(self)
db.session.commit()
return res
def get_schema(self):
schema = []
tables = TableMetadata.query.filter(
TableMetadata.data_source_id == self.id).all()
for table in tables:
if not table.exists:
continue
table_info = {
'name': table.name,
'exists': table.exists,
'hasColumnMetadata': table.column_metadata,
'columns': []
}
columns = ColumnMetadata.query.filter(
ColumnMetadata.table_id == table.id)
table_info['columns'] = sorted([{
'key': column.id,
'name': column.name,
'type': column.type,
'exists': column.exists,
'example': column.example
} for column in columns if column.exists == True],
key=itemgetter('name'))
schema.append(table_info)
return sorted(schema, key=itemgetter('name'))
def _pause_key(self):
return 'ds:{}:pause'.format(self.id)
@property
def paused(self):
return redis_connection.exists(self._pause_key())
@property
def pause_reason(self):
return redis_connection.get(self._pause_key())
def pause(self, reason=None):
redis_connection.set(self._pause_key(), reason or '')
def resume(self):
redis_connection.delete(self._pause_key())
def add_group(self, group, view_only=False):
dsg = DataSourceGroup(group=group,
data_source=self,
view_only=view_only)
db.session.add(dsg)
return dsg
def remove_group(self, group):
DataSourceGroup.query.filter(
DataSourceGroup.group == group,
DataSourceGroup.data_source == self).delete()
db.session.commit()
def update_group_permission(self, group, view_only):
dsg = DataSourceGroup.query.filter(
DataSourceGroup.group == group,
DataSourceGroup.data_source == self).one()
dsg.view_only = view_only
db.session.add(dsg)
return dsg
@property
def query_runner(self):
return get_query_runner(self.type, self.options)
@classmethod
def get_by_name(cls, name):
return cls.query.filter(cls.name == name).one()
# XXX examine call sites to see if a regular SQLA collection would work better
@property
def groups(self):
groups = DataSourceGroup.query.filter(
DataSourceGroup.data_source == self)
return dict(map(lambda g: (g.group_id, g.view_only), groups))
name='John Doe',
email=Sequence('test{}@example.com'),
group_ids=[2],
org_id=1)
org_factory = ModelFactory(redash.models.Organization,
name=Sequence("Org {}"),
slug=Sequence("org{}.example.com"),
settings={})
data_source_factory = ModelFactory(
redash.models.DataSource,
name=Sequence('Test {}'),
type='pg',
# If we don't use lambda here it will reuse the same options between tests:
options=lambda: ConfigurationContainer.from_json('{"dbname": "test"}'),
org_id=1)
dashboard_factory = ModelFactory(redash.models.Dashboard,
name='test',
user=user_factory.create,
layout='[]',
is_draft=False,
org=1)
api_key_factory = ModelFactory(redash.models.ApiKey,
object=dashboard_factory.create)
query_factory = ModelFactory(redash.models.Query,
name='Query',
description='',
def new(name=None, type=None, options=None, organization="default"):
"""Create new data source."""
if name is None:
name = click.prompt("Name")
if type is None:
print("Select type:")
for i, query_runner_name in enumerate(query_runners.keys()):
print("{}. {}".format(i + 1, query_runner_name))
idx = 0
while idx < 1 or idx > len(list(query_runners.keys())):
idx = click.prompt("[{}-{}]".format(1, len(query_runners.keys())), type=int)
type = list(query_runners.keys())[idx - 1]
else:
validate_data_source_type(type)
query_runner = query_runners[type]
schema = query_runner.configuration_schema()
if options is None:
types = {"string": str, "number": int, "boolean": bool}
options_obj = {}
for k, prop in schema["properties"].items():
required = k in schema.get("required", [])
default_value = "<<DEFAULT_VALUE>>"
if required:
default_value = None
prompt = prop.get("title", k.capitalize())
if required:
prompt = "{} (required)".format(prompt)
else:
prompt = "{} (optional)".format(prompt)
value = click.prompt(
prompt,
default=default_value,
type=types[prop["type"]],
show_default=False,
)
if value != default_value:
options_obj[k] = value
options = ConfigurationContainer(options_obj, schema)
else:
options = ConfigurationContainer(json_loads(options), schema)
if not options.is_valid():
print("Error: invalid configuration.")
exit()
print(
"Creating {} data source ({}) with options:\n{}".format(
type, name, options.to_json()
)
)
data_source = models.DataSource.create_with_group(
name=name,
type=type,
options=options,
org=models.Organization.get_by_slug(organization),
)
models.db.session.commit()
print("Id: {}".format(data_source.id))
def process_result_value(self, value, dialect):
return ConfigurationContainer.from_json(value)
class DataSource(BelongsToOrgMixin, db.Model):
id = Column(db.Integer, primary_key=True)
org_id = Column(db.Integer, db.ForeignKey('organizations.id'))
org = db.relationship(Organization, backref="data_sources")
name = Column(db.String(255))
type = Column(db.String(255))
options = Column(ConfigurationContainer.as_mutable(Configuration))
queue_name = Column(db.String(255), default="queries")
scheduled_queue_name = Column(db.String(255), default="scheduled_queries")
created_at = Column(db.DateTime(True), default=db.func.now())
data_source_groups = db.relationship("DataSourceGroup", back_populates="data_source",
cascade="all")
__tablename__ = 'data_sources'
__table_args__ = (db.Index('data_sources_org_id_name', 'org_id', 'name'),)
def __eq__(self, other):
return self.id == other.id
def to_dict(self, all=False, with_permissions_for=None):
d = {
'id': self.id,
'name': self.name,
'type': self.type,
'syntax': self.query_runner.syntax,
'paused': self.paused,
'pause_reason': self.pause_reason
}
if all:
schema = get_configuration_schema_for_query_runner_type(self.type)
self.options.set_schema(schema)
d['options'] = self.options.to_dict(mask_secrets=True)
d['queue_name'] = self.queue_name
d['scheduled_queue_name'] = self.scheduled_queue_name
d['groups'] = self.groups
if with_permissions_for is not None:
d['view_only'] = db.session.query(DataSourceGroup.view_only).filter(
DataSourceGroup.group == with_permissions_for,
DataSourceGroup.data_source == self).one()[0]
return d
def __str__(self):
return text_type(self.name)
@classmethod
def create_with_group(cls, *args, **kwargs):
data_source = cls(*args, **kwargs)
data_source_group = DataSourceGroup(
data_source=data_source,
group=data_source.org.default_group)
db.session.add_all([data_source, data_source_group])
return data_source
@classmethod
def all(cls, org, group_ids=None):
data_sources = cls.query.filter(cls.org == org).order_by(cls.id.asc())
if group_ids:
data_sources = data_sources.join(DataSourceGroup).filter(
DataSourceGroup.group_id.in_(group_ids))
return data_sources.distinct()
@classmethod
def get_by_id(cls, _id):
return cls.query.filter(cls.id == _id).one()
def delete(self):
Query.query.filter(Query.data_source == self).update(dict(data_source_id=None, latest_query_data_id=None))
QueryResult.query.filter(QueryResult.data_source == self).delete()
res = db.session.delete(self)
db.session.commit()
return res
def get_schema(self, refresh=False):
key = "data_source:schema:{}".format(self.id)
cache = None
if not refresh:
cache = redis_connection.get(key)
if cache is None:
query_runner = self.query_runner
schema = sorted(query_runner.get_schema(get_stats=refresh), key=lambda t: t['name'])
redis_connection.set(key, json_dumps(schema))
else:
schema = json_loads(cache)
return schema
def _pause_key(self):
return 'ds:{}:pause'.format(self.id)
@property
def paused(self):
return redis_connection.exists(self._pause_key())
@property
def pause_reason(self):
return redis_connection.get(self._pause_key())
def pause(self, reason=None):
redis_connection.set(self._pause_key(), reason or '')
def resume(self):
redis_connection.delete(self._pause_key())
def add_group(self, group, view_only=False):
dsg = DataSourceGroup(group=group, data_source=self, view_only=view_only)
db.session.add(dsg)
return dsg
def remove_group(self, group):
DataSourceGroup.query.filter(
DataSourceGroup.group == group,
DataSourceGroup.data_source == self
).delete()
db.session.commit()
def update_group_permission(self, group, view_only):
dsg = DataSourceGroup.query.filter(
DataSourceGroup.group == group,
DataSourceGroup.data_source == self).one()
dsg.view_only = view_only
db.session.add(dsg)
return dsg
@property
def query_runner(self):
return get_query_runner(self.type, self.options)
@classmethod
def get_by_name(cls, name):
return cls.query.filter(cls.name == name).one()
# XXX examine call sites to see if a regular SQLA collection would work better
@property
def groups(self):
groups = DataSourceGroup.query.filter(
DataSourceGroup.data_source == self
)
return dict(map(lambda g: (g.group_id, g.view_only), groups))
def test_adds_data_source_to_default_group(self):
data_source = DataSource.create_with_group(org=self.factory.org, name='test', options=ConfigurationContainer.from_json('{"dbname": "test"}'), type='pg')
self.assertIn(self.factory.org.default_group.id, data_source.groups)
user_factory = ModelFactory(redash.models.User,
name='John Doe',
email=Sequence('test{}@example.com'),
groups=[2],
org=1)
org_factory = ModelFactory(redash.models.Organization,
name=Sequence("Org {}"),
slug=Sequence("org{}.example.com"),
settings={})
data_source_factory = ModelFactory(
redash.models.DataSource,
name=Sequence('Test {}'),
type='pg',
options=ConfigurationContainer.from_json('{"dbname": "test"}'),
org=1)
dashboard_factory = ModelFactory(redash.models.Dashboard,
name='test',
user=user_factory.create,
layout='[]',
org=1)
api_key_factory = ModelFactory(redash.models.ApiKey,
object=dashboard_factory.create)
query_factory = ModelFactory(redash.models.Query,
name='New Query',
description='',
query='SELECT 1',
def process_result_value(self, value, dialect):
return ConfigurationContainer.from_json(
super(EncryptedConfiguration,
self).process_result_value(value, dialect))
def python_value(self, value):
return ConfigurationContainer.from_json(value)
def setUp(self):
self.config = {"a": 1, "b": "test"}
self.container = ConfigurationContainer(self.config, configuration_schema)
class NotificationDestination(BelongsToOrgMixin, db.Model):
id = primary_key("NotificationDestination")
org_id = Column(key_type("Organization"), db.ForeignKey("organizations.id"))
org = db.relationship(Organization, backref="notification_destinations")
user_id = Column(key_type("User"), db.ForeignKey("users.id"))
user = db.relationship(User, backref="notification_destinations")
name = Column(db.String(255))
type = Column(db.String(255))
options = Column(
"encrypted_options",
ConfigurationContainer.as_mutable(
EncryptedConfiguration(
db.Text, settings.DATASOURCE_SECRET_KEY, FernetEngine
)
),
)
created_at = Column(db.DateTime(True), default=db.func.now())
__tablename__ = "notification_destinations"
__table_args__ = (
db.Index(
"notification_destinations_org_id_name", "org_id", "name", unique=True
),
)
def __str__(self):
return str(self.name)
def to_dict(self, all=False):
d = {
"id": self.id,
"name": self.name,
"type": self.type,
"icon": self.destination.icon(),
}
if all:
schema = get_configuration_schema_for_destination_type(self.type)
self.options.set_schema(schema)
d["options"] = self.options.to_dict(mask_secrets=True)
return d
@property
def destination(self):
return get_destination(self.type, self.options)
@classmethod
def all(cls, org):
notification_destinations = cls.query.filter(cls.org == org).order_by(
cls.id.asc()
)
return notification_destinations
def notify(self, alert, query, user, new_state, app, host):
schema = get_configuration_schema_for_destination_type(self.type)
self.options.set_schema(schema)
return self.destination.notify(
alert, query, user, new_state, app, host, self.options
)
def test_works_for_schema_without_secret(self):
secretless = configuration_schema.copy()
secretless.pop("secret")
container = ConfigurationContainer({"a": 1, "b": "test", "e": 3}, secretless)
container.update({"a": 2})
self.assertEqual(container["a"], 2)
def process_result_value(self, value, dialect):
return ConfigurationContainer.from_json(value)
def setUp(self):
self.config = {'a': 1, 'b': 'test'}
self.container = ConfigurationContainer(self.config,
configuration_schema)
def python_value(self, value):
return ConfigurationContainer.from_json(value)
def test_works_for_schema_without_secret(self):
secretless = configuration_schema.copy()
secretless.pop('secret')
container = ConfigurationContainer({'a': 1, 'b': 'test', 'e': 3}, secretless)
container.update({'a': 2})
self.assertEqual(container['a'], 2)
