def _compare_and_update(latest_version):
# TODO: support alpha channel (allow setting which channel to check & parse build number)
is_newer = semver.compare(current_version, latest_version) == -1
logging.info("Latest version: %s (newer: %s)", latest_version, is_newer)
if is_newer:
redis_connection.set(REDIS_KEY, latest_version)
else:
redis_connection.delete(REDIS_KEY)
def setUp(self):
self.list = "test_list"
redis_connection.delete(self.list)
self.keys = []
for score in range(0, 100):
key = 'k:{}'.format(score)
self.keys.append(key)
redis_connection.zadd(self.list, score, key)
redis_connection.set(key, 1)
def get_databricks_databases(data_source_id, redis_key):
try:
data_source = models.DataSource.get_by_id(data_source_id)
databases = data_source.query_runner.get_databases()
redis_connection.set(redis_key, json_dumps(databases))
redis_connection.expire(redis_key, DATABRICKS_REDIS_EXPIRATION_TIME)
return databases
except Exception:
return {"error": {"code": 2, "message": "Error retrieving database list."}}
def setUp(self):
self.list = "test_list"
redis_connection.delete(self.list)
self.keys = []
for score in range(0, 100):
key = 'k:{}'.format(score)
self.keys.append(key)
redis_connection.zadd(self.list, {key: score})
redis_connection.set(key, 1)
def _compare_and_update(latest_version):
# TODO: support alpha channel (allow setting which channel to check & parse build number)
is_newer = semver.compare(current_version, latest_version) == -1
logging.info("Latest version: %s (newer: %s)", latest_version, is_newer)
if is_newer:
redis_connection.set(REDIS_KEY, latest_version)
else:
redis_connection.delete(REDIS_KEY)
def get_schema(self, refresh=False):
cache = None
if not refresh:
cache = redis_connection.get(self._schema_key)
if cache is None:
query_runner = self.query_runner
schema = sorted(query_runner.get_schema(get_stats=refresh), key=lambda t: t['name'])
redis_connection.set(self._schema_key, json_dumps(schema))
else:
schema = json_loads(cache)
return schema
def get_database_tables_with_columns(data_source_id, database_name, redis_key):
try:
data_source = models.DataSource.get_by_id(data_source_id)
tables = data_source.query_runner.get_database_tables_with_columns(
database_name
)
# check for tables since it doesn't return an error when the requested database doesn't exist
if tables or redis_connection.exists(redis_key):
redis_connection.set(redis_key, json_dumps(tables))
redis_connection.expire(
redis_key, DATABRICKS_REDIS_EXPIRATION_TIME,
)
return {"schema": tables, "has_columns": True}
except Exception:
return {"error": {"code": 2, "message": "Error retrieving schema."}}
def store_health_status(data_source_id, data_source_name, query_text, data):
key = "data_sources:health"
cache = json.loads(redis_connection.get(key) or "{}")
if data_source_id not in cache:
cache[data_source_id] = {"metadata": {"name": data_source_name}, "queries": {}}
cache[data_source_id]["queries"][query_text] = data
cache[data_source_id]["status"] = "SUCCESS"
for query_status in cache[data_source_id]["queries"].values():
if query_status["status"] == "FAIL":
cache[data_source_id]["status"] = "FAIL"
break
redis_connection.set(key, json.dumps(cache))
def get_schema(self, refresh=False):
key = "data_source:schema:{}".format(self.id)
cache = None
if not refresh:
cache = redis_connection.get(key)
if cache is None:
query_runner = self.query_runner
schema = sorted(query_runner.get_schema(get_stats=refresh), key=lambda t: t['name'])
redis_connection.set(key, json.dumps(schema))
else:
schema = json.loads(cache)
return schema
def get_schema(self, refresh=False):
out_schema = None
if not refresh:
out_schema = self.get_cached_schema()
if out_schema is None:
query_runner = self.query_runner
schema = query_runner.get_schema(get_stats=refresh)
try:
out_schema = self._sort_schema(schema)
except Exception:
logging.exception(
"Error sorting schema columns for data_source {}".format(self.id)
)
out_schema = schema
finally:
redis_connection.set(self._schema_key, json_dumps(out_schema))
return out_schema
def _compare_and_update(latest_version):
# http://taobaofed.org/blog/2016/08/05/instructions-of-semver/
# Semantic Versioning, 语义化版本
# semver
#常规版本号
# 0.1.0
# 大版本（不兼容），小版本（向后兼容），修订（一些小更新）
# 预发版本号
# "1.0.0-beta.1"< stage > 一般选用：alpha、beta、rc。
# 因此在版本的大小比较上，仍然先比较常规版本号部分；对于预发标记部分的比较，则是根据 ASCII 字母表中的顺序来进行。
is_newer = semver.compare(current_version, latest_version) == -1
logging.info("Latest version: %s (newer: %s)", latest_version, is_newer)
if is_newer:
redis_connection.set(REDIS_KEY, latest_version)
else:
redis_connection.delete(REDIS_KEY)
def pause(self, reason=None):
redis_connection.set(self._pause_key(), reason)
