def _compare_and_update(latest_version): # TODO: support alpha channel (allow setting which channel to check & parse build number) is_newer = semver.compare(current_version, latest_version) == -1 logging.info("Latest version: %s (newer: %s)", latest_version, is_newer) if is_newer: redis_connection.set(REDIS_KEY, latest_version) else: redis_connection.delete(REDIS_KEY)
def setUp(self): self.list = "test_list" redis_connection.delete(self.list) self.keys = [] for score in range(0, 100): key = 'k:{}'.format(score) self.keys.append(key) redis_connection.zadd(self.list, score, key) redis_connection.set(key, 1)
def get_databricks_databases(data_source_id, redis_key): try: data_source = models.DataSource.get_by_id(data_source_id) databases = data_source.query_runner.get_databases() redis_connection.set(redis_key, json_dumps(databases)) redis_connection.expire(redis_key, DATABRICKS_REDIS_EXPIRATION_TIME) return databases except Exception: return {"error": {"code": 2, "message": "Error retrieving database list."}}
def setUp(self): self.list = "test_list" redis_connection.delete(self.list) self.keys = [] for score in range(0, 100): key = 'k:{}'.format(score) self.keys.append(key) redis_connection.zadd(self.list, {key: score}) redis_connection.set(key, 1)
def get_schema(self, refresh=False): cache = None if not refresh: cache = redis_connection.get(self._schema_key) if cache is None: query_runner = self.query_runner schema = sorted(query_runner.get_schema(get_stats=refresh), key=lambda t: t['name']) redis_connection.set(self._schema_key, json_dumps(schema)) else: schema = json_loads(cache) return schema
def get_database_tables_with_columns(data_source_id, database_name, redis_key): try: data_source = models.DataSource.get_by_id(data_source_id) tables = data_source.query_runner.get_database_tables_with_columns( database_name ) # check for tables since it doesn't return an error when the requested database doesn't exist if tables or redis_connection.exists(redis_key): redis_connection.set(redis_key, json_dumps(tables)) redis_connection.expire( redis_key, DATABRICKS_REDIS_EXPIRATION_TIME, ) return {"schema": tables, "has_columns": True} except Exception: return {"error": {"code": 2, "message": "Error retrieving schema."}}
def store_health_status(data_source_id, data_source_name, query_text, data): key = "data_sources:health" cache = json.loads(redis_connection.get(key) or "{}") if data_source_id not in cache: cache[data_source_id] = {"metadata": {"name": data_source_name}, "queries": {}} cache[data_source_id]["queries"][query_text] = data cache[data_source_id]["status"] = "SUCCESS" for query_status in cache[data_source_id]["queries"].values(): if query_status["status"] == "FAIL": cache[data_source_id]["status"] = "FAIL" break redis_connection.set(key, json.dumps(cache))
def get_schema(self, refresh=False): key = "data_source:schema:{}".format(self.id) cache = None if not refresh: cache = redis_connection.get(key) if cache is None: query_runner = self.query_runner schema = sorted(query_runner.get_schema(get_stats=refresh), key=lambda t: t['name']) redis_connection.set(key, json.dumps(schema)) else: schema = json.loads(cache) return schema
def get_schema(self, refresh=False): out_schema = None if not refresh: out_schema = self.get_cached_schema() if out_schema is None: query_runner = self.query_runner schema = query_runner.get_schema(get_stats=refresh) try: out_schema = self._sort_schema(schema) except Exception: logging.exception( "Error sorting schema columns for data_source {}".format(self.id) ) out_schema = schema finally: redis_connection.set(self._schema_key, json_dumps(out_schema)) return out_schema
def _compare_and_update(latest_version): # http://taobaofed.org/blog/2016/08/05/instructions-of-semver/ # Semantic Versioning, 语义化版本 # semver #常规版本号 # 0.1.0 # 大版本(不兼容),小版本(向后兼容),修订(一些小更新) # 预发版本号 # "1.0.0-beta.1"< stage > 一般选用:alpha、beta、rc。 # 因此在版本的大小比较上,仍然先比较常规版本号部分;对于预发标记部分的比较,则是根据 ASCII 字母表中的顺序来进行。 is_newer = semver.compare(current_version, latest_version) == -1 logging.info("Latest version: %s (newer: %s)", latest_version, is_newer) if is_newer: redis_connection.set(REDIS_KEY, latest_version) else: redis_connection.delete(REDIS_KEY)
def pause(self, reason=None): redis_connection.set(self._pause_key(), reason)