class ServiceHub:
"""
Contains a list of all available Story hub services
"""
def __init__(self, hub=None):
if hub is not None:
self.hub = hub
return
cache_dir = get_cache_dir()
self.hub_path = path.join(cache_dir, "hub.json")
if path.exists(self.hub_path):
hub = self.read_hub_from_json()
# local JSON storage doesn't exist or reading it failed
if hub is None:
makedirs(cache_dir, exist_ok=True)
self.hub = ServiceWrapper()
self.update_service_wrapper()
self.hub = hub
self.update_thread = AutoUpdateThread(self.update_service_wrapper,
initial_update=False)
def read_hub_from_json(self):
"""
Try loading an existing service JSON blob from storage.
Returns an initialized ServiceWrapper if successful, None otherwise.
"""
try:
return ServiceWrapper.from_json_file(self.hub_path)
except JSONDecodeError:
# local JSON blob might be invalid
# see e.g. https://github.com/storyscript/sls/issues/191
return None
except OSError:
# reading local JSON blob might fail
# see e.g. https://github.com/storyscript/sls/issues/195
return None
def update_service_wrapper(self):
"""
Update the in-memory ServiceWrapper and save a snapshot into
the cache_dir.
"""
services = self.hub.fetch_services()
self.hub.reload_services(services)
self.hub.as_json_file(self.hub_path)
def find_services(self, keyword):
for service_name in self.hub.get_all_service_names():
if service_name.startswith(keyword):
try:
service = Service(self.get_service_data(service_name))
service.set_name(service_name)
yield service
except BaseException:
# ignore all invalid services
log.warning("Service '%s' has an invalid config",
service_name)
def get_service_data(self, service_name):
return self.hub.get(alias=service_name)
class ArchivedStoryscriptHub:
update_thread = None
retry_lock = Lock()
update_lock = Lock()
ttl_cache_for_services = TTLCache(maxsize=128, ttl=1 * 60)
ttl_cache_for_service_names = TTLCache(maxsize=1, ttl=1 * 60)
@staticmethod
def get_cache_dir():
return user_cache_dir("storyscript", "hub-sdk")
def __init__(self, db_path: str = None, auto_update: bool = True):
"""
StoryscriptHub - a utility to access Storyscript's hub service data.
:param db_path: The path for the database caching file
:param auto_update: Will automatically pull services from the hub
every 30 seconds
"""
if db_path is None:
db_path = StoryscriptHub.get_cache_dir()
os.makedirs(db_path, exist_ok=True)
self.db_path = db_path
# We are not updating cache over here since, it makes
# startup slow. If need arises in case of missing service
# we will update the cache. For long running service
# cache will be updated automatically as well via the
# AutoUpdateThread.
self._service_wrapper = ServiceWrapper()
if auto_update:
self.update_thread = AutoUpdateThread(
update_function=self.update_cache)
@cached(cache=ttl_cache_for_service_names)
def get_all_service_names(self) -> [str]:
"""
Get all service names and aliases from the database.
:return: An array of strings, which might look like:
["hello", "universe/hello"]
"""
services = []
with Database(self.db_path):
for s in Service.select(Service.name, Service.alias,
Service.username):
if s.alias:
services.append(s.alias)
services.append(f"{s.username}/{s.name}")
return services
@cached(cache=ttl_cache_for_services)
def get(self,
alias=None,
owner=None,
name=None) -> Union[Service, ServiceData]:
"""
Get a service from the database.
:param alias: Takes precedence when specified over owner/name
:param owner: The owner of the service
:param name: The name of the service
:return: Returns a :py:class:~.sdk.service.ServiceData.ServiceData
object instance.
"""
service = None
service = self._service_wrapper.get(alias=alias,
owner=owner,
name=name)
if service is not None:
return service
if service is None:
service = self._get(alias, owner, name)
if service is None:
# Maybe it's new in the Hub?
with self.retry_lock:
service = self._get(alias, owner, name)
if service is None:
self.update_cache()
service = self._get(alias, owner, name)
if service is not None:
# ensures test don't break
if isinstance(service, MagicMock):
return service
assert isinstance(service, Service)
# we can safely convert this object since it was probably loaded
# from the cache
return ServiceData.from_dict(
data={"service_data": json.loads(service.raw_data)})
return service
def _get(self, alias: str = None, owner: str = None, name: str = None):
try:
if alias is not None and alias.count("/") == 1:
owner, name = alias.split("/")
alias = None
with Database(self.db_path):
if alias:
service = Service.select().where(Service.alias == alias)
else:
service = Service.select().where(
(Service.username == owner) & (Service.name == name))
return service.get()
except DoesNotExist:
return None
def update_cache(self):
services = GraphQL.get_all()
# tell the service wrapper to reload any services from the cache.
if self._service_wrapper is not None:
self._service_wrapper.reload_services(services)
with Database(self.db_path) as db:
with db.atomic(lock_type="IMMEDIATE"):
Service.delete().execute()
for service in services:
Service.create(
service_uuid=service["serviceUuid"],
name=service["service"]["name"],
alias=service["service"]["alias"],
username=service["service"]["owner"]["username"],
description=service["service"]["description"],
certified=service["service"]["isCertified"],
public=service["service"]["public"],
topics=json.dumps(service["service"]["topics"]),
state=service["state"],
configuration=json.dumps(service["configuration"]),
readme=service["readme"],
raw_data=json.dumps(service),
)
with self.update_lock:
self.ttl_cache_for_service_names.clear()
self.ttl_cache_for_services.clear()
return True
class StoryscriptHub:
update_thread = None
retry_lock = Lock()
update_lock = Lock()
ttl_cache_for_services = TTLCache(maxsize=128, ttl=1 * 60)
ttl_cache_for_service_names = TTLCache(maxsize=1, ttl=1 * 60)
@staticmethod
def get_config_dir(app):
if sys.platform == 'win32':
p = os.getenv('APPDATA')
else:
p = os.getenv('XDG_DATA_HOME', os.path.expanduser('~/'))
return os.path.join(p, app)
def __init__(self, db_path: str = None, auto_update: bool = True,
service_wrapper=False):
"""
StoryscriptHub - a utility to access Storyscript's hub service data.
:param db_path: The path for the database caching file
:param auto_update: Will automatically pull services from the hub
every 30 seconds
:param service_wrapper: Allows you to utilize safe ServiceData objects
"""
if db_path is None:
db_path = StoryscriptHub.get_config_dir('.storyscript')
os.makedirs(db_path, exist_ok=True)
self.db_path = db_path
self._service_wrapper = None
if service_wrapper:
self._service_wrapper = ServiceWrapper()
# we need to update the cache immediately for the
# service wrapper to initialize data.
self.update_cache()
if auto_update:
self.update_thread = AutoUpdateThread(
update_function=self.update_cache)
@cached(cache=ttl_cache_for_service_names)
def get_all_service_names(self) -> [str]:
"""
Get all service names and aliases from the database.
:return: An array of strings, which might look like:
["hello", "universe/hello"]
"""
services = []
with Database(self.db_path):
for s in Service.select(Service.name, Service.alias,
Service.username):
if s.alias:
services.append(s.alias)
services.append(f'{s.username}/{s.name}')
return services
@cached(cache=ttl_cache_for_services)
def get(self, alias=None, owner=None, name=None,
wrap_service=False) -> Union[Service, ServiceData]:
"""
Get a service from the database.
:param alias: Takes precedence when specified over owner/name
:param owner: The owner of the service
:param name: The name of the service
:param wrap_service: When set to true, it will return a
@ServiceData object
:return: Returns a Service instance, with all fields populated
"""
service = None
# check if the service_wrapper was initialized for automatic
# wrapping
if self._service_wrapper is not None:
service = self._service_wrapper.get(alias=alias, owner=owner,
name=name)
if service is None:
service = self._get(alias, owner, name)
if service is None:
# Maybe it's new in the Hub?
with self.retry_lock:
service = self._get(alias, owner, name)
if service is None:
self.update_cache()
service = self._get(alias, owner, name)
if service is not None:
# ensures test don't break
if isinstance(service, MagicMock):
return service
assert isinstance(service, Service) or \
isinstance(service, ServiceData)
# if the service wrapper is set, and the service doesn't exist
# we can safely convert this object since it was probably loaded
# from the cache
if wrap_service or self._service_wrapper is not None:
return ServiceData.from_dict(data={
"service_data": json.loads(service.raw_data)
})
if service.topics is not None:
service.topics = json.loads(service.topics)
if service.configuration is not None:
service.configuration = json.loads(service.configuration)
return service
def _get(self, alias: str = None, owner: str = None, name: str = None):
try:
if alias is not None and alias.count("/") == 1:
owner, name = alias.split("/")
alias = None
with Database(self.db_path):
if alias:
service = Service.select().where(Service.alias == alias)
else:
service = Service.select().where(
(Service.username == owner) & (Service.name == name))
return service.get()
except DoesNotExist:
return None
def update_cache(self):
services = GraphQL.get_all()
# tell the service wrapper to reload any services from the cache.
if self._service_wrapper is not None:
self._service_wrapper.reload_services(services)
with Database(self.db_path) as db:
with db.atomic(lock_type='IMMEDIATE'):
Service.delete().execute()
for service in services:
Service.create(
service_uuid=service['serviceUuid'],
name=service['service']['name'],
alias=service['service']['alias'],
username=service['service']['owner']['username'],
description=service['service']['description'],
certified=service['service']['isCertified'],
public=service['service']['public'],
topics=json.dumps(service['service']['topics']),
state=service['state'],
configuration=json.dumps(service['configuration']),
readme=service['readme'],
raw_data=json.dumps(service))
with self.update_lock:
self.ttl_cache_for_service_names.clear()
self.ttl_cache_for_services.clear()
return True