def __init__(self, db_path: str = None, auto_update: bool = True,
service_wrapper=False):
"""
StoryscriptHub - a utility to access Storyscript's hub service data.
:param db_path: The path for the database caching file
:param auto_update: Will automatically pull services from the hub
every 30 seconds
:param service_wrapper: Allows you to utilize safe ServiceData objects
"""
if db_path is None:
db_path = StoryscriptHub.get_config_dir('.storyscript')
os.makedirs(db_path, exist_ok=True)
self.db_path = db_path
self._service_wrapper = None
if service_wrapper:
self._service_wrapper = ServiceWrapper()
# we need to update the cache immediately for the
# service wrapper to initialize data.
self.update_cache()
if auto_update:
self.update_thread = AutoUpdateThread(
update_function=self.update_cache)
def __init__(self, db_path: str = None, auto_update: bool = True):
"""
StoryscriptHub - a utility to access Storyscript's hub service data.
:param db_path: The path for the database caching file
:param auto_update: Will automatically pull services from the hub
every 30 seconds
"""
if db_path is None:
db_path = StoryscriptHub.get_cache_dir()
os.makedirs(db_path, exist_ok=True)
self.db_path = db_path
# We are not updating cache over here since, it makes
# startup slow. If need arises in case of missing service
# we will update the cache. For long running service
# cache will be updated automatically as well via the
# AutoUpdateThread.
self._service_wrapper = ServiceWrapper()
if auto_update:
self.update_thread = AutoUpdateThread(
update_function=self.update_cache)
def test_reload_services_with_list_dict(mocker):
expected_service_datas = ["microservice/not_python", "npython"]
mocker.patch.object(GraphQL, "get_all", return_value=service_data_fixture)
hub = ServiceWrapper([not_python_fixture])
assert hub.get_all_service_names() == expected_service_datas
def test_dynamic_loading_with_deserialization(mocker):
expected_service_datas = ['microservice/not_python', 'npython', 'test/helloworld', 'hello', 'microservice/hashes']
mocker.patch.object(GraphQL, 'get_all', return_value=service_data_fixture)
hub = ServiceWrapper([not_python_fixture,
"hello",
"microservice/hashes"
])
assert hub.get_all_service_names() == expected_service_datas
def test_dynamic_loading_with_list_service_names(mocker):
expected_service_datas = [
"test/helloworld",
"hello",
"microservice/hashes",
]
mocker.patch.object(GraphQL, "get_all", return_value=service_data_fixture)
hub = ServiceWrapper(["hello", "microservice/hashes"])
assert hub.get_all_service_names() == expected_service_datas
def test_serialization(mocker):
expected_service_datas = [
"test/helloworld",
"hello",
"microservice/hashes",
"microservice/not_python",
"npython",
]
mocker.patch.object(GraphQL, "get_all", return_value=service_data_fixture)
hub = ServiceWrapper(["hello", "microservice/hashes"])
hub.update_service(not_python_fixture, hub.services)
temp_file = tempfile.mktemp(suffix=".json")
hub.as_json_file(temp_file)
assert hub.get_all_service_names() == expected_service_datas
test_hub = ServiceWrapper.from_json_file(path=temp_file)
assert test_hub.get_all_service_names() == expected_service_datas
os.remove(temp_file)
def load_services_from_file(cls):
"""
Loads services from files into the _fixed_service_wrapper
each time it is called.
"""
cls._fixed_service_wrapper = ServiceWrapper.from_json_file(
hub_fixtures_file)
def test_deserialization_from_file(mocker):
expected_service_datas = [
"microservice/python",
"python",
"microservice/hashes",
"storyscript/http",
"http",
"test/helloworld",
"hello",
]
temp_file = tempfile.mktemp(suffix=".json")
with open(temp_file, "w") as outfile:
json.dump(service_data_fixture, outfile)
hub = ServiceWrapper.from_json_file(path=temp_file)
mocker.patch.object(ServiceData, "from_dict")
assert hub.get_all_service_names() == expected_service_datas
assert hub.get("python") is not None
ServiceData.from_dict.assert_called_with(
data={"service_data": service_data_fixture[0]})
os.remove(path=temp_file)
def test_init_hub_path(patch):
"""
Tests that an SLS App with a Hub Path gets properly initialized.
"""
patch.object(ServiceWrapper, "from_json_file")
hub_path = ".hub.path."
app = App(hub_path=hub_path)
ServiceWrapper.from_json_file.assert_called_with(hub_path)
assert app.hub is ServiceWrapper.from_json_file()
def test_api_loads_with_hub():
"""
Ensures Api.load functions return errors
"""
hub = ServiceWrapper(services=None)
story = Api.loads("http server", backend="semantic", hub=hub)
e = story.errors()[0]
assert (e.short_message() ==
"E0139: Service `http` does not exist on the hub.")
def __init__(self, hub=None):
if hub is not None:
self.hub = hub
return
cache_dir = get_cache_dir()
self.hub_path = path.join(cache_dir, "hub.json")
if path.exists(self.hub_path):
hub = self.read_hub_from_json()
# local JSON storage doesn't exist or reading it failed
if hub is None:
makedirs(cache_dir, exist_ok=True)
self.hub = ServiceWrapper()
self.update_service_wrapper()
self.hub = hub
self.update_thread = AutoUpdateThread(self.update_service_wrapper,
initial_update=False)
def test_deserialization_from_json(mocker):
expected_service_datas = ["microservice/python", "python"]
jsonstr = json.dumps([service_data_fixture[0]])
hub = ServiceWrapper.from_json(jsonstr)
assert hub.get_all_service_names() == expected_service_datas
mocker.patch.object(ServiceData, "from_dict")
assert hub.get("python") is not None
ServiceData.from_dict.assert_called_with(
data={"service_data": service_data_fixture[0]})
def test_init_hub_path(patch):
"""
Tests that an SLS App with a Hub Path gets properly initialized.
"""
patch.init(Workspace)
patch.object(ServiceWrapper,
"from_json_file",
return_value="ConstServiceHub")
hub_path = ".hub.path."
app = App(hub_path=hub_path)
ServiceWrapper.from_json_file.assert_called_with(hub_path)
Workspace.__init__.assert_called_with(".root.",
hub=ServiceWrapper.from_json_file())
assert isinstance(app.ws, Workspace)
assert app.hub == "ConstServiceHub"
def read_hub_from_json(self):
"""
Try loading an existing service JSON blob from storage.
Returns an initialized ServiceWrapper if successful, None otherwise.
"""
try:
return ServiceWrapper.from_json_file(self.hub_path)
except JSONDecodeError:
# local JSON blob might be invalid
# see e.g. https://github.com/storyscript/sls/issues/191
return None
except OSError:
# reading local JSON blob might fail
# see e.g. https://github.com/storyscript/sls/issues/195
return None
def test_deserialization_from_file(mocker):
expected_service_datas = ['microservice/python', 'python', 'microservice/hashes', 'storyscript/http', 'http', 'test/helloworld', 'hello']
temp_file = tempfile.mktemp(suffix=".json")
with open(temp_file, 'w') as outfile:
json.dump(service_data_fixture, outfile)
hub = ServiceWrapper.from_json_file(path=temp_file)
mocker.patch.object(ServiceData, 'from_dict')
assert hub.get_all_service_names() == expected_service_datas
assert hub.get('python') is not None
ServiceData.from_dict.assert_called_with(data={"service_data": service_data_fixture[0]})
os.remove(path=temp_file)
def test_serialization(mocker):
expected_service_datas = ['microservice/not_python', 'npython', 'test/helloworld', 'hello', 'microservice/hashes']
mocker.patch.object(GraphQL, 'get_all', return_value=service_data_fixture)
hub = ServiceWrapper([not_python_fixture,
"hello",
"microservice/hashes"
])
temp_file = tempfile.mktemp(suffix=".json")
hub.as_json_file(temp_file)
assert hub.get_all_service_names() == expected_service_datas
test_hub = ServiceWrapper.from_json_file(path=temp_file)
assert test_hub.get_all_service_names() == expected_service_datas
os.remove(temp_file)
def __init__(self, hub=None, hub_path=None):
self.language_server = LanguageServer
if isinstance(hub_path, str):
hub = ServiceWrapper.from_json_file(hub_path)
self.hub = hub
self.ws = Workspace(".root.", hub=hub)
def update_hub_fixture():
fixture_dir = path.dirname(path.realpath(__file__))
out_file = path.join(fixture_dir, "hub.fixed.json")
services = StoryscriptHub().get_all_service_names()
ServiceWrapper(services).as_json_file(out_file)
def update_hub_fixture():
fixture_dir = path.dirname(path.realpath(__file__))
out_file = path.join(fixture_dir, "hub.fixed.json")
ServiceWrapper(subset).as_json_file(out_file)
class StoryscriptHub:
update_thread = None
retry_lock = Lock()
update_lock = Lock()
ttl_cache_for_services = TTLCache(maxsize=128, ttl=1 * 60)
ttl_cache_for_service_names = TTLCache(maxsize=1, ttl=1 * 60)
@staticmethod
def get_config_dir(app):
if sys.platform == 'win32':
p = os.getenv('APPDATA')
else:
p = os.getenv('XDG_DATA_HOME', os.path.expanduser('~/'))
return os.path.join(p, app)
def __init__(self, db_path: str = None, auto_update: bool = True,
service_wrapper=False):
"""
StoryscriptHub - a utility to access Storyscript's hub service data.
:param db_path: The path for the database caching file
:param auto_update: Will automatically pull services from the hub
every 30 seconds
:param service_wrapper: Allows you to utilize safe ServiceData objects
"""
if db_path is None:
db_path = StoryscriptHub.get_config_dir('.storyscript')
os.makedirs(db_path, exist_ok=True)
self.db_path = db_path
self._service_wrapper = None
if service_wrapper:
self._service_wrapper = ServiceWrapper()
# we need to update the cache immediately for the
# service wrapper to initialize data.
self.update_cache()
if auto_update:
self.update_thread = AutoUpdateThread(
update_function=self.update_cache)
@cached(cache=ttl_cache_for_service_names)
def get_all_service_names(self) -> [str]:
"""
Get all service names and aliases from the database.
:return: An array of strings, which might look like:
["hello", "universe/hello"]
"""
services = []
with Database(self.db_path):
for s in Service.select(Service.name, Service.alias,
Service.username):
if s.alias:
services.append(s.alias)
services.append(f'{s.username}/{s.name}')
return services
@cached(cache=ttl_cache_for_services)
def get(self, alias=None, owner=None, name=None,
wrap_service=False) -> Union[Service, ServiceData]:
"""
Get a service from the database.
:param alias: Takes precedence when specified over owner/name
:param owner: The owner of the service
:param name: The name of the service
:param wrap_service: When set to true, it will return a
@ServiceData object
:return: Returns a Service instance, with all fields populated
"""
service = None
# check if the service_wrapper was initialized for automatic
# wrapping
if self._service_wrapper is not None:
service = self._service_wrapper.get(alias=alias, owner=owner,
name=name)
if service is None:
service = self._get(alias, owner, name)
if service is None:
# Maybe it's new in the Hub?
with self.retry_lock:
service = self._get(alias, owner, name)
if service is None:
self.update_cache()
service = self._get(alias, owner, name)
if service is not None:
# ensures test don't break
if isinstance(service, MagicMock):
return service
assert isinstance(service, Service) or \
isinstance(service, ServiceData)
# if the service wrapper is set, and the service doesn't exist
# we can safely convert this object since it was probably loaded
# from the cache
if wrap_service or self._service_wrapper is not None:
return ServiceData.from_dict(data={
"service_data": json.loads(service.raw_data)
})
if service.topics is not None:
service.topics = json.loads(service.topics)
if service.configuration is not None:
service.configuration = json.loads(service.configuration)
return service
def _get(self, alias: str = None, owner: str = None, name: str = None):
try:
if alias is not None and alias.count("/") == 1:
owner, name = alias.split("/")
alias = None
with Database(self.db_path):
if alias:
service = Service.select().where(Service.alias == alias)
else:
service = Service.select().where(
(Service.username == owner) & (Service.name == name))
return service.get()
except DoesNotExist:
return None
def update_cache(self):
services = GraphQL.get_all()
# tell the service wrapper to reload any services from the cache.
if self._service_wrapper is not None:
self._service_wrapper.reload_services(services)
with Database(self.db_path) as db:
with db.atomic(lock_type='IMMEDIATE'):
Service.delete().execute()
for service in services:
Service.create(
service_uuid=service['serviceUuid'],
name=service['service']['name'],
alias=service['service']['alias'],
username=service['service']['owner']['username'],
description=service['service']['description'],
certified=service['service']['isCertified'],
public=service['service']['public'],
topics=json.dumps(service['service']['topics']),
state=service['state'],
configuration=json.dumps(service['configuration']),
readme=service['readme'],
raw_data=json.dumps(service))
with self.update_lock:
self.ttl_cache_for_service_names.clear()
self.ttl_cache_for_services.clear()
return True
class ArchivedStoryscriptHub:
update_thread = None
retry_lock = Lock()
update_lock = Lock()
ttl_cache_for_services = TTLCache(maxsize=128, ttl=1 * 60)
ttl_cache_for_service_names = TTLCache(maxsize=1, ttl=1 * 60)
@staticmethod
def get_cache_dir():
return user_cache_dir("storyscript", "hub-sdk")
def __init__(self, db_path: str = None, auto_update: bool = True):
"""
StoryscriptHub - a utility to access Storyscript's hub service data.
:param db_path: The path for the database caching file
:param auto_update: Will automatically pull services from the hub
every 30 seconds
"""
if db_path is None:
db_path = StoryscriptHub.get_cache_dir()
os.makedirs(db_path, exist_ok=True)
self.db_path = db_path
# We are not updating cache over here since, it makes
# startup slow. If need arises in case of missing service
# we will update the cache. For long running service
# cache will be updated automatically as well via the
# AutoUpdateThread.
self._service_wrapper = ServiceWrapper()
if auto_update:
self.update_thread = AutoUpdateThread(
update_function=self.update_cache)
@cached(cache=ttl_cache_for_service_names)
def get_all_service_names(self) -> [str]:
"""
Get all service names and aliases from the database.
:return: An array of strings, which might look like:
["hello", "universe/hello"]
"""
services = []
with Database(self.db_path):
for s in Service.select(Service.name, Service.alias,
Service.username):
if s.alias:
services.append(s.alias)
services.append(f"{s.username}/{s.name}")
return services
@cached(cache=ttl_cache_for_services)
def get(self,
alias=None,
owner=None,
name=None) -> Union[Service, ServiceData]:
"""
Get a service from the database.
:param alias: Takes precedence when specified over owner/name
:param owner: The owner of the service
:param name: The name of the service
:return: Returns a :py:class:~.sdk.service.ServiceData.ServiceData
object instance.
"""
service = None
service = self._service_wrapper.get(alias=alias,
owner=owner,
name=name)
if service is not None:
return service
if service is None:
service = self._get(alias, owner, name)
if service is None:
# Maybe it's new in the Hub?
with self.retry_lock:
service = self._get(alias, owner, name)
if service is None:
self.update_cache()
service = self._get(alias, owner, name)
if service is not None:
# ensures test don't break
if isinstance(service, MagicMock):
return service
assert isinstance(service, Service)
# we can safely convert this object since it was probably loaded
# from the cache
return ServiceData.from_dict(
data={"service_data": json.loads(service.raw_data)})
return service
def _get(self, alias: str = None, owner: str = None, name: str = None):
try:
if alias is not None and alias.count("/") == 1:
owner, name = alias.split("/")
alias = None
with Database(self.db_path):
if alias:
service = Service.select().where(Service.alias == alias)
else:
service = Service.select().where(
(Service.username == owner) & (Service.name == name))
return service.get()
except DoesNotExist:
return None
def update_cache(self):
services = GraphQL.get_all()
# tell the service wrapper to reload any services from the cache.
if self._service_wrapper is not None:
self._service_wrapper.reload_services(services)
with Database(self.db_path) as db:
with db.atomic(lock_type="IMMEDIATE"):
Service.delete().execute()
for service in services:
Service.create(
service_uuid=service["serviceUuid"],
name=service["service"]["name"],
alias=service["service"]["alias"],
username=service["service"]["owner"]["username"],
description=service["service"]["description"],
certified=service["service"]["isCertified"],
public=service["service"]["public"],
topics=json.dumps(service["service"]["topics"]),
state=service["state"],
configuration=json.dumps(service["configuration"]),
readme=service["readme"],
raw_data=json.dumps(service),
)
with self.update_lock:
self.ttl_cache_for_service_names.clear()
self.ttl_cache_for_services.clear()
return True
def update_hub_fixtures(cls, services):
cls._fixed_service_wrapper = ServiceWrapper(services)
cls._fixed_service_wrapper.as_json_file(hub_fixtures_file)
class ServiceHub:
"""
Contains a list of all available Story hub services
"""
def __init__(self, hub=None):
if hub is not None:
self.hub = hub
return
cache_dir = get_cache_dir()
self.hub_path = path.join(cache_dir, "hub.json")
if path.exists(self.hub_path):
hub = self.read_hub_from_json()
# local JSON storage doesn't exist or reading it failed
if hub is None:
makedirs(cache_dir, exist_ok=True)
self.hub = ServiceWrapper()
self.update_service_wrapper()
self.hub = hub
self.update_thread = AutoUpdateThread(self.update_service_wrapper,
initial_update=False)
def read_hub_from_json(self):
"""
Try loading an existing service JSON blob from storage.
Returns an initialized ServiceWrapper if successful, None otherwise.
"""
try:
return ServiceWrapper.from_json_file(self.hub_path)
except JSONDecodeError:
# local JSON blob might be invalid
# see e.g. https://github.com/storyscript/sls/issues/191
return None
except OSError:
# reading local JSON blob might fail
# see e.g. https://github.com/storyscript/sls/issues/195
return None
def update_service_wrapper(self):
"""
Update the in-memory ServiceWrapper and save a snapshot into
the cache_dir.
"""
services = self.hub.fetch_services()
self.hub.reload_services(services)
self.hub.as_json_file(self.hub_path)
def find_services(self, keyword):
for service_name in self.hub.get_all_service_names():
if service_name.startswith(keyword):
try:
service = Service(self.get_service_data(service_name))
service.set_name(service_name)
yield service
except BaseException:
# ignore all invalid services
log.warning("Service '%s' has an invalid config",
service_name)
def get_service_data(self, service_name):
return self.hub.get(alias=service_name)
from glob import glob
from os import path
from storyhub.sdk.ServiceWrapper import ServiceWrapper
test_dir = path.dirname(path.dirname(path.realpath(__file__)))
def find_test_files(relative=False):
"""
Returns all available test files.
"""
files = glob(path.join(test_dir, "**", "*.story"), recursive=True)
if relative:
return list(map(lambda e: path.relpath(e, test_dir), files))
return files
fixture_dir = path.join(test_dir, "..", "fixtures", "hub")
fixture_file = path.join(fixture_dir, "hub.fixed.json")
hub = ServiceWrapper.from_json_file(fixture_file)
def test_deserialization():
service_datas = JsonFixtureHelper.load_fixture("hello_services")
hub = ServiceWrapper.from_dict(service_datas)
assert hub.get_all_service_names() == ["test/helloworld", "hello"]