def start_autoregister(self, local_ip):
"""Start the autoregister procedure to introduce ourselves to the LogicModule."""
self.execution_environment.prepareThread()
logger.info("Start Autoregister with %s local_ip", local_ip)
lm_client = self.execution_environment.get_runtime().ready_clients["@LM"]
success = False
retries = 0
max_retries = Configuration.MAX_RETRY_AUTOREGISTER
sleep_time = Configuration.RETRY_AUTOREGISTER_TIME / 1000
execution_environment_id = self.execution_environment.get_execution_environment_id()
while not success:
try:
storage_location_id = lm_client.autoregister_ee(execution_environment_id,
settings.dataservice_name,
local_ip,
settings.dataservice_port,
LANG_PYTHON)
except Exception as e:
logger.debug("Catched exception of type %s. Message:\n%s", type(e), e)
if retries > max_retries:
logger.warn("Could not create channel, aborting (reraising exception)")
raise
else:
logger.info("Could not create channel, retry #%d of %i in %i seconds", retries, max_retries, sleep_time)
# TODO: Not Very performing, find a better way
time.sleep(sleep_time)
retries += 1
else:
success = True
logger.info("Current DataService autoregistered. Associated StorageLocationID: %s",
storage_location_id)
settings.storage_id = storage_location_id
settings.environment_id = execution_environment_id
# Retrieve the storage_location connection data
storage_location = lm_client.get_storage_location_for_ds(storage_location_id)
logger.debug("StorageLocation data: {name: '%s', hostname: '%s', port: %d}",
storage_location.name,
storage_location.hostname,
storage_location.storageTCPPort)
logger.info("Starting client to StorageLocation {%s} at %s:%d",
storage_location_id, storage_location.hostname, storage_location.storageTCPPort)
storage_client = EEClient(storage_location.hostname, storage_location.storageTCPPort)
# Leave the ready client to the Storage Location globally available
self.execution_environment.get_runtime().ready_clients["@STORAGE"] = storage_client
storage_client.associate_execution_environment(execution_environment_id)
settings.logicmodule_dc_instance_id = lm_client.get_dataclay_id()
logger.verbose("DataclayInstanceID is %s, storing client in cache", settings.logicmodule_dc_instance_id)
self.execution_environment.get_runtime().ready_clients[settings.logicmodule_dc_instance_id] = self.execution_environment.get_runtime().ready_clients["@LM"]
def deploy_class(namespace,
full_name,
source,
imports,
source_deploy_path,
ds_deploy=False):
"""Deploy a class source to the filesystem.
:param namespace: The namespace of the class (first part of package name).
:param full_name: The full name (including package) of the class.
:param source: The Python's source code for the class.
:param source_deploy_path: The root for deployed source code.
:param ds_deploy: If true, that means that the deploy is for a DataService (not client)
"""
_ensure_package(source_deploy_path, ds_deploy)
package, klass = ("%s.%s" % (namespace, full_name)).rsplit('.', 1)
logger.info("Going to deploy class %s in path %s/__init__.py", klass,
package.replace(".", "/"))
current_path = source_deploy_path
for p in package.split("."):
current_path = os.path.join(current_path, p)
_ensure_package(current_path, ds_deploy)
class_path = os.path.join(current_path, "__init__.py")
logger.info("Class destination file: %s", class_path)
if not os.path.exists(class_path):
raise IOError(
"__init__.py file in package %s should have been already initialized"
% package)
else:
# Because we need to insert imports in a nice place, read & write approach
with open(class_path, 'rt') as f:
contents = f.readlines()
contents = contents[:MAGIC_LINE_NUMBER_FOR_IMPORTS] \
+ [imports] \
+ contents[MAGIC_LINE_NUMBER_FOR_IMPORTS:] \
+ [source]
with open(class_path, 'wt') as f:
f.write("".join(contents))
def preface_autoregister(self):
"""Perform a pre-initialization of stuff (prior to the autoregister call)."""
self.execution_environment.prepareThread()
# logger.info("Preface Autoregister")
# Check if there is an explicit IP for autoregistering
local_ip = os.getenv("DATASERVICE_HOST", "")
if not local_ip:
local_ip = socket.gethostbyname(socket.gethostname())
logger.info("Starting client to LogicModule at %s:%d",
settings.logicmodule_host, settings.logicmodule_port)
lm_client = LMClient(settings.logicmodule_host, settings.logicmodule_port)
# Leave the ready client to the LogicModule globally available
self.execution_environment.get_runtime().ready_clients["@LM"] = lm_client
# logger.info("local_ip %s returned", local_ip)
return local_ip
def persist_and_exit(self):
logger.info("Performing exit hook --persisting files")
self.execution_environment.prepareThread()
self.execution_environment.get_runtime().stop_gc()
logger.info("Flushing all objects to disk")
self.execution_environment.get_runtime().flush_all()
logger.info("Stopping runtime")
logger.info("Notifying LM, current EE left")
self.execution_environment.notify_execution_environment_shutdown()
from dataclay.api import finish
finish()
clean_runtime()
def reset_caches(self):
logger.info("Received SIGHUP --proceeding to reset caches")
ClassLoader.cached_metaclass_info.clear()
ClassLoader.cached_metaclasses.clear()
def start(self):
"""Start the dataClay server (Execution Environment).
Keep in mind that the configuration in both dataClay's global ConfigOptions
and the server-specific one called ServerConfigOptions should be accurate.
Furthermore, this function expects that the caller will take care of the
dataClay library initialization.
This function does not return (by itself), so feel free to spawn it inside
a greenlet or a subprocess (typical in testing)
"""
set_defaults()
# Create the deployment folder and add it to the path
try:
os.makedirs(settings.deploy_path_source)
except OSError as e:
if e.errno != 17:
# Not the "File exists" expected error, reraise it
raise
sys.path.insert(1, settings.deploy_path_source)
self.execution_environment = ExecutionEnvironment(settings.dataservice_name)
# 0 or undefined should become None, which becomes a default of 5x number of cores (see futures' docs)
max_workers = Configuration.THREAD_POOL_WORKERS or None
self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=max_workers),
options=(('grpc.max_send_message_length', -1,),
('grpc.max_receive_message_length', -1,),
))
ee = DataServiceEE(self.execution_environment)
from dataclay.communication.grpc.generated.dataservice import dataservice_pb2_grpc as ds
ds.add_DataServiceServicer_to_server(ee, self.server)
address = str(settings.server_listen_addr) + ":" + str(settings.server_listen_port)
logger.info("Starting DataServiceEE on %s", address)
try:
# ToDo: Better way for start server?
self.server.add_insecure_port(address)
self.server.start()
# ToDo: Check that the server is correctly started
# ToDo: -> aka "if the port was in use, fail tremendously and loudly
self.local_ip = self.preface_autoregister()
self.start_autoregister(self.local_ip)
ee.ass_client()
self.running = True
signal.signal(signal.SIGINT, self.exit_gracefully_signal)
signal.signal(signal.SIGTERM, self.exit_gracefully_signal)
logger.info("Started Python Execution environment on %s", address)
# write state file
try:
f = open("state.txt", "w")
f.write("READY")
f.close()
except:
logger.info("State file not writable. Skipping file creation.")
try:
while self.running:
time.sleep(SERVER_TIME_CHECK_SECONDS)
except RuntimeError:
logger.info("Runtime Error")
except:
traceback.print_exc()
logger.info("** Finished Python Execution Environment on %s", address)