def generate_certificates(base_dir, client_only=False):
''' Generate client and server CURVE certificate files'''
logger = StratusLogger.getLogger()
keys_dir = os.path.join(base_dir, 'certificates')
if not os.path.exists(keys_dir): os.makedirs(keys_dir)
logger.info(f"Generating ZMQ key and certificate files in {base_dir}")
public_keys_dir = os.path.join(base_dir, 'public_keys')
secret_keys_dir = os.path.join(base_dir, 'private_keys')
# Create directories for certificates, remove old content if necessary
for d in [keys_dir, public_keys_dir, secret_keys_dir]:
if os.path.exists(d):
shutil.rmtree(d)
os.mkdir(d)
# create new keys in certificates dir
server_public_file, server_secret_file = zmq.auth.create_certificates(
keys_dir, "server")
client_public_file, client_secret_file = zmq.auth.create_certificates(
keys_dir, "client")
# move public keys to appropriate directory
for key_file in os.listdir(keys_dir):
if key_file.endswith(".key"):
shutil.move(os.path.join(keys_dir, key_file),
os.path.join(public_keys_dir, '.'))
# move secret keys to appropriate directory
for key_file in os.listdir(keys_dir):
if key_file.endswith(".key_secret"):
shutil.move(os.path.join(keys_dir, key_file),
os.path.join(secret_keys_dir, '.'))
def __init__(self, htype: str, **kwargs):
self.logger = StratusLogger.getLogger()
self.parms = kwargs
self.name = self['name']
self.type: str = htype
htype1 = self.parms.pop("type")
assert htype1 == htype, "Sanity check of Handler type failed: {} vs {}".format(htype1,htype)
def __init__(self, rid: str, cid: str, manager: ResponseManager, **kwargs):
super(RestTask,
self).__init__(rid=rid, cid=cid,
**kwargs) # **{ "rid":rid, "cid":cid, **kwargs }
self.logger = StratusLogger.getLogger()
self.manager: ResponseManager = manager
self._exception = None
def __init__( self, **kwargs ):
self.logger = StratusLogger.getLogger()
self.nodes: Dict[str, DGNode] = {}
self.graph = kwargs.get( "graph", nx.DiGraph( ) )
for node in kwargs.get( "nodes", [] ): self.add( node )
self._allow_multiple_outputs = kwargs.get( "allow_multiple_outputs", False )
self._connected = False
def __init__(self, rid: str, cid: str, wpsRequest: WPSExecution, **kwargs):
TaskHandle.__init__(self, rid=rid, cid=cid, **kwargs)
self.logger = StratusLogger.getLogger()
self.execution: WPSExecution = wpsRequest
self._statMessage = None
self._status = Status.UNKNOWN
self.cacheDir: str = self.createCache(**kwargs)
def __init__( self, _core: StratusCoreBase, **kwargs ):
Thread.__init__( self )
self.logger = StratusLogger.getLogger()
self.core = _core
self.registeredRequests = set()
self.requestQueue = queue.Queue()
self.active_workflows: Dict[str, StratusWorkflow] = {}
self.completed_workflows: Dict[str, StratusWorkflow] = {}
self._active = True
def __init__( self, _context: zmq.Context, _response_port: int, **kwargs ):
super(StratusZMQResponder, self).__init__()
self.logger = StratusLogger.getLogger()
self.context: zmq.Context = _context
self.response_port = _response_port
self.executing_jobs: Dict[str, StratusResponse] = {}
self.status_reports: Dict[str,str] = {}
self.client_address = kwargs.get( "client_address", "*" )
self.getKeyDir( **kwargs )
self.socket: zmq.Socket = self.initSocket()
def __init__(self, core: StratusCore, **kwargs):
StratusServerApp.__init__(self, core, **kwargs)
self.logger = StratusLogger.getLogger()
self.active = True
self.parms = self.getConfigParms('stratus')
self.client_address = self.parms.get("client_address", "*")
self.request_port = self.parms.get("request_port", 4556)
self.response_port = self.parms.get("response_port", 4557)
self.active_handlers = {}
self.getCertDirs()
def __init__(self, core: Optional[StratusCoreBase],
settings: Dict[str, Dict], **kwargs):
self.logger = StratusLogger.getLogger()
self._core = core
self._handlers: Dict[str, StratusFactory] = {}
self._app_handler: StratusFactory = None
self._internal_clients = kwargs.get("internal_clients", True)
self._parms = kwargs
self._constructors: Dict[str, Callable[[], StratusFactory]] = {}
self.configSpec: Dict[str, Dict] = settings
self._init()
def __init__(self, **kwargs):
self.logger = StratusLogger.getLogger()
self.cert_dir = os.path.expanduser(
kwargs.get("certificate_path", "~/.stratus/zmq"))
self.public_keys_dir = os.path.join(self.cert_dir, 'public_keys')
self.secret_keys_dir = os.path.join(self.cert_dir, 'private_keys')
if not (os.path.exists(self.public_keys_dir)
and os.path.exists(self.secret_keys_dir)):
raise Exception(
f"Must copy the contents of the zmq server certificates directory to {self.cert_dir}"
)
def __init__(self, configSpec: Union[str, Dict[str, Dict]], **kwargs):
self._workers: Dict[str, TaskManager] = {}
self._flower = None
self.baseDir = os.path.dirname(__file__)
StratusCore.__init__(self,
configSpec,
internal_clients=False,
**kwargs)
self.parms.update(type="celery")
self.logger = StratusLogger.getLogger()
self.logger.info(f"Starting CeleryCore with parms: {self.parms}")
if self.parm('flower', False): self._startFlower()
def __init__(self, cid: str, host_address: str, **kwargs):
Thread.__init__(self)
self.logger = StratusLogger.getLogger()
self.host_address = host_address
self.active = True
self.cid = cid
self.debug = False
self.setName('STRATUS zeromq client Response Thread')
self.setDaemon(True)
self.poll_freq = kwargs.get("poll_freq", 0.5)
self.timeout = kwargs.get("timeout", 60.0)
self.statusMap: Dict[str, Status] = {}
self.active_requests = set()
def __init__(self, rid: str, cid: str, refs: Dict,
wpsRequest: WPSExecuteRequest, **kwargs):
super(RestTask, self).__init__(rid=rid, cid=cid, **kwargs)
self.logger = StratusLogger.getLogger()
self.statusUrl: str = refs.get("status", None)
self.fileUrl: str = refs.get("file", None)
self.dataUrl: str = refs.get("data", None)
self.dapUrl: str = refs.get("dap", None)
self.wpsRequest: WPSExecuteRequest = wpsRequest
self._exception = None
self._statMessage = None
self._status = Status.UNKNOWN
self.cacheDir: str = self.createCache(**kwargs)
def __init__(self, context: zmq.Context, connector: ConnectionMode,
rid: str, host: str, port: int, status: Status,
cache_dir: str, **kwargs):
Thread.__init__(self)
self.context = context
self._connector = connector
self.logger = StratusLogger.getLogger()
self.host = host
self.port = port
self.requestId = rid
self.active = True
self.mstate = MessageState.RESULT
self.setName('STRATUS zeromq client Response Thread')
self.cached_results: queue.Queue[TaskResult] = queue.Queue()
self.setDaemon(True)
self.cacheDir = os.path.expanduser(cache_dir)
self.log("Created RM, cache dir = " + self.cacheDir)
self._status = status
self._exception = None
def __init__(self):
self.logger = StratusLogger.getLogger()
self.app = connexion.FlaskApp("stratus",
specification_dir='api/',
debug=True)
self.app.add_error_handler(500, self.render_server_error)
self.app.app.register_error_handler(TypeError,
self.render_server_error)
settings = os.environ.get('STRATUS_SETTINGS', self.SETTINGS)
config_file = Config(settings)
flask_parms = config_file.get_map('flask')
flask_parms['SQLALCHEMY_DATABASE_URI'] = flask_parms['DATABASE_URI']
self.app.app.config.update(flask_parms)
self.parms = config_file.get_map('stratus')
api = self.getParameter('API')
handler = self.getParameter('HANDLER')
celery_parms = config_file.get_map('celery')
self.app.app.config.update(celery_parms)
self.celery = self.make_celery(self.app.app)
self.db = SQLAlchemy(self.app.app)
self.app.add_api(api + ".yaml", resolver=StratusResolver(api))
def __init__(self, cid: str, manager: ResponseManager, **kwargs):
super(zmqTask, self).__init__(rid=manager.requestId, cid=cid, **kwargs)
self.logger = StratusLogger.getLogger()
self.manager = manager
def __init__(self, name: str, app: StratusAppBase, **kwargs):
self.logger = StratusLogger.getLogger()
self.parms = kwargs
self.name = name
self.app: StratusAppBase = app
def __init__(self, epaSpec: str):
self.logger = StratusLogger.getLogger()
self._epaSpec = epaSpec
def __init__(self):
Thread.__init__(self)
self._completedProcess = None
self.logger = StratusLogger.getLogger()
def __init__( self, core: StratusCoreBase ):
StratusAppBase.__init__(self, core)
self.logger = StratusLogger.getLogger()
self.parms = self.getConfigParms('stratus')
def __init__(self, configSpec: Union[str,Dict[str,Dict]], **kwargs ):
self.logger = StratusLogger.getLogger()
self.config = self.getSettings( configSpec )
self.parms = self.getConfigParms('stratus')
self.id = UID.randomId(8)
def __init__(self, manager: AsyncResult, **kwargs):
TaskHandle.__init__(self, **kwargs)
self.logger = StratusLogger.getLogger()
self.manager: AsyncResult = manager
self._exception = None
class StratusClient:
__metaclass__ = abc.ABCMeta
logger = StratusLogger.getLogger()
def __init__(self, type: str, **kwargs):
cid = kwargs.get("cid")
self.cid = cid if cid else UID.randomId(6)
self.type: str = type
self.name: str = kwargs.get("name")
self.cache_dir: str = kwargs.get("cache_dir", "~/.edas/cache")
self.parms = {**kwargs, "cid": self.cid, "type": type}
self.priority: float = float(self.parm("priority", "0"))
self.active = False
self._endpointSpecs: List[EndpointSpec] = None
self.clients = {self.cid}
@property
def handle(self):
return f"{self.name}:{self.cid}"
def activate(self):
self.active = True
self.init()
def init(self):
if self._endpointSpecs is None:
endPointData = self.capabilities("epas")
if "error" in endPointData:
raise Exception("Error accessing endpoint data: " +
endPointData["message"])
self.logger.info("EndpointSpecs: " + str(endPointData))
self._endpointSpecs = [
EndpointSpec(epaSpec) for epaSpec in endPointData["epas"]
]
self.activate()
@abc.abstractmethod
@stratusrequest
def request(self,
request: Dict,
inputs: List[TaskResult] = None,
**kwargs) -> TaskHandle:
pass
@abc.abstractmethod
def status(self, **kwargs) -> Status:
pass
@abc.abstractmethod
def capabilities(self, type: str, **kwargs) -> Dict:
pass
def __del__(self):
self.shutdown()
def shutdown(self):
if self.active:
self.active = False
@property
def endpointSpecs(self) -> List[str]:
return [str(eps) for eps in self._endpointSpecs]
def handles(self, epa: str, **kwargs) -> bool:
for endpointSpec in self._endpointSpecs:
if endpointSpec.handles(epa, **kwargs): return True
return False
def __getitem__(self, key: str) -> str:
result = self.parms.get(key, None)
assert result is not None, "Missing required parameter in {}: {}, params: {}\n * {} ".format(
self.__class__.__name__, key, str(self.parms),
"\n * ".join(traceback.format_stack()))
return result
def parm(self, key: str, default: str = None) -> Optional[str]:
return self.parms.get(key, default)
def updateMetadata(self, requestSpec: Dict) -> Dict:
requestDict = dict(requestSpec)
source_client = requestDict.get("cid")
requestDict["rid"] = requestSpec.get("rid", UID.randomId(6))
if source_client: self.clients.add(source_client)
requestDict["cid"] = self.cid
requestDict["clients"] = ",".join(self.clients)
return requestDict
def hasClient(self, cid: str) -> bool:
return cid in self.clients
def __init__(self, result: TaskResult, **kwargs):
TaskHandle.__init__(self, **kwargs)
self.logger = StratusLogger.getLogger()
self.result: TaskResult = result
self._exception = None
def __init__(self, inputs: List[str], outputs: List[str] = None, **kwargs):
self.logger = StratusLogger.getLogger()
self.params: Dict[str, Any] = kwargs
self.id = self.get("id", UID.randomId(6))
self._inputs = inputs
self._outputs = outputs if outputs is not None else [UID.randomId(6)]
def __init__( self, host_address ):
self.logger = StratusLogger.getLogger()
self._host_address = host_address
self.ns = {'wps': "http://www.opengis.net/wps/1.0.0", "ows": "http://www.opengis.net/ows/1.1"}