async def put(self, uuid):
request_data = tornado.escape.json_decode(self.request.body)
params = request_data['dataset']['params']
params["type"] = ds_type = params["type"].upper()
cls = get_dataset_cls(ds_type)
ConverterCls = cls.get_msg_converter()
converter = ConverterCls()
try:
dataset_params = converter.to_python(params)
executor = self.data.get_executor()
ds = await load(filetype=cls,
executor=executor,
enable_async=True,
**dataset_params)
await executor.run_function(ds.check_valid)
self.data.register_dataset(
uuid=uuid,
dataset=ds,
params=request_data['dataset'],
)
details = await self.data.serialize_dataset(dataset_id=uuid)
msg = Message(self.data).create_dataset(dataset=uuid,
details=details)
log_message(msg)
self.write(msg)
self.event_registry.broadcast_event(msg)
except Exception as e:
if self.data.has_dataset(uuid):
await self.data.remove_dataset(uuid)
msg = Message(self.data).create_dataset_error(uuid, str(e))
log_message(msg, exception=True)
self.write(msg)
return
async def put(self, uuid):
request_data = tornado.escape.json_decode(self.request.body)
params = request_data['dataset']['params']
params["type"] = ds_type = params["type"].lower()
cls = get_dataset_cls(ds_type)
ConverterCls = cls.get_msg_converter()
converter = ConverterCls()
dataset_params = converter.to_python(params)
try:
executor = self.data.get_executor()
ds = await executor.run_function(load,
filetype=cls,
**dataset_params)
ds = await executor.run_function(ds.initialize)
available_workers = await executor.get_available_workers()
ds.set_num_cores(len(available_workers))
await executor.run_function(ds.check_valid)
self.data.register_dataset(
uuid=uuid,
dataset=ds,
params=request_data['dataset'],
)
details = await self.data.serialize_dataset(dataset_id=uuid)
msg = Message(self.data).create_dataset(dataset=uuid,
details=details)
log_message(msg)
self.write(msg)
self.event_registry.broadcast_event(msg)
except Exception as e:
msg = Message(self.data).create_dataset_error(uuid, str(e))
log_message(msg, exception=True)
self.write(msg)
return
def get(self):
try:
ds_type = self.request.arguments['type'][0].decode("utf8")
cls = get_dataset_cls(ds_type)
ConverterCls = cls.get_msg_converter()
converter = ConverterCls()
schema = converter.SCHEMA
msg = Message(self.data).dataset_schema(ds_type, schema)
log_message(msg)
self.write(msg)
except Exception as e:
msg = Message(self.data).dataset_schema_failed(ds_type, str(e))
log_message(msg, exception=True)
self.write(msg)
return
async def put(self, uuid):
request_data: Dict = tornado.escape.json_decode(self.request.body)
params = request_data['dataset']['params']
params["type"] = ds_type = params["type"].upper()
cls = get_dataset_cls(ds_type)
ConverterCls = cls.get_msg_converter()
converter = ConverterCls()
try:
dataset_params = converter.to_python(params)
executor = self.state.executor_state.get_executor()
ds = await load(filetype=cls,
executor=executor,
enable_async=True,
**dataset_params)
await self.prime_numba_caches(ds)
self.dataset_state.register(
uuid=uuid,
dataset=ds,
params=request_data['dataset'],
converted=dataset_params,
)
details = await self.dataset_state.serialize(dataset_id=uuid)
msg = Message(self.state).create_dataset(dataset=uuid,
details=details)
log_message(msg)
self.write(msg)
self.event_registry.broadcast_event(msg)
except Exception as e:
if uuid in self.dataset_state:
await self.dataset_state.remove(uuid)
msg = Message(self.state).create_dataset_error(uuid, str(e))
log_message(msg, exception=True)
self.write(msg)
return