def drop_udf_by_name(self, name: str):
"""Drop a udf entry from the catalog udfmetadata
Arguments:
name (str): udf name to be deleted
Returns:
True if successfully deleted else True
"""
try:
udf_record = self.udf_by_name(name)
udf_record.delete()
except Exception:
logger.exception("Delete udf failed for name {}".format(name))
return False
return True
def connection_lost(self, exc, exc2=None):
logger.debug("[ " + str(self.id) + " ]" + " Disconnected from server")
try:
self.transport.abort() # free sockets early, free sockets often
self.transport = None
except Exception as e:
logger.exception(e)
exc2 = e
finally:
if exc or exc2:
EvaClient.__errors__ += 1
self.done.set_exception(exc or exc2)
self.done.exception() # remove _tb_logger
else:
self.done.set_result(None)
def petastorm_type_cast(schema: Unischema, df: pd.DataFrame) \
-> pd.DataFrame:
"""
Try to cast the type if schema defined in UnischemeField for
Petastorm is not consistent with panda DataFrame provided.
"""
for unischema in schema.fields.values():
if not isinstance(unischema.codec, NdarrayCodec):
continue
# We only care when the cell data is np.ndarray
col = unischema.name
dtype = unischema.numpy_dtype
try:
df[col] = df[col].apply(lambda x: x.astype(dtype, copy=False))
except Exception:
logger.exception('Failed to cast %s to %s for Petastorm' %
(col, dtype))
return df
def exec(self):
"""Drop table executor"""
catalog_manager = CatalogManager()
if len(self.node.table_refs) > 1:
logger.exception('Drop supports only single table')
table_ref = self.node.table_refs[0]
if not catalog_manager.check_table_exists(
table_ref.table.database_name, table_ref.table.table_name):
err_msg = "Table: {} does not exsits".format(table_ref)
if self.node.if_exists:
logger.warn(err_msg)
else:
logger.exception(err_msg)
if table_ref.table.table_obj.is_video:
VideoStorageEngine.drop(table=table_ref.table.table_obj)
else:
StorageEngine.drop(table=table_ref.table.table_obj)
success = catalog_manager.drop_dataset_metadata(
table_ref.table.database_name, table_ref.table.table_name)
if not success:
err_msg = "Failed to drop {}".format(table_ref)
logger.exception(err_msg)
yield Batch(
pd.DataFrame(
{
"Table Successfully dropped: {}".format(
table_ref.table.table_name)
},
index=[0],
))
def drop(self, table: DataFrameMetadata):
dir_path = Path(table.file_url)
try:
shutil.rmtree(str(dir_path))
except Exception as e:
logger.exception(f'Failed to drop the video table {e}')