def set_size(self, size=None, scale_factor=1.0):
"""
Set data size of this node
"""
if scale_factor != 1.0:
logger.warning(
"Trying to set scale factor for ShuffleNode, ignore")
self._size = self.__from_node.size()
return super(LogicalPlan.ShuffleNode,
self).set_size(size, scale_factor)
def _force_delete_file(self, path):
toft_path = self._toft_path(path)
if toft_path.startswith('/hdfs/'):
self._hadoop_client().fs_rmr(path, self._hadoop_config)
return True
else:
try:
logger.debug('rmtree %s' % path)
shutil.rmtree(path)
except Exception as e:
logger.warning('%s' % e)
return not os.path.exists(path)
def _force_delete_file(self, path):
toft_path = self._toft_path(path)
if toft_path.startswith('/hdfs/'):
self._hadoop_client().fs_rmr(path, self._hadoop_config)
return True
else:
try:
logger.debug('rmtree %s' % path)
shutil.rmtree(path)
except Exception as e:
logger.warning('%s' % e)
return not os.path.exists(path)
def set_size(self, size=None, scale_factor=1.0):
"""
Set data size of this node
"""
logger.warning("Trying to set size for UnionNode, ignore")
return self