def __exit__(self, *args, **kwargs):
if self.write_status:
HDFStore.__exit__(self, *args, **kwargs)
os.close(self._flock)
h5_size = os.path.getsize(self.fname) / 1000 / 1000
new_limit = ((h5_size / self.big_H5_Size_limit + 1) *
self.big_H5_Size_limit)
# global Compress_Count
# if Compress_Count == 1 and self.h5_size_org > self.big_H5_Size_limit:
# cct.get_config_value(self.config_ini,self.fname_o,h5_size,new_limit)
# log.info("Compress_Count init:%s h5_size_org:%s" % (Compress_Count, self.h5_size_org))
log.info("fname:%s h5_size:%s big:%s" %
(self.fname, h5_size, self.big_H5_Size_limit))
# if h5_size >= self.big_H5_Size_limit:
if cct.get_config_value(self.config_ini, self.fname_o, h5_size,
new_limit):
time_pt = time.time()
if os.path.exists(self.fname) and os.path.exists(
self.temp_file):
log.error("remove tmpfile is exists:%s" % (self.temp_file))
os.remove(self.temp_file)
os.rename(self.fname, self.temp_file)
if cct.get_os_system() == 'mac':
p = subprocess.Popen(
self.ptrepack_cmds %
(self.complib, self.temp_file, self.fname),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
else:
back_path = os.getcwd()
os.chdir(self.basedir)
log.info('current path is: %s after change dir' %
os.getcwd())
pt_cmd = self.ptrepack_cmds % (
self.complib, self.temp_file.split(self.basedir)[1],
self.fname.split(self.basedir)[1])
p = subprocess.Popen(pt_cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
os.chdir(back_path)
p.wait()
# ptrepack --chunkshape=auto --complevel=9 --complib=zlib tdx_all_df_300.h5_tmp tdx_all_df_300.h5
if p.returncode != 0:
# log.error("ptrepack hdf Error:%s for tmp_file:%s Er:%s" % (self.fname,self.temp_file,p.stderr))
log.error("ptrepack hdf Error:%s tofile:%s Er:%s" %
(self.temp_file, self.fname,
p.stdout.read().decode("gbk")))
# return -1
else:
if os.path.exists(self.temp_file):
os.remove(self.temp_file)
# log.error("fname:%s h5_size:%sM Limit:%s t:%.1f" % (self.fname, h5_size, new_limit , time_pt - time.time()))
if os.path.exists(self._lock):
os.remove(self._lock)
def __exit__(self, *args, **kwargs):
if self.write_status:
HDFStore.__exit__(self, *args, **kwargs)
os.close(self._flock)
h5_size = os.path.getsize(self.fname) / 1000 / 1000
if h5_size > self.big_H5_Size:
log.info("h5_size:%sM" % (h5_size))
os.rename(self.fname, self.temp_file)
p = subprocess.Popen(self.ptrepack_cmds %
(self.temp_file, self.fname),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
p.wait()
if p.returncode != 0:
log.error("ptrepack hdf Error.:%s" % (self.fname))
# return -1
else:
os.remove(self.temp_file)
os.remove(self._lock)
gc.collect()
# !/usr/bin/env python
def __exit__(self, *args, **kwargs):
logger.debug("Exit SafeHDFStore")
HDFStore.__exit__(self, *args, **kwargs)
os.close(self._flock)
os.remove(self._lock)
def __exit__(self, *args, **kwargs):
HDFStore.__exit__(self, *args, **kwargs)
os.close(self._flock)
os.remove(self._lock)
def __exit__(self, *args, **kwargs):
HDFStore.__exit__(self, *args, **kwargs)
if IS_PARALLEL:
os.close(self._flock)
os.remove(self._lock)
def __exit__(self, *args, **kwargs):
HDFStore.__exit__(self, *args, **kwargs)
os.close(self._flock)
os.remove(self._lock)
