def __call__(self, t_path, name_info, i_str):
'''
Load chunk from t_path and put it into the right place in s3
using the output_name template from the config
'''
name_info.update( get_name_info(t_path, i_str=i_str) )
if name_info['num'] == 0:
o_path = None
return o_path
o_fname = self.config['output_name'] % name_info
o_path = os.path.join(self.config['s3_path_prefix'], o_fname + '.tar.gz')
logger.info('to_s3_tarballs: \n\t%r\n\tfrom: %r\n\tby way of %r ' % (o_path, i_str, t_path))
## forcibly collect dereferenced objects
#gc.collect()
t_path2 = tarball_export(t_path, name_info)
data = open(t_path2).read()
name_info['md5'] = hashlib.md5(data).hexdigest() # pylint: disable=E1101
self.upload(o_path, data, name_info)
self.cleanup(t_path)
self.cleanup(t_path2)
logger.info('to_s3_tarballs finished:\n\t input: %s\n\toutput: %s' % (i_str, o_path))
## return the final output path
return o_path
def __call__(self, t_path, name_info, i_str):
'''
Load chunk from t_path and put it into the right place in s3
using the output_name template from the config
'''
name_info.update(get_name_info(t_path, i_str=i_str))
if name_info['num'] == 0:
o_path = None
return o_path
o_fname = self.config['output_name'] % name_info
o_path = os.path.join(self.config['s3_path_prefix'],
o_fname + '.tar.gz')
logger.info('to_s3_tarballs: \n\t%r\n\tfrom: %r\n\tby way of %r ' %
(o_path, i_str, t_path))
## forcibly collect dereferenced objects
#gc.collect()
t_path2 = tarball_export(t_path, name_info)
data = open(t_path2).read()
name_info['md5'] = hashlib.md5(data).hexdigest() # pylint: disable=E1101
self.upload(o_path, data, name_info)
self.cleanup(t_path)
self.cleanup(t_path2)
logger.info('to_s3_tarballs finished:\n\t input: %s\n\toutput: %s' %
(i_str, o_path))
## return the final output path
return o_path
def __call__(self, t_path, name_info, i_str):
name_info.update( get_name_info( t_path, i_str=i_str ) )
if name_info['num'] == 0:
return None
o_fname = self.config['output_name'] % name_info
o_dir = self.config['output_path']
o_path = os.path.join(o_dir, o_fname + '.tar.gz')
## if dir is missing make it
dirname = os.path.dirname(o_path)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
t_path2 = tarball_export(self.config, t_path, name_info)
## do an atomic renaming
try:
logger.debug('attemping os.rename(%r, %r)' % (t_path2, o_path))
os.rename(t_path2, o_path)
except OSError, exc:
if exc.errno==18:
patient_move(t_path2, o_path)
else:
msg = 'failed shutil.copy2(%r, %r) and/or os.remove(t_path)\n%s'\
% (t_path2, o_path, traceback.format_exc(exc))
logger.critical(traceback.format_exc(exc))
raise
