def __call__(self, t_path, name_info, i_str): ''' Load chunk from t_path and put it into the right place in s3 using the output_name template from the config ''' name_info.update( get_name_info(t_path, i_str=i_str) ) if name_info['num'] == 0: o_path = None return o_path o_fname = self.config['output_name'] % name_info o_path = os.path.join(self.config['s3_path_prefix'], o_fname + '.tar.gz') logger.info('to_s3_tarballs: \n\t%r\n\tfrom: %r\n\tby way of %r ' % (o_path, i_str, t_path)) ## forcibly collect dereferenced objects #gc.collect() t_path2 = tarball_export(t_path, name_info) data = open(t_path2).read() name_info['md5'] = hashlib.md5(data).hexdigest() # pylint: disable=E1101 self.upload(o_path, data, name_info) self.cleanup(t_path) self.cleanup(t_path2) logger.info('to_s3_tarballs finished:\n\t input: %s\n\toutput: %s' % (i_str, o_path)) ## return the final output path return o_path
def __call__(self, t_path, name_info, i_str): ''' Load chunk from t_path and put it into the right place in s3 using the output_name template from the config ''' name_info.update(get_name_info(t_path, i_str=i_str)) if name_info['num'] == 0: o_path = None return o_path o_fname = self.config['output_name'] % name_info o_path = os.path.join(self.config['s3_path_prefix'], o_fname + '.tar.gz') logger.info('to_s3_tarballs: \n\t%r\n\tfrom: %r\n\tby way of %r ' % (o_path, i_str, t_path)) ## forcibly collect dereferenced objects #gc.collect() t_path2 = tarball_export(t_path, name_info) data = open(t_path2).read() name_info['md5'] = hashlib.md5(data).hexdigest() # pylint: disable=E1101 self.upload(o_path, data, name_info) self.cleanup(t_path) self.cleanup(t_path2) logger.info('to_s3_tarballs finished:\n\t input: %s\n\toutput: %s' % (i_str, o_path)) ## return the final output path return o_path
def __call__(self, t_path, name_info, i_str): name_info.update( get_name_info( t_path, i_str=i_str ) ) if name_info['num'] == 0: return None o_fname = self.config['output_name'] % name_info o_dir = self.config['output_path'] o_path = os.path.join(o_dir, o_fname + '.tar.gz') ## if dir is missing make it dirname = os.path.dirname(o_path) if dirname and not os.path.exists(dirname): os.makedirs(dirname) t_path2 = tarball_export(self.config, t_path, name_info) ## do an atomic renaming try: logger.debug('attemping os.rename(%r, %r)' % (t_path2, o_path)) os.rename(t_path2, o_path) except OSError, exc: if exc.errno==18: patient_move(t_path2, o_path) else: msg = 'failed shutil.copy2(%r, %r) and/or os.remove(t_path)\n%s'\ % (t_path2, o_path, traceback.format_exc(exc)) logger.critical(traceback.format_exc(exc)) raise