def putter(self, put, put_queue, stat_queue, return_dict):
self.pid=current_process().pid
log = logging.getLogger(os.path.basename(sys.argv[0]))
connection, bucket = None, None
file_object_cache = FileObjectCache()
while True:
args = put_queue.get()
if args is None:
put_queue.task_done()
break
#pp(args.fpath)
value_kwargs ={'path': args.fpath}
key_name=None
value_kwargs.update(dict(opt=self.opt))
value = Value(file_object_cache, **value_kwargs)
try:
content = value.get_content()
total_ins = self.insert_content(content) if not self.opt.dry_run else 0
log.debug('Source row count = %d' %args.source_cnt)
log.debug('0'*100)
args.total_ins=total_ins
return_dict[value.path] = total_ins
stat_queue.put(dict(key_name=key_name, key_stats=dict(size=value.get_size(),source_cnt = args.source_cnt, total_inserted=total_ins, filename=value.filename, path=value.path)))
except IOError as exc:
log.error('%s (%s)' % (value.path, exc))
put_queue.task_done()
