def run(self):
files = {}
for d in self.dirs:
files[d] = list(helper.walk_files(os.path.normpath(d), yield_folders = True))
if len(files) > 0:
self.qm.open()
self.logger.info('Queue path ...')
for i in files:
if len(files[i]) > 0:
for path in files[i]:
relpath = os.path.relpath(path, i) # 相对于root的相对路径
el = self.bucket + '::' + i+ '::' + relpath + '::C'
hashcode = helper.calc_el_md5(i, relpath, self.bucket)
if not self.is_el_queued(hashcode):
data={"root": i, "relpath": relpath, "bucket": self.bucket, "action": 'C', "status": 0, "retries" : 0}
self.qm.save(data)
'''queue el, el: element of queue , formated as "bucket::root::path"'''
try:
self.queue.put(el, block = True, timeout = 1)
msg = 'queue element:' + el
#print msg
self.logger.info(msg)
except Full as e:
self.queue.put(None)
self.logger.error(e.message)
self.qm.close()
self.queue.put(None)
#self.queue.join()
return
def save_el(self, root, relpath, bucket, action):
hashcode = helper.calc_el_md5(root, relpath, bucket)
self.qm.open()
if self.is_el_existed(hashcode):
self.qm.update_action(hashcode, action)
self.qm.update_status(hashcode, 0)
else:
data={"root": root, "relpath": relpath, "bucket": bucket, "action": action, "status": 0, "retries": 0}
self.qm.save(data)
self.qm.close()
def save_el(self, root, relpath, bucket, action):
hashcode = helper.calc_el_md5(root, relpath, bucket)
self.qm.open()
if self.is_el_existed(hashcode):
self.qm.update_action(hashcode, action)
self.qm.update_status(hashcode, 0)
else:
data = {
"root": root,
"relpath": relpath,
"bucket": bucket,
"action": action,
"status": 0,
"retries": 0
}
self.qm.save(data)
self.qm.close()
def run(self):
self.logger.info('Now starting sync thread ...')
self.qm.open()
while True:
if self._terminate:
break
item = self.queue.get()
if item is None:
break
(bucket, root, relpath, action) = item.split('::')
if len(bucket) > 0 and len(root) > 0 and len(relpath) > 0 and len(action) > 0:
hashcode = helper.calc_el_md5(root, relpath, bucket)
if not self.is_el_processed(hashcode):
oss_obj_name = os.path.join(os.path.basename(root), relpath)
if len(oss_obj_name) > 0:
if(action == 'M' or action == 'C'):
success = self.upload(bucket, oss_obj_name, os.path.join(root, relpath))
msg = 'put object ' + oss_obj_name + ' to bucket ' + bucket
if(action == 'D'):
success = self.delete_oss_objects(bucket, oss_obj_name)
msg = 'delete object ' + oss_obj_name + ' of bucket ' + bucket
if success:
msg += ' success'
self.logger.info(msg)
self.qm.update_status(hashcode, 1)
else:
if success == False:
msg += ' failure'
self.logger.error(msg)
"""requeue losing element"""
row = self.qm.get(hashcode)
if row:
retries = int(row['retries'] )
if retries < MAX_RETRIES:
self.queue_el(item)
self.qm.update_retries(hashcode, retries + 1)
else:
self.logger.critical(msg + ' exceed max retries')
else:
self.logger.critical(msg + ' failure, resource may not exists.')
pass
self.qm.close()
self.queue.task_done()
return
def queue_el(self, bucket, root, path):
"""根据bucket和root以及路径生成队列元素"""
relpath = os.path.relpath(path, root) # 相对于root的相对路径
filehash = ""
if os.path.isfile(path):
filehash = helper.calc_file_md5(path)
hashcode = helper.calc_el_md5(root, relpath, bucket, filehash)
el = bucket + '::' + root + '::' + relpath + '::C' + '::' + hashcode
if not self.is_el_queued(hashcode):
data={"root": root, "relpath": relpath, "bucket": bucket, "action": 'C', "status": 0, "hashcode": hashcode, "retries" : 0}
try:
self.qm.save(data)
self.queue.put(el, block = True, timeout = 1)
msg = 'queue element:' + el
#print msg
self.logger.info(msg)
except Full as e:
self.logger.error(e.message)
def run(self):
files = {}
for d in self.dirs:
files[d] = list(
helper.walk_files(os.path.normpath(d), yield_folders=True))
if len(files) > 0:
self.qm.open()
self.logger.info('Queue path ...')
for i in files:
if len(files[i]) > 0:
for path in files[i]:
relpath = os.path.relpath(path, i) # 相对于root的相对路径
el = self.bucket + '::' + i + '::' + relpath + '::C'
hashcode = helper.calc_el_md5(i, relpath, self.bucket)
if not self.is_el_queued(hashcode):
data = {
"root": i,
"relpath": relpath,
"bucket": self.bucket,
"action": 'C',
"status": 0,
"retries": 0
}
self.qm.save(data)
'''queue el, el: element of queue , formated as "bucket::root::path"'''
try:
self.queue.put(el, block=True, timeout=1)
msg = 'queue element:' + el
#print msg
self.logger.info(msg)
except Full as e:
self.queue.put(None)
self.logger.error(e.message)
self.qm.close()
self.queue.put(None)
#self.queue.join()
return
