def run(self):
files = {}
for d in self.dirs:
files[d] = list(helper.walk_files(os.path.normpath(d), yield_folders = True))
if len(files) > 0:
self.qm.open()
self.logger.info('Queue path ...')
for i in files:
if len(files[i]) > 0:
for path in files[i]:
relpath = os.path.relpath(path, i) # 相对于root的相对路径
el = self.bucket + '::' + i+ '::' + relpath + '::C'
hashcode = helper.calc_el_md5(i, relpath, self.bucket)
if not self.is_el_queued(hashcode):
data={"root": i, "relpath": relpath, "bucket": self.bucket, "action": 'C', "status": 0, "retries" : 0}
self.qm.save(data)
'''queue el, el: element of queue , formated as "bucket::root::path"'''
try:
self.queue.put(el, block = True, timeout = 1)
msg = 'queue element:' + el
#print msg
self.logger.info(msg)
except Full as e:
self.queue.put(None)
self.logger.error(e.message)
self.qm.close()
self.queue.put(None)
#self.queue.join()
return
def queue_dir(self, queue_path):
files = list(helper.walk_files(queue_path, yield_folders=True))
if len(files) > 0:
for path in files:
relpath = os.path.relpath(path, self.monitered_dir)
self.save_el(self.monitered_dir, relpath, self.bucket, 'C')
el = self.bucket + '::' + self.monitered_dir + '::' + relpath + '::' + 'C'
self.queue_el(el)
def queue_dir(self, queue_path): files = list(helper.walk_files(queue_path, yield_folders = True)) if len(files) > 0: for path in files: relpath = os.path.relpath(path, self.monitered_dir) self.save_el(self.monitered_dir, relpath, self.bucket,'C') el = self.bucket + '::' + self.monitered_dir + '::' + relpath + '::' + 'C' self.queue_el(el)
def queue_folders(self, bucket, folders):
"""将目录中的文件解析成队列元素并压入队列"""
files = {}
elements = []
for d in folders:
files[d] = list(helper.walk_files(os.path.normpath(d), yield_folders = True))
if len(files) > 0:
for k in files:
if len(files[k]) > 0:
for path in files[k]:
self.queue_el(bucket, k, path)
def run(self):
files = {}
for d in self.dirs:
files[d] = list(
helper.walk_files(os.path.normpath(d), yield_folders=True))
if len(files) > 0:
self.qm.open()
self.logger.info('Queue path ...')
for i in files:
if len(files[i]) > 0:
for path in files[i]:
relpath = os.path.relpath(path, i) # 相对于root的相对路径
el = self.bucket + '::' + i + '::' + relpath + '::C'
hashcode = helper.calc_el_md5(i, relpath, self.bucket)
if not self.is_el_queued(hashcode):
data = {
"root": i,
"relpath": relpath,
"bucket": self.bucket,
"action": 'C',
"status": 0,
"retries": 0
}
self.qm.save(data)
'''queue el, el: element of queue , formated as "bucket::root::path"'''
try:
self.queue.put(el, block=True, timeout=1)
msg = 'queue element:' + el
#print msg
self.logger.info(msg)
except Full as e:
self.queue.put(None)
self.logger.error(e.message)
self.qm.close()
self.queue.put(None)
#self.queue.join()
return
