def run(self):
q = queue.Queue()
# 启动U盘监控模块
monitor = self._get_monitor(q)
monitor.run()
self._threads.append(monitor)
# 启动node_manager，获取本地节点信息；刷新本地文件相关信息
# 初始节点信息大多无效，当有节点通信以后，需要再次保存节点信息
node_manager = self._node_manager
local_node = node_manager.load_node(self.config.local_path)
local_node.load_filelists()
self.local_node = local_node
# 启动文件监控模块
self.watcher = file_watch.Watcher(q)
self.watcher.add_node(local_node)
self.watcher.start()
self._threads.append(self.watcher)
klog.info("Start monitor job.")
# 主线程监控queue，然后处理任务
while True:
msg = q.get()
self._do_work(msg)
q.task_done()
for t in self._threads:
t.join()
def store_fileinfo(self, file_name):
h_utf8 = kutil.map_to_utf8(self.to_map())
json_str = json.dumps(h_utf8)
klog.info(u"store file info to %s: %s", file_name, json_str)
with open(file_name, "wb") as f:
f.write(json_str)
def remove_node_by_path(self, disk_path):
for name, n in self.all_nodes.iteritems():
if n.path == disk_path:
if n.status == NodeStatus.Syncing:
klog.warn(u"Sync failed, because of you remove the disk")
klog.info(u"Remove node, name=%s, path=%s", name, disk_path)
self.all_nodes.pop(name)
def on_created(self, event):
super(FileHandler, self).on_created(event)
klog.info(u"file created, %s", event.src_path)
if not event.is_directory:
klog.info(u"created name:[%s]", event.src_path)
m = message.FileMessage(message.Type.FileChange, event.src_path, self.node)
self.queue.put(m)
def _disk_remove(self, msg):
driver_path = msg.disk_path
udisk_path = os.path.join(driver_path, self.config.removable_disk_path)
if not os.path.exists(udisk_path):
klog.info(u"path is not exist: %s", udisk_path)
return
self.watcher.remove_path(udisk_path)
self._node_manager.remove_node_by_path(udisk_path)
def _disk_arrival(self, msg):
driver_path = msg.disk_path
udisk_path = os.path.join(driver_path, self.config.removable_disk_path)
if not os.path.exists(udisk_path):
klog.info(u"path is not exist: %s", udisk_path)
return
new_node = self._node_manager.load_node(udisk_path)
new_node.load_filelists()
self.local_node.sync_file(new_node)
self.watcher.add_node(new_node)
def sync_file(self, other_node):
self.status = NodeStatus.Syncing
other_node.status = NodeStatus.Syncing
(to_other, from_other, conflict) = self._diff_file(other_node)
self.sync(other_node, to_other)
other_node.sync(self, from_other)
# @todo conflict的文件处理
self.status = NodeStatus.Idle
other_node.status = NodeStatus.Idle
klog.info(u"Sync files finished.")
def _file_walk(path, dir_name, level):
total_size = 0
md5_str = ""
subfiles_map = {}
subfiles = os.listdir(path)
for file in subfiles:
fullname = os.path.join(path, file)
file_stat = os.stat(fullname)
if level == 0 \
and (file == u".node.txt" or file == u".filelists.txt"):
klog.info(u"Skip file, %s", file)
continue
file_info = FileInfo()
sub = {}
mode = file_stat.st_mode
if stat.S_ISDIR(mode):
sub_type = FileType.Dir
(sub, sub_size, sub_md5) = _file_walk(fullname, file, level + 1)
sub_md5 = kutil.str_md5(sub_md5)
elif stat.S_ISREG(mode):
sub_type = FileType.File
sub_size = file_stat.st_size
sub_md5 = kutil.file_md5(fullname)
else:
continue
file_info.name = file
file_info.parent_dir = dir_name
file_info.relative_path = path
file_info.type = sub_type
file_info.create_time = file_stat.st_ctime
file_info.size = sub_size
file_info.md5 = sub_md5
file_info.subfiles = sub
subfiles_map[file] = file_info
md5_str += sub_md5
total_size += sub_size
return (subfiles_map, total_size, md5_str)
def _on_device_change(self, hwnd, msg, wparam, lparam):
dev_broadcast_hdr = DevBroadcastHeader.from_address(lparam)
if wparam == DeviceEvent.Arrival:
if dev_broadcast_hdr.dbch_devicetype == DeviceType.Volume:
klog.info(u"It's a volume!")
driver_path = WinDiskMonitor._get_driver(lparam)
dev_broadcast_volume = DevBroadcastVolume.from_address(lparam)
if dev_broadcast_volume.dbcv_flags == DeviceVolume.Media:
pass
super(WinDiskMonitor, self).on_disk_arrive(driver_path)
elif wparam == DeviceEvent.RemoveComplete:
driver_path = WinDiskMonitor._get_driver(lparam)
super(WinDiskMonitor, self).on_disk_remove(driver_path)
return 1
def update_version(self):
filelist_name = os.path.join(self.base_path, Node.FileListsName)
root_file_info = file_info.parse_filelists(filelist_name)
if not root_file_info:
self.root_file_info.store_fileinfo(filelist_name)
return
fmap_record = {}
root_file_info.create_file_map(fmap_record)
for k, f in self.root_file_map.iteritems():
if k in fmap_record:
f.version = fmap_record[k].version
if f.md5 != fmap_record[k].md5:
klog.info(
u"Update version exception, md5 is not matched for file=%s, %s",
f.relative_path, f.name)
else:
klog.info(u"Update version new file=%s, %s", f.relative_path,
f.name)
def sync(self, to_node, fileinfo_list):
cwd = os.getcwd()
src_base_path = self.base_path
dst_base_path = to_node.base_path
try:
os.chdir(src_base_path)
for finfo in fileinfo_list:
src_relative_path = finfo.relative_path
dst_path = os.path.join(dst_base_path, src_relative_path)
if not os.path.exists(dst_path):
os.makedirs(dst_path)
src_file = os.path.join(src_relative_path, finfo.name)
dst_file = os.path.join(dst_path, finfo.name)
if finfo.type == file_info.FileType.Dir:
# shutil.copytree(local_src_file, local_dst_file)
# 只能一个文件一个文件的copy
if not os.path.exists(dst_file):
klog.info(u"Sync file, make dirs=%s", dst_file)
os.makedirs(dst_file)
else:
klog.info(u"Sync file, filename=%s", src_file)
# @todo 考虑copy失败的情况，如磁盘满
shutil.copyfile(src_file, dst_file)
Node.flush_version(finfo, self, to_node)
klog.info(u"Sync files finished from node/%s to node/%s",
self.name, to_node.name)
except:
klog.error(u"sync file error, %s", traceback.format_exc())
finally:
os.chdir(cwd)
def on_disk_remove(self, driver_path):
klog.info(u"Removable disk leave: %s", driver_path)
m = message.Message(message.Type.DiskRemove)
m.disk_path = driver_path
self.queue.put(m)
def on_disk_arrive(self, driver_path):
klog.info(u"Removable disk arrive: %s", driver_path)
m = message.Message(message.Type.DiskArrival)
m.disk_path = driver_path
self.queue.put(m)