示例#1
0
 def get_File_Stat(self, filename, hashmode=False):
   self.m_lock.acquire()
   self.file_stat_count.value += 1
   self.m_lock.release()
   
   self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "called(" + str(self.file_stat_count.value) + "/" + str(len(self.file_queue.mQueue_list)) + "):[" + filename + "]")
 
   filepath = os.path.join(self.rootpath, filename)
   try:
     path, f = os.path.split(filepath)
     realpath=os.path.realpath(filepath)
     relpath=os.path.relpath(self.rootpath, filepath)
     mode = os.stat(filepath).st_mode
     dev = os.stat(filepath).st_dev
     hardlink = os.stat(filepath).st_nlink
     uid = os.stat(filepath).st_uid
     gid = os.stat(filepath).st_gid
     size = os.stat(filepath).st_size
     atime = os.stat(filepath).st_atime
     mtime = os.stat(filepath).st_mtime
     ctime = os.stat(filepath).st_ctime
     backup = "backup"
 
     #self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "acquired stat of file:[" + filepath + "]")
   except Exception, e:
     self.logger.error(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "ailed to acquire stat of file:[" + filepath + "]", e)
     return False
示例#2
0
 def start(self):
   self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "called")
   
   self.inject_Queues_From_a_Directory(self.rootpath)
   
   worker_list = []
   while not self.dir_queue.empty():
     if len(multiprocessing.active_children()) <= self.CONST.MAX_WORKER_PROCESS:
       p = multiprocessing.Process(target=self.worker, args=())
       worker_list.append(p)
       p.start()
     else:
       self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "waiting for " + str(self.CONST.SLEEP_TIME_WORKER) + " seconds to start new worker process.")
       time.sleep(self.CONST.SLEEP_TIME_WORKER)
     
   for job in worker_list:
     job.join()
     self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "waiting for children processes being completed: cpid=[" + str(job.pid) + "]")
   
   self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), " finished with " 
             + str(len(self.dir_list)) + " directories, " 
             + str(len(self.file_list)) + " files, "
             + str(len(self.link_list)) + " links, "
             + str(len(self.other_list)) + " others, "
             + str(len(self.error_list)) + " errors"
             + str(len(self.file_stat_dict.keys())) + "filestat keys"
             )
   
   
   self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "finished:" + str(time.time() - self.begin_time) + " seconds were elapsed..")
   
   return True
示例#3
0
 def hashfileSHA256(self, filename, blocksize=2*1024*1024):
   #self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "called:[" + filename + "]")
 
   try:
     f = open(filename, 'rb')
     buf = [1]
     shasum = hashlib.sha256()
     while len(buf)>0:
       buf = f.read(blocksize)
       shasum.update(buf)
   except Exception, e:
     self.logger.error(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "can't be hashed with SHA256:[" + filename + "]", e)
     return "oops"
示例#4
0
 def worker(self):
   #self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "called")
   
   task_list = []
   while not self.dir_queue.empty():
     if threading.active_count() <= self.CONST.MAX_TASK_THREAD:
       t = threading.Thread(target=self.task, args=())
       task_list.append(t)
       t.start()
     else:
       self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "waiting for " + str(self.CONST.SLEEP_TIME_WORKER) + " seconds to start new task thread.")
       time.sleep(self.CONST.SLEEP_TIME_TASK)
     
   for job in task_list:
     job.join()
     self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "waiting for children threads being completed: " + str(job.name))
   
   return True
示例#5
0
 def unit_test(self):
   self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "called")
   
   self.inject_Queues_From_a_Directory(self.rootpath)
   
   for filename in self.file_queue.mQueue_list:
     self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "filename:[" + filename + "]")
   
   for dirname in self.dir_queue.mQueue_list:
     self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "dirname:[" + dirname + "]")
         
   for link in self.link_queue.mQueue_list:
     self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "link:[" + link + "]")   
      
   for other in self.other_queue.mQueue_list:
     self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "other:[" + other + "]")
         
   for error in self.error_queue.mQueue_list:
     self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "error:[" + error + "]")
   
   self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "finished")
   return True
示例#6
0
                 'm':mode,
                 'd':dev,
                 'h':hardlink,
                 'u':uid,
                 'g':gid,
                 's':size,
                 'a':atime,
                 't':mtime,
                 'c':ctime,
                 'h':sha256,
                 'b':backup
                 }
     #self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "add file stats:[" + filepath + "]")
   except Exception, e:
     item_dict={}
     self.logger.error(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "failed to add file stats:[" + filepath + "]", e)
 
   self.file_stat_dict[filename] = item_dict
   self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "SHA256:["+ self.file_stat_dict[filename].get('h') + "]:" + filename)
   return
   
 def inject_Queues_From_a_Directory(self, dirname):
   #self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "called:[" + dirname + "]")
   
   if os.path.exists(dirname):
     if os.path.isdir(dirname)==True:
       #self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "path:[" + dirname + "] is a directory")
       pass
     else:
       #self.logger.debug(str(time.time() - self.begin_time) + ": " +__name__ + "." + mLogger.func_name(), "path:[" + dirname + "] isn't a directory")
       raise Exception("IOError", "path:[" + self.path + "] isn't directory")