def __init__(self, file_copy_spec, file_copied_callback=None):
'''
file_copy_spec is dictionary indexed by full source path containing list of full
destination paths
file_copied_callback is called for each attempted file copy and passed a CopyStatus
'''
self.file_copy_spec = file_copy_spec
self.file_copied_callback = file_copied_callback
self.pool = multiprocessing.Pool(processes=COPY_POOL_SIZE)
self.copy_processes_active = 0
mgr = multiprocessing.Manager()
self.queue = mgr.Queue()
self.copied_files = FileCopyHistoryStore()
def __init__(self,file_copy_spec,file_copied_callback = None):
'''
file_copy_spec is dictionary indexed by full source path containing list of full
destination paths
file_copied_callback is called for each attempted file copy and passed a CopyStatus
'''
self.file_copy_spec = file_copy_spec
self.file_copied_callback = file_copied_callback
self.pool = multiprocessing.Pool(processes=COPY_POOL_SIZE)
self.copy_processes_active = 0
mgr = multiprocessing.Manager()
self.queue = mgr.Queue()
self.copied_files = FileCopyHistoryStore()
def test_init_no_path(self):
fchs = FileCopyHistoryStore()
self.assertEqual(fchs.files_seen, set())
def test_save_and_load(self):
(dummy, path) = tempfile.mkstemp()
fchs = FileCopyHistoryStore(path)
fchs.add('testpath', 1)
fchs1 = FileCopyHistoryStore(path)
self.assertTrue(fchs1.contains('testpath', 1))
def test_init_with_path(self):
(dummy, path) = tempfile.mkstemp()
fchs = FileCopyHistoryStore(path)
self.assertEqual(fchs.files_seen, set())
class FileCopier(object):
def __init__(self,file_copy_spec,file_copied_callback = None):
'''
file_copy_spec is dictionary indexed by full source path containing list of full
destination paths
file_copied_callback is called for each attempted file copy and passed a CopyStatus
'''
self.file_copy_spec = file_copy_spec
self.file_copied_callback = file_copied_callback
self.pool = multiprocessing.Pool(processes=COPY_POOL_SIZE)
self.copy_processes_active = 0
mgr = multiprocessing.Manager()
self.queue = mgr.Queue()
self.copied_files = FileCopyHistoryStore()
def _copy_file(self,src,dst):
if not dst.exists():
_logger.debug('Queuing copy from %s to %s',src,dst)
self.copy_processes_active += 1
self.pool.apply_async(copy_file_task, (src, dst, self.queue))
# copy_file_task(src,dst,self.queue)
def _find_copy_spec(self,source_path):
for source_spec in self.file_copy_spec:
if fnmatch.fnmatch(source_path, source_spec):
return self.file_copy_spec[source_spec], source_spec
return None, None
def _process_one_destination(self,dest_path,source_file,source_spec,dest_is_history_path):
if dest_path is not None:
_logger.debug('Processing destination path: %s for source file: %s', dest_path, source_file)
destination = Destination(source_spec,source_file.path,dest_path,dest_is_history_path)
self._copy_file(source_file,File(destination.path))
def _file_has_already_been_copied(self,aFile):
return self.copied_files.contains(aFile.path,aFile.mtime())
def _file_is_not_too_old(self,aFile):
retval = aFile.mtime() > (time.time() - MAX_AGE_TO_COPY)
# print '_src_file_is_not_too_old ', retval
return retval
def _file_is_not_too_young(self,aFile):
retval = aFile.mtime() < (time.time() - MIN_AGE_TO_COPY)
return retval
def _record_that_file_has_been_copied(self, aFile):
self.copied_files.add(aFile.path,aFile.mtime())
def _process_all_destinations(self,source_file,source_spec,copy_spec):
if not self._file_has_already_been_copied(source_file):
for idx, dest_path in enumerate(copy_spec):
dest_is_history_path = idx == 1
self._process_one_destination(dest_path,source_file,source_spec,dest_is_history_path)
self._record_that_file_has_been_copied(source_file)
def _check_copy_status(self):
'''
Drain the queue of status messages and process each.
Call the callback function (if specified) for each status message.
Decrement the count of outstanding copy processes for each status message received.
Log each status.
'''
while not self.queue.empty():
result = self.queue.get()
self.copy_processes_active -= 1
result.log(_logger)
if self.file_copied_callback is not None:
self.file_copied_callback(result)
def flush(self):
'''
Wait for all copy processes to complete
'''
while self.copy_processes_active > 0:
self._check_copy_status()
time.sleep(0.01)
def poll(self):
'''
Check whether there are files to copy and if so copy them
'''
_logger.debug('FileCopier.poll() called')
for destination in self.file_copy_spec:
matching_files = glob.glob(destination)
for aFile in map(File,matching_files):
if self._file_is_not_too_old(aFile) and self._file_is_not_too_young(aFile):
self._process_all_destinations(aFile,destination,self.file_copy_spec[destination])
self._check_copy_status()
def test_save_and_load(self):
(dummy, path) = tempfile.mkstemp()
fchs = FileCopyHistoryStore(path)
fchs.add("testpath", 1)
fchs1 = FileCopyHistoryStore(path)
self.assertTrue(fchs1.contains("testpath", 1))
class FileCopier(object):
def __init__(self, file_copy_spec, file_copied_callback=None):
'''
file_copy_spec is dictionary indexed by full source path containing list of full
destination paths
file_copied_callback is called for each attempted file copy and passed a CopyStatus
'''
self.file_copy_spec = file_copy_spec
self.file_copied_callback = file_copied_callback
self.pool = multiprocessing.Pool(processes=COPY_POOL_SIZE)
self.copy_processes_active = 0
mgr = multiprocessing.Manager()
self.queue = mgr.Queue()
self.copied_files = FileCopyHistoryStore()
def _copy_file(self, src, dst):
if not dst.exists():
_logger.debug('Queuing copy from %s to %s', src, dst)
self.copy_processes_active += 1
self.pool.apply_async(copy_file_task, (src, dst, self.queue))
# copy_file_task(src,dst,self.queue)
def _find_copy_spec(self, source_path):
for source_spec in self.file_copy_spec:
if fnmatch.fnmatch(source_path, source_spec):
return self.file_copy_spec[source_spec], source_spec
return None, None
def _process_one_destination(self, dest_path, source_file, source_spec,
dest_is_history_path):
if dest_path is not None:
_logger.debug(
'Processing destination path: %s for source file: %s',
dest_path, source_file)
destination = Destination(source_spec, source_file.path, dest_path,
dest_is_history_path)
self._copy_file(source_file, File(destination.path))
def _file_has_already_been_copied(self, aFile):
return self.copied_files.contains(aFile.path, aFile.mtime())
def _file_is_not_too_old(self, aFile):
retval = aFile.mtime() > (time.time() - MAX_AGE_TO_COPY)
# print '_src_file_is_not_too_old ', retval
return retval
def _file_is_not_too_young(self, aFile):
retval = aFile.mtime() < (time.time() - MIN_AGE_TO_COPY)
return retval
def _record_that_file_has_been_copied(self, aFile):
self.copied_files.add(aFile.path, aFile.mtime())
def _process_all_destinations(self, source_file, source_spec, copy_spec):
if not self._file_has_already_been_copied(source_file):
for idx, dest_path in enumerate(copy_spec):
dest_is_history_path = idx == 1
self._process_one_destination(dest_path, source_file,
source_spec,
dest_is_history_path)
self._record_that_file_has_been_copied(source_file)
def _check_copy_status(self):
'''
Drain the queue of status messages and process each.
Call the callback function (if specified) for each status message.
Decrement the count of outstanding copy processes for each status message received.
Log each status.
'''
while not self.queue.empty():
result = self.queue.get()
self.copy_processes_active -= 1
result.log(_logger)
if self.file_copied_callback is not None:
self.file_copied_callback(result)
def flush(self):
'''
Wait for all copy processes to complete
'''
while self.copy_processes_active > 0:
self._check_copy_status()
time.sleep(0.01)
def poll(self):
'''
Check whether there are files to copy and if so copy them
'''
_logger.debug('FileCopier.poll() called')
for destination in self.file_copy_spec:
matching_files = glob.glob(destination)
for aFile in map(File, matching_files):
if self._file_is_not_too_old(
aFile) and self._file_is_not_too_young(aFile):
self._process_all_destinations(
aFile, destination, self.file_copy_spec[destination])
self._check_copy_status()
