def _sync_paths(self, source, target): """Non-recursive directory sync. """ if not os.path.isdir(target): os.mkdir(target) self.changes.append(('mkdir', source, target)) for fname in os.listdir(source): src_path = os.path.join(source, fname) if os.path.isfile(src_path): dst_path = os.path.join(target, fname) # Skip large files: # - pxp > 10GB # - others > 5GB src_stat = os.stat(src_path) if (src_stat.st_size > 5e9 and not src_path.endswith('.pxp') ) or (src_stat.st_size > 15e9): self.log(" err! %s => %s" % (src_path, dst_path)) self.changes.append(('error', src_path, 'file too large')) continue status = sync_file(src_path, dst_path) if status == 'skip': self.skipped += 1 elif status == 'copy': self.log(" copy %s => %s" % (src_path, dst_path)) self.changes.append(('copy', src_path, dst_path)) elif status == 'update': self.log(" updt %s => %s" % (src_path, dst_path)) self.changes.append(('update', src_path, dst_path))
def _sync_paths(source, target, changes): """Non-recursive directory sync. Return the number of skipped files. """ skipped = 0 if not os.path.isdir(target): os.mkdir(target) changes.append(('mkdir', source, target)) # Leave a note about the source of this data open(os.path.join(target, 'sync_source'), 'wb').write(source.encode('utf8')) for fname in os.listdir(source): src_path = os.path.join(source, fname) if os.path.isfile(src_path): dst_path = os.path.join(target, fname) # Skip Igor temporary files if fname.endswith('.pxpT0'): continue # Skip large files: # - pxp > 20GB # - others > 5GB src_size = os.stat(src_path).st_size # extension may be buried behind a backup date like "somefile.pxp_2018-11-20_02-01-20_0" m = re.match(r'.*(.[a-z]{3})(_2.*)?', os.path.split(src_path)[1]) ext = '' if m is None else m.groups()[0] max_size = {'.pxp': 20e9, '.nwb': 7e9}.get(ext, 5e9) if src_size > max_size: log(" err! %s => %s" % (src_path, dst_path)) changes.append(('error', src_path, 'file too large')) continue status = sync_file(src_path, dst_path) if status == 'skip': skipped += 1 elif status == 'copy': log(" copy %s => %s" % (src_path, dst_path)) changes.append(('copy', src_path, dst_path)) elif status == 'update': log(" updt %s => %s" % (src_path, dst_path)) changes.append(('update', src_path, dst_path)) return skipped
def _sync_paths(source, target, changes): """Non-recursive directory sync. Return the number of skipped files. """ skipped = 0 if not os.path.isdir(target): os.mkdir(target) changes.append(('mkdir', source, target)) # Leave a note about the source of this data open(os.path.join(target, 'sync_source'), 'wb').write(source) for fname in os.listdir(source): src_path = os.path.join(source, fname) if os.path.isfile(src_path): dst_path = os.path.join(target, fname) # Skip Igor temporary files if fname.endswith('.pxpT0'): continue # Skip large files: # - pxp > 20GB # - others > 5GB src_size = os.stat(src_path).st_size ext = os.path.splitext(src_path)[1] max_size = {'.pxp': 20e9, '.nwb': 7e9}.get(ext, 5e9) if src_size > max_size: log(" err! %s => %s" % (src_path, dst_path)) changes.append(('error', src_path, 'file too large')) continue status = sync_file(src_path, dst_path) if status == 'skip': skipped += 1 elif status == 'copy': log(" copy %s => %s" % (src_path, dst_path)) changes.append(('copy', src_path, dst_path)) elif status == 'update': log(" updt %s => %s" % (src_path, dst_path)) changes.append(('update', src_path, dst_path)) return skipped