def _sync_paths(self, source, target):
"""Non-recursive directory sync.
"""
if not os.path.isdir(target):
os.mkdir(target)
self.changes.append(('mkdir', source, target))
for fname in os.listdir(source):
src_path = os.path.join(source, fname)
if os.path.isfile(src_path):
dst_path = os.path.join(target, fname)
# Skip large files:
# - pxp > 10GB
# - others > 5GB
src_stat = os.stat(src_path)
if (src_stat.st_size > 5e9 and not src_path.endswith('.pxp')
) or (src_stat.st_size > 15e9):
self.log(" err! %s => %s" % (src_path, dst_path))
self.changes.append(('error', src_path, 'file too large'))
continue
status = sync_file(src_path, dst_path)
if status == 'skip':
self.skipped += 1
elif status == 'copy':
self.log(" copy %s => %s" % (src_path, dst_path))
self.changes.append(('copy', src_path, dst_path))
elif status == 'update':
self.log(" updt %s => %s" % (src_path, dst_path))
self.changes.append(('update', src_path, dst_path))
def _sync_paths(source, target, changes):
"""Non-recursive directory sync.
Return the number of skipped files.
"""
skipped = 0
if not os.path.isdir(target):
os.mkdir(target)
changes.append(('mkdir', source, target))
# Leave a note about the source of this data
open(os.path.join(target, 'sync_source'),
'wb').write(source.encode('utf8'))
for fname in os.listdir(source):
src_path = os.path.join(source, fname)
if os.path.isfile(src_path):
dst_path = os.path.join(target, fname)
# Skip Igor temporary files
if fname.endswith('.pxpT0'):
continue
# Skip large files:
# - pxp > 20GB
# - others > 5GB
src_size = os.stat(src_path).st_size
# extension may be buried behind a backup date like "somefile.pxp_2018-11-20_02-01-20_0"
m = re.match(r'.*(.[a-z]{3})(_2.*)?', os.path.split(src_path)[1])
ext = '' if m is None else m.groups()[0]
max_size = {'.pxp': 20e9, '.nwb': 7e9}.get(ext, 5e9)
if src_size > max_size:
log(" err! %s => %s" % (src_path, dst_path))
changes.append(('error', src_path, 'file too large'))
continue
status = sync_file(src_path, dst_path)
if status == 'skip':
skipped += 1
elif status == 'copy':
log(" copy %s => %s" % (src_path, dst_path))
changes.append(('copy', src_path, dst_path))
elif status == 'update':
log(" updt %s => %s" % (src_path, dst_path))
changes.append(('update', src_path, dst_path))
return skipped
def _sync_paths(source, target, changes):
"""Non-recursive directory sync.
Return the number of skipped files.
"""
skipped = 0
if not os.path.isdir(target):
os.mkdir(target)
changes.append(('mkdir', source, target))
# Leave a note about the source of this data
open(os.path.join(target, 'sync_source'), 'wb').write(source)
for fname in os.listdir(source):
src_path = os.path.join(source, fname)
if os.path.isfile(src_path):
dst_path = os.path.join(target, fname)
# Skip Igor temporary files
if fname.endswith('.pxpT0'):
continue
# Skip large files:
# - pxp > 20GB
# - others > 5GB
src_size = os.stat(src_path).st_size
ext = os.path.splitext(src_path)[1]
max_size = {'.pxp': 20e9, '.nwb': 7e9}.get(ext, 5e9)
if src_size > max_size:
log(" err! %s => %s" % (src_path, dst_path))
changes.append(('error', src_path, 'file too large'))
continue
status = sync_file(src_path, dst_path)
if status == 'skip':
skipped += 1
elif status == 'copy':
log(" copy %s => %s" % (src_path, dst_path))
changes.append(('copy', src_path, dst_path))
elif status == 'update':
log(" updt %s => %s" % (src_path, dst_path))
changes.append(('update', src_path, dst_path))
return skipped
def _sync_paths(source, target, changes):
"""Non-recursive directory sync.
Return the number of skipped files.
"""
skipped = 0
if not os.path.isdir(target):
os.mkdir(target)
changes.append(('mkdir', source, target))
# Leave a note about the source of this data
open(os.path.join(target, 'sync_source'), 'wb').write(source)
for fname in os.listdir(source):
src_path = os.path.join(source, fname)
if os.path.isfile(src_path):
dst_path = os.path.join(target, fname)
# Skip Igor temporary files
if fname.endswith('.pxpT0'):
continue
# Skip large files:
# - pxp > 20GB
# - others > 5GB
src_size = os.stat(src_path).st_size
ext = os.path.splitext(src_path)[1]
max_size = {'.pxp': 20e9, '.nwb': 7e9}.get(ext, 5e9)
if src_size > max_size:
log(" err! %s => %s" % (src_path, dst_path))
changes.append(('error', src_path, 'file too large'))
continue
status = sync_file(src_path, dst_path)
if status == 'skip':
skipped += 1
elif status == 'copy':
log(" copy %s => %s" % (src_path, dst_path))
changes.append(('copy', src_path, dst_path))
elif status == 'update':
log(" updt %s => %s" % (src_path, dst_path))
changes.append(('update', src_path, dst_path))
return skipped