def pull(self):
self.logger.debug(f"Pulling {self.dest} from {self.source}")
# self.sync_filelist()
# cwd = os.getcwd()
# os.chdir(self.path)
# self.pullfiles(".")
args = []
# TODO: parse out multiple options
options = self.config.getConfig(self.context, "rsync options")
# if options is not None and "delete" in options:
# self.logger.debug("DELETING")
# args = [ "--delete" ]
# # , "--delete-excluded" ] # this will screw with replicas
args = []
for option in options:
args += [f"--{option}"]
# elif self.verbose:
# self.logger.info("Probably not deleting")
ignorals = self.build_ignorals()
if len(ignorals) > 0:
args += [f"--exclude={item}" for item in ignorals]
if self.testing:
# test mode: strip out hostnames for the rsync
source = config.path_for(self.source)
else:
source = self.source
dest = config.path_for(self.dest)
self.logger.info("Starting rsync ...")
file_state.rsync(source, dest, args)
def scan(self, gen_checksums=False):
self.logger.info("Scanning...")
changed = super().scan(gen_checksums)
if changed:
self.logger.info("Pushing states to source")
file_state.rsync(self.states_filename,
f"{self.source}/.ghetto_cluster",
stfu=True)
return changed
def retrieve(self, source_context, filename, counter=0):
self.logger.debug(
f"retrieving {source_context}:{filename} to {self.path}/{source_context}/{filename}"
)
# 0: do I have it?
if self.scanners[source_context].contains_p(filename):
self.logger.debug(f"I already have {filename}")
# just send the one; inform() will handle the rest
self.claim(source_context, filename, dropping=True)
return
# 1: build the filenames (full path) for source + dest
source = self.config.get(source_context, "source") + "/" + filename
src_host = config.host_for(source)
hostname = config.host_for(self.config.get(self.context, "backup"))
if src_host == hostname: # a local copy, just use path
source = config.path_for(source)
dest_path = f"{self.path}/{source_context}"
dest = f"{dest_path}/{filename}"
# 2: make the transfer
self.logger.debug(f"rsync {source} {dest}")
self.makedirs(dest)
rsync_stat = file_state.rsync(source, dest)
self.logger.debug(f"rsync returned {rsync_stat}")
if rsync_stat == 0:
# 3: record it
self.claim(source_context, filename, dropping=True) # no retry
else:
self.logger.error("Failed to rsync???")
raise FileNotFoundError
def retrieve(self, source_context, filename):
self.logger.debug(f"retrieving {source_context}:{filename}" + \
f" to {self.path}/{source_context}/{filename}")
# 0: do I have it?
if self.scanners[source_context].contains_p(filename):
claimed = self.claim(source_context, filename, dropping=True)
self.logger.debug(
f"I already have {filename}; claimed = {claimed}")
if claimed in ("ack", "keep"):
return claimed
else:
self.logger.debug(f"Something's wrong, trying again")
# 1: build the filenames (full path) for source + dest
source = self.config.get(source_context, "source") + "/" + filename
src_host = config.host_for(source)
hostname = config.host_for(self.config.get(self.context, "backup"))
if src_host == hostname: # a local copy, just use path
source = config.path_for(source)
dest = f"{self.path}/{source_context}/{filename}"
# 2: make the transfer
self.logger.debug(f"rsync {source} {dest}")
self.makedirs(dest)
rsync_stat = file_state.rsync(source, dest)
self.logger.debug(f"rsync returned {rsync_stat}")
if rsync_stat == 0:
# 3: record it
self.claim(source_context, filename, dropping=True)
else:
self.logger.error("Failed to rsync???")
raise FileNotFoundError
def pull_master_config(self):
if self.master_config is None:
return
self.logger.error(f"ERROR: {self.filename} MUST contain a " \
f"\"master config:\" line")
sys.exit(1)
host = host_for(self.master_config)
if host == self.hostname:
self.logger.debug("I am master, not pulling the config")
return
self.logger.debug(f"pulling config {self.master_config}" \
f" -> {self.filename}")
if self.testing:
self.logger.debug(self.master_config)
master_config = path_for(self.master_config)
else:
master_config = self.master_config
file_state.rsync(master_config, self.filename, stfu=True)
def test_rsync(self):
return
global tempdir
cfg = config.Config.instance()
cfg.set("global", "verbose", "yes")
source = f"{tempdir}/file_0.1k"
dest = f"{tempdir}/copy_file_0.1k"
exitvalue = file_state.rsync(source, dest)
self.assertEquals(exitvalue, 0)
self.assertTrue(os.path.exists(dest))
dest = f"{tempdir}/copy' file_0.1k"
exitvalue = file_state.rsync(source, dest)
self.assertEquals(exitvalue, 0)
self.assertTrue(os.path.exists(dest))
# 2018-08-25 15:20:14,904 [rsync] ['rsync', '-a', '--inplace', '--partial', '--timeout', '180', "mini:/Volumes/Docs_ZFS/BitTorrent\\ Sync/hacking/Drew's\\ things/things/3d\\ printer/cuttlefish/M_cuttlefish_upright_80pct.gcode.gz", "/mnt/data/austin/cluster-backups/a55fde13/BitTorrent Sync/hacking/Drew's things/things/3dprinter/cuttlefish/M_cuttlefish_upright_80pct.gcode.gz", '-v', '--progress']
source = "mini:/Volumes/Docs_ZFS/BitTorrent Sync/hacking/Drew's things/things/3d printer/cuttlefish/M_cuttlefish_upright_80pct.gcode.gz"
dest = "/tmp" # "/mnt/data/austin/cluster-backups/a55fde13/BitTorrent Sync/hacking/Drew's things/things/3dprinter/cuttlefish/M_cuttlefish_upright_80pct.gcode.gz"
exitvalue = file_state.rsync(source, dest)
self.assertEquals(exitvalue, 0)
self.assertTrue(os.path.exists(dest))