def run(self): # do regress the target directory if necessary if self._operate_regress(): # regress was necessary and failed return 1 previous_time = self.repo.get_mirror_time(refresh=True) if previous_time < 0 or previous_time >= Time.getcurtime(): log.Log( "Either there is more than one current_mirror or " "the last backup is not in the past. Aborting.", log.ERROR) return 1 if Globals.get_api_version() < 201: # compat200 if previous_time: Time.setprevtime_compat200(previous_time) self.repo.base_dir.conn.Main.backup_touch_curmirror_local( self.dir.base_dir, self.repo.base_dir) backup.mirror_and_increment_compat200(self.dir.base_dir, self.repo.base_dir, self.repo.incs_dir) self.repo.base_dir.conn.Main.backup_remove_curmirror_local() else: backup.mirror_compat200(self.dir.base_dir, self.repo.base_dir) self.repo.base_dir.conn.Main.backup_touch_curmirror_local( self.dir.base_dir, self.repo.base_dir) self.repo.base_dir.conn.Main.backup_close_statistics(time.time()) else: # API 201 and higher self._operate_backup(previous_time) return 0
def testStringtotime(self): """Test converting string to time""" timesec = int(time.time()) assert timesec == int(Time.stringtotime(Time.timetostring(timesec))) assert not Time.stringtotime("2001-18-83T03:03:03Z") assert not Time.stringtotime("2001-01-23L03:03:03L") assert not Time.stringtotime("2001_01_23T03:03:03Z")
def _list_increments_sizes(self): """ Print out a summary of the increments with their size and cumulative size """ triples = self.repo.get_increments_sizes() if self.values.parsable_output: print( yaml.safe_dump(triples, explicit_start=True, explicit_end=True)) else: stat_obj = statistics.StatsObj() # used for byte summary string print("{: ^24} {: ^17} {: ^17}".format("Time", "Size", "Cumulative size")) print("{:-^24} {:-^17} {:-^17}".format("", "", "")) # print the normal increments then the mirror for triple in triples[:-1]: print("{: <24} {: >17} {: >17}".format( Time.timetopretty(triple["time"]), stat_obj.get_byte_summary_string(triple["size"]), stat_obj.get_byte_summary_string(triple["total_size"]))) print("{: <24} {: >17} {: >17} (current mirror)".format( Time.timetopretty(triples[-1]["time"]), stat_obj.get_byte_summary_string(triples[-1]["size"]), stat_obj.get_byte_summary_string(triples[-1]["total_size"])))
def _writer_helper(self, typestr, time, meta_class, force=False): """ Returns a writer class or None if the meta class isn't active. For testing purposes, the force option allows to skip the activity validation. """ if time is None: timestr = Time.getcurtimestr() else: timestr = Time.timetobytes(time) triple = map(os.fsencode, (meta_class.get_prefix(), timestr, typestr)) filename = b'.'.join(triple) rp = self.data_dir.append(filename) assert not rp.lstat(), "File '{rp}' shouldn't exist.".format(rp=rp) assert rp.isincfile(), ( "Path '{irp}' must be an increment file.".format(irp=rp)) if meta_class.is_active() or force: # Before API 201, metafiles couldn't be compressed return meta_class(rp, 'w', compress=(Globals.compression or Globals.get_api_version() < 201), callback=self._add_incrp) else: return None
def testPrettyTimes(self): """Convert seconds to pretty and back""" now = int(time.time()) for i in [1, 200000, now]: assert Time.prettytotime(Time.timetopretty(i)) == i, i assert Time.prettytotime("now") is None assert Time.prettytotime("12314") is None
def run(self): # do regress the target directory if necessary if self.target.needs_regress(): ret_code = self.target.regress() if ret_code != 0: return ret_code previous_time = self.target.get_mirror_time() if previous_time < 0 or previous_time >= Time.curtime: self.log("Either there is more than one current_mirror or " "the last backup is not in the past. Aborting.", self.log.ERROR) return 1 elif previous_time: Time.setprevtime(previous_time) self.target.base_dir.conn.Main.backup_touch_curmirror_local( self.source.base_dir, self.target.base_dir) backup.Mirror_and_increment(self.source.base_dir, self.target.base_dir, self.target.incs_dir) self.target.base_dir.conn.Main.backup_remove_curmirror_local() else: backup.Mirror(self.source.base_dir, self.target.base_dir) self.target.base_dir.conn.Main.backup_touch_curmirror_local( self.source.base_dir, self.target.base_dir) self.target.base_dir.conn.Main.backup_close_statistics(time.time()) return 0
def testPrettyTimes(self): """Convert seconds to pretty and back""" now = int(time.time()) for i in [1, 200000, now]: self.assertEqual(Time.prettytotime(Time.timetopretty(i)), i) self.assertIsNone(Time.prettytotime("now")) self.assertIsNone(Time.prettytotime("12314"))
def connect(self): """ Connect to potentially provided locations arguments, remote or local. Defines the current time as being the time of a potentially upcoming backup. Returns self, to be used as context manager. """ if 'locations' in self.values: # TODO encapsulate the following lines into one # connections/connections_mgr construct, so that the action doesn't # need to care about cmdpairs and Security (which would become a # feature of the connection). cmdpairs = SetConnections.get_cmd_pairs( self.values.locations, remote_schema=self.remote_schema, ssh_compression=self.values.ssh_compression, remote_tempdir=self.remote_tempdir, term_verbosity=log.Log.term_verbosity) Security.initialize(self.get_security_class(), cmdpairs) self.connected_locations = list( map(SetConnections.get_connected_rpath, cmdpairs)) else: Security.initialize(self.get_security_class(), []) self.connected_locations = [] # once the connection is set, we can define "now" as being the current # time, unless the user defined a fixed a current time. Time.set_current_time(self.values.current_time) return self
def testBytestotime(self): """Test converting byte string to time""" timesec = int(time.time()) assert timesec == int( Time.bytestotime(Time.timetostring(timesec).encode('ascii'))) # assure that non-ascii byte strings return None and that they don't # throw an exception (issue #295) assert Time.bytestotime(b'\xff') is None
def testStringtotime(self): """Test converting string to time""" timesec = int(time.time()) self.assertEqual(timesec, int(Time.stringtotime(Time.timetostring(timesec)))) # stringtotime returns None if the time string is invalid self.assertIsNone(Time.stringtotime("2001-18-83T03:03:03Z")) self.assertIsNone(Time.stringtotime("2001-01-23L03:03:03L")) self.assertIsNone(Time.stringtotime("2001_01_23T03:03:03Z"))
def testConversion(self): """test timetostring and stringtotime""" Time.setcurtime() assert type(Time.curtime) is float or int, Time.curtime assert type(Time.curtimestr) is str, Time.curtimestr assert (cmp_times(int(Time.curtime), Time.curtimestr) == 0 or cmp_times(int(Time.curtime) + 1, Time.curtimestr) == 0) time.sleep(1.05) assert cmp_times(time.time(), Time.curtime) == 1 assert cmp_times(Time.timetostring(time.time()), Time.curtimestr) == 1
def setup(self): # in setup we return as soon as we detect an issue to avoid changing # too much return_code = super().setup() if return_code != 0: return return_code return_code = self._set_no_compression_regexp() if return_code != 0: return return_code return_code = self.dir.setup() if return_code != 0: return return_code owners_map = { "users_map": self.values.user_mapping_file, "groups_map": self.values.group_mapping_file, "preserve_num_ids": self.values.preserve_numerical_ids } return_code = self.repo.setup(self.dir, owners_map=owners_map) if return_code != 0: return return_code # TODO validate how much of the following lines and methods # should go into the directory/repository modules if Globals.get_api_version() < 201: # compat200 SetConnections.BackupInitConnections(self.dir.base_dir.conn, self.repo.base_dir.conn) self.repo.base_dir.conn.fs_abilities.backup_set_globals( self.dir.base_dir, self.values.force) self.repo.setup_quoting() previous_time = self.repo.get_mirror_time() if previous_time >= Time.getcurtime(): log.Log("The last backup is not in the past. Aborting.", log.ERROR) return 1 if log.Log.verbosity > 0: try: # the target repository must be writable log.Log.open_logfile(self.repo.data_dir.append("backup.log")) except (log.LoggerError, Security.Violation) as exc: log.Log("Unable to open logfile due to '{ex}'".format(ex=exc), log.ERROR) return 1 log.ErrorLog.open(Time.getcurtimestr(), compress=self.values.compression) (select_opts, select_data) = selection.get_prepared_selections( self.values.selections) self.dir.set_select(select_opts, select_data) self._warn_if_infinite_recursion(self.dir.base_dir, self.repo.base_dir) return 0
def testConversion(self): """test timetostring and stringtotime""" Time.set_current_time() self.assertIsInstance(Time.curtime, (float, int)) self.assertIsInstance(Time.curtimestr, str) self.assertTrue( self.cmp_times(int(Time.curtime), Time.curtimestr) == 0 or self.cmp_times(int(Time.curtime) + 1, Time.curtimestr) == 0) time.sleep(1.05) self.assertEqual(self.cmp_times(time.time(), Time.curtime), 1) self.assertEqual( self.cmp_times(Time.timetostring(time.time()), Time.curtimestr), 1)
def test_action_regress(self): """test different ways of regressing""" # regressing a successful backup doesn't do anything self.assertEqual( comtst.rdiff_backup_action(False, None, self.bak_path, None, ("--api-version", "201"), b"regress", ()), 0) # we again simulate a crash _repo_shadow.ShadowRepo.touch_current_mirror( rpath.RPath(Globals.local_connection, self.bak_path, ("rdiff-backup-data", )), Time.timetostring(20000)) # the current process (the test) is still running, hence it fails self.assertNotEqual( comtst.rdiff_backup_action(True, None, self.bak_path, None, ("--api-version", "201"), b"regress", ()), 0) # but it runs with --force self.assertEqual( comtst.rdiff_backup_action(True, None, self.bak_path, None, ("--api-version", "201", "--force"), b"regress", ()), 0) # we restore and compare self.assertEqual( comtst.rdiff_backup_action(True, True, self.bak_path, self.to2_path, ("--api-version", "201"), b"restore", ()), 0) self.assertFalse(fileset.compare_paths(self.from2_path, self.to2_path)) # we again simulate a crash _repo_shadow.ShadowRepo.touch_current_mirror( rpath.RPath(Globals.local_connection, self.bak_path, ("rdiff-backup-data", )), Time.timetostring(10000)) # and then try to backup, which fails because without force self.assertNotEqual( comtst.rdiff_backup_action( True, True, self.from4_path, self.bak_path, ("--api-version", "201", "--current-time", "40000"), b"backup", ()), 0) # now with --force, it can't be exactly the same time or it fails # on error_log already existing self.assertEqual( comtst.rdiff_backup_action( True, True, self.from4_path, self.bak_path, ("--api-version", "201", "--current-time", "40001", "--force"), b"backup", ()), 0) # we restore and compare self.assertEqual( comtst.rdiff_backup_action(True, True, self.bak_path, self.to4_path, ("--api-version", "201"), b"restore", ()), 0) self.assertFalse(fileset.compare_paths(self.from4_path, self.to4_path)) # all tests were successful self.success = True
def get_correct(self, mirror_rp, test_time): """Return correct version with base mirror_rp at time test_time""" assert -1 < test_time < 2000000000, test_time dirname, basename = mirror_rp.dirsplit() for filename in restore_base_filenames: comps = filename.split(b".") base = b".".join(comps[:-1]) t = Time.bytestotime(comps[-1]) if t == test_time and basename == base: return restore_base_rp.append(filename) # Correct rp must be empty return restore_base_rp.append(b"%b.%b" % (basename, Time.timetobytes(test_time)))
def testGenericString(self): """Test genstrtotime, conversion of arbitrary string to time""" g2t = Time.genstrtotime assert g2t('now', 1000) == 1000 assert g2t('2h3s', 10000) == 10000 - 2*3600 - 3 assert g2t('2001-09-01T21:49:04Z') == \ Time.stringtotime('2001-09-01T21:49:04Z') assert g2t('2002-04-26T04:22:01') == \ Time.stringtotime('2002-04-26T04:22:01' + Time.gettzd()) t = Time.stringtotime('2001-05-12T00:00:00' + Time.gettzd()) assert g2t('2001-05-12') == t assert g2t('2001/05/12') == t assert g2t('5/12/2001') == t assert g2t('123456') == 123456
def testGenericString(self): """Test genstrtotime, conversion of arbitrary string to time""" g2t = Time.genstrtotime assert g2t('now', 1000) == 1000 assert g2t('2h3s', 10000) == 10000 - 2 * 3600 - 3 assert g2t('2001-09-01T21:49:04Z') == \ Time.stringtotime('2001-09-01T21:49:04Z') assert g2t('2002-04-26T04:22:01') == \ Time.stringtotime('2002-04-26T04:22:01' + Time.gettzd()) t = Time.stringtotime('2001-05-12T00:00:00' + Time.gettzd()) assert g2t('2001-05-12') == t assert g2t('2001/05/12') == t assert g2t('5/12/2001') == t assert g2t('123456') == 123456
def testGenericString(self): """Test genstrtotime, conversion of arbitrary string to time""" g2t = Time.genstrtotime self.assertEqual(g2t('now', 1000), 1000) self.assertEqual(g2t('2h3s', 10000), 10000 - 2 * 3600 - 3) self.assertEqual(g2t('2001-09-01T21:49:04Z'), Time.stringtotime('2001-09-01T21:49:04Z')) self.assertEqual( g2t('2002-04-26T04:22:01'), Time.stringtotime('2002-04-26T04:22:01' + Time._get_tzd())) t = Time.stringtotime('2001-05-12T00:00:00' + Time._get_tzd()) self.assertEqual(g2t('2001-05-12'), t) self.assertEqual(g2t('2001/05/12'), t) self.assertEqual(g2t('5/12/2001'), t) self.assertEqual(g2t('123456'), 123456)
def cmp_times(self, time1, time2): """Compare time1 and time2 and return -1, 0, or 1""" if type(time1) is str: time1 = Time.stringtotime(time1) self.assertIsNotNone(time1) if type(time2) is str: time2 = Time.stringtotime(time2) self.assertIsNotNone(time2) if time1 < time2: return -1 elif time1 == time2: return 0 else: return 1
def setup(self): """ Prepare the execution of the action. """ # Set default change ownership flag, umask, relay regexps os.umask(0o77) Time.setcurtime(Globals.current_time) SetConnections.UpdateGlobal("client_conn", Globals.local_connection) Globals.postset_regexp('no_compression_regexp', Globals.no_compression_regexp_string) for conn in Globals.connections: conn.robust.install_signal_handlers() conn.Hardlink.initialize_dictionaries() return 0
def cmp_times(time1, time2): """Compare time1 and time2 and return -1, 0, or 1""" if type(time1) is str: time1 = Time.stringtotime(time1) assert time1 is not None if type(time2) is str: time2 = Time.stringtotime(time2) assert time2 is not None if time1 < time2: return -1 elif time1 == time2: return 0 else: return 1
def recreate_attr(self, regress_time): """ Make regress_time mirror_metadata snapshot by patching We write to a tempfile first. Otherwise, in case of a crash, it would seem we would have an intact snapshot and partial diff, not the reverse. """ temprp = [self.data_dir.get_temp_rpath()] def callback(rp): temprp[0] = rp writer = self._meta_main_class(temprp[0], 'wb', check_path=0, callback=callback) for rorp in self._get_meta_main_at_time(regress_time, None): writer.write_object(rorp) writer.close() finalrp = self.data_dir.append(b"mirror_metadata.%b.snapshot.gz" % Time.timetobytes(regress_time)) assert not finalrp.lstat(), ( "Metadata path '{mrp}' shouldn't exist.".format(mrp=finalrp)) rpath.rename(temprp[0], finalrp) if Globals.fsync_directories: self.data_dir.fsync()
def mark_incomplete(self, curtime, rp): """Check the date of current mirror Return 1 if there are two current_mirror incs and last one has time curtime. Return 0 if only one with time curtime, and then add a current_mirror marker. Return -1 if only one and time is not curtime. """ rbdir = rp.append_path("rdiff-backup-data") inclist = restore.get_inclist(rbdir.append("current_mirror")) self.assertIn( len(inclist), (1, 2), "There must be 1 or 2 elements in '{paths_list}'.".format( paths_list=str([x.path for x in inclist]))) inc_date_pairs = [(inc.getinctime(), inc) for inc in inclist] inc_date_pairs.sort() if len(inclist) == 2: self.assertEqual(inc_date_pairs[-1][0], curtime) return 1 if inc_date_pairs[-1][0] == curtime: result = 0 marker_time = curtime - 10000 else: self.assertEqual(inc_date_pairs[-1][0], curtime - 10000) marker_time = curtime result = -1 cur_mirror_rp = rbdir.append("current_mirror.%s.data" % (Time.timetostring(marker_time), )) self.assertFalse(cur_mirror_rp.lstat()) cur_mirror_rp.touch() return result
def mark_incomplete(self, curtime, rp): """Check the date of current mirror Return 1 if there are two current_mirror incs and last one has time curtime. Return 0 if only one with time curtime, and then add a current_mirror marker. Return -1 if only one and time is not curtime. """ rbdir = rp.append_path("rdiff-backup-data") inclist = restore.get_inclist(rbdir.append("current_mirror")) assert 1 <= len(inclist) <= 2, str([x.path for x in inclist]) inc_date_pairs = [(inc.getinctime(), inc) for inc in inclist] inc_date_pairs.sort() if len(inclist) == 2: assert inc_date_pairs[-1][0] == curtime, \ (inc_date_pairs[-1][0], curtime) return 1 if inc_date_pairs[-1][0] == curtime: result = 0 marker_time = curtime - 10000 else: assert inc_date_pairs[-1][0] == curtime - 10000 marker_time = curtime result = -1 cur_mirror_rp = rbdir.append("current_mirror.%s.data" % (Time.timetostring(marker_time), )) assert not cur_mirror_rp.lstat() cur_mirror_rp.touch() return result
def __init__(self, start_time=None): """StatFileObj initializer - zero out file attributes""" StatsObj.__init__(self) for attr in self._stat_file_attrs: self.set_stat(attr, 0) if start_time is None: start_time = Time.getcurtime() self.StartTime = start_time self.Errors = 0
def write_active_statfileobj(end_time=None): """Write active StatFileObj object to session statistics file""" global _active_statfileobj assert _active_statfileobj, "Stats object must be set before writing." rp_base = Globals.rbdir.append(b"session_statistics") session_stats_rp = increment.get_inc(rp_base, 'data', Time.getcurtime()) _active_statfileobj.finish(end_time) _active_statfileobj.write_stats_to_rp(session_stats_rp) _active_statfileobj = None
def _yield_metadata(): """Iterate rorps from metadata file, if any are available""" meta_manager = meta_mgr.get_meta_manager(True) metadata_iter = meta_manager.get_metas_at_time(regress_time) if metadata_iter: return metadata_iter log.Log.FatalError("No metadata for time {pt} ({rt}) found, " "cannot regress".format( pt=Time.timetopretty(regress_time), rt=regress_time))
def _get_timestats_string(self): """Return portion of statistics string dealing with time""" timelist = [] if self.StartTime is not None: timelist.append( "StartTime %.2f (%s)\n" % (self.StartTime, Time.timetopretty(self.StartTime))) if self.EndTime is not None: timelist.append("EndTime %.2f (%s)\n" % (self.EndTime, Time.timetopretty(self.EndTime))) if self.ElapsedTime or (self.StartTime is not None and self.EndTime is not None): if self.ElapsedTime is None: self.ElapsedTime = self.EndTime - self.StartTime timelist.append( "ElapsedTime %.2f (%s)\n" % (self.ElapsedTime, Time.inttopretty(self.ElapsedTime))) return "".join(timelist)
def test_symlink_popple(self): """Test for Popple's symlink bug Earlier, certain symlinks could cause data loss in _source_ directory when regressing. See mailing lists around 4/2/05 for more info. """ self.delete_tmpdirs() # Make directories rp1 = Local.get_tgt_local_rp('sym_in1') if rp1.lstat(): rp1.delete() rp1.mkdir() rp1_d = rp1.append('subdir') rp1_d.mkdir() rp1_d_f = rp1_d.append('file') rp1_d_f.touch() rp2 = Local.get_tgt_local_rp('sym_in2') if rp2.lstat(): rp2.delete() rp2.mkdir() rp2_s = rp2.append('subdir') rp2_s.symlink("%s/%s" % (abs_test_dir, rp1_d.path)) # Backup rdiff_backup(True, True, rp1.path, Local.rpout.path, current_time=10000) rdiff_backup(True, True, rp2.path, Local.rpout.path, current_time=20000) # Make failed backup rbdir = Local.rpout.append('rdiff-backup-data') curmir = rbdir.append('current_mirror.%s.data' % (Time.timetostring(30000), )) curmir.touch() # Regress rdiff_backup(True, True, Local.rpout.path, None, current_time=30000, extra_options=b'--check-destination-dir') # Check to see if file still there rp1_d_f.setdata() assert rp1_d_f.isreg(), 'File %a corrupted' % (rp1_d_f.path, )
def _get_parsed_time(self, timestr, ref_rp=None): """ Parse time string, potentially using the given remote path as reference Returns None if the time string couldn't be parsed, else the time in seconds. The reference remote path is used when the time string consists in a number of past backups. """ try: if Globals.get_api_version() < 201: # compat200 return Time.genstrtotime(timestr, rp=ref_rp) else: sessions = self.repo.get_increment_times(ref_rp) return Time.genstrtotime(timestr, session_times=sessions) except Time.TimeException as exc: log.Log("Time string '{ts}' couldn't be parsed " "due to '{ex}'".format(ts=timestr, ex=exc), log.ERROR) return None
def _list_increments(self): """ Print out a summary of the increments and their times """ incs = self.repo.get_increments() if self.values.parsable_output: if Globals.get_api_version() < 201: for inc in incs: print("{ti} {it}".format(ti=inc["time"], it=inc["type"])) else: print(yaml.safe_dump(incs, explicit_start=True, explicit_end=True)) else: print("Found {ni} increments:".format(ni=len(incs) - 1)) for inc in incs[:-1]: print(" {ib} {ti}".format( ib=inc["base"], ti=Time.timetopretty(inc["time"]))) print("Current mirror: {ti}".format( ti=Time.timetopretty(incs[-1]["time"]))) # time of the mirror
def testConversion(self): """test timetostring and stringtotime""" Time.setcurtime() assert type(Time.curtime) is types.FloatType or types.LongType assert type(Time.curtimestr) is types.StringType assert (Time.cmp(int(Time.curtime), Time.curtimestr) == 0 or Time.cmp(int(Time.curtime) + 1, Time.curtimestr) == 0) time.sleep(1.05) assert Time.cmp(time.time(), Time.curtime) == 1 assert Time.cmp(Time.timetostring(time.time()), Time.curtimestr) == 1
def testPrettyIntervals(self): """Test printable interval conversion""" assert Time.inttopretty(3600) == "1 hour" assert Time.inttopretty(7220) == "2 hours 20 seconds" assert Time.inttopretty(0) == "0 seconds" assert Time.inttopretty(353) == "5 minutes 53 seconds" assert Time.inttopretty(3661) == "1 hour 1 minute 1 second" assert Time.inttopretty(353.234234) == "5 minutes 53.23 seconds"
rf = getrp("regular_file") rf2 = getrp("two_hardlinked_files1") exec1 = getrp("executable") exec2 = getrp("executable2") sig = getrp("regular_file.sig") hl1, hl2 = map(getrp, ["two_hardlinked_files1", "two_hardlinked_files2"]) test = getrp("test") dir = getrp(".") sym = getrp("symbolic_link") nothing = getrp("nothing") target = rpath.RPath(lc, "testfiles/output/out") out2 = rpath.RPath(lc, "testfiles/output/out2") out_gz = rpath.RPath(lc, "testfiles/output/out.gz") Time.setcurtime(1000000000) Time.setprevtime(999424113) prevtimestr = "2001-09-02T02:48:33-07:00" t_pref = "testfiles/output/out.2001-09-02T02:48:33-07:00" t_diff = "testfiles/output/out.2001-09-02T02:48:33-07:00.diff" Globals.no_compression_regexp = \ re.compile(Globals.no_compression_regexp_string, re.I) class inctest(unittest.TestCase): """Test the incrementRP function""" def setUp(self): Globals.set('isbackup_writer',1) MakeOutputDir() def check_time(self, rp):
def add_current_mirror(self, time): """Add current_mirror marker at given time""" cur_mirror_rp = self.output_rbdir_rp.append( "current_mirror.%s.data" % (Time.timetostring(time),)) cur_mirror_rp.touch()