def main(**kwargs): """ Main control function for the script """ inputDirs = kwargs.pop('inputDirs') useRsync = kwargs.pop('useRsync', False) rsyncConfig = kwargs.pop('rsyncConfig', None) overwrite = kwargs.pop('overwrite', False) removeSource = kwargs.pop('removeSource', False) for inputDir in inputDirs: print(">>> Processing {0} dir".format(inputDir)) if useRsync: # add trailing slash for rsync path inputDir = os.path.join(inputDir, '') # set the rsync config if passed if rsyncConfig: rsync.load_config(input_json = rsyncConfig) # first copy all remote files to local input dir print(">>> Syncing {0} directory with {1}".format(inputDir, rsync.config['targetDir'])) rsync.rsync(source = inputDir, target = rsync.config['targetDir'], dryRun = False, swap = True) # then process all files in local input dir make_reports(inputDir = inputDir, overwrite = overwrite) # then copy everything back to remote destination; remove local copies print(">>> Syncing {0} directory with {1}".format(rsync.config['targetDir'], inputDir)) if removeSource: flags = ['--remove-source-files'] else: flags = [] rsync.rsync(source = inputDir, target = rsync.config['targetDir'], dryRun = False, flags = flags) else: # process all files in local input dir make_reports(inputDir = inputDir)
def push(self): self.logger.info(" Pushing replica state to source") if self.verbose: args = ["-v"] else: args = [] rsync.rsync(self.states_filename, f"{self.source}.gc", args, stfu=False)
def __init__(self, debug=False): self.debug = debug if self.debug: logLevel = logging.DEBUG else: logLevel = logging.INFO logging.basicConfig(format='%(levelname)s: %(message)s', level=logLevel) if not os.path.isdir(self.OUTPUT_DIR): os.mkdir(self.OUTPUT_DIR) if not os.path.isdir(self.UNPROCESSED_DIR): os.mkdir(self.UNPROCESSED_DIR) if not os.path.isdir(self.PROCESSED_DIR): os.mkdir(self.PROCESSED_DIR) if not os.path.isdir(self.PROCESSING_DIR): os.mkdir(self.PROCESSING_DIR) self.db = DB(debug = self.debug) self.rsync = rsync( self.debug ) if socket.gethostname() == 'cet-sif': self.OnServer = True # self.host = socket.gethostbyname( 'cet-research.colorado.edu' ) self.host = '128.138.248.205' else: self.OnServer = False self.host = socket.gethostbyname(socket.gethostname())
def __init__(self, debug=False): self.debug = debug # Set debug mode if self.debug: logLevel = logging.DEBUG else: logLevel = logging.INFO # Set logging level logging.basicConfig(format='%(levelname)s: %(message)s', level=logLevel) if not os.path.isdir(self.OUTPUT_DIR): os.mkdir(self.OUTPUT_DIR) if not os.path.isdir(self.UNPROCESSED_DIR): os.mkdir(self.UNPROCESSED_DIR) if not os.path.isdir(self.PROCESSED_DIR): os.mkdir(self.PROCESSED_DIR) if not os.path.isdir(self.PROCESSING_DIR): os.mkdir(self.PROCESSING_DIR) self.db = DB(debug = self.debug) self.rsync = rsync( self.debug ) if socket.gethostname() == 'cet-sif': self.OnServer = True self.host = '128.138.248.205' # Static ip resolution of cet-sif.colorado.edu else: self.OnServer = False self.host = socket.gethostbyname(socket.gethostname()) # Set to localhost
def sshSync(self): from osx_notifier import Notifier from rsync import rsync from files import GetFiles from files import ReadFilesRsync get = GetFiles() Send = Notifier() user = ReadFilesRsync(get.GetUser()) server = (ReadFilesRsync(get.GetServer())) mode = ReadFilesRsync(get.GetMode()) print "Hole Verzeichnisse" directory = [] i = 0 fobj = open(get.GetDir()) for line in fobj: directory.append(line.rstrip()) i = i + 1 fobj.close() SRC = [] DST = [] check = mode.ReadFile() if check == "-p": for i in range(len(directory)): tmp = directory[i].split(":") tmp1 = tmp[0] tmp2 = tmp1.split("/") tmp3 = "/%s" % tmp2[-1] tmp4 = tmp[1] tmp2.pop() tmpSRC = ("/").join(tmp2) tmpDST = tmp[1] + tmp3 DST.append(tmpDST) SRC.append(tmpSRC) else: for i in range(len(directory)): tmp = directory[i].split(":") SRC.append(tmp[0]) DST.append(tmp[1]) print "Starte Sync..." Send.SendMessage("Synchronisation wird gestartet") for i in range(len(SRC)): print "DST: %s SRC: %s" % (SRC[i], DST[i]) Send.SendMessage("Synchronisiere %s zu %s" % (SRC[i], DST[i])) sync = rsync(SRC[i], DST[i], user.ReadFile(), server.ReadFile(), mode.ReadFile()) sync.ssh() Send.SendMessage("Synchronisation beendet!") exit()
def pull_source_config(self): if self.source_config is None: self.logger.error(f"ERROR: {self.filename} MUST contain a " \ f"\"source config:\" line") sys.exit(1) host = host_for(self.source_config) if host == self.hostname: self.logger.debug("I am source, not pulling the config") return self.logger.debug(f"pulling config {self.source_config}" \ f" -> {self.filename}") if self.testing: self.logger.debug(self.source_config) source_config = path_for(self.source_config) else: source_config = self.source_config rsync(source_config, self.filename, stfu=True)
def sshSync(self): from osx_notifier import Notifier from rsync import rsync from files import GetFiles from files import ReadFilesRsync get = GetFiles() Send = Notifier() user = ReadFilesRsync(get.GetUser()) server = (ReadFilesRsync(get.GetServer())) mode = ReadFilesRsync(get.GetMode()) print"Hole Verzeichnisse" directory = [] i = 0 fobj = open(get.GetDir()) for line in fobj: directory.append(line.rstrip()) i = i + 1 fobj.close() SRC = [] DST = [] check = mode.ReadFile() if check == "-p": for i in range(len(directory)): tmp = directory[i].split(":") tmp1 = tmp[0] tmp2 = tmp1.split("/") tmp3 = "/%s" % tmp2[-1] tmp4 = tmp[1] tmp2.pop() tmpSRC = ("/").join(tmp2) tmpDST = tmp[1] + tmp3 DST.append(tmpDST) SRC.append(tmpSRC) else: for i in range(len(directory)): tmp = directory[i].split(":") SRC.append(tmp[0]) DST.append(tmp[1]) print "Starte Sync..." Send.SendMessage("Synchronisation wird gestartet") for i in range(len(SRC)): print "DST: %s SRC: %s" % (SRC[i], DST[i]) Send.SendMessage("Synchronisiere %s zu %s" % (SRC[i], DST[i])) sync = rsync(SRC[i], DST[i], user.ReadFile(), server.ReadFile(), mode.ReadFile()) sync.ssh() Send.SendMessage("Synchronisation beendet!") exit()
def pull(self, deleting=False): self.logger.debug("Pulling") # print(f"running for {self.path} <= {self.source}") self.logger.info(f"pulling {self.path} <= {self.source}") args = ["-a"] if self.verbose: args.append("-v") options = self.config.getConfig(self.context, "rsync options") if len(options) > 0: if not deleting and "delete" in options: options.remove("delete") args += [f"--{option}" for option in options] ignorals = self.config.get_ignorals(self.context) if len(ignorals) > 0: args += [f"--exclude={item}" for item in ignorals] self.logger.debug("Starting rsync pull...") self.logger.debug(f"rsync {' '.join(args)} {self.source} {self.path}") rsync.rsync(self.source, self.path, args) self.pull_states()
check = mode.ReadFile() if check == "-p": for i in range (len(directory)): tmp = directory[i].split(":") tmp1 = tmp[0] tmp2 = tmp1.split("/") tmp3 = "/%s" % tmp2[-1] tmp4 = tmp[1] tmp2.pop() tmpSRC = ("/").join(tmp2) tmpDST = tmp[1] + tmp3 DST.append(tmpDST) SRC.append(tmpSRC) else: for i in range (len(directory)): tmp = directory[i].split(":") SRC.append(tmp[0]) DST.append(tmp[1]) print "Starte Sync..." for i in range (len(SRC)): print "DST: %s SRC: %s" % (SRC[i], DST[i]) sync = rsync(SRC[i], DST[i], mode.ReadFile()) sync.Sync() exit()
def process(self): """Process the arguments, distribute the work. """ # Load the config file, setup logging and read the arguments. super().process() #ready to go. #This is where we do what we need based on the config file and arguments. # Everything we need should be in self.ARGS. Except for our project # section. We can ask our super to merge the config section in as # soon as we know it's name. project_base = self.config['default']['ProjectBase'] project_root = self.config['default']['ProjectRoot'] self.project_root = os.path.join(project_base, project_root) self.excludes = [] if 'file' in self.args: current_directory = os.path.abspath('') self.file = self.args.file if 'p' in self.args: self.name = self.args.p if 'new' in self.args: self.name = self.args.new # Directory is the local location of the project. # The default is the name of the project. if 'directory' in self.args and self.args.directory is not None: self.directory = self.args.directory else: self.directory = self.name self.find_working_project() self.logger.debug('%s : %s' % (self.name, self.directory)) # We should know our project name by now. Merge the section into # our ARGS dictionary. self.merge_section(self.name) # if a filename was given, get it's relative path in the project. if self.file: self.set_file(current_directory) # Try to load the Project manager's config file and get # the default settings and project definitions. self.set_project_config() # be ready to create or use a local git. self.set_local_git(project_base) #Get the open command if we have one. self.set_opencmd() #get the excludes for the rsync. self.setup_excludes() # set up the rsync connection self.rsync = rsync(self.cmd, self.logger, self.host_path, self.host, self.abs_project_path, self.excludes) # Need some debug for understanding???? Here it is. #self.logger.info(self.args) #self.logger.info(self.config) #self.print_config() # run our default functions for our subparsers. # It's either this, or process them all the hard way. # self.args.func() will farm out the arguments to each # subparser's default function. # surround it all with a try to handle errors nicely. #try: if 'func' in self.args: self.args.func()
def pull_states(self): self.logger.info(" Pulling source state") args = ["--delete", "-a"] if self.verbose: args.append("-v") rsync.rsync(f"{self.source}.gc", self.path, args, stfu=False)