def main(**kwargs):
"""
Main control function for the script
"""
inputDirs = kwargs.pop('inputDirs')
useRsync = kwargs.pop('useRsync', False)
rsyncConfig = kwargs.pop('rsyncConfig', None)
overwrite = kwargs.pop('overwrite', False)
removeSource = kwargs.pop('removeSource', False)
for inputDir in inputDirs:
print(">>> Processing {0} dir".format(inputDir))
if useRsync:
# add trailing slash for rsync path
inputDir = os.path.join(inputDir, '')
# set the rsync config if passed
if rsyncConfig:
rsync.load_config(input_json = rsyncConfig)
# first copy all remote files to local input dir
print(">>> Syncing {0} directory with {1}".format(inputDir, rsync.config['targetDir']))
rsync.rsync(source = inputDir, target = rsync.config['targetDir'], dryRun = False, swap = True)
# then process all files in local input dir
make_reports(inputDir = inputDir, overwrite = overwrite)
# then copy everything back to remote destination; remove local copies
print(">>> Syncing {0} directory with {1}".format(rsync.config['targetDir'], inputDir))
if removeSource:
flags = ['--remove-source-files']
else:
flags = []
rsync.rsync(source = inputDir, target = rsync.config['targetDir'], dryRun = False, flags = flags)
else:
# process all files in local input dir
make_reports(inputDir = inputDir)
def push(self):
self.logger.info(" Pushing replica state to source")
if self.verbose:
args = ["-v"]
else:
args = []
rsync.rsync(self.states_filename,
f"{self.source}.gc",
args,
stfu=False)
def __init__(self, debug=False): self.debug = debug if self.debug: logLevel = logging.DEBUG else: logLevel = logging.INFO logging.basicConfig(format='%(levelname)s: %(message)s', level=logLevel) if not os.path.isdir(self.OUTPUT_DIR): os.mkdir(self.OUTPUT_DIR) if not os.path.isdir(self.UNPROCESSED_DIR): os.mkdir(self.UNPROCESSED_DIR) if not os.path.isdir(self.PROCESSED_DIR): os.mkdir(self.PROCESSED_DIR) if not os.path.isdir(self.PROCESSING_DIR): os.mkdir(self.PROCESSING_DIR) self.db = DB(debug = self.debug) self.rsync = rsync( self.debug ) if socket.gethostname() == 'cet-sif': self.OnServer = True # self.host = socket.gethostbyname( 'cet-research.colorado.edu' ) self.host = '128.138.248.205' else: self.OnServer = False self.host = socket.gethostbyname(socket.gethostname())
def __init__(self, debug=False):
self.debug = debug # Set debug mode
if self.debug:
logLevel = logging.DEBUG
else:
logLevel = logging.INFO
# Set logging level
logging.basicConfig(format='%(levelname)s: %(message)s', level=logLevel)
if not os.path.isdir(self.OUTPUT_DIR):
os.mkdir(self.OUTPUT_DIR)
if not os.path.isdir(self.UNPROCESSED_DIR):
os.mkdir(self.UNPROCESSED_DIR)
if not os.path.isdir(self.PROCESSED_DIR):
os.mkdir(self.PROCESSED_DIR)
if not os.path.isdir(self.PROCESSING_DIR):
os.mkdir(self.PROCESSING_DIR)
self.db = DB(debug = self.debug)
self.rsync = rsync( self.debug )
if socket.gethostname() == 'cet-sif':
self.OnServer = True
self.host = '128.138.248.205' # Static ip resolution of cet-sif.colorado.edu
else:
self.OnServer = False
self.host = socket.gethostbyname(socket.gethostname()) # Set to localhost
def sshSync(self):
from osx_notifier import Notifier
from rsync import rsync
from files import GetFiles
from files import ReadFilesRsync
get = GetFiles()
Send = Notifier()
user = ReadFilesRsync(get.GetUser())
server = (ReadFilesRsync(get.GetServer()))
mode = ReadFilesRsync(get.GetMode())
print "Hole Verzeichnisse"
directory = []
i = 0
fobj = open(get.GetDir())
for line in fobj:
directory.append(line.rstrip())
i = i + 1
fobj.close()
SRC = []
DST = []
check = mode.ReadFile()
if check == "-p":
for i in range(len(directory)):
tmp = directory[i].split(":")
tmp1 = tmp[0]
tmp2 = tmp1.split("/")
tmp3 = "/%s" % tmp2[-1]
tmp4 = tmp[1]
tmp2.pop()
tmpSRC = ("/").join(tmp2)
tmpDST = tmp[1] + tmp3
DST.append(tmpDST)
SRC.append(tmpSRC)
else:
for i in range(len(directory)):
tmp = directory[i].split(":")
SRC.append(tmp[0])
DST.append(tmp[1])
print "Starte Sync..."
Send.SendMessage("Synchronisation wird gestartet")
for i in range(len(SRC)):
print "DST: %s SRC: %s" % (SRC[i], DST[i])
Send.SendMessage("Synchronisiere %s zu %s" % (SRC[i], DST[i]))
sync = rsync(SRC[i], DST[i], user.ReadFile(), server.ReadFile(),
mode.ReadFile())
sync.ssh()
Send.SendMessage("Synchronisation beendet!")
exit()
def pull_source_config(self):
if self.source_config is None:
self.logger.error(f"ERROR: {self.filename} MUST contain a " \
f"\"source config:\" line")
sys.exit(1)
host = host_for(self.source_config)
if host == self.hostname:
self.logger.debug("I am source, not pulling the config")
return
self.logger.debug(f"pulling config {self.source_config}" \
f" -> {self.filename}")
if self.testing:
self.logger.debug(self.source_config)
source_config = path_for(self.source_config)
else:
source_config = self.source_config
rsync(source_config, self.filename, stfu=True)
def sshSync(self):
from osx_notifier import Notifier
from rsync import rsync
from files import GetFiles
from files import ReadFilesRsync
get = GetFiles()
Send = Notifier()
user = ReadFilesRsync(get.GetUser())
server = (ReadFilesRsync(get.GetServer()))
mode = ReadFilesRsync(get.GetMode())
print"Hole Verzeichnisse"
directory = []
i = 0
fobj = open(get.GetDir())
for line in fobj:
directory.append(line.rstrip())
i = i + 1
fobj.close()
SRC = []
DST = []
check = mode.ReadFile()
if check == "-p":
for i in range(len(directory)):
tmp = directory[i].split(":")
tmp1 = tmp[0]
tmp2 = tmp1.split("/")
tmp3 = "/%s" % tmp2[-1]
tmp4 = tmp[1]
tmp2.pop()
tmpSRC = ("/").join(tmp2)
tmpDST = tmp[1] + tmp3
DST.append(tmpDST)
SRC.append(tmpSRC)
else:
for i in range(len(directory)):
tmp = directory[i].split(":")
SRC.append(tmp[0])
DST.append(tmp[1])
print "Starte Sync..."
Send.SendMessage("Synchronisation wird gestartet")
for i in range(len(SRC)):
print "DST: %s SRC: %s" % (SRC[i], DST[i])
Send.SendMessage("Synchronisiere %s zu %s" % (SRC[i], DST[i]))
sync = rsync(SRC[i], DST[i], user.ReadFile(), server.ReadFile(), mode.ReadFile())
sync.ssh()
Send.SendMessage("Synchronisation beendet!")
exit()
def pull(self, deleting=False):
self.logger.debug("Pulling")
# print(f"running for {self.path} <= {self.source}")
self.logger.info(f"pulling {self.path} <= {self.source}")
args = ["-a"]
if self.verbose:
args.append("-v")
options = self.config.getConfig(self.context, "rsync options")
if len(options) > 0:
if not deleting and "delete" in options:
options.remove("delete")
args += [f"--{option}" for option in options]
ignorals = self.config.get_ignorals(self.context)
if len(ignorals) > 0:
args += [f"--exclude={item}" for item in ignorals]
self.logger.debug("Starting rsync pull...")
self.logger.debug(f"rsync {' '.join(args)} {self.source} {self.path}")
rsync.rsync(self.source, self.path, args)
self.pull_states()
check = mode.ReadFile()
if check == "-p":
for i in range (len(directory)):
tmp = directory[i].split(":")
tmp1 = tmp[0]
tmp2 = tmp1.split("/")
tmp3 = "/%s" % tmp2[-1]
tmp4 = tmp[1]
tmp2.pop()
tmpSRC = ("/").join(tmp2)
tmpDST = tmp[1] + tmp3
DST.append(tmpDST)
SRC.append(tmpSRC)
else:
for i in range (len(directory)):
tmp = directory[i].split(":")
SRC.append(tmp[0])
DST.append(tmp[1])
print "Starte Sync..."
for i in range (len(SRC)):
print "DST: %s SRC: %s" % (SRC[i], DST[i])
sync = rsync(SRC[i], DST[i], mode.ReadFile())
sync.Sync()
exit()
def process(self):
"""Process the arguments, distribute the work. """
# Load the config file, setup logging and read the arguments.
super().process()
#ready to go.
#This is where we do what we need based on the config file and arguments.
# Everything we need should be in self.ARGS. Except for our project
# section. We can ask our super to merge the config section in as
# soon as we know it's name.
project_base = self.config['default']['ProjectBase']
project_root = self.config['default']['ProjectRoot']
self.project_root = os.path.join(project_base, project_root)
self.excludes = []
if 'file' in self.args:
current_directory = os.path.abspath('')
self.file = self.args.file
if 'p' in self.args:
self.name = self.args.p
if 'new' in self.args:
self.name = self.args.new
# Directory is the local location of the project.
# The default is the name of the project.
if 'directory' in self.args and self.args.directory is not None:
self.directory = self.args.directory
else:
self.directory = self.name
self.find_working_project()
self.logger.debug('%s : %s' % (self.name, self.directory))
# We should know our project name by now. Merge the section into
# our ARGS dictionary.
self.merge_section(self.name)
# if a filename was given, get it's relative path in the project.
if self.file:
self.set_file(current_directory)
# Try to load the Project manager's config file and get
# the default settings and project definitions.
self.set_project_config()
# be ready to create or use a local git.
self.set_local_git(project_base)
#Get the open command if we have one.
self.set_opencmd()
#get the excludes for the rsync.
self.setup_excludes()
# set up the rsync connection
self.rsync = rsync(self.cmd, self.logger,
self.host_path, self.host,
self.abs_project_path,
self.excludes)
# Need some debug for understanding???? Here it is.
#self.logger.info(self.args)
#self.logger.info(self.config)
#self.print_config()
# run our default functions for our subparsers.
# It's either this, or process them all the hard way.
# self.args.func() will farm out the arguments to each
# subparser's default function.
# surround it all with a try to handle errors nicely.
#try:
if 'func' in self.args:
self.args.func()
def pull_states(self):
self.logger.info(" Pulling source state")
args = ["--delete", "-a"]
if self.verbose:
args.append("-v")
rsync.rsync(f"{self.source}.gc", self.path, args, stfu=False)
