def up(self):
remote_dir = self.remote_dir
if not os.path.exists(remote_dir):
os.mkdir(remote_dir)
tries = 5
while tries > 0:
self.mount(remote_dir)
sleep(1)
try:
sh.ls(remote_dir)
except sh.ErrorReturnCode:
pstor.umount(remote_dir)
else:
break
tries -= 1
else:
raise exceptions.PstorException("Can't ls in mounted webdav directory")
remote_dir = os.path.join(self.remote_dir, 'pstor/')
if not os.path.exists(remote_dir):
os.mkdir(remote_dir)
#TODO: Check if not rewrites files (e.g. existing repo in new repo)
sh.rsync(remote_dir, '.pstor/encrypted', recursive=True)
def fetch_remote_book_to_local_path(self):
sh.rsync(
'-rvhz',
'[email protected]::gutenberg/{0}'.format(self.book.remote_path),
self.book.local_path + '/',
'--exclude-from=exclude.txt'
)
def rsync(self, src, dest, link_dest):
rsync(self.OPT,
src + "/",
dest,
link_dest=link_dest,
exclude_from=self.EXCLUDES_FILE,
_out=sys.stdout.buffer)
def _update_packages(self):
''' update the package cache '''
remote = CONFIG.parabola.mirror
local = self._pkgfiles_dir
os.makedirs(local, exist_ok=True)
sh.rsync('-a', '--delete-after', '--filter', 'P *.*info', remote,
local)
def rsync(self, src, dest, link_dest):
rsync(
self.OPT, src + "/", dest,
link_dest=link_dest,
exclude_from=self.EXCLUDES_FILE,
_out=sys.stdout.buffer
)
def copy_openvpn_cert(openvpn_cert_path_, device_mount_):
"""Copy OpenVPN certificate(s) (entire directory) to directory in mounted casper-rw file system."""
dest_ = "{0}/{1}".format(device_mount_,
os.path.basename(openvpn_cert_path_))
if mem.options.verbose:
print "[+] Copying OpenVPN cert(s) from {0} to {1})".format(
openvpn_cert_path_, dest_)
if not os.path.exists(dest_):
try:
sudo.mkdir(dest_)
except Exception as exception_:
print u"[!] Exception: {0}".format(exception_)
print "[!] Can't create directory {0}".format(dest_)
# Using SSHFS causes problems for root/user interaction. Copy everything
# to /tmp first, then use sudo to copy to casper-rw.
tmp_ = "/tmp/{0}".format(os.path.split(dest_)[1])
if tmp_ == "/tmp/":
print "[!] Refusing to possibly delete /tmp/"
sys.exit(1)
rsync("-a", "--delete", "{0}/".format(openvpn_cert_path_),
"{0}/".format(tmp_))
try:
sudo.rsync("-a", "--delete", "--no-owner", "{0}/".format(tmp_),
"{0}/".format(dest_))
rm("-rf", tmp_)
except Exception as exception_:
print u"[!] Exception: {0}".format(exception_)
print "[!] Failed: copy_openvpn_cert()"
rm("-rf", tmp_)
cleanexit(1)
def backupDir(self, top_tree, backup_dir, dry_run):
'''
Method to copy recursively to a backup directory
'''
try:
from sh import rsync
except ImportError:
print "Unable to import rsync package"
top_dir = os.path.basename(top_tree)
top_tree += os.sep
self.checkDirExists(top_tree)
if 'sh' in sys.modules:
# If rsync is available
rsync("-auhv", "--log-file={}".format(self.logger), top_tree, backup_dir)
else:
try:
if dry_run:
self.logger.info('Will copy from {} to {}'.format(top_tree, backup_dir))
sys.exit(1)
shutil.copytree(top_tree, backup_dir)
except shutil.Error as e:
print('Directory not copied. Error: %s' % e)
except OSError as e:
print('Directory not copied. Error: %s' % e)
def sync_repo(commit):
config_options = getConfigOptions()
rsyncdest = config_options.rsyncdest
rsyncport = config_options.rsyncport
datadir = os.path.realpath(config_options.datadir)
if rsyncdest != '':
# We are only rsyncing the current repo dir to rsyncdest
rsyncpaths = []
# We are inserting a dot in the path after repos, this is used by
# rsync -R (see man rsync)
commitdir_abs = os.path.join(datadir, "repos", ".",
commit.getshardedcommitdir())
rsyncpaths.append(commitdir_abs)
# We also need report.html, status_report.html, queue.html,
# styles.css and the consistent and current symlinks
for filename in [
'report.html', 'status_report.html', 'styles.css', 'queue.html'
]:
filepath = os.path.join(datadir, "repos", ".", filename)
rsyncpaths.append(filepath)
rsh_command = 'ssh -p %s -o StrictHostKeyChecking=no' % rsyncport
try:
sh.rsync('-avzR', '--delete-delay', '-e', rsh_command, rsyncpaths,
rsyncdest)
except Exception as e:
logger.warn('Failed to rsync content to %s ,'
'got error %s' % (rsyncdest, e))
# Raise exception, so it can be treated as an error
raise e
def main():
if os.getcwd() == os.path.expanduser('~'):
print('NO')
exit(1)
addr = sys.argv[1]
del sys.argv[1]
path = '/tmp/{}'.format(uuid.uuid4())
env = []
while '=' in sys.argv[1]:
env.append(sys.argv[1].split('='))
del sys.argv[1]
env = dict(env)
print(
sh.rsync('-a', '-v', '-z', '--exclude', '__pycache__', '--exclude',
'*.swp', '--exclude', '.git', '{}/'.format(os.getcwd()),
'{}:{}'.format(addr, path)))
cmd = 'cd {} && {}'.format(path, ' '.join(sys.argv[1:]))
connection = fabric.Connection(addr, inline_ssh_env=True)
connection.client.load_system_host_keys()
connection.run(cmd, env=env)
print(
sh.rsync('-a', '-v', '-z', '{}:{}/logs/'.format(addr, path),
os.getcwd() + '/logs'))
def fetch_remote_book_to_local_path(self):
sh.rsync(
'-rvhz',
'{}{}'.format(PG_RSYNC, self.book.remote_path),
self.book.local_path + '/',
'--exclude-from=exclude.txt'
)
def delete_file(node): server = get_server_for_node(node) for line in rsync(POLLY_FILES, server, archive=True, compress=True, relative=True, delete=True, itemize_changes=True, dry_run=True, _iter=True): print line parts = line.split() if "deleting" in parts[0]: fileid = get_file_id(parts[1]) add_packet(parts[1]) add_port(node, "deleting", fileid) add_port(current_node, "deleting", fileid) else: print "Not deleting:", line save_status_update() server = POLLY_USER + "@" + node + ":/" for line in rsync(POLLY_FILES, server, archive=True, compress=True, relative=True, delete=True, itemize_changes=True, _iter=True): print line parts = line.split() if "deleting" in parts[0]: fileid = get_file_id(parts[1]) add_packet(parts[1]) add_port(node, "deleted", fileid) add_port(current_node, "deleted", fileid) else: print "Not deleting:", line save_status_update()
def sync_repo(commit):
config_options = getConfigOptions()
rsyncdest = config_options.rsyncdest
rsyncport = config_options.rsyncport
datadir = os.path.realpath(config_options.datadir)
if rsyncdest != '':
# We are only rsyncing the current repo dir to rsyncdest
rsyncpaths = []
# We are inserting a dot in the path after repos, this is used by
# rsync -R (see man rsync)
commitdir_abs = os.path.join(datadir, "repos", ".",
commit.getshardedcommitdir())
rsyncpaths.append(commitdir_abs)
# We also need report.html, status_report.html, queue.html,
# styles.css and the consistent and current symlinks
for filename in ['report.html', 'status_report.html', 'styles.css',
'queue.html', 'status_report.csv']:
filepath = os.path.join(datadir, "repos", ".", filename)
rsyncpaths.append(filepath)
rsh_command = 'ssh -p %s -o StrictHostKeyChecking=no' % rsyncport
try:
sh.rsync('-avzR', '--delete-delay',
'-e', rsh_command,
rsyncpaths, rsyncdest)
except Exception as e:
logger.warn('Failed to rsync content to %s ,'
'got error %s' % (rsyncdest, e))
# Raise exception, so it can be treated as an error
raise e
def sync_repo(commit):
rsyncdest = config_options.rsyncdest
rsyncport = config_options.rsyncport
datadir = os.path.realpath(config_options.datadir)
if rsyncdest != '':
# We are only rsyncing the current repo dir to rsyncdest
rsyncpaths = []
# We are inserting a dot in the path after repos, this is used by
# rsync -R (see man rsync)
commitdir_abs = os.path.join(datadir, "repos", ".",
commit.getshardedcommitdir())
rsyncpaths.append(commitdir_abs)
# We also need report.html, status_report.html and styles.css
for filename in ['report.html', 'status_report.html', 'styles.css']:
filepath = os.path.join(datadir, "repos", ".", filename)
rsyncpaths.append(filepath)
rsh_command = 'ssh -p %s -o StrictHostKeyChecking=no' % rsyncport
for dirname in rsyncpaths:
try:
sh.rsync('-avzR', '--delete',
'-e', rsh_command,
dirname, rsyncdest)
except Exception as e:
logger.warn('Failed to sync directory %s to %s ,'
'got error %s' % (dirname, rsyncdest, e))
def _sync(sync_mappings):
for from_path, to_path in sync_mappings:
to_path.mkdir(parents=True, exist_ok=True)
exclude_params = zip(len(excludes) * ['--exclude'], excludes)
rsync('-ad', from_path.as_posix() + '/', to_path.as_posix() + '/',
delete=True, *exclude_params)
def install(src_path_str: str, dst_path_str: str):
try:
print(f'{src_path_str} -> {dst_path_str}')
sh.rsync('-r', src_path_str, dst_path_str)
except sh.ErrorReturnCode_23:
print(f'src not found: {src_path_str}')
except sh.ErrorReturnCode_11:
print(f'dst not found: {dst_path_str}')
def insert_app(path_to_app, appname, blacklist=None, whitelist=None):
# insert appname into our source_app
params = ['-r', path_to_app + '/', appname + '/Contents/Resources/myapp']
if whitelist:
params.append('--include-from={}'.format(whitelist))
if blacklist:
params.append('--exclude-from={}'.format(blacklist))
sh.rsync(*params)
def insert_app(path_to_app, appname, blacklist=None, whitelist=None):
# insert appname into our source_app
params = ['-r', path_to_app + '/', appname + '/Contents/Resources/myapp']
if whitelist:
params.append('--include-from={}'.format(whitelist))
if blacklist:
params.append('--exclude-from={}'.format(blacklist))
sh.rsync(*params)
def format(self, project: Project):
"""
Take the original code from archive, extract it to a temp directory,
run the formatter, and then copy the formatter result to the project
root
"""
with tempfile.TemporaryDirectory() as temp_dir:
sh.tar("-C", temp_dir, "-x", "-f", str(project.archive_path))
self.do_format(temp_dir, project)
sh.rsync("-a", temp_dir + "/", str(project.root))
def sync(self):
if not pstor.mounted(self.remote_dir):
print "<Not mounted>"
return
remote_dir = os.path.join(self.remote_dir, 'pstor')
if not os.path.exists(remote_dir):
os.mkdir(remote_dir)
sh.rsync('.pstor/encrypted/', remote_dir, recursive=True, delete=True)
def _submit(self, execution):
self.log(
f"Rsyncing project {execution.project.directory_path} -> {self.host}:{self.directory}"
)
path = execution.project.directory_path
if path[-1] != "/":
path += "/"
sh.rsync(
"-azL",
path,
f"{self.host}:{self.directory}"
f"{' --delete' if self.sync.get('delete_on_remote', False) else ''}",
)
self.log("Rsync complete. Executing ...")
if execution.storage["url"].startswith("ssh://" + self.host):
# rewrite SSH storage path as relative path on remote host
execution.storage["url"] = execution.storage["url"].replace(
"ssh://" + self.host.rstrip("/") + ":", ""
)
url = os.path.join(
execution.storage["url"],
execution.storage.get("directory", ""),
execution.submission_id,
)
engine = self.engine.to_json().replace('"', '\\"')
remote_project = copy.deepcopy(execution.project.options)
remote_project["directory"] = self.directory
project = json.dumps(remote_project).replace('"', '\\"')
code = f"""
import machinable as ml
e = ml.Execution.from_storage('{url}')
e.set_engine(ml.Engine.from_json('{engine}'))
e.set_storage({execution.storage})
e.set_project(ml.Project.from_json('{project}'))
e.submit()
""".replace(
"\n ", ";"
)[
1:-1
]
command = f'`cd {self.directory}; {self.python} -c "{code}"`'
try:
p = self.shell(command, _bg=True)
execution.set_result(p)
except sh.ErrorReturnCode as ex:
execution.set_result(
ExecutionException(ex.stderr.decode("utf-8"), reason="engine_failure")
)
self.log(f"Execution failed: {str(ex)}", level="error")
return execution
def sync(source, target, delete=False):
try:
if delete:
opt = options + ["--delete-after"]
else:
opt = options
msgstat("Syncing {} with {}".format(source, target))
rsync(opt, source, target, _out=debugSh, _err=debugSh)
except:
msgerr("Failed syncing {} with {}".format(source, target))
else:
msgstat("Success syncing {} with {}".format(source, target))
def rsyncTimecoursesForTask(task):
directory = getTimecourseDirectory(task)
to_path = directory + '/'
if not os.path.exists(to_path):
os.makedirs(to_path)
for ip in IPS:
host = 'mkoenig@' + ip
from_path = host + ":" + directory + '/'
print host
if exists_remote(host, to_path):
print 'rsync : ', from_path, '->', to_path
#rsync -ravzX --delete mkoenig@ip:directory directory
rsync("-ravzX", from_path + '*.Rdata', to_path)
rsync("-ravzX", from_path + '*.gz', to_path)
def sync(remote_dir, target_dir):
if '/' not in remote_dir[-1]:
remote_dir += '/'
output = rsync('-a', remote_dir, target_dir)
return output.exit_code
def expand(self, file_id, path=''):
logger.info('Expanding %s', file_id)
if path:
path += '/'
with self.working:
self._stop()
conf = self._config[file_id]
try:
out = sh.rsync('--no-motd', '--no-h', conf['source'] + '/' + path + '/').split('\n')
except sh.ErrorReturnCode_23:
raise cherrypy.HTTPError(400, 'The path {!r} in the rsync target {} is not a directory'.format(path, file_id))
files = []
for line in out:
line = line.strip(' ')
if line is '':
continue
match = self.LISTING_LINE.match(line)
if match is None:
logger.critical('Error parsing rsync listing (line %r)', line)
else:
data = match.groupdict()
if data['file'] != '.':
files.append((data['directory'] == 'd', data['file']))
logger.info('Retrieved a list of %d files', len(files))
return files
def run(self, ctx={}):
working_dir = ctx.get("current_dir", self.local_dir)
log_file = self.get_log_file(ctx)
new_env = os.environ.copy()
if self.password is not None:
new_env["RSYNC_PASSWORD"] = self.password
if self.env is not None and isinstance(self.env, dict):
for k, v in self.env.items():
new_env[k] = v
with open(log_file, 'w', buffering=1) as f:
def log_output(line):
f.write(line)
for stage in (1, 2):
_args = self.options(stage)
_args.append(self.upstream_url)
_args.append(working_dir)
f.write("==== Stage {} Begins ====\n\n".format(stage))
self.p = sh.rsync(
*_args, _env=new_env, _out=log_output,
_err_to_out=True, _out_bufsize=1, _bg=False
)
self.p.wait()
def run(self, ctx={}):
working_dir = ctx.get("current_dir", self.local_dir)
log_file = self.get_log_file(ctx)
new_env = os.environ.copy()
if self.password is not None:
new_env["RSYNC_PASSWORD"] = self.password
if self.env is not None and isinstance(self.env, dict):
for k, v in self.env.items():
new_env[k] = v
with open(log_file, 'w', buffering=1) as f:
def log_output(line):
f.write(line)
for stage in (1, 2):
_args = self.options(stage)
_args.append(self.upstream_url)
_args.append(working_dir)
f.write("==== Stage {} Begins ====\n\n".format(stage))
self.p = sh.rsync(*_args,
_env=new_env,
_out=log_output,
_err_to_out=True,
_out_bufsize=1,
_bg=False)
self.p.wait()
def cache_files(files, cache_dir=None, min_free_space_ratio=2):
if cache_dir is None: return None
import os, sh, psutil, glob, time, logging
from pathlib import Path
pc = Path(cache_dir)
pc.mkdir(parents=True, exist_ok=True)
available_disk_space = psutil.disk_usage(cache_dir).free
total_file_size = 0
for filename in files:
total_file_size += os.path.getsize(filename)
if available_disk_space < total_file_size * min_free_space_ratio: # only when there is enough free disk space
return None
mapping = {}
todel = set()
for f in files:
f1 = Path(f)
f2 = pc / f1.name
mapping[f] = f2.as_posix()
todel.add(f2.as_posix())
if f2.exists() and f2.stat().st_size == f1.stat().st_size:
continue
# check if it is already being rsync'ed by another process
ftmp = glob.glob(cache_dir + os.sep + "." + f + ".*")
if len(ftmp) == 1:
time0 = time.time()
while 1:
if os.path.exists(ftmp[0]):
if time.time(
) - time0 > 10 * 60: # wait for 10 minutes at most
break
time.sleep(5)
else:
break
if f2.exists() and f2.stat().st_size == f1.stat().st_size:
continue
logging.info(f"Caching {f1} to {f2}")
sh.rsync(["-a", f1.as_posix(), f2.as_posix()])
if mapping:
import atexit
atexit.register(remove_cached_files, cached_files=todel)
return mapping
def _synchronize_mappings(sync_mappings):
for from_path, to_path in sync_mappings:
if ssh_connection_uri:
ssh(ssh_connection_uri, 'mkdir', '-p', to_path.as_posix())
to_uri = ssh_connection_uri + ':' + to_path.as_posix()
else:
to_path.mkdir(parents=True, exist_ok=True)
to_uri = to_path.as_posix()
exclude_params = zip(len(excludes) * ['--exclude'], excludes)
rsync('-rltv',
from_path.as_posix() + '/',
to_uri + '/',
delete=True,
*exclude_params)
def main():
logging.info("Starting rsync.")
if logfile and exclusions and args.quiet:
rsync("-auhv", exclusions, "--log-file={}".format(logfile), backupdir, destinationdir)
elif logfile and exclusions:
print(rsync("-auhv", exclusions, "--log-file={}".format(logfile), backupdir, destinationdir))
elif args.quiet and exclusions:
rsync("-av", exclusions, backupdir, destinationdir)
elif logfile and args.quiet:
rsync("-av", "--log-file={}".format(logfile), backupdir, destinationdir)
else:
rsync("-av", backupdir, destinationdir)
logging.info("done.")
def send_file(node): server = get_server_for_node(node) ci = ALL_NODES.index(node) if ci > 0: current_node = ALL_NODES[ci - 1] else: current_node = "127.0.0.1" for line in rsync(POLLY_FILES, server, archive=True, compress=True, relative=True, update=True, itemize_changes=True, dry_run=True, _iter=True): print line if line[1] == 'f': parts = line.split() fileid = get_file_id(parts[1]) if line[0] == "<": #sent add_packet(parts[1]) add_port(current_node, "broadcasting", fileid) add_port(node, "downloading", fileid) elif line[0] == ">": #received add_packet(parts[1]) add_port(node, "downloading", fileid) add_port(current_node, "broadcasting", fileid) elif line[0] == ".": #nothing pass save_status_update() server = POLLY_USER + "@" + node + ":/" for line in rsync(POLLY_FILES, server, archive=True, compress=True, relative=True, update=True, itemize_changes=True, _iter=True): print line if line[1] == 'f': parts = line.split() fileid = get_file_id(parts[1]) if line[0] == "<": #sent add_packet(parts[1]) add_port(current_node, "waiting", fileid) add_port(node, "waiting", fileid) elif line[0] == ">": #received add_packet(parts[1]) add_port(current_node, "waiting", fileid) add_port(node, "waiting", fileid) elif line[0] == ".": #nothing pass save_status_update()
def upload_files_to_server(addr, path):
# exclude some files
files_to_exclude = [
'__pycache__', '*.swp', '.git', '.DS_Store', '.gitignore'
]
files_to_exclude = ' --exclude '.join(files_to_exclude).split()
print(
sh.rsync('-a', '-v', '-z', '--exclude', *files_to_exclude,
'{}/'.format(os.getcwd()), '{}:{}'.format(addr, path)))
def save_status_update():
global PORTS, PACKETS
data = {
'packets': PACKETS,
'ports': PORTS,
}
encoded = json.dumps(data)
with open(POLLY_STATUS, "w+") as sfile:
sfile.write(encoded)
for node in POLLY_NODES:
print "Saving status", node
server = get_server_for_node(node)
rsync(POLLY_STATUS, server, archive=True, compress=True, relative=True, update=True, itemize_changes=True)
PORTS = []
PACKETS = []
def rsync_to(src: pathlib.Path, dst: pathlib.Path, filetype: str,
dry_run: bool):
if filetype == 'file':
dst.parent.mkdir(parents=True, exist_ok=True)
src_string = str(src) + '/' if filetype == 'folder' else str(src)
dst_string = str(dst) if filetype == 'folder' else str(dst.parent) + '/'
click.secho(f'Syncing path from {src_string} to {dst_string}', fg='green')
exclude_params = zip(len(excludes) * ['--exclude'], excludes)
if not dry_run:
rsync('-rlt',
'--max-size=200m',
'--stats',
src_string,
dst_string,
delete=True,
*exclude_params,
_out=process_output)
def sync_download(self, sourcepath, targetpath):
#download PDB
try:
updatelog = rsync("-rlpt", "-v", "-z", "--delete", "--port=33444",
"-i", "--log-file=pdbrsync.log", sourcepath,
targetpath)
logging.info("Update %s is done." % self.getTargetDB())
except:
logging.info("download %s is err." % self.getTargetDB())
return updatelog
def mass_storage_backup(source_path):
if source_path is None:
return
unique_id = get_unique_name(source_path)
destination_path = os.path.join(BACKUP_PATH, unique_id) + os.sep
# Create the folder
os.makedirs(destination_path, exist_ok=True)
log.info("Starting backup for %s to %s", source_path, destination_path)
# File synchronization
sh.rsync("-a", "--chmod=Du=rwx,Dgo=rwx,Fu=rw,Fog=rw", source_path + os.sep,
destination_path)
# Flush disk buffers
sh.sync()
log.info("Finished backup for %s", source_path)
# Schedule a lychee sync for now
global next_lychee_sync
next_lychee_sync = datetime.now()
def execute(self):
logger = logging.getLogger('backup.rsync')
with self.mount:
print("Performing backup")
logger.debug("Initiating backup")
super(RSync, self).execute()
if self.configuration['backup'].get('incremental', False):
rsync_args = [
'-abs', '--no-links', '--delete',
'--backup-dir=%s' % os.path.join(
self.configuration['destination']['incremental_destination'], self.date_stamp)]
else:
logger.debug("Not incremental backup")
rsync_args = ['-as', '--no-links']
if self.configuration['backup'].get('ignore_existing', False):
logger.debug("Ignoring existing files")
rsync_args.append('--delete')
if self.arguments.verbosity:
rsync_args.append('-v')
if self.arguments.dry_run:
rsync_args.append('-n')
if self.configuration['destination']['type'] == 'ssh':
rsync_args.extend(('-e', 'ssh'))
for exclude in self.configuration['backup'].get('excludes', []):
rsync_args.append('--exclude=%s' % exclude)
rsync_args.append(self.configuration['backup']['directories'][0] + '/')
rsync_args.append(self.configuration['destination']['destination'])
logger.debug(rsync_args)
try:
# noinspection PyUnresolvedReferences
sh.rsync(*rsync_args)
except sh.ErrorReturnCode as exc:
logger.critical("Backup failure: %s", exc.stderr)
else:
print("Backup finished")
logger.info("Backup successfully completed.")
logger.debug("Elapsed time: %.3f sec", self.stat())
def skeletonize():
'''Update Skeleton HTML5-Boilerplate.'''
print green("Skeletonizing the project directory...")
# Skeleton
print blue("Installing skeleton HTML5 Boilerplate.")
os.chdir(PROJ_DIR)
sh.git.submodule.update(init=True)
os.chdir(PROJ_DIR + "/skeleton")
sh.git.pull("origin", "master")
sh.rsync("-av", "images", "{0}/{1}/static/".format(PROJ_DIR,APP_NAME))
sh.rsync("-av", "stylesheets", "{0}/{1}/static/".format(PROJ_DIR,APP_NAME))
sh.rsync("-av", "index.html", "{0}/{1}/templates/base_t.html".format(PROJ_DIR,APP_NAME))
os.chdir(PROJ_DIR)
# Patch the base template with templating tags
print blue("Patching the base template.")
os.chdir(PROJ_DIR + "/{0}/templates/".format(APP_NAME))
template_patch = open("base_t.patch".format(APP_NAME))
sh.patch(strip=0, _in=template_patch)
template_patch.close()
os.chdir(PROJ_DIR)
# Jquery
print blue("Installing jquery 1.9.0.")
os.chdir(PROJ_DIR + "/" + APP_NAME + "/static/js")
sh.curl("http://code.jquery.com/jquery-1.9.0.min.js", O=True)
os.chdir(PROJ_DIR)
def rsync(self, src, dest):
# enable arcfour and no compression for faster speed
ssh_options = 'ssh -T -c arcfour -o Compression=no -x ' \
'-i "%s" -l vagrant' % self.ssh_key()
# basic args for rsync
args = ['--delete', '--archive', '--hard-links',
'--one-file-system', '--compress-level=0',
'--omit-dir-times', '-e', ssh_options]
if verbosity():
args.append('-v')
if verbosity() > 1:
args.append('--progress')
if verbosity() > 2:
args.append('--stats')
# check for ignore
conf = self.project.config()
if 'rsync_ignores' in conf:
# TODO: check format
args += map(lambda x: '--exclude="%s"' % x,
conf['rsync_ignores'])
# then add sec and dest
args += [src, dest]
self._logger.debug('running: rsync %s' % ' '.join(args))
try:
rsync(*args,
_out=sys.stdout, _err=sys.stderr,
_out_bufsize=0, _err_bufsize=0)
except ErrorReturnCode:
return False
return True
def copy_site_files(source, target):
"""
Rsync all non-regeneratable Plone files.
We also *must* not copy some files as they would mess up
running the buildout on the new target.
http://plone.org/documentation/kb/copying-a-plone-site
"""
# Make sure we can SSH into to the box without interaction
ssh_handshake(source)
print "Syncing site files from old site %s" % source
from sh import rsync
# XXX: --progress here is too verbose, rsync having multiple file transfer indicator?
rsync("-a", "--compress-level=9", "--inplace",
"--exclude", "*.lock", # Data.fs.lock, instance.lock
"--exclude", "*.pid",
"--exclude", "*.log", # A lot of text data we probably don't need on the new server
"--exclude", "var/log/*", # Old rotated log files
"--exclude", "eggs",
"--exclude", "downloads", # Redownload
"--exclude", "parts", # Buildout always regenerates this folder
"--exclude", "bin", # old bin/ scripts may not regenereate, point to Py interpreter on the old server
"--exclude", ".installed.cfg", # Otherwise does not regenerate zeoserver
"--exclude", ".mr.developer.cfg",
"--exclude", "*.old", # Data.fs.old
"%s/*" % source, target,
_out=_unbuffered_stdout,
_err=_unbuffered_stdout,
_out_bufsize=0).wait()
# Rercreate regeneratable folders
install("-d", "%s/eggs" % target)
install("-d", "%s/parts" % target)
install("-d", "%s/downloads" % target)
install("-d", "%s/bin" % target)
def doBackup(logfile, exclusions, is_quiet, backupdir, destinationdir):
# Do the actual backup
logging.info("Starting rsync.")
if logfile and exclusions and is_quiet:
rsync("-auhv", exclusions, "--log-file={}".format(logfile), backupdir,
destinationdir)
elif logfile and exclusions:
print(
rsync("-auhv", exclusions, "--log-file={}".format(logfile),
backupdir, destinationdir))
elif is_quiet and exclusions:
rsync("-av", exclusions, backupdir, destinationdir)
elif logfile and is_quiet:
rsync("-av", "--log-file={}".format(logfile), backupdir,
destinationdir)
else:
rsync("-av", backupdir, destinationdir)
logging.info("done.")
def doInstallerCopy():
options = setup()
# Put mapr-setup.sh on root for installer folder
output = rsync(RSYNC_EXTRA_OPTS,
'%s/yum/qa/%s/mapr-setup.sh' % (RELEASE_DIR, INSTALLER_VER),
'%s/installer/' % BASE_DIR)
# Redhat
RSYNC_SRC = '%s/yum/qa/%s/mapr-*' % (RELEASE_DIR, INSTALLER_VER)
output = rsync(RSYNC_EXTRA_OPTS, '--delete', sh.glob(RSYNC_SRC),
'%s/installer/redhat/' % BASE_DIR)
# Ubuntu
RSYNC_SRC = '%s/qa/%s/mapr-*' % (RELEASE_DIR, INSTALLER_VER)
output = rsync(RSYNC_EXTRA_OPTS, sh.glob(RSYNC_SRC),
'%s/installer/ubuntu/' % BASE_DIR)
RSYNC_SRC = '%s/qa/%s/dists/binary/mapr-*' % (RELEASE_DIR, INSTALLER_VER)
output = rsync(RSYNC_EXTRA_OPTS, '--delete', sh.glob(RSYNC_SRC),
'%s/installer/ubuntu/dists/binary/' % BASE_DIR)
print 'Reindexing Installer redhat and ubuntu directories'
doInstallerReindex()
def sync_symlinks(commit):
config_options = getConfigOptions()
rsyncdest = config_options.rsyncdest
rsyncport = config_options.rsyncport
datadir = os.path.realpath(config_options.datadir)
if rsyncdest != '':
# We want to sync the symlinks in a second pass, once all content
# has been copied, to avoid a race condition it they are copied first
rsyncpaths = []
for filename in ['consistent', 'current']:
filepath = os.path.join(datadir, "repos", ".", filename)
rsyncpaths.append(filepath)
rsh_command = 'ssh -p %s -o StrictHostKeyChecking=no' % rsyncport
try:
sh.rsync('-avzR', '--delete-delay', '-e', rsh_command, rsyncpaths,
rsyncdest)
except Exception as e:
# We are not raising exceptions for symlink rsyncs, these will
# be fixed after another build
logger.warn('Failed to rsync symlinks to %s ,'
'got error %s' % (rsyncdest, e))
def run(self):
self.ensure_log_dir()
_args = self.options
_args.append(self.upstream_url)
_args.append(self.local_dir)
now = datetime.now().strftime("%Y-%m-%d_%H")
log_file = self.log_file.format(date=now)
new_env = os.environ.copy()
if self.password is not None:
new_env["RSYNC_PASSWORD"] = self.password
self.p = sh.rsync(*_args, _env=new_env, _out=log_file, _err=log_file,
_out_bufsize=1, _bg=True)
def doComponentCopy():
options = setup()
print('Doing component copy: %s' % options.component)
cname, cver = options.component.split('-')
ecoList = options.dest.split(',')
for eco in ecoList:
destEco = eco[0]
destOS = eco[1:]
pattern = '*%s*%s*' % (cname, cver)
for operSys in destOS:
print('Copying %s-%s to Ecosystem %s.x -> %s' %
(cname, cver, destEco, OS_LIST[operSys]))
# Remove existing files of the same component/version
WORKDIR = '%s/%s' % (ECO_DIR_DICT[destEco],
PACKAGETYPE_DICT[OS_LIST[operSys]])
os.chdir(WORKDIR)
rm('-fv', sh.glob(pattern), _out=log)
# Rsync over the new files
RSYNC_SRC = '%s/%s/%s' % (RELEASE_DIR, PKG_DICT[operSys], pattern)
rsync(sh.glob('%s' % RSYNC_SRC), '.')
return 0
def run(self, ctx={}):
_args = self.options
_args.append(self.upstream_url)
working_dir = ctx.get("current_dir", self.local_dir)
_args.append(working_dir)
log_file = self.get_log_file(ctx)
new_env = os.environ.copy()
if self.password is not None:
new_env["RSYNC_PASSWORD"] = self.password
self.p = sh.rsync(*_args, _env=new_env, _out=log_file,
_err_to_out=True, _out_bufsize=1, _bg=True)
def sync(self, local_branch, remote_branch, root=""):
"""
Sync/ update/ deploy a repository
"""
src_path = "{}/{}".format(self.git.get_path(), root)
dst_path = "{}/{}/{}/".format(self.get("target"), local_branch, self.get("name"))
if not os.path.isdir(dst_path):
os.makedirs(dst_path)
if not src_path.endswith("/"):
src_path += "/"
if not dst_path.endswith("/"):
dst_path += "/"
self.logger.debug("Preparing repo sync %s by checking out ref %s", self["name"], remote_branch)
self.git.checkout(remote_branch)
self.logger.info("Synchronizing repo using source path %s and destination path %s", src_path, dst_path)
rsync_args = ["--verbose", "--archive", "--checksum", "--delete", "--exclude=.git/", src_path, dst_path]
self.logger.debug("Running rsync with the arguments %s", rsync_args)
rsync(rsync_args)
def _stage_in_data(self, experimentCfg):
"""Move the input files to the remote system."""
self.logger.info('Staging data into the HPC system..')
experiment_dir = experimentCfg.get_input_data()
job_script = experimentCfg.get_job_script()
# input data dir ?
if experiment_dir:
try:
# transfer input data dir
self.logger.debug(
"Staging experiment dir '{}' to HPC system..".format(
experimentCfg.get_input_data()))
rsync_output = rsync(
"-pzvr", experimentCfg.get_input_data(), '{}@{}:{}'.format(
self.hpcConfig.get_value('user_name'),
self.hpcConfig.get_value('host'),
self.hpcConfig.get_value('execution_dir')))
self.logger.debug("rsync output:\n{}".format(rsync_output))
except ErrorReturnCode as e:
self.logger.error('Staging input data failed:\n{}'.format(
e.stderr))
raise e
try:
# transfer job script
rsync_output = rsync(
"-pzvr", job_script,
'{}@{}:{}'.format(self.hpcConfig.get_value('user_name'),
self.hpcConfig.get_value('host'),
self.hpcConfig.get_value('execution_dir')))
self.logger.debug("rsync output:\n{}".format(rsync_output))
except ErrorReturnCode as e:
self.logger.error('Staging job script failed:\n{}'.format(
e.stderr))
raise e
def sync_symlinks(commit):
config_options = getConfigOptions()
rsyncdest = config_options.rsyncdest
rsyncport = config_options.rsyncport
datadir = os.path.realpath(config_options.datadir)
if rsyncdest != '':
# We want to sync the symlinks in a second pass, once all content
# has been copied, to avoid a race condition it they are copied first
rsyncpaths = []
for filename in ['consistent', 'current']:
filepath = os.path.join(datadir, "repos", ".", filename)
rsyncpaths.append(filepath)
rsh_command = 'ssh -p %s -o StrictHostKeyChecking=no' % rsyncport
try:
sh.rsync('-avzR', '--delete-delay',
'-e', rsh_command,
rsyncpaths, rsyncdest)
except Exception as e:
# We are not raising exceptions for symlink rsyncs, these will
# be fixed after another build
logger.warn('Failed to rsync symlinks to %s ,'
'got error %s' % (rsyncdest, e))
def _vim_environment(self):
sh.mkdir('-p', hpath('.vim/colors'), **SHARG)
sh.cp(wpath('vim/molokai.vim'), hpath('.vim/colors/molokai.vim'),
**SHARG)
sh.cp(wpath('vim/solarized.vim'), hpath('.vim/colors/solarized.vim'),
**SHARG)
sh.rsync('-rzvaP', wpath('vim/UltiSnips'), hpath('.vim'), **SHARG)
vundle_dir = hpath('.vim/bundle/Vundle.vim')
if os.path.isdir(vundle_dir):
os.chdir(vundle_dir)
sh.git('pull', **SHARG)
os.chdir(working_dir)
else:
sh.git('clone', 'https://github.com/VundleVim/Vundle.vim.git',
vundle_dir, **SHARG)
template = self.temp_env.get_template(wpath('vim/vimrc.template'))
with open(hpath('.vimrc'), 'w') as vimrc:
vimrc.write(template.render(python_path=self.python_path))
# sh.cp(wpath('vim/vimrc'), hpath('.vimrc'), **SHARG)
sh.vim('+PluginInstall', '+qall', **SHARG)
ycm_dir = hpath('.vim/bundle/YouCompleteMe')
if not os.path.isdir(ycm_dir):
sh.git('clone', 'https://github.com/Valloric/YouCompleteMe.git',
ycm_dir, **SHARG)
if not os.path.isfile(
hpath(
'.vim/bundle/YouCompleteMe/third_party/ycmd/ycm_core.so')):
os.chdir(ycm_dir)
sh.git('submodule', 'update', '--init', '--recursive', **SHARG)
sh.zsh('install.sh', '--clang-completer', **SHARG)
os.chdir(working_dir)
def LicheePiImage(workdir, boot_files, kernel_files, rootfs_files):
mkdir('-p', workdir)
IMAGE_NAME = 'sdcard.img'
IMAGE_PATH = str(Path(workdir).joinpath(IMAGE_NAME))
dd('if=/dev/zero', 'of={}'.format(IMAGE_PATH), 'bs=1M', 'count=300')
loop_dev = str(losetup('-f')).split()[0]
losetup(loop_dev, IMAGE_PATH)
sfdisk(cat(_in='1M,16M,c\n,,L'), loop_dev)
partx('-u', loop_dev)
mkfsvfat('{}p1'.format(loop_dev))
mkfsext4('{}p2'.format(loop_dev))
dd('if=/dev/zero', 'of={}'.format(loop_dev), 'bs=1K', 'seek=1',
'count=1023')
dd('if={}'.format(boot_files['bootstrap']), 'of={}'.format(loop_dev),
'bs=1K', 'seek=8')
sync()
mkdir('-p', '/tmp/p1')
mkdir('-p', '/tmp/p2')
mount('{}p1'.format(loop_dev), '/tmp/p1')
mount('{}p2'.format(loop_dev), '/tmp/p2')
cp(boot_files['bin'], '/tmp/p1/')
cp(kernel_files['bin'], '/tmp/p1/')
cp(kernel_files['dtb'], '/tmp/p1/')
mkimage('-C', 'none', '-A', 'arm', '-T', 'script', '-d',
'./resources/boot.cmd', '/tmp/p1/boot.scr')
rsync('-r', '--links', rootfs_files['rootdir'] + '/', '/tmp/p2/')
mkdir('-p', '/tmp/p2/etc/init.d')
mkdir('-p', '/tmp/p2/proc')
mkdir('-p', '/tmp/p2/dev')
mkdir('-p', '/tmp/p2/sys')
mkdir('-p', '/tmp/p2/var')
touch('/tmp/p2/etc/init.d/rcS')
chown('-R', 'root:root', '/tmp/p2/')
def sync(self, local_branch, remote_branch, root=''):
"""
Sync/ update/ deploy a repository
"""
src_path = '{}/{}'.format(self.git.get_path(), root)
dst_path = '{}/{}/{}/'.format(self.get('target'), local_branch, self.get('name'))
if not os.path.isdir(dst_path):
os.makedirs(dst_path)
if not src_path.endswith('/'):
src_path += '/'
if not dst_path.endswith('/'):
dst_path += '/'
self.git.checkout(remote_branch)
print(rsync('--verbose', '--archive', '--checksum', '--delete', '--exclude=.git/', src_path, dst_path)) # TODO
def sync(synObj):
"""Setup rsync with options from different sources and run rsync"""
rsync_opts = []
rsync_opts.append("--archive") # archive
# rsync_opts.append("--update") # skip files that are newer on the receiver
rsync_opts.append("--human-readable") # output numbers in a human-readable format
rsync_opts.append("--verbose") # increase verbosity
rsync_opts.append("--recursive") # recurse into directories
rsync_opts.append("--compress") # compress file data during the transfer
rsync_opts.append("--cvs-exclude") # auto-ignore files in the same way CVS does
# rsync_opts.append("--delete") # delete extraneous files from dest dirs
# rsync_opts.append('--filter=\"dir-merge,- .gitignore\"')
rsync_opts.append('--exclude=*.bin')
rsync_opts.append('--exclude=' + synObj.config_file)
if synObj.dryrun:
if synObj.verbose:
print("# --dryrun is turned ON!")
rsync_opts.append("--dry-run") # no transfer, just report
if 'rsync_options' in synObj.host_toml:
for option in synObj.host_toml['rsync_options']:
rsync_opts.append(option)
for option in synObj.rsync_options:
rsync_opts.append(str(option))
if synObj.verbose:
print("# All rsync options: {}".format(rsync_opts))
sourceDir = synObj.localDir + "/" # make sure it has a trailing slash, for rsync
if 'hostname' in synObj.host_toml:
destDir = str(synObj.host_toml['hostname']) + ":" + synObj.destDir
if synObj.gather:
sourceDir, destDir = destDir, sourceDir
if synObj.multihost:
print("Syncing with {}".format(synObj.entry))
print(rsync(rsync_opts, sourceDir, destDir))
exit(1)
check_dir_exist(backupdir)
check_dir_exist(destinationdir)
# Core rsync arguments
rsync_args = "-ahv"
# Add rsync arguments based on user input
if args.dryrun:
rsync_args += "n"
if logfile:
log_args = args.logfile
else:
log_args = ""
# Do the actual backup
logging.info("Starting rsync.")
if args.echo:
if args.trash:
del_args = "--del"
else:
del_args = ""
print("The command is rsync %s %s %s %s %s %s" % (rsync_args, del_args, backupdir, destinationdir, ">>", log_args))
else:
if args.trash:
rsync(rsync_args, "--del", backupdir, destinationdir, _out=log_args)
else:
rsync(rsync_args, backupdir, destinationdir, _out=log_args)
def main():
# Read and parse configuration.
configfile = os.path.join(sys.path[0], 'config.yml')
with open(configfile, 'r') as config:
y = yaml.load(config)
local_path = y['local_path']
remote_path = y['remote_path']
gps_url = y['gps_url']
# Parse args
parser = ArgumentParser()
parser.add_argument(
'-f', '--filename', nargs = '?',
help='The input status update filename.'
)
parser.add_argument(
'--prod', action='store_true',
help=('Set flag to publish to live site via '
'rsync to configured remote_path')
)
args = parser.parse_args()
# Read and convert jot file
if args.filename:
jotsrc = os.path.join(sys.path[0], args.filename)
else:
draftpath = os.path.abspath(os.path.join(sys.path[0], 'draft'))
filen = max(os.listdir(draftpath),
key = lambda f: os.path.getmtime(os.path.join(draftpath, f)))
jotsrc = os.path.join(sys.path[0], 'draft', filen)
with codecs.open(jotsrc, encoding='utf-8') as jotfile:
jot_md = jotfile.read()
md = markdown.Markdown(['meta'])
jot_html = md.convert(jot_md)
# Set timestamp from meta or from current:
if md.Meta.has_key('timestamp'):
timestamp = md.Meta['timestamp'][0]
timestamp = u'**%s** \\- ' % timestamp
else:
timestamp = strftime('%Y-%m-%d %H:%M:%S')
timestamp = u'**%s** \\- ' % timestamp
timestamp = markdown.markdown(timestamp)[:-4] # remove trailing p tag
# Set geotag from meta or Whereami (or blank)
if md.Meta.has_key('mapquery'):
qs = md.Meta['mapquery'][0]
geotag = (u' <a href="http://maps.google.com/maps?q=%s">'
'<i class="fa fa-map-marker"></i></a></p>') % qs
elif gps_url:
url = gps_url
locreq = requests.get(gps_url)
locdata = locreq.json()
lat = locdata['latitude']
lng = locdata['longitude']
geotag = (u' <a href="http://maps.google.com/maps?q=%s,%s">'
'<i class="fa fa-map-marker"></i></a></p>') % (lat,lng)
else:
geotag = '</p>'
# Remove leading and closing p tags from post,
# we will wrap in timestamp and geotag
jot_html = jot_html[3:-4]
# Apply the jot_html to the template w/ jinja2
PWDIR = os.path.join(sys.path[0])
j = Environment(loader=FileSystemLoader(PWDIR), trim_blocks=True)
content = j.get_template('template.html').render(jotting = jot_html,
timestamp = timestamp, geotag = geotag)
# Write rendered content to index file
indexsrc = os.path.join(sys.path[0], 'static', 'index.html')
with codecs.open(indexsrc, encoding='utf-8', mode='w') as j:
j.write(content)
# only rsync to server if Prod arg is passed
if args.prod:
local_path = os.path.join(local_path, 'static/')
rsync("-ae", "ssh", local_path, remote_path)
else:
pass
DeleteFiles(file_type, sourcedir)
# Empty trash can
try:
rmtree(os.path.expanduser("~/.local/share/Trash/files"))
except OSError:
logging.warning("Could not empty the trash or trash already empty.")
pass
# Handle exclusions
exclusions = []
if args.exclude:
for argument in args.exclude:
exclusions.append("--exclude={}".format(argument))
# Rsync files
logging.info("Starting rsync.")
if logfile and exclusions and args.quiet:
rsync("-auhv", exclusions, "--log-file={}".format(logfile), sourcedir,
targetdir)
elif logfile and exclusions:
print(rsync("-auhv", exclusions, "--log-file={}".format(logfile),
sourcedir, targetdir))
elif args.quiet and exclusions:
rsync("-av", exclusions, sourcedir, targetdir)
elif logfile and args.quiet:
rsync("-av", "--log-file={}".format(logfile), sourcedir, targetdir)
else:
rsync("-av", sourcedir, targetdir)
logging.info("done.")
def main():
config = get_config()
dest = config.destination
src = config.source
# ensure the destination directory exists
os.makedirs(dest, exist_ok=True)
# lock it so only we can use it
unlock_dest = None
try:
unlock_dest = lock_dest(dest)
except IOError as e:
logging.info('Backup already in progress, exiting.')
return 0
# remove the lock when exiting under normal circumstances
atexit.register(unlock_dest)
# make sure we remove the lock on exit, now that we've acquired it. we catch
# these signals explicitly since it virtually guarantees that we'll remove the
# lock on exit, unless something catastrophic happens. we have to wrap the
# function since handler functions must take two arguments, otherwise they
# error.
unlock_dest_handler = lambda a, b: unlock_dest()
signal.signal(signal.SIGABRT, unlock_dest_handler)
signal.signal(signal.SIGINT, unlock_dest_handler)
signal.signal(signal.SIGSEGV, unlock_dest_handler)
signal.signal(signal.SIGTERM, unlock_dest_handler)
# get a timestamp for the backup directory
backup_timestamp = time.strftime(TIME_FORMAT)
# get names for our backup directories
incomplete_backup_dir = os.path.join(dest,
INCOMPLETE_PREFIX + BACKUP_PREFIX + backup_timestamp)
complete_backup_dir = os.path.join(dest, BACKUP_PREFIX + backup_timestamp)
current_link = os.path.join(dest, CURRENT_LINK_NAME)
logging.info("Backing up '%s' to '%s'...", src, dest)
# start the backup
rsync_result = rsync(
'--exclude', '/dev/*',
'--exclude', '/dev/*',
'--exclude', '/home/*/.cache',
'--exclude', '/home/*/.config/google-chrome-*/Default/File System/*',
'--exclude', '/home/*/.config/google-chrome-*/Default/GPUCache/*',
'--exclude', '/home/*/.dropbox/logs/*',
'--exclude', '/home/*/.local/share/Trash',
'--exclude', '/home/*/.mozilla/firefox/*/Cache',
'--exclude', '/home/*/.thumbnails',
'--exclude', '/mnt/*',
'--exclude', '/proc/*',
'--exclude', '/run/*',
'--exclude', '/sys/*',
'--exclude', '/tmp/*',
'--exclude', '/var/lock/*',
'--exclude', '/var/log/journal/*',
'--exclude', '/var/run/*',
'--exclude', '/var/tmp/*',
'--exclude', dest,
'--include', '/home',
# backup from the source to our 'incomplete' directory
src, incomplete_backup_dir,
# this does the incremental magic
link_dest=current_link,
# prettify output a bit
itemize_changes=True,
human_readable=True,
# look through all subdirectories of the given one
recursive=True,
# include all file types and duplicate all permissions
links=True,
perms=True,
times=True,
group=True,
owner=True,
devices=True,
specials=True,
executability=True,
# log all rsync output through our logger
_out=logging.info,
_err=logging.error,
)
# bail if the backup didn't succeed
if rsync_result.exit_code != 0:
logging.error('rsync process exited with code %d, backup failed!',
rsync_result.exit_code)
return rsync_result.exit_code
else:
logging.info('Backup was successful')
# mark the backup as 'complete'
logging.info('Marking the backup as complete...')
os.rename(incomplete_backup_dir, complete_backup_dir)
# remove any existing symlink and create a new one
logging.info('Updating link to point at the current backup...')
current_link_path = os.path.join(dest, CURRENT_LINK_NAME)
if os.path.lexists(current_link_path):
logging.debug("Removing existing link at '%s'", current_link_path)
os.unlink(current_link_path)
# makes sure the link is relative, so we can move the backup folder without
# breaking the link.
os.symlink(os.path.basename(complete_backup_dir), current_link_path)
# remove old backup folders
keep_duration_seconds = 60 * 60 * 24 * config.days_to_keep
remove_old_backups(dest, time.time() - keep_duration_seconds)
# prune incomplete backup folders once a newer backup exists
prune_incomplete_backups(dest)
return 0
parser.print_help()
sys.exit(1)
#Defining variables
backup_path = args.backup
backup_to_path = args.destination
#backup_path = raw_input("What do you want backed up today?\n")
if not os.path.exists(backup_path):
print backup_path, "does not exist"
exit(1)
if os.path.islink(backup_path):
print backup_path, "contains a symlink."
exit(1)
#backup_to_path = raw_input("Where would you like to back this up to?\n")
if not os.path.exists(backup_to_path):
print backup_to_path, "does not exist"
exit(1)
if os.path.islink(backup_to_path):
print backup_to_path, "contains a symlink."
exit(1)
rsync("-Pavhu", "--exclude=lost+found", "--exclude=/sys", "--exclude=/tmp", "--exclude=/proc", "--exclude=/mnt", "--exclude=/dev", "--exclude=/backup", backup_path, backup_to_path)
