def clone(self, force=False, commit_hook=True): if os.path.exists(self.repo_path): print('Path {0} already exists'.format(self.repo_path)) if force: pass else: return args = [ 'clone', self.url, self.repo_path ] print('Cloning repository {0} to {1}'.format(self.project, self.repo_path)) git(*args) if commit_hook: curr_dir = os.getcwd() try: os.chdir(self.repo_path) git_dir = str(git('rev-parse', '--git-dir')).strip() scp( '-p', '-P', self.gerrit_port, '{0}@{1}:hooks/commit-msg'.format(self.gerrit_user, self.gerrit_host), os.path.join(git_dir, 'hooks'), ) finally: os.chdir(curr_dir)
def script(host, name): h = Host.objects.get(hostname=host) sh.scp(scripts_dir+name, "root@"+h.ip+":/tmp/"+name) cmd = "ssh root@"+h.ip+" "+'"sh /tmp/{}"'.format(name) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) data = p.communicate() return data
def script(host, name): h = Host.objects.filter(hostname=host).first() sh.scp(SCRIPTS_DIR + name, "root@{0} /tem/{1}".format(h.ip, name)) cmd = "ssh root@" + h.ip + "sh /tem/{0}".format(name) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) data = p.communicate() return data
def host_sync(request): # backup old ansible hosts file now = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M') try: old_hosts = ansible_dir + "/hosts" sh.scp(old_hosts, ansible_dir + "/hosts_monster_backup_{0}".format(now)) except: pass group = HostGroup.objects.all() ansible_file = open(ansible_dir + "/hosts", "wb") all_host = Host.objects.all() for host in all_host: #gitlab ansible_host=10.100.1.76 host_name=gitlab host_item = host.hostname + " " + "ansible_host=" + host.ip + " " + "host_name=" + host.hostname + "\n" ansible_file.write(host_item) for g in group: group_name = "[" + g.name + "]" + "\n" ansible_file.write(group_name) get_member = HostGroup.objects.get(name=g) members = get_member.serverList.all() for m in members: group_item = m.hostname + "\n" ansible_file.write(group_item) ansible_file.close() logging.info("==========ansible tasks start==========") logging.info("User:"******"Task: sync cmdb info to ansible hosts") logging.info("==========ansible tasks end============") return HttpResponse("ok")
def ssh_push_file(src_file, dst_dir, username, address, silent=False): if not os.path.isfile(src_file): print("Not file, skip pushing " + src_file) return src_checksum = bench_utils.file_checksum(src_file) dst_file = os.path.join(dst_dir, os.path.basename(src_file)) stdout_buff = [] try: sh.ssh('%s@%s' % (username, address), "md5sum", dst_file, _out=lambda line: stdout_buff.append(line)) except sh.ErrorReturnCode_1: print("Scp %s to %s" % (src_file, dst_dir)) sh.ssh('%s@%s' % (username, address), "mkdir -p %s" % dst_dir) sh.scp(src_file, '%s@%s:%s' % (username, address, dst_dir)) else: dst_checksum = stdout_buff[0].split()[0] if src_checksum == dst_checksum: if not silent: print("Equal checksum with %s and %s" % (src_file, dst_file)) else: if not silent: print("Scp %s to %s" % (src_file, dst_dir)) sh.scp(src_file, '%s@%s:%s' % (username, address, dst_dir))
def send_to_matlab(model, averages): fd = tempfile.NamedTemporaryFile() fd.write('\n'.join('%s,%f' % a for a in averages)) fd.flush() logger.debug('Sending query list and GoogleScores to fmedia13') scp(fd.name, 'fmedia13:/tmp/fludetector_google_matlab_input') fd.close() fmedia13 = ssh.bake('fmedia13') run = ';'.join([ "fin='/tmp/fludetector_google_matlab_input'", "fout='/tmp/fludetector_google_matlab_output'", "cd /home/vlampos/website_v2", "run('gpml/startup.m')", "%s(fin,fout)" % model.get_data()['matlab_function'], "exit"]) logger.debug('Running matlab function over scores') fmedia13('matlab', '-nodisplay', '-nojvm', '-r', '"%s"' % run) logger.debug('Reading matlab results back') value = float(fmedia13('cat', '/tmp/fludetector_google_matlab_output').strip()) logger.debug('Cleaning up temp files') fmedia13('rm', '/tmp/fludetector_google_matlab_input', '/tmp/fludetector_google_matlab_output') return value
def put(self, local: str, remote: str) -> str: """Copy local to remote on the remote host. Return remote.""" if self.is_localhost() and local != remote: sh.cp(local, remote) elif not self.is_localhost(): sh.scp(local, f"{self.node_address}:{remote}") return remote
def push(self, src_path, dst_path): if self.system == SystemType.android: sh_commands.adb_push(src_path, dst_path, self.address) elif self.system == SystemType.arm_linux: try: sh.scp(src_path, '%s@%s:%s' % (self.username, self.address, dst_path)) except sh.ErrorReturnCode_1 as e: six.print_('Error msg {}'.format(e), file=sys.stderr)
def ensure_org_repo(name: str, user: str) -> str: if os.path.exists(os.path.join(name, ".git")): # git_reset_upstream(name) pass else: git.clone(gerrit_url(name, user), name) scp_src = "{}@gerrit.openbmc-project.xyz:hooks/commit-msg".format(user) scp_dst = "{}/.git/hooks/".format(name) sh.scp("-p", "-P", 29418, scp_src, scp_dst) return name
def ssh_push(src_path, dst_dir, username, address, silent=False): if os.path.isdir(src_path): print("Start scp dir %s=>%s, basename=%s" % (src_path, dst_dir, os.path.basename(src_path))) sh.scp("-r", src_path, '%s@%s:%s' % (username, address, dst_dir)) tmp_dst_dir = os.path.join(dst_dir, os.path.basename(src_path)) sh.ssh('%s@%s' % (username, address), "mv %s/* %s" % (tmp_dst_dir, dst_dir)) else: ssh_push_file(src_path, dst_dir, username, address, silent)
def script(host, name): h = Asset.objects.get(name=host) nicinfo = NIC.objects.get(asset_id=h.id) ip = nicinfo.ipaddress sh.scp(scripts_dir + name, "chenqiufei@" + ip + ":/tmp/" + name) cmd = "ssh chenqiufei@" + ip + " " + '"sh /tmp/{}"'.format(name) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) data = p.communicate() data = str(data) return data
def upload_report(path): sh.jupyter("nbconvert", path) root, ext = os.path.splitext(path) html = root + ".html" filename = os.path.basename(html) url = os.path.join( "http://atmos.washington.edu/~nbren12/reports/uw-machine-learning/", filename) sh.scp(html, "olympus:~/public_html/reports/uw-machine-learning/") print("Report uploaded to {}".format(url))
def script(host, name): h = Host.objects.get(hostname=host) sh.scp(scripts_dir+name, "admin@"+h.ip+":/tmp/"+name) file_suffix = os.path.splitext(name)[1] cmd = "ssh admin@" + h.ip + " " + '"sh /tmp/{}"'.format(name) if file_suffix == '.sh': cmd = "ssh admin@" + h.ip + " " + '"sh /tmp/{}"'.format(name) elif file_suffix == '.py': cmd = "ssh admin@" + h.ip + " " + '"python /tmp/{}"'.format(name) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) data = p.communicate() return data
def copy_to_opsman(self, opts, source, target=""): host = urlparse.urlparse(self.url).netloc from sh import scp try: scp("-oStrictHostKeyChecking=no", "-i {} ".format(opts['ssh_private_key_path']), source, "ubuntu@" + host + ":" + target) except Exception as ex: print "Error copying", source, target print ex.stdout print ex.stderr raise
def pull(self, src_path, dst_path='.'): if os.path.isdir(dst_path): exist_file = dst_path + '/' + src_path.split('/')[-1] if os.path.exists(exist_file): sh.rm('-rf', exist_file) elif os.path.exists(dst_path): sh.rm('-f', dst_path) try: sh.scp('-r', '%s@%s:%s' % (self.username, self.address, src_path), dst_path) except sh.ErrorReturnCode_1 as e: six.print_('Error msg {}'.format(e), file=sys.stderr) return
def submit(self, scriptPath): """.. function:: submit(scriptPath): Submits and runs a given local script with given parameters on cluster :param scriptPath: path of script on local machine""" #transfer script file to remote host scpHost = self.host + ":~" scp(scriptPath, scpHost) result = ssh(self.host, "qsub", script) #Return an id... return result
def push(self, src_path, dst_path): mace_check(os.path.exists(src_path), "Device", '{} not found'.format(src_path)) six.print_("Push %s to %s" % (src_path, dst_path)) if self.system == SystemType.android: sh_commands.adb_push(src_path, dst_path, self.address) elif self.system == SystemType.arm_linux: try: sh.scp( src_path, '{}@{}:{}'.format(self.username, self.address, dst_path)) except sh.ErrorReturnCode_1 as e: six.print_('Push Failed !', e, file=sys.stderr) raise e
def backup_database(output): """ This action will create a backup of the SQLite3 database into specified output. It then will try to scp the backup to any available worker Pioreactors as a futher backup. A cronjob is set up as well to run this action every 12 hours. """ import sqlite3 from sh import scp, ErrorReturnCode def progress(status, remaining, total): logger.debug(f"Copied {total-remaining} of {total} pages.") logger.debug(f"Starting backup of database to {output}") con = sqlite3.connect(config.get("storage", "database")) bck = sqlite3.connect(output) with bck: con.backup(bck, pages=-1, progress=progress) bck.close() con.close() logger.debug( f"Completed backup of database to {output}. Attempting distributed backup..." ) n_backups = 2 backups_complete = 0 available_workers = get_active_workers_in_inventory() while (backups_complete < n_backups) and (len(available_workers) > 0): backup_unit = available_workers.pop() if backup_unit == get_unit_name(): continue try: scp(output, f"{backup_unit}:{output}") except ErrorReturnCode: logger.debug( f"Unable to backup database to {backup_unit}. Is it online?", exc_info=True, ) logger.warning(f"Unable to backup database to {backup_unit}.") else: logger.debug(f"Backed up database to {backup_unit}:{output}.") backups_complete += 1 return
def write_kafka_config(self): f = NamedTemporaryFile(delete=False) f.write( jinja2.Template(KAFKA_CONFIG).render( self.node_config["kafka_config"]).encode("utf-8")) f.close() scp( "-i", self.node_config["ssh_key"], f.name, self.node_config["ssh_user"] + "@" + self.node_config["host"] + ":kafka.cfg") os.unlink(f.name) path = self.node_config["kafka_config"]["path"] ssh("-i", self.node_config["ssh_key"], self.node_config["ssh_user"] + "@" + self.node_config["host"], f"sudo mv kafka.cfg {path}")
def send_scripts_to_server(config: dict, destination: str, tar_filename: str, scripts_dir: str): click.echo('{} Creating tar of local scripts'.format(log_prefix())) with tarfile.open(tar_filename, "w:gz") as tar: for script_path in config['scripts']: tar.add(script_path) tar.list(verbose=True) click.echo('{} Sending scripts to instance'.format(log_prefix())) destination_with_location = destination + ':.' # scp -i ~/keys/hello.key local_scripts.tar.gz [email protected]:. scp('-i', config['private_key_path'], tar_filename, destination_with_location) click.echo('{} Extracting tar on instance'.format(log_prefix())) # ssh -i ~/keys/hello.key [email protected] tar xzf local_scripts.tar.gz ssh('-i', config['private_key_path'], destination, 'tar', 'xzf', tar_filename)
def transfer(self, filePath, remote=False): """.. function:: transfer(files, remote) Transfer files, local or remote, to host specified on command line :param filePath: path/address of files to be transferred to host :param remote: boolean dictating if files are remote (true) or local (false)""" if remote: sh.wget("-N", "-P", "./files", filePath) #Places files within a local directory named "files" scpHost = self.host + ":~" scp("-r", "./files", scpHost) else: scpHost = self.host + ":~" scp(filePath, scpHost) #May need to edit command to recursively handle directories
def download_rc_files(userid): for hostname in rc_file_locations: host = rc_file_locations[hostname] host["userid"] = userid host["dest"] = os.path.expanduser(host["dest"]) print "fetching from ", host["hostname"], pprint(host) print "create directory:", "%(dest)s" % host os.system("mkdir -p %(dest)s" % host) copy_cmd = "scp -o StricthostKeyChecking=no %(userid)s@%(hostname)s:%(source)s %(dest)s" % host print " <-", copy_cmd print " ", result = None try: from sh import scp result = scp("-o", "StrictHostKeyChecking=no", "%(userid)s@%(hostname)s:%(source)s" % host, "%(dest)s" % host) print "ok", result except Exception, e: print "failed", result, e
def get_text(self, filename: str, _iter=None): """Get the text from filename on the node. The argument _iter is the same as in the sh library. """ if self.is_localhost(): return sh.cat(filename, _iter=_iter) return sh.scp(f"{self.node_address}:{filename}", "/dev/stdout", _iter=_iter)
def __call__(self, local_file, remote_file, **kwargs): self.process = scp( '-P', self.port, local_file, '{}@{}:{}'.format(self.user, self.host, remote_file), _out=self.out_iteract, _out_bufsize=0, _tty_in=True, _err=self.err_iteract, **kwargs) super().__call__(local_file, remote_file, **kwargs)
def pull(self, src_path, file_name, dst_path='.'): if not os.path.exists(dst_path): sh.mkdir("-p", dst_path) src_file = "%s/%s" % (src_path, file_name) dst_file = "%s/%s" % (dst_path, file_name) if os.path.exists(dst_file): sh.rm('-f', dst_file) six.print_("Pull %s to %s" % (src_path, dst_path)) if self.system == SystemType.android: sh_commands.adb_pull(src_file, dst_file, self.address) elif self.system == SystemType.arm_linux: try: sh.scp('-r', '%s@%s:%s' % (self.username, self.address, src_file), dst_file) except sh.ErrorReturnCode_1 as e: six.print_("Pull Failed !", file=sys.stderr) raise e
def copy_from(self, remote_path, local_path, recursive=True, as_user='******'): recursive_param = '' if recursive: recursive_param ='-r' command = ['-i', str(self._identityfile), '-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null', recursive_param, str(as_user) + "@" + str(self._ip) + ":" + str(remote_path), str(local_path)] print(" ".join(command)) ret = scp(command) return ret
def copy_from(self, remote_path, local_path, recursive=True, as_user='******'): recursive_param = '' if recursive: recursive_param ='-r' command = ['-i', str(self._identityfile), '-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null', recursive_param, str(as_user) + "@" + str(self._ip) + ":" + str(remote_path), str(local_path)] print " ".join(command) ret = scp(command) return ret
def ensure_repo(name): if not os.path.exists(WORKING_DIR): sh.mkdir('-p', WORKING_DIR) logger.info('working directory %s created' % WORKING_DIR) fs_name = name.replace('/', '-') clone_folder = os.path.join(WORKING_DIR, fs_name) if is_git_repo(clone_folder): sh.cd(clone_folder) logger.info("Found Repo. Updating") sh.git.fetch('gerrit') logger.info("Repo updated") else: logger.info("Repo not found. Cloning") sh.cd(WORKING_DIR) sh.git.clone(GERRIT_TEMPLATE % name, fs_name) logger.info("Clone completed. Setting up git review") sh.cd(fs_name) sh.git.remote('add', 'gerrit', GERRIT_TEMPLATE % name) sh.scp('-p', '-P', '29418', '[email protected]:hooks/commit-msg', '.git/hooks/') logger.info("git review setup")
def write_zookeeper_configs(self): f = NamedTemporaryFile(delete=False) f.write( jinja2.Template(ZOOKEEPER_CONFIG).render( self.node_config["zookeeper_config"]).encode("utf-8")) f.close() scp( "-i", self.node_config["ssh_key"], f.name, self.node_config["ssh_user"] + "@" + self.node_config["host"] + ":zoo.cfg") os.unlink(f.name) path = self.node_config["zookeeper_config"]["path"] myid_path = self.node_config["zookeeper_config"]["dataDir"] + "/myid" myid = self.node_config["zookeeper_config"]["id"] ssh("-i", self.node_config["ssh_key"], self.node_config["ssh_user"] + "@" + self.node_config["host"], f"sudo mv zoo.cfg {path}") ssh("-i", self.node_config["ssh_key"], self.node_config["ssh_user"] + "@" + self.node_config["host"], f"sudo bash -c \"echo {myid} > {myid_path}\"")
def main() -> None: count_new_dirs = 0 start_time = time.time() src_dirs = set(sh.ssh(host, f'ls {src_dir}').split()) dst_dirs = set(sh.ls(f'{dst_dir}').split()) new_dirs = src_dirs - dst_dirs if new_dirs: count_new_dirs = len(new_dirs) log('backup_img', f'На сервере обнаружено новых каталогов: {count_new_dirs}. Воссоздание дерева каталогов') for new_dir in new_dirs: sh.mkdir('-p', f'{dst_dir}/{new_dir}') del src_dirs, dst_dirs, new_dirs src_files = set(sh.ssh(host, f'cd {src_dir}; find -type f').split()) sh.cd(dst_dir) dst_files = set(sh.find('-type', 'f').split()) new_files = src_files - dst_files size_before_copying = int(sh.du('-s').split()[0]) if new_files: count_new_files = len(new_files) log('backup_img', f'На сервере обнаружено новых файлов: {count_new_files}. Начинаю копирование...') for cnt, new_file in enumerate(new_files): if (cnt + 1) % 10 == 0: log('backup_img', f'Скопировано файлов: {cnt}...') current_dir, current_file = new_file.split('/')[1:] sh.scp('-p', '-r', f'{host}:{src_dir}/{current_dir}/{current_file}', f'{dst_dir}/{current_dir}/{current_file}') size_after_copying = int(sh.du('-s').split()[0]) size_delta = round((size_after_copying - size_before_copying) / 1024, 1) log('backup_img', f'Размер скопированных файлов {size_delta} Мб') time_delta = int(time.time() - start_time) time_delta = f'{str(time_delta // 3600).zfill(2)}:{str(time_delta // 60 % 60).zfill(2)}:{str(time_delta % 60).zfill(2)}' log('backup_img', f'Время копирования: {time_delta}') message = f'{get_time_stamp()} New images copying completed.\n' \ f'Copied file size: {size_delta}Mb\n' \ f'New files: {count_new_files}\n' \ f'New dirs: {count_new_dirs}\n' \ f'Time spent copying: {time_delta}' send_message_telegram(message) del src_files, dst_files, new_files return None
def __call__(self, local_file, remote_file, **kwargs): self.process = scp( '-o UserKnownHostsFile=/dev/null', '-o StrictHostKeyChecking=no', '-o LogLevel=quiet', '-P', self.port, local_file, '{0}@{1}:{2}'.format(self.user, self.host, remote_file), _out=self.out_iteract, _out_bufsize=0, _tty_in=True, **kwargs) super(SCPController, self).__call__(local_file, remote_file, **kwargs)
def run_trial(fs, trial_num, run_type): print "%s: trial %d, %s" % (fs, trial_num, run_type) trace_filename = fs + '.' + run_type + '.' + str(trial_num) if run_type == 'write': while True: try: sh.sudo('hadoop fs -rmr -skipTrash /benchmarks'.split()) break except sh.ErrorReturnCode_255: print "failed to delete benchmark data; retrying in 5 seconds..." sh.sleep(5) for server in servers: server.start_blktrace(server.hostname + '.' + trace_filename) #server.start_blktrace('data/blktrace/%s/%s.%d' % (fs, server.hostname, trial_num)) sh.sleep(1) try: """ sh.time( 'sudo hadoop jar /usr/share/hadoop/hadoop-examples-1.0.4.jar teragen -Dmapred.map.tasks=10 1000000000 /tera'.split(), _err = 'data/%s.gen.%d' % (fs, trial_num) ) sh.time( 'sudo hadoop jar /usr/share/hadoop/hadoop-examples-1.0.4.jar terasort -Dmapred.map.tasks=10 -Dmapred.reduce.tasks=10 /tera /sorted'.split(), _err = 'data/%s.sort.%d' % (fs, trial_num) ) """ sh.time( ('sudo hadoop jar /usr/share/hadoop/hadoop-test-1.0.4.jar TestDFSIO -%s -fileSize 100 -nrFiles 100' % run_type).split(), _err = 'data/%s/%s.%d' % (run_type, fs, trial_num) ) except sh.ErrorReturnCode_255: print "WARNING: test failed" for server in servers: server.end_blktrace() sh.scp(('root@%s:%s.* %s' % (server.hostname, server.hostname + '.' + trace_filename, 'data/blktrace/%s/' % fs)).split())
def __call__(self, local_file, remote_file, **kwargs): self.process = scp('-P', self.port, local_file, '{}@{}:{}'.format(self.user, self.host, remote_file), _out=self.out_iteract, _out_bufsize=0, _tty_in=True, _err=self.err_iteract, **kwargs) super().__call__(local_file, remote_file, **kwargs)
def __call__(self, local_file, remote_file, **kwargs): self.process = scp('-o UserKnownHostsFile=/dev/null', '-o StrictHostKeyChecking=no', '-o LogLevel=quiet', '-P', self.port, local_file, '{0}@{1}:{2}'.format(self.user, self.host, remote_file), _out=self.out_iteract, _out_bufsize=0, _tty_in=True, **kwargs) super(SCPController, self).__call__(local_file, remote_file, **kwargs)
#!/usr/bin/env python import os import sh import sys import arcomm _, local, host = sys.argv[:3] sh.scp(local, "{}:/tmp/".format(host)) package = os.path.basename(local) script = """ no extension {0} delete extension:{0} """.format(package) print arcomm.execute('eapi://{}'.format(host), script.splitlines()) script = """ copy file:/tmp/{0} extension: extension {0} copy installed-extensions boot-extensions configure no snmp-server extension .1.3.6.1.4.1.8072.1.3.1.5 snmp-server extension .1.3.6.1.4.1.8072.1.3.1.5 file:/var/tmp/snmpext end write """.format(package) print arcomm.execute('eapi://{}'.format(host), script.splitlines())
if os.path.exists(baseBuildDir): shutil.rmtree(baseBuildDir) sh.mkdir(baseBuildDir) sh.cd(baseBuildDir) print "\nChecking out repositories" for b in builds: repository = b['repository'] url = b['url'] try: print "\tCloning " + b['name'] sh.git("clone", url, repository) sh.cd(repository) sh.git("config", "remote.origin.push", "refs/heads/*:refs/for/*") sh.scp("-p", "-P", "29418", "{0}@review.motechproject.org:hooks/commit-msg".format(gerritUsername), ".git/hooks/") sh.cd('..') except Exception, e: print "\t\t*** Unable to continue: exception pulling %s ***" % b['repository'] print e return 1 # run mvn release:branch command print "\nBranching repositories" for b in builds: repository = b['repository'] print "\tBranching %s" % repository sh.cd(repository) branchNameArg = "-DbranchName={0}".format(branchName)
def main(): default_path = '.futuregrid/novarc' arguments = docopt(__doc__, version='0.8') DEBUG("arguments", arguments) home = os.environ['HOME'] DEBUG("home", home) ###################################################################### # This secion deals with handeling "cm config" related commands ###################################################################### is_config = arguments['config'] != None if is_config: DEBUG('Arguments', arguments) file = arguments['--file'] try: config = cm_config(file) DEBUG("config", config) except IOError: print "%s: Configuration file '%s' not found" % ("CM ERROR", file) sys.exit(1) except (yaml.scanner.ScannerError, yaml.parser.ParserError) as yamlerror: print "%s: YAML error: %s, in configuration file '%s'" % ("CM ERROR", yamlerror, file) sys.exit(1) except: print "Unexpected error:", sys.exc_info()[0] sys.exit(1) name = arguments['NAME'] if arguments['fetch'] or name == 'fetch': DEBUG('Arguments', arguments) # get user var = {} var['user'] = arguments['--user'] var['host'] = arguments['--remote'] var['file'] = ".futuregrid/cloudmesh.yaml" if var['user'] == None: var['user'] = getpass.getuser() from_location = "%(user)s@%(host)s:%(file)s" % var to_location = os.path.expanduser("~/%(file)s" % var) if os.path.isfile(to_location): print "WARNING: The file %s exists" % to_location if not yn_choice("Would you like to replace the file", default='y'): sys.exit(1) print from_location print to_location print "Copy cloudmesh file from %s to %s" % (from_location, to_location) result = scp(from_location, to_location) print result sys.exit(0) if arguments['projects'] and arguments['list']: projects = config.data['cloudmesh']['projects'] print yaml.dump(projects, default_flow_style=False, indent=4) sys.exit(0) if arguments['projects'] and arguments['?']: projects = config.data['cloudmesh']['projects']['active'] print "Please select from the following:" print "0 - Cancel" selected = False choices = [] while not selected: counter = 1 for name in projects: print counter, "-" "%20s" % name choices.append(name) counter += 1 print "Please select:" input = int(sys.stdin.readline()) if input == 0: print "Selection terminated" sys.exit(0) selected = (input > 0) and (input < counter) print "Selected: ", input name = choices[input-1] print name sys.exit(0) if arguments['init'] or name == 'init': output = arguments['--out'] username = arguments['--user'] or os.getenv('USER') config.userdata_handler = ldap_user config.cloudcreds_handler = openstack_grizzly_cloud ########### for testing ############################################################# # config.cloudcreds_handler._client = mock_keystone.Client # config.cloudcreds_handler._client.mockusername = username # config.cloudcreds_handler._client.mocktenants = config.data['cloudmesh']['active'] ##################################################################################### config.initialize(username) try: config.write(output) except OSError as oserr: if oserr.errno == 17: print "'%s' exists, please rename or remove it and try again." % oserr.filename sys.exit(0) if arguments['list'] or name == 'list': for name in config.keys(): if 'cm_type' in config.data['cloudmesh']['clouds'][name]: print name, "(%s)" % config.data['cloudmesh']['clouds'][name]['cm_type'] sys.exit(0) if arguments['dump'] or name =='dump': format = arguments['--format'] if format == 'yaml': print yaml.dump(config, default_flow_style=False, indent=4) elif format == 'dict' or format ==None: print config sys.exit(0) if name == '?': if file == None: arguments['--out'] = "%s/%s" % (home, default_path) print "Please select from the following:" print "0 - Cancel" selected = False choices = [] while not selected: counter = 1 for name in config.keys(): if 'cm_type' in config.data['cloudmesh']['clouds'][name]: print counter, "-" "%20s" % name, "(%s)" % config.data['cloudmesh']['clouds'][name]['cm_type'] choices.append(name) counter += 1 print "Please select:" input = int(sys.stdin.readline()) if input == 0: print "Selection terminated" sys.exit(0) selected = (input > 0) and (input < counter) print "Selected: ", input name = choices[input-1] output = arguments['--out'] if name != None: cloud = config.cloud(name) if not cloud: print "%s: The cloud '%s' is not defined." % ("CM ERROR", name) print "Try instead:" for keyname in config.keys(): print " ", keyname sys.exit(1) if cloud['cm_type'] == 'eucalyptus': if arguments['--project']: project = arguments['--project'] if not project in cloud: print "No such project %s defined in cloud %s." % (project, name) sys.exit(1) else: project = config.cloud_default(name, 'project') or config.projects('default') if not project in cloud: print "Default project %s not defined in cloud %s." % (project, name) sys.exit(1) rc_func = lambda name: config.rc_euca(name, project) else: rc_func = config.rc result = rc_func(name) if arguments["-"]: print result else: if output == None: arguments['--out'] = "%s/%s" % (home, default_path) output = arguments['--out'] out = False try: # First we try to open the file assuming it doesn't exist out = os.open(output, os.O_CREAT | os.O_EXCL | os.O_WRONLY, stat.S_IRUSR | stat.S_IWUSR) except OSError as oserr: # If file exists, offer to replace it if oserr.errno == 17: delete = raw_input("'%s' exists; Overwrite it [N|y]? " % output) if delete.strip().lower() == 'y': out = os.open(output, os.O_TRUNC | os.O_WRONLY) if out: os.write(out,result) os.close(out)
def exec_scripts(request): # This 总体记录功能 tasktype = '命令执行' taskuser = request.user.name tasktime = datetime.datetime.now() ret = [] if request.method == 'POST': server = request.POST.getlist('mserver', []) group = request.POST.getlist('mgroup', []) scripts = request.POST.getlist('mscripts', []) args = request.POST.getlist('margs') command = request.POST.get('mcommand') if server: if scripts: for name in server: host = Asset.objects.get(name=name) nicinfo = NIC.objects.get(asset_id=host.id) ip = nicinfo.ipaddress ret.append(host.name) logging.info("==========Script Start==========") logging.info("User:"******"Host:" + host.name) for s in scripts: try: sh.scp(scripts_dir + s, "chenqiufei@{}:/tmp/".format(ip) + s) except: pass cmd = "ssh chenqiufei@" + ip + " " + '"sh /tmp/{} {}"'.format( s, args) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) data = p.communicate() ret.append(data) logging.info("Scripts:" + s) for d in data: logging.info(d) logging.info("==========Script End============") # This 局部记录功能 taskinfo = '在服务器:{} 上执行脚本 :{}'.format(server, scripts) else: for name in server: host = Asset.objects.get(name=name) nicinfo = NIC.objects.get(asset_id=host.id) ip = nicinfo.ipaddress ret.append(host.name) logging.info("==========Shell Start==========") logging.info("User:"******"Host:" + host.name) command_list = command.split('\n') for cmd in command_list: cmd = "ssh chenqiufei@" + ip + " " + '"{}"'.format(cmd) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) data = p.communicate() ret.append(data) logging.info("command:" + cmd) for d in data: logging.info(d) logging.info("==========Shell End============") # This 局部记录功能 taskinfo = '在服务器:{} 上执行命令: {}'.format(server, command) if group: if scripts: for g in group: logging.info("==========Script Start==========") logging.info("User:"******"Group:" + g) get_group = HostGroup.objects.get(name=g) hosts = get_group.serverList.all() serverlist = [] #f表结构适配,从groud里面获取到assetid,serverlist是整理后的资产列表 for server in hosts: obj = Asset.objects.get(id=server.id) serverlist.append(server) ret.append(g) for asset in serverlist: nicinfo = NIC.objects.get(asset_id=asset.id) ip = nicinfo.ipaddress ret.append(asset.name) for s in scripts: try: sh.scp(scripts_dir + s, "chenqiufei@{}:/tmp/".format(ip) + s) except: pass cmd = "ssh chenqiufei@" + ip + " " + '"sh /tmp/{} {}"'.format( s, args) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) data = p.communicate() ret.append(data) logging.info("command:" + cmd) for d in data: logging.info(d) logging.info("==========Script End============") # This 局部记录功能 taskinfo = '在服务器组:{} 上执行脚本 :{}'.format(group, scripts) else: command_list = [] command_list = command.split('\n') for g in group: logging.info("==========Shell Start==========") logging.info("User:"******"Group:" + g) get_group = HostGroup.objects.get(name=g) hosts = get_group.serverList.all() serverlist = [] for server in hosts: obj = Asset.objects.get(id=server.id) serverlist.append(server) ret.append(g) for asset in serverlist: nicinfo = NIC.objects.get(asset_id=asset.id) ip = nicinfo.ipaddress ret.append(asset.name) for cmd in command_list: cmd = "ssh chenqiufei@" + ip + " " + '"{}"'.format( cmd) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) data = p.communicate() ret.append(data) logging.info("command:" + cmd) for d in data: logging.info(d) logging.info("==========Shell End============") # This 局部记录功能 taskinfo = '在服务器组:{} 上执行命令: {}'.format(group, command) # This 总体记录功能 status = p.returncode if status == 0: taskstatus = True else: taskstatus = False TaskRecord.objects.create(tasktype=tasktype, taskuser=taskuser, tasktime=tasktime, taskstatus=taskstatus, taskinfo=taskinfo) return render(request, 'setup/shell_result.html', locals())
#!/usr/bin/env python3 from sh import scp, glob import yaml import logger f = open("deploy.yaml") deploy_data = yaml.load(f) logger.info("Deploying to " + deploy_data["host"]["address"] + ":" + deploy_data["host"]["path"]) try: scp("-r", glob("html/*"), deploy_data["host"]["address"] + ":" + deploy_data["host"]["path"]) logger.info("Deployment complete.") except Exception: logger.error("Deployment failed.")
def transfer(local, remote): login, remote_path = remote.split(':') local_fingerprint = local + '.fingerprint' fingerprint_name = os.path.basename(local_fingerprint) local_patch = local + '.rawpatch' remote_fingerprint = os.path.join('/tmp', fingerprint_name) fingerprint_script = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'make-fingerprint') rawpatch_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'rawpatch') print("Sending the rawpatch script to the remote system.") sh.scp(rawpatch_script, f"{login}:/tmp/") print("Sending the fingerprint script to the remote system.") sh.scp(fingerprint_script, f"{login}:/tmp/") print("Computing the master fingerprint on the remote machine…") remote_fingerprint_output = sh.ssh( login, f"/tmp/make-fingerprint --master {remote_path}", _bg=True) print("|| Meanwhile, computing the local fingerprint", os.path.basename(local)) bsync.save_fingerprint(local) print( f"|| Sending the fingerprint to the remote system: {login}:/tmp/{fingerprint_name}" ) output = sh.scp(local_fingerprint, f"{login}:/tmp/") print_streams(output) print("|| waiting for the master fingerprint to be computed…") remote_fingerprint_output.wait() print_streams(remote_fingerprint_output) print("Creating the rawpatch on the remote machine.") output = sh.ssh( login, f"/tmp/rawpatch --master {remote_path} --fingerprint {remote_fingerprint}" ) remote_patch = output.stdout.decode('utf8') print_streams(output) if 'NoChanges' in remote_patch: print("The files are already identical.") print("Cleaning up the remote files") output = sh.ssh(login, f"rm {remote_fingerprint}") print_streams(output) else: print("Retrieving the patch from the remote machine.") output = sh.scp(f"{login}:{remote_patch}", f"{local_patch}") print("Cleaning up the remote files") output = sh.ssh(login, f"rm {remote_fingerprint} {remote_patch}") print_streams(output) print("Applying the patch") bsync.apply_rawpatch(local, local_patch) print("The patch has been applied! Deleting the patch file…") sh.rm(local_patch) print("Done!")
def exec_scripts(request): ret = [] temp_name = "setup/setup-header.html" if request.method == 'POST': server = request.POST.getlist('mserver', []) group = request.POST.getlist('mgroup', []) scripts = request.POST.getlist('mscripts', []) command = request.POST.get('mcommand') if server: if scripts: for name in server: host = Host.objects.get(hostname=name) ret.append(host.hostname) logging.info("==========Shell Start==========") logging.info("User:"******"Host:" + host.hostname) for s in scripts: try: sh.scp(scripts_dir + s, "root@{}:/tmp/".format(host.ip) + s) except: pass cmd = "ssh root@" + host.ip + " " + '"sh /tmp/{}"'.format( s) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) data = p.communicate() ret.append(data) logging.info("Scripts:" + s) for d in data: logging.info(d) logging.info("==========Shell End============") else: for name in server: host = Host.objects.get(hostname=name) ret.append(host.hostname) logging.info("==========Shell Start==========") logging.info("User:"******"Host:" + host.hostname) command_list = command.split('\n') for cmd in command_list: cmd = "ssh root@" + host.ip + " " + '"{}"'.format(cmd) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) data = p.communicate() ret.append(data) logging.info("command:" + cmd) for d in data: logging.info(d) logging.info("==========Shell End============") if group: if scripts: for g in group: logging.info("==========Shell Start==========") logging.info("User:"******"Group:" + g) hosts = Host.objects.filter(group__name=g) ret.append(g) for host in hosts: ret.append(host.hostname) for s in scripts: try: sh.scp(scripts_dir + s, "root@{}:/tmp/".format(host.ip) + s) except: pass cmd = "ssh root@" + host.ip + " " + '"sh /tmp/{}"'.format( s) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) data = p.communicate() ret.append(data) logging.info("command:" + cmd) for d in data: logging.info(d) logging.info("==========Shell End============") else: command_list = [] command_list = command.split('\n') for g in group: logging.info("==========Shell Start==========") logging.info("User:"******"Group:" + g) hosts = Host.objects.filter(group__name=g) ret.append(g) for host in hosts: ret.append(host.hostname) for cmd in command_list: cmd = "ssh root@" + host.ip + " " + '"{}"'.format( cmd) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) data = p.communicate() ret.append(data) logging.info("command:" + cmd) for d in data: logging.info(d) logging.info("==========Shell End============") return render(request, 'setup/shell_result.html', locals())
def cm_manage(): """Usage: cm-manage config projects list cm-manage config projects cm-manage config [-f FILE] [-o OUT] [-p PROJECT] cloud NAME [-] cm-manage config dump [--format=(yaml|dict)] cm-manage config init [-o OUT] [-u USER] cm-manage config list cm-manage config password NAME cm-manage config show passwords cm-manage config fetch [-u USER] [-r HOST] cm-manage --version cm-manage --help Arguments: NAME name of the cloud Options: -h --help show this help message and exit -v --version show version and exit -f NAME --file=NAME the Name of the cloud to be specified, if ? a selection is presented -o OUT --out=OUT writes the result in the specifide file -p PROJECT --project=PROJECT selects a project (e.g. for eucalyptus which has project-specific environments) -u USER --user=USER the user (login) name -r HOST --remote=HOST the host machine on which the yaml file is located in the ~/.futuregrid directory [default: sierra.futuregrid.org] -d --debug debug - this option is a - at the end of the command. If data is written to a file it is also put out to stdout Description: Command to generate rc files from our cloudmesh configuration files. This program generates form a YAML file containing the login information for a cloud an rc file that can be used to later source it. Example: we assume the yaml file has an entry india-openstack:: cm-manage config -o novarc india-openstack source novarc This will create a novarc file and than you can source it:: cm-manage config ? - Presents a selction of cloud choices and writes the choice into a file called ~/.futuregrid/novarc """ default_path = '.futuregrid/novarc' arguments = docopt(cm_manage.__doc__) DEBUG("arguments", arguments) home = os.environ['HOME'] # DEBUG("home", home) # # This secion deals with handeling "cm config" related commands ###################################################################### is_config = arguments['config'] != None if is_config: # DEBUG('Arguments', arguments) file = arguments['--file'] try: config = cm_config(file) # DEBUG("config", config) except IOError: print "{0}: Configuration file '{1}' not found".format("CM ERROR", file) sys.exit(1) except (yaml.scanner.ScannerError, yaml.parser.ParserError) as yamlerror: print "{0}: YAML error: {1}, in configuration file '{2}'".format("CM ERROR", yamlerror, file) sys.exit(1) except: print "Unexpected error:", sys.exc_info()[0] sys.exit(1) name = arguments['NAME'] # # NOT TESTED # if arguments['fetch'] or name == 'fetch': DEBUG('Arguments', arguments) # get user var = {} var['user'] = arguments['--user'] var['host'] = arguments['--remote'] var['file'] = ".futuregrid/cloudmesh.yaml" if var['user'] is None: var['user'] = getpass.getuser() from_location = "%(user)s@%(host)s:%(file)s" % var to_location = config_file("/%(file)s" % var) if os.path.isfile(to_location): print "WARNING: The file %s exists" % to_location if not yn_choice("Would you like to replace the file", default='y'): sys.exit(1) print from_location print to_location print "Copy cloudmesh file from %s to %s" % (from_location, to_location) result = scp(from_location, to_location) print result sys.exit(0) # # ok # # if (arguments['projects'] and arguments['list']) : if arguments['projects'] and arguments['list']: projects = config.get('cloudmesh.projects') print yaml.dump(projects, default_flow_style=False, indent=4) sys.exit(0) # # OK, needs setting # if arguments['projects']: projects = config.projects('active') print "Please select from the following:" print "0 - Cancel" selected = False choices = [] while not selected: counter = 1 for name in projects: print counter, "-" "%20s" % name choices.append(name) counter += 1 print "Please select:" input = int(sys.stdin.readline()) if input == 0: print "Selection terminated" sys.exit(0) selected = (input > 0) and (input < counter) print "Selected: ", input name = choices[input - 1] print name print "ERROR: THIS JUST SELECTS A PROJECT ID BUT DOES NOT SET IT" sys.exit(0) if arguments['init'] or name == 'init': output = arguments['--out'] username = arguments['--user'] or os.getenv('USER') location = path_expand(output) new_yaml_file = open(location, 'w+') user_yaml = cm_user().generate_yaml(username, 'cloudmesh') print >> new_yaml_file, yaml.dump(user_yaml, default_flow_style=False) new_yaml_file.close() print "Written new yaml file in " + location sys.exit(0) # # OK # if arguments['list'] or name == 'list': for name in config.cloudnames(): if 'cm_type' in config.cloud(name): print name, "(%s)" % config.cloud(name)['cm_type'] sys.exit(0) # # NOT TESTED # if arguments['password']: oldpass = getpass("Current password: "******"New password: "******"New password (again): ") # TODO: some kind of password strength checking? if newpass1 == newpass2: config.change_own_password(name, oldpass, newpass1) else: print "New passwords did not match; password not changed." sys.exit(0) # # OK, but does not display the username # if arguments['show'] or name == 'show' and arguments['passwords']: warning = "Your passwords will appear on the screen. Continue?" if yn_choice(warning, 'n'): me = ConfigDict(filename=config_file("/.futuregrid/me.yaml")) banner("PASSWORDS") for name in me['password']: print "{0}: {1}".format(name, me['password'][name]) sys.exit(0) # # OK # if arguments['dump'] or name == 'dump': format = arguments['--format'] if format == 'yaml': d = dict(config) print yaml.dump(d, default_flow_style=False) elif format == 'dict' or format is None: print config sys.exit(0) # # NOT TESTED # if name in ['?', 'x']: if file is None: arguments['--out'] = "%s/%s" % (home, default_path) print "Please select from the following:" print "0 - Cancel" selected = False choices = [] while not selected: counter = 1 for name in config.cloudnames(): if 'cm_type' in config.cloud(name): print "{0} - {1:<30} ({2})".format(counter, name, config.cloud(name)['cm_type']) choices.append(name) counter += 1 print "Please select:" input = int(sys.stdin.readline()) if input == 0: print "Selection terminated" sys.exit(0) selected = (input > 0) and (input < counter) print "Selected: ", input name = choices[input - 1] output = arguments['--out'] # # OK # if name is not None: cloud = config.cloud(name) if not cloud: print "%s: The cloud '%s' is not defined." % ("CM ERROR", name) print "Try instead:" for keyname in config.cloudnames(): print " ", keyname sys.exit(1) if cloud['cm_type'] == 'eucalyptus': if arguments['--project']: project = arguments['--project'] if not project in cloud: print "No such project %s defined in cloud %s." % (project, name) sys.exit(1) else: project = config.cloud_default( name, 'project') or config.projects('default') if not project in cloud: print "Default project %s not defined in cloud %s." % (project, name) sys.exit(1) rc_func = lambda name: config.rc_euca(name, project) else: rc_func = config.rc result = rc_func(name) # # OK # if arguments["-"]: print result else: if output is None: arguments['--out'] = "%s/%s" % (home, default_path) output = arguments['--out'] out = False if yn_choice("Would you like to review the information", default="y"): banner("WARNING: FIle will be written to " + output) print result print banner("") try: # First we try to open the file assuming it doesn't exist out = os.open( output, os.O_CREAT | os.O_EXCL | os.O_WRONLY, stat.S_IRUSR | stat.S_IWUSR) except OSError as oserr: # If file exists, offer to replace it if oserr.errno == 17: delete = raw_input( "'%s' exists; Overwrite it [N|y]? " % output) if delete.strip().lower() == 'y': out = os.open(output, os.O_TRUNC | os.O_WRONLY) if out: os.write(out, result) os.close(out)
#!/usr/bin/env python3 from sh import scp, glob import yaml import logger f = open("deploy.yaml") deploy_data = yaml.load(f) logger.info("Deploying to " + deploy_data["host"]["address"] + ":" + deploy_data["host"]["path"]); try: scp("-r", glob("html/*"), deploy_data["host"]["address"] + ":" + deploy_data["host"]["path"]); logger.info("Deployment complete.") except Exception: logger.error("Deployment failed.")
def exec_scripts(request): ret = [] temp_name = "setup/setup-header.html" if request.method == 'POST': server = request.POST.getlist('mserver', []) group = request.POST.getlist('mgroup', []) scripts = request.POST.getlist('mscripts', []) args = request.POST.getlist('margs') command = request.POST.get('mcommand') if server: if scripts: for name in server: host = Host.objects.get(hostname=name) ret.append(host.hostname) logging.info("==========Shell Start==========") logging.info("User:"******"Host:"+host.hostname) for s in scripts: try: sh.scp(scripts_dir+s, "root@{}:/tmp/".format(host.ip)+s) except: pass cmd = "ssh root@"+host.ip+" "+'"sh /tmp/{} {}"'.format(s, args) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) data = p.communicate() ret.append(data) logging.info("Scripts:"+s) for d in data: logging.info(d) logging.info("==========Shell End============") else: for name in server: host = Host.objects.get(hostname=name) ret.append(host.hostname) logging.info("==========Shell Start==========") logging.info("User:"******"Host:"+host.hostname) command_list = command.split('\n') for cmd in command_list: cmd = "ssh root@"+host.ip+" "+'"{}"'.format(cmd) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) data = p.communicate() ret.append(data) logging.info("command:"+cmd) for d in data: logging.info(d) logging.info("==========Shell End============") if group: if scripts: for g in group: logging.info("==========Shell Start==========") logging.info("User:"******"Group:"+g) get_group = HostGroup.objects.get(name=g) hosts = get_group.serverList.all() ret.append(g) for host in hosts: ret.append(host.hostname) for s in scripts: try: sh.scp(scripts_dir+s, "root@{}:/tmp/".format(host.ip)+s) except: pass cmd = "ssh root@"+host.ip+" "+'"sh /tmp/{} {}"'.format(s, args) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) data = p.communicate() ret.append(data) logging.info("command:"+cmd) for d in data: logging.info(d) logging.info("==========Shell End============") else: command_list = [] command_list = command.split('\n') for g in group: logging.info("==========Shell Start==========") logging.info("User:"******"Group:"+g) get_group = HostGroup.objects.get(name=g) hosts = get_group.serverList.all() ret.append(g) for host in hosts: ret.append(host.hostname) for cmd in command_list: cmd = "ssh root@"+host.ip+" "+'"{}"'.format(cmd) p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) data = p.communicate() ret.append(data) logging.info("command:"+cmd) for d in data: logging.info(d) logging.info("==========Shell End============") return render(request, 'setup/shell_result.html', locals())