def revoke_access(user, ip, cnt, op_user, is_admin): # we allow admin to op on all | or user to operate on himself if not (is_admin or user == op_user): raise Exception("auth violation") users = get_users() r, aips = get_fw_rules(users) assert "." in ip, "illegal ip %s" % ip ur = r[user] print "walking rules for user %s" % user for r in ur: # print 'comparing %s with ip %s , cnt %s'%(r,ip,cnt) if r["source"] == ip and str(r["cnt"]) == str(cnt): # delegated erase for dfw in DELEGATED_FIREWALLS: dcmd = "del " dcmd += " --user=%s" % escapeshellarg(user) dcmd += " %s" % escapeshellarg(r["source"]) fcmd = "ssh " + dfw["ssh"] + " " + escapeshellarg(dfw["cmd"] % dcmd) print fcmd st, op = gso(fcmd) assert st == 0, "%s => %s" % (fcmd, op) # local erase cmd = "sudo iptables -DINPUT %s" % r["cnt"] st, op = gso(cmd) assert st == 0 print cmd pass
def revoke_access(user, ip, cnt, op_user, is_admin): #we allow admin to op on all | or user to operate on himself if not (is_admin or user == op_user): raise Exception('auth violation') users = get_users() r, aips = get_fw_rules(users) assert '.' in ip, "illegal ip %s" % ip ur = r[user] print 'walking rules for user %s' % user for r in ur: #print 'comparing %s with ip %s , cnt %s'%(r,ip,cnt) if r['source'] == ip and str(r['cnt']) == str(cnt): #delegated erase for dfw in DELEGATED_FIREWALLS: dcmd = 'del ' dcmd += ' --user=%s' % escapeshellarg(user) dcmd += ' %s' % escapeshellarg(r['source']) fcmd = 'ssh ' + dfw['ssh'] + ' ' + escapeshellarg( dfw['cmd'] % dcmd) print fcmd st, op = gso(fcmd) assert st == 0, "%s => %s" % (fcmd, op) #local erase cmd = 'sudo iptables -DINPUT %s' % r['cnt'] st, op = gso(cmd) assert st == 0 print cmd pass
def pack(comment): # rename the desired file folder in form of 20010101-comment date = time.strftime('%Y%m%d', time.localtime(time.time())) # des_folder: destination of status, output files des_folder = date + '-' + comment # remove then creat the folder gso('rm -rf %s' % des_folder) gso('mkdir -p %s/orbit_results' % des_folder) gso('cp {orbit,orbit.F} ./%s' % des_folder) gso('cp {*.plt,orbit.out,configuration.py} ./%s/orbit_results' % (des_folder)) gso('rm -rf q63887*')
def upload_arp_table(self): "Responsible for sending everything of value to the server" arp_data = run_or_die('arp -an') fd, file_name = tempfile.mkstemp(prefix="arp_") open(file_name, 'w').write(arp_data) output = self._upload_file(file_name) if output == OK: gso('rm -f %s' % file_name) os.close(fd) return OK os.close(fd) self.log("Error uploading") raise UploadException(self.options.server_url, output)
def kill_pro(): cmd = "ps aux|grep tomcat|grep -v grep|awk '{print $2}'" status, result = gso(cmd) if result: cmd_kill = "kill -9 %s" % result status, result = gso(cmd_kill) if status: print( '<<<<<<<<<<<<<<<<<<<< tomcat process is dead! >>>>>>>>>>>>>>>>>>>>' ) return 0 else: print('<<<<<<<<<<<<<<<<<<<< killed abort! >>>>>>>>>>>>>>>>>>>>') return 1
def hunt_and_explode(self): ''' Used to find all type-5 package data in the repository and untar the files properly. ''' base_path = make_path(get_spkg_path(), "repos") output = "" if sys.platform == "cli": tfn = "c:/spkg/tmp/tarballs.txt" cmd = 'bash -c "find %s -name \"*.tar.gz\"" > %s' % (base_path, tfn) #Logger.info("CMD: (%s)" % cmd) os.system(cmd) output = open(tfn).read() else: _status, output = gso('find %s -name "*.tar.gz"' % base_path) start_dir = get_slash_cwd() for full_tar_file_name in output.split('\n'): tmp_list = full_tar_file_name.split('/') tar_file_name = tmp_list[-1] base_name = tar_file_name.split('.tar.gz')[0] tar_file_dir = make_path(tmp_list[:-1] + [base_name]) if not os.path.isdir(tar_file_dir): #Logger.info("Exploding %s..." % base_name) cmd = 'bash -c "mkdir -p %s"' % tar_file_dir #Logger.info("tar_file_dir: %s" % tar_file_dir) status = os.system(cmd) if status == OK: cmd = 'bash -c "cd %s && tar -mxzf ../%s"' cmd = cmd % (tar_file_dir, tar_file_name) #Logger.info("Cmd: %s" % cmd) if os.system(cmd) != OK: msg = "could not untar %s" % (tar_file_name) raise Exceptions.BadPackage(full_tar_file_name, msg) os.chdir(start_dir)
def _sync_restore_data(self, package_name, target_list): ''' Prior to a restore command, sends all necessary data to the remote machine to $SPKG_DIR/archive package_name -- name of package being restored target_list -- list of (one) name of a backup dataset ''' if len(target_list) != 1: msg = "One and only one Restore Target can be specified (received %s)" raise RestoreFailure(msg % target_list) target = target_list[0] remote_dir = os.path.join(self.spkg_dir, "archive", package_name, target) self.run_cmd(" mkdir -p %s" % remote_dir) archive_dir = os.path.join(self.server_home, "archive", self.machine_name, package_name, target) if not os.path.isdir(archive_dir): reason = "{0} does not exist".format(archive_dir) errmsg = "Cannot restore target {0} for package {1}/machine {2}"\ ": {3}" errmsg = errmsg.format(target, package_name, self.machine_name, reason) self.polling_log.error(errmsg) raise RestoreFailure(reason, target, package_name, self.machine_name) cmd = "rsync -La {0}/* {1}@{2}:{3}" cmd = cmd.format(archive_dir, self.username, self.ip_address, remote_dir) self.server_log.info("Running: %s" % cmd) self.polling_log.debug(cmd) self.polling_log.info("Copying archive to remote machine...") status, output = gso(cmd) if status != OK: msg = "Error rsyncing restore data: %s" % output raise RestoreFailure(msg)
def dumpDTF(videoPath): outputPath = os.path.splitext(videoPath)[0] + '_features.txt' command = './DenseTrack ' + videoPath + ' > ' + outputPath out = gso(command) if out[0]!=0: raise Exception("Error while getting dense trajectory features : " + out[1])
def perform_fastforward(repo,rev,push=False): print 'FAST FORWARDING %s TO %s'%(repo,rev) am = AutoMerger() am.completed = [] am.args = getargs(JENKINS_TARGET_BRANCH,rev,push) am.merge(repo,rev,JENKINS_TARGET_BRANCH,'_') assert len(am.completed)==1,"completed %s for %s"%(am.completed,repo) for sm,pth in JENKINS_REPO_SUBMODULES.get(repo,{}).items(): st,op = gso('cd repos/%s && git submodule | grep %s'%(repo,pth)) ; assert st==0 cid = re.compile('^\-([0-9a-f]+) ').search(op).group(1) print 'UPDATING SUBMODULE %s at %s with rev %s'%(sm,pth,cid) ams = AutoMerger() ams.completed = [] ams.args = getargs(JENKINS_TARGET_BRANCH,cid,push) ams.merge(sm,cid,JENKINS_TARGET_BRANCH,'_') assert len(ams.completed)==1,"completed %s for %s"%(ams.completed,sm) if ams.completed[0]['prev_rev']==ams.completed[0]['new_rev']: print 'NO CHANGES DETECTED IN %s'%sm else: print '%s : SUBMODULE UPDATED COMPLETE'%sm if am.completed[0]['prev_rev']==am.completed[0]['new_rev']: print 'NO CHANGES DETECTED in %s'%repo else: print 'SUCCESFULLY UPDATED: %s'%repo
def monitor(): # a program to monitor running status if sys.version[0] == '2': if nplot == 2: snap_time = 45 if nplot in [1,3]: snap_time = 3 else: snap_time = 15 status, output = gso('qstat') while output != '': print(output) time.sleep(snap_time) status, output = gso('qstat') print('\n\n\n\n\n') print('job is done')
def run_or_die(cmd): try: status, output = gso(cmd) if status != OK: raise CommandException(cmd, output) except OSError: raise CommandException(cmd, str(OSError)) return output
def dumpDTF(videoPath): outputPath = os.path.splitext(videoPath)[0] + '_features.txt' command = './DenseTrack ' + videoPath + ' > ' + outputPath out = gso(command) if out[0] != 0: raise Exception("Error while getting dense trajectory features : " + out[1])
def _check_dispatcher_running(cls, uri): "Check for tcp port listener" tcp_port = uri.split(':')[-1].split('/')[0] cmd = 'lsof -i tcp:%s | grep python | grep -v grep' % tcp_port _status, output = gso(cmd) if output: return True return False
def _build_component(self, component_dict, section_name, svn_user, svn_password, tmp_path, debug, prepare): '''Export one item from SVN and create an appropriate TAR file component_dict -- dictionary of item data section_name -- either 'libs' or 'injectors' svn_user -- account to use for logging in to svn svn_password -- password for said account ''' tar_file_path = component_dict.get("path", '') svn_url = component_dict["svn"] if svn_url.endswith('/'): svn_url = svn_url[:-1] base = svn_url.split('/')[-1] if prepare: version = "HEAD" else: version = component_dict.get("version", "HEAD") expected_file_path = os.path.join(self.server_home, "repos", section_name, "%s-%s.tar.gz" % (base, version)) if tar_file_path == expected_file_path: if os.path.isfile(tar_file_path): msg = "No need to build %s: already exists" % base self.polling_log.info(msg) return None start_path = os.getcwd() checkout_dir = os.path.join(tmp_path, section_name, base) if not os.path.isdir(checkout_dir): os.system("mkdir -p %s" % checkout_dir) msg = "Checking out and building %s..." % base self.polling_log.info(msg) version = self._svn_checkout(version, svn_user, svn_password, svn_url, checkout_dir, debug) os.chdir( checkout_dir ) section_dir = os.path.join(self.server_home, "repos", section_name) if not os.path.isdir(section_dir): status = os.system("mkdir -p %s" % section_dir) if status != OK: msg = "Could not create required directory: %s" % section_dir raise CnmServerException(msg) else: msg = "Created required directory: %s" % section_dir self.server_log.warning(msg) tar_file_path = os.path.join(section_dir, "%s-%s.tar.gz" % (base, version)) cmd = "tar -czf %s *" % ( tar_file_path ) status, output = gso(cmd) if status != OK: self.server_log.error("Command: %s" % cmd) self.server_log.error("Current Directory: %s" % os.getcwd()) self.server_log.error("Output from command: %s" % output) msg = "Could not create tarball" os.chdir( start_path ) raise CnmServerException(msg) os.chdir( start_path ) return tar_file_path, version
def get_fw_rules(users=None, by_user=True): st, op = gso('sudo iptables -tfilter -nvL %s' % IPT_CHAIN) assert st == 0 if by_user: rt = {} else: rt = [] all_allowed = [] cnt = 0 for row in op.split('\n'): row = row.strip() res = matchre.search(row) rule_params = strre.search(row) if not rule_params: continue cnt += 1 #print 'cnt %s : %s'%(cnt,rule_params.groups()) if res: #print 'searching %s\nwith\n%s'%(strrestr,row) dptre = re.compile('dpt\:(\d+)') dptres = dptre.search(row) if dptres: dport = dptres.group(1) else: dport = None source = rule_params.group('source') dtraw = res.group('data') dt = json.loads(base64.b64decode(dtraw)) user = dt['u'] stamp = dt['s'] if users and user not in users: raise Exception('unknown user %s' % user) if by_user and user not in rt: #raise Exception('adding user %s because not in %s'%(user,rt.keys())) rt[user] = [] row = { 'source': source, 'cnt': cnt, 'pkts': rule_params.group('pkts'), 'age': (datetime.datetime.now() - datetime.datetime.fromtimestamp(stamp)), 'note': dt['n'], 'dport': dport } if by_user: rt[user].append(row) else: row['user'] = user rt.append(row) aaw = source if dport: aaw += '=>:' + dport all_allowed.append(aaw) return rt, all_allowed
def get_version(): "Get the SVN revision number for this build" status, output = gso("git rev-parse --short HEAD") if status != OK: raise BuildConfigurationException("Unable to obtain SVN version") version_info = output.strip() print ">>> (%s)" % version_info version_python_code = VERSION_TEMPLATE % version_info open("src/_version.py", 'w').write(version_python_code)
def check_pro(): cmd = 'ps aux|grep tomcat|grep -v grep' status, result = gso(cmd) if result: print('<<<<<<<<<<<<<<<<<<<< tomcat running >>>>>>>>>>>>>>>>>>>>') return 0 else: print('<<<<<<<<<<<<<<<<<<<< tomcat stopped >>>>>>>>>>>>>>>>>>>>') return 1
def monitor(): # a program to monitor running status if sys.version[0] == '2': snap_time = 45 status, output = gso('qstat') pattern = '(\d*).service' try: jobid = re.findall(pattern, output)[-1] except IndexError: print('no jobid found, check the grammer') print(output) return print('jobid is ', jobid, '\n') while re.findall(jobid, output): print(output) time.sleep(snap_time) status, output = gso('qstat') print('\n\n\n\n\n') print('job is done')
def sub_task(comment, submit): claim() # make file print('making files, please wait') print('making equilibrium files') status, output = gso('make FC=pgf90 eqs') print(output) status, output = gso('./eqs') print(output) status, output = gso('make FC=pgf90') print(output) # let the user choose weather to submit the job # make sure it is 1 when using batch test if submit == 1: status, output = gso('qsub ./job.pbs') print(output) use.monitor() use.pack(comment) else: print('not submitted')
def get_fw_rules(users=None, by_user=True): st, op = gso("sudo iptables -tfilter -nvL %s" % IPT_CHAIN) assert st == 0 if by_user: rt = {} else: rt = [] all_allowed = [] cnt = 0 for row in op.split("\n"): row = row.strip() res = matchre.search(row) rule_params = strre.search(row) if not rule_params: continue cnt += 1 # print 'cnt %s : %s'%(cnt,rule_params.groups()) if res: # print 'searching %s\nwith\n%s'%(strrestr,row) dptre = re.compile("dpt\:(\d+)") dptres = dptre.search(row) if dptres: dport = dptres.group(1) else: dport = None source = rule_params.group("source") dtraw = res.group("data") dt = json.loads(base64.b64decode(dtraw)) user = dt["u"] stamp = dt["s"] if users and user not in users: raise Exception("unknown user %s" % user) if by_user and user not in rt: # raise Exception('adding user %s because not in %s'%(user,rt.keys())) rt[user] = [] row = { "source": source, "cnt": cnt, "pkts": rule_params.group("pkts"), "age": (datetime.datetime.now() - datetime.datetime.fromtimestamp(stamp)), "note": dt["n"], "dport": dport, } if by_user: rt[user].append(row) else: row["user"] = user rt.append(row) aaw = source if dport: aaw += "=>:" + dport all_allowed.append(aaw) return rt, all_allowed
def sub_task(comment, pdist, numeric, submit, monitor): print(ying) # make file print('making files, please wait') print('making equilibrium files') status, output = gso('make FC=pgf90 eqs') print(output) status, output = gso('./eqs') print(output) status, output = gso('make FC=pgf90') print(output) # let the user choose weather to submit the job # make sure it is 'y' when using batch test if submit == 'y': status, output = gso('qsub ./job.pbs') print(output) if monitor == 'y': use.monitor() use.pack(comment, pdist, numeric) else: print('not submitted')
def _rsync_repository(self, tmp_path, dest): """Performing an RSYNC command to get all repos files to the remote machine""" cmd = "rsync -La %s %s@%s:%s" cmd = cmd % (tmp_path, self.username, self.ip_address, dest) self.server_log.info("Running: %s" % cmd) files = glob.glob("%s/*" % tmp_path) self.polling_log.debug(cmd) status, output = gso(cmd) if status != OK: msg = "Error rsyncing repository: %s" % output raise MachineUnavailableException(self.machine_name, msg)
def modem_terminal(): status, dmesg_out = gso("dmesg") if status != 0: raise Exception('dmesg returned ' + status) tty_lines = [line for line in dmesg_out.split('\n') if 'GSM modem' in line] if len(tty_lines) == 0: raise Exception('No lines for GSM found in dmesg!') return sorted([line.split(' ')[-1] for line in tty_lines if 'tty' in line])[0]
def runSvn(self, tokens, action): if len(tokens) < 3: return FAIL, ["Incomplete command"] fileName = os.path.join(mode.serverHome, tokens[2]) if not os.path.isfile(fileName): return FAIL, ["Unknown file: %s" % fileName] cmd = "%s %s %s" % (self.svnCmd, action, fileName) status, output = gso(cmd) if status != OK: errmsg = ["Repository is corrupt. Please examine it manually"] errmsg += output.split('\n') return FAIL, errmsg return OK, output.split('\n')
def update_tomcat(filename, version): now_project_path = '/usr/local/tomcat/webapps/%s' % ln_name # TODO 先删除上一个版本解压之后的文件再更新软连接. shutil.rmtree(now_project_path) os.remove('/usr/local/tomcat/webapps/publish.war') cmd_ln = 'ln -s /usr/local/tomcat/webapps/publish.war /data/version/%s/%s' % ( version, filename) status, result = gso(cmd_ln) if result: print('SoftConnect create success!') return 0 else: print('SoftConnect create failed!') exit(1)
def implement_rules(self, cmds): "Actually run the rules that we put together" if self.options.test: print "This script would implement the following rules:" for cmd in cmds: print " %s" % cmd else: for cmd in cmds: if self.options.verbose: self.log("Running: [%s]..." % cmd) status, output = gso(cmd) if status != OK: self.log("COMMAND FAILED: [%s]" % cmd) raise DownloadException(self.options.server_url, output) if self.options.verbose: print "SUCCESS"
def push(self, *args, **kw): config = get_config() if not self.cli_opts.label: raise SatCLIArgumentError, 'channel -l/--label required' channels = self.cli_opts.label.split(',') for channel in channels: if self.cli_opts.rpms: rpms = glob(str(self.cli_opts.rpms)) for rpm in rpms: nosig_txt = '' if config['allow_nosig']: nosig_txt = '--nosig' cmd = "%s %s -u %s -p %s --server %s -c %s %s" % \ (config['cmd_rhnpush'], rpm, config['user'], config['password'], config['server'], channel, nosig_txt) res = gso(cmd) if res[0] != 0: log.warn(res[1]) if self.cli_opts.srpms: srpms = glob(str(self.cli_opts.srpms)) for srpm in srpms: if os.path.exists(srpm): nosig_txt = '' if config['allow_nosig']: nosig_txt = '--nosig' cmd = "%s %s --source -u %s -p %s --server %s %s" % \ (config['cmd_rhnpush'], srpm, config['user'], config['password'], config['server'], nosig_txt) res = gso(cmd) if res[0] != 0: log.warn(res[1]) else: log.warn("SRPM '%s' doesn't exist!" % srpm)
def allow_access(user, ip=None, note=None, dport=None): assert ip or dport, "at least ip or dport have to be specified." users = get_users() rules, all_allowed = get_fw_rules(users) if ip and dport: cond = ip + "=>:" + dport in all_allowed elif dport: cond = "0.0.0.0/0=>:" + dport in all_allowed else: cond = ip in all_allowed if not cond: dt = base64.b64encode(json.dumps({"u": user, "s": time.time(), "n": note})) cmd = "sudo iptables -IINPUT %s" % IPT_INSPOS if ip: cmd += " -s %s" % (ip) if dport: cmd += " -p tcp --dport %s" % dport cmd += ' -j ACCEPT -m comment --comment="ServerAccess d:%s"' % (dt) # print cmd st, op = gso(cmd) assert st == 0, "%s => %s" % (cmd, op) for dfw in DELEGATED_FIREWALLS: dcmd = "add " if note: dcmd += " --note %s" % escapeshellarg(note) if dport: dcmd += " --dport=%s" % escapeshellarg(dport) if user: dcmd += " --user=%s" % escapeshellarg(user) if ip: dcmd += " %s" % escapeshellarg(ip) fcmd = "ssh " + dfw["ssh"] + " " + escapeshellarg(dfw["cmd"] % dcmd) print fcmd st, op = gso(fcmd) assert st == 0, "%s => %s" % (fcmd, op)
def runSvnMult(self, action, argString='', parseFunction=statusAndFile): directories = ["include", "bom", "client"] cmdOutput = {} for directory in directories: cmdOutput[directory] = [] directoryPath = os.path.join(mode.serverHome, directory) cmd = "%s %s %s %s" % (self.svnCmd, action, directoryPath, argString) status, output = gso(cmd) if status != OK: errmsg = ["Repository is corrupt. Please examine it manually"] errmsg += [cmd] errmsg += output.split('\n') return FAIL, errmsg cmdOutput = parseFunction(output, cmdOutput, directory) return OK, cmdOutput
def update_job(): #TODO 把版本文件拷到指定目录下 version = time.strftime('%Y-%m-%d-%H', time.localtime(time.time())) cmd_mkdir = "mkdir /data/version/%s" % version status, result = gso(cmd_mkdir) if not result: print('Create Version of the directory success!') file_paths = os.path.isdir(file_path) if not file_path: mkdir_req = os.mkdir(file_path) else: file = os.listdir(file_path) filename = file[0] cmd_cp = 'mv /data/file/%s /data/version/%s' % (filename, version) status, result = gso(cmd_cp) if result: print('copy file success!') return filename, version else: print('copy file filed!') exit(1) else: print('Create Version of the directory failed!') exit(1)
def start_pro(): path = tomcat_pah + '/bin' print(path) os.chdir(path) cmd = "sh startup.sh" status, result = gso(cmd) print(result) if not result: print( "<<<<<<<<<<<<<<<<<<<< tomcat startuo failed >>>>>>>>>>>>>>>>>>>>") return 1 else: print( "<<<<<<<<<<<<<<<<<<<< tomcat startup success >>>>>>>>>>>>>>>>>>>>") return 0
def allow_access(user, ip=None, note=None, dport=None): assert ip or dport, "at least ip or dport have to be specified." users = get_users() rules, all_allowed = get_fw_rules(users) if ip and dport: cond = ip + '=>:' + dport in all_allowed elif dport: cond = '0.0.0.0/0=>:' + dport in all_allowed else: cond = ip in all_allowed if not cond: dt = base64.b64encode( json.dumps({ 'u': user, 's': time.time(), 'n': note })) cmd = 'sudo iptables -IINPUT %s' % IPT_INSPOS if ip: cmd += ' -s %s' % (ip) if dport: cmd += ' -p tcp --dport %s' % dport cmd += ' -j ACCEPT -m comment --comment="ServerAccess d:%s"' % (dt) #print cmd st, op = gso(cmd) assert st == 0, "%s => %s" % (cmd, op) for dfw in DELEGATED_FIREWALLS: dcmd = 'add ' if note: dcmd += ' --note %s' % escapeshellarg(note) if dport: dcmd += ' --dport=%s' % escapeshellarg(dport) if user: dcmd += ' --user=%s' % escapeshellarg(user) if ip: dcmd += ' %s' % escapeshellarg(ip) fcmd = 'ssh ' + dfw['ssh'] + ' ' + escapeshellarg(dfw['cmd'] % dcmd) print fcmd st, op = gso(fcmd) assert st == 0, "%s => %s" % (fcmd, op)
def _process_backup(self, backup_dict, package_name): ''' We've just instructed a package to back itself up. We need to haul the backup data back to archive it and collect some statistics on it. backup_dict -- Detailed information about the backup Example: {'__BACKUP_DIR__': '/tmp/tmp3GHMRw', '__POST_BACKUP__': 'start', '__PRE_BACKUP__': 'stop', '__START_TIME__': 1284766556.062264, 'main_file': {'status': 0, 'elapsed_time': 0.0013320446014404297, 'file_name': '/tmp/foogazi/test_type_5', 'size': 18L, 'md5': {'test_type_5': 'a670f8128b02d00725cff2b4973fb47e', 'test_type_5.bz2': '17e6a319e8e0b496dab7eb9b3ce03994' } } } ''' remote_dir = backup_dict.get("__BACKUP_DIR__") if not remote_dir: msg = "Package does not request data be copied back from machine." self.polling_log.warning(msg) self.server_log.warning(msg) return backup_dict start_time = str(int(backup_dict.get("__START_TIME__"))) archive_dir = os.path.join(self.server_home, "archive", self.machine_name, package_name, start_time) os.system("mkdir -p %s" % archive_dir) cmd = "rsync -La %s@%s:%s/* %s" cmd = cmd % (self.username, self.ip_address, remote_dir, archive_dir) self.polling_log.debug(cmd) status, output = gso(cmd) if status != OK: msg = "Error rsyncing remote backup: %s" % output backup_dict["__SYNC__"] = FAIL self.status = FAIL else: backup_dict["__SYNC__"] = OK self.run_cmd("rm -rf %s" % remote_dir) del backup_dict["__BACKUP_DIR__"] backup_summary = yaml.dump(backup_dict) summary_file = os.path.join(archive_dir, "backup_info.yml") open(summary_file, 'w').write(backup_summary) return backup_dict
def roll_code(ver): version = int(ver) + 2 # TODO 按时间排序,然后重建软连接 cmd_sort = "ls -lt /data/version|sed -n %sp|awk '{print $9}'" % version print(cmd_sort) status, result = gso(cmd_sort) filename = os.listdir('/data/version/%s' % result) if not status: print('准备回滚到版本号为%s的版本' % result) status = check_pro() if status == 0: kill = kill_pro() if kill == 0: code = update_tomcat(filename, result) if code == 0: print( '<<<<<<<<<<<<<<<<<<<< Remote release success! >>>>>>>>>>>>>>>>>>>>' ) start_pro() else: print( '<<<<<<<<<<<<<<<<<<<< Remote release failed! >>>>>>>>>>>>>>>>>>>>' ) exit(1) else: exit(1) else: filename, version = update_job() code = update_tomcat(filename, version) if code == 0: print( '<<<<<<<<<<<<<<<<<<<< Remote release success! >>>>>>>>>>>>>>>>>>>>' ) start_pro() else: print( '<<<<<<<<<<<<<<<<<<<< Remote release failed! >>>>>>>>>>>>>>>>>>>>' ) exit(1) else: print('获取版本号失败...') exit(1)
def runParallel(self, ncpus) : if self.mainLoop: self.log.fatal("Cannot use custom main loop in multi-process mode, check your options") return 1 self.setupParallelLogging( ) from Gaudi.Configuration import Configurable import GaudiMP.GMPBase as gpp c = Configurable.allConfigurables self.log.info('-'*80) self.log.info('%s: Parallel Mode : %i '%(__name__, ncpus)) from commands import getstatusoutput as gso metadataCommands = [ 'uname -a', 'echo $CMTCONFIG', 'echo $GAUDIAPPNAME', 'echo $GAUDIAPPVERSION'] for comm in metadataCommands : s, o = gso( comm ) if s : o = "Undetermined" string = '%s: %30s : %s '%(__name__, comm, o) self.log.info( string ) try : events = str(c['ApplicationMgr'].EvtMax) except : events = "Undetermined" self.log.info('%s: Events Specified : %s '%(__name__,events)) self.log.info('-'*80) # Parall = gpp.Coordinator(ncpus, shared, c, self.log) Parall = gpp.Coord( ncpus, c, self.log ) sysStart = time() sc = Parall.Go() self.log.info('MAIN.PY : received %s from Coordinator'%(sc)) if sc.isFailure() : return 1 sysTime = time()-sysStart self.log.name = 'Gaudi/Main.py Logger' self.log.info('-'*80) self.log.info('%s: parallel system finished, time taken: %5.4fs', __name__, sysTime) self.log.info('-'*80) return 0
# -*-coding:utf-8 -*-
def pack(comment,pdist,numeric): # rename the desired file folder as 20010101-comment date = time.strftime('%Y%m%d',time.localtime(time.time())) # des_folder: destination of output files des_folder = date + '-' + comment # remove than creat the folder gso('rm -rf %s'%des_folder) gso('mkdir -p %s/orbit_results'%des_folder) # cp certain files based the value of pdist and numeric if pdist*numeric == 2: # numeric balance and distribution gso('cp {orbit,orbit.F,spdata,fbm_dist.dat,job.pbs} ./%s'%des_folder) elif pdist == 2: # numeric distribution gso('cp {orbit,orbit.F,fbm_dist.dat,job.pbs,spdata} ./%s'%des_folder) else: gso('cp {orbit,orbit.F,spdata,job.pbs} ./%s'%des_folder) # a program to package file gso('cp {*.plt,orbit.out,configuration.py} ./%s/orbit_results'%(des_folder)) gso('cp -r ./plot_functions ./%s'%(des_folder))
def run(start_date=None,end_date=None,makereport=False): usermap = {} conf = json.loads(open('config.json','r').read().replace('\n','')) GITHUB_USER = conf['user'] #open('githubuser.txt','r').read().strip() GITHUB_PASSWORD = conf['user']+':'+conf['password'] #open('githubpw.txt','r').read().strip() GITHUB_PROJECTS = conf['projects'] #open('githubprojects.txt','r').read().strip().split('\n') usermapjson = conf['usermap'] for k,v in usermapjson.items(): usermap[re.compile(re.escape(k))]=v by_story={} by_user={} by_project={} by_date={} user_project={} project_user={} user_date={} user_project_date={} project_date={} for proj in GITHUB_PROJECTS: ofn = os.path.join('data','%s.json'%proj) fetchnew=True if os.path.exists(ofn): st = os.stat(ofn) fmt = datetime.datetime.fromtimestamp(st.st_mtime) oneh = (datetime.datetime.now()-datetime.timedelta(hours=1)) #print 'fmt = %s; oneh = %s'%(fmt,oneh) if fmt>=oneh: fetchnew=False else: os.unlink(ofn) if fetchnew: print 'fetching project %s'%proj pagenum=1 wr={'commits':[]} while True: print 'taking page %s'%pagenum cmd = "curl -s -u '%s' 'http://github.com/api/v2/json/commits/list/%s/%s/master?page=%s'"%(GITHUB_PASSWORD,GITHUB_USER,proj,pagenum) st,op=gso(cmd) ; assert st==0 pagenum+=1 dt = json.loads(op) if 'commits' in dt: print '%s entries'%len(dt['commits']) if 'commits' not in dt or not len(dt['commits']): break wr['commits']+=dt['commits'] fp = open(ofn,'w') ; fp.write(json.dumps(wr)); fp.close() else: print '%s is recent enough'%proj def initarr(): return {'times':0,'diff':0,'removed':0,'added':0,'ids':[]} if start_date: fr,to = start_date,end_date else: if len(sys.argv)>1: fr,to = [datetime.datetime.strptime(it,'%Y-%m-%d').date() for it in sys.argv[1].split(':')] print 'report is for range %s - %s'%(fr,to) else: fr,to = None,None if len(sys.argv)>2: if sys.argv[2][0]=='@': rcptraw = open(sys.argv[2][1:],'r').read() else: rcptraw = rcpt = sys.argv[2] rcpt = rcptraw.split(',') else: rcpt = None comre = re.compile('([a-f0-9]{16})') for fn in glob.glob('data/*.json'): if comre.search(fn): continue obj = json.loads(open(fn,'r').read()) print 'going over proj %s'%fn assert 'commits' in obj,'cannot find commits in %s'%fn for c in obj['commits']: projfn = os.path.basename(fn).replace('.json','') comfn = os.path.join('data','%s.%s.json'%(projfn,c['id'])) storyre = re.compile('#(\d+)') stories=[] for stres in re.finditer(storyre,c['message']): storyid = stres.group(1) if storyid not in stories: stories.append(storyid) user = c['committer']['email'] for um,uv in usermap.items(): if um.search(user): user = uv break assert user,c proj = os.path.basename(fn).replace('.json','') date = datetime.datetime.strptime(c['authored_date'][0:-5],'%Y-%m-%dT%H:%M:%S-') #dateutil.parser.parse(c['authored_date']) if fr and date.date()<fr: continue if to and date.date()>to: continue #print '%s -> %s on %s'%(user,proj,date) comid = '/%s/%s/commit/%s'%(GITHUB_USER,projfn,c['id']) commsg = c['message'] if not os.path.exists(comfn): print 'fetching commit %s'%c['id'] comurl = 'http://github.com/api/v2/json/commits/show/%s/%s/%s'%(GITHUB_USER,projfn,c['id']) curlcmd = 'curl -u %s %s > %s'%(GITHUB_PASSWORD,comurl,comfn) st,op = gso(curlcmd) ; assert st==0 assert os.path.exists(comfn) try: comdt = json.loads(open(comfn).read()) except: raise Exception('could not load from %s'%comfn) if 'modified' in comdt['commit']: try: dfsum = sum([len(mod['diff'].split('\n')) for mod in comdt['commit']['modified'] if 'diff' in mod]) except: raise Exception(comdt['commit']['modified']) else: dfsum =0 if 'removed' in comdt['commit']: removed = len(comdt['commit']['removed']) else: removed = 0 if 'added' in comdt['commit']: added = len(comdt['commit']['added']) else: added = 0 if proj not in project_date: project_date[proj]={} if proj not in project_user: project_user[proj]={} if user not in project_user[proj]: project_user[proj][user]=initarr() if date.date() not in project_date[proj]: project_date[proj][date.date()] = initarr() if user not in by_user: user_project_date[user]={} user_date[user]={} user_project[user]={} by_user[user]=initarr() if len(stories): for storyid in stories: if storyid not in by_story: by_story[storyid]=initarr() incr(by_story[storyid]) def idsort(i1,i2): return cmp(i1[3],i2[3]) def incr(o): #global dfsum,added,removed,comid,commsg o['times']+=1 o['diff']+=dfsum o['removed']+=removed o['added']+=added o['ids'].append([comid,commsg,user,date]) o['ids'].sort(idsort) incr(by_user[user]) incr(project_user[proj][user]) if proj not in by_project: by_project[proj]=initarr() if proj not in user_project[user]: user_project[user][proj]=initarr() incr(user_project[user][proj]) incr(by_project[proj]) if date.date() not in by_date: by_date[date.date()]=initarr() if date.date() not in user_date[user]: user_date[user][date.date()]=initarr() if proj not in user_project_date[user]: user_project_date[user][proj]={} if date.date() not in user_project_date[user][proj]: user_project_date[user][proj][date.date()]=initarr() incr(project_date[proj][date.date()]) incr(user_project_date[user][proj][date.date()]) incr(by_date[date.date()]) incr(user_date[user][date.date()]) if not makereport: return {'by_user':by_user,'by_story':by_story} def srtit(a1,a2): return cmp(a1[0],a2[0]) def srt2(i1,i2): return cmp(i1[1]['diff'],i2[1]['diff']) def srt3(i1,i2): return cmp(i1[0],i2[0]) import time def totimestamp(dt): return time.mktime(dt.timetuple()) + 0/1e6 #dt.microsecond/1e6 jsexp=[] for user in user_date: items = user_date[user].items() #print([totimestamp(it[0]) for it in items]) for item in items: #if user not in ['*****@*****.**','*****@*****.**']: continue #'*****@*****.**': continue tm = int(totimestamp(item[0])) curi = item[1]['diff'] jsexp.append({'action':user,'time':curi,'curitems':int(tm)}) def srtjsexp(i1,i2): return cmp(i1['curitems'],i2['curitems']) jsexp.sort(srtjsexp) op="""<doctype !html> <html> <head> <script type='text/javascript' src='jquery-1.6.1.min.js'></script> <script type='text/javascript' src='raphael-min.js'></script> <script type='text/javascript'> var data = %s; </script> <script type='text/javascript' src='plotgraph.js'></script> <style type='text/css'> #info { width:920px; height:200px; } thead { background-color:#abc; } </style> </head> <body> <div id='info'></div> """%(json.dumps(jsexp)) dtpat = "<table><thead><tr><th>date<th>commits<th>added<th>removed<th>difflines<th>links</tr></thead><tbody>\n" dtendpat = "</tbody></table>" rowpat = "<tr><td><nobr>%s</nobr></td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>\n" def mkrow(date,dt,commits=True): if commits: cm = ', '.join(["<a href='https://github.com%s' title='%s by %s on %s'>%s</a>"%(com,commsg,user,stamp,com.split('/')[4][0:4]) for com,commsg,user,stamp in dt['ids']]) else: cm='' rt= rowpat%(date,dt['times'],dt['added'],dt['removed'],dt['diff'],cm) return rt opa=[] if fr:opa.append("commits are starting from %s"%(fr)) if to:opa.append("commits are until %s"%(to)) opa.append("generated on %s"%datetime.datetime.now()) op+=' :: '.join(['<small>%s</small>'%ope for ope in opa])+'<br />' op+="<h1>user totals</h1>" op+=dtpat uitems = by_user.items() uitems.sort(srt2,reverse=True) for user,commits in uitems: op+=mkrow(user,commits,commits=False) op+=dtendpat op+="<h1>project totals by date</h1>" for proj,dates in project_date.items(): op+="<h2>%s</h2>"%proj op+=dtpat commits = dates.items() commits.sort(srt3) for date,commit in commits: op+=mkrow(date,commit,commits=True) op+=dtendpat op+="<h1>project totals by commiter</h1>" for proj,commiters in project_user.items(): op+="<h2>%s</h2>"%proj op+=dtpat.replace('date','commiter') commits = commiters.items() commits.sort(srt2,reverse=True) for user,commit in commits: op+=mkrow(user,commit,commits=True) op+=dtendpat for user,projects in user_project_date.items(): op+="<h1>%s commits by %s into %s project(s)</h1>\n"%(by_user[user]['times'],user,len(user_project[user])) op+="<h2>by dates</h2>\n" dates = user_date[user].items() dates.sort(srtit) op+=dtpat for date,commits in dates: op+=mkrow(date,commits,commits=True) op+="</tbody></table>" for proj,dates in projects.items(): op+="<h3>into project %s</h3>"%proj commits = dates.items() commits.sort(srtit) op+=dtpat for date,commits in commits: op+=mkrow(date,commits) op+="</tbody></table>" op+="</body></html>" ofn = 'commits' if fr: ofn+='-%s'%fr if to: ofn+=':%s'%to ofn+='.html' if not rcpt: fp = open(ofn,'w') ; fp.write(op) ; fp.close() print 'written to %s'%ofn else: srvr = conf['smtp_server'] port = conf['smtp_port'] un = conf['smtp_user'] pw = conf['smtp_pw'] sender = conf['smtp_sender'] import smtplib from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText fromaddr = sender msg = MIMEMultipart('alternative') msg['Subject'] = 'project commits for %s - %s'%(fr,to) msg['From'] = sender #msg['Reply-To'] = sender part2 = MIMEText(op, 'html') msg.attach(part2) # Credentials (if needed) username = un password = pw # The actual mail send server = smtplib.SMTP('%s:%s'%(srvr,port)) server.starttls() server.login(username,password) for rc in rcpt: print 'mailing to %s -> %s'%(fromaddr,rc) toaddrs = rc msg['To']=rc server.sendmail(fromaddr, toaddrs, msg.as_string()) server.quit()
def create(self, *args, **kw): config = get_config() errors = [] channels = [] if not self.cli_opts.advisory: errors.append(('SatCLIArgumentError', 'errata -a/--advisory required.')) if not self.cli_opts.rpms: errors.append(('SatCLIArgumentError', 'errata --rpms required.')) if not self.cli_opts.channel and not self.cli_opts.channels_file: errors.append(('SatCLIArgumentError', 'errata -c/--channel or --channels-file required.')) if self.cli_opts.channel: _channels = self.cli_opts.channel.split(',') for _c in _channels: channels.append(_c) if self.cli_opts.channels_file: if os.path.exists(self.cli_opts.channels_file): f = open(self.cli_opts.channels_file, 'r') for line in f.readlines(): channels.append(line.strip('\n')) else: log.warn("channels file '%s' doesn't exist!" % \ self.cli_opts.channels_file) if not self.cli_opts.synopsis: errors.append(('SatCLIArgumentError', 'errata --synopsis required.')) if not self.cli_opts.product: errors.append(('SatCLIArgumentError', 'errata --product required.')) if not self.cli_opts.advisory_type: errors.append(('SatCLIArgumentError', 'errata --type required.')) if not self.cli_opts.advisory_type in ['bug', 'enhancement', 'security']: errors.append(('SatCLIArgumentError', 'errata --type must be one of bug, enhancement, security.')) if not self.cli_opts.description: errors.append(('SatCLIArgumentError', 'errata --description required.')) abort_on_error(errors) if not self.cli_opts.topic: self.cli_opts.topic = "%s update." % self.cli_opts.advisory_type.capitalize() if not self.cli_opts.solution: self.cli_opts.solution = config['errata']['solution'] if self.cli_opts.keywords: self.cli_opts.keywords = self.cli_opts.keywords.split(',') else: self.cli_opts.keywords = [] rpms = glob(str(self.cli_opts.rpms)) rpms_data = [] package_ids = [] for r in rpms: nosig_txt = '' if config['allow_nosig']: nosig_txt = '--nosig' cmd = "%s %s -u %s -p %s --server %s %s" % \ (config['cmd_rhnpush'], r, config['user'], config['password'], config['server'], nosig_txt) gso(cmd) rpm = RPM(file(r)) package = g.proxy.query(model.Package, just_one=True, name=rpm[rpmdefs.RPMTAG_NAME], version=rpm[rpmdefs.RPMTAG_VERSION], release=rpm[rpmdefs.RPMTAG_RELEASE], arch=rpm[rpmdefs.RPMTAG_ARCH]) rpms_data.append(package) if self.cli_opts.srpm: if os.path.exists(self.cli_opts.srpm): rpm = RPM(file(self.cli_opts.srpm)) nosig_txt = '' if config['allow_nosig']: nosig_txt = '--nosig' cmd = "%s %s --source -u %s -p %s --server %s %s" % \ (config['cmd_rhnpush'], self.cli_opts.srpm, config['user'], config['password'], config['server'], nosig_txt) gso(cmd) else: log.warn("SRPM '%s' doesn't exist!" % self.cli_opts.srpm) for p in rpms_data: package_ids.append(p.id) if self.cli_opts.advisory_type == 'bug': self.cli_opts.advisory_type = 'Bug Fix Advisory' elif self.cli_opts.advisory_type == 'enhancement': self.cli_opts.advisory_type = 'Product Enhancement Advisory' elif self.cli_opts.advisory_type == 'security': self.cli_opts.advisory_type = 'Security Advisory' e = model.Errata() e.synopsis = self.cli_opts.synopsis e.advisory_name = self.cli_opts.advisory e.advisory_release = 1 e.advisory_type = self.cli_opts.advisory_type e.product = self.cli_opts.product e.topic = self.cli_opts.topic e.description = self.cli_opts.description e.references = self.cli_opts.references or '' e.notes = self.cli_opts.notes or '' e.solution = self.cli_opts.solution e.bug_ids = [] e.keywords = self.cli_opts.keywords or [] e.package_ids = package_ids e.publish = self.cli_opts.publish e.channels = channels g.proxy.create(e) res = g.proxy.query(model.Errata, just_one=True, all_data=True, advisory=self.cli_opts.advisory) return dict(errata=res)
# -*- coding: utf-8 -*-
def file_count(): """Count the number of files within specified directory.""" count = gso('ls | wc -l')[1] return count
def insert_1m_worker(request, amt, count): count = int(count) if amt == "1m": amt = 1000000 elif amt == "100k": amt = 100000 elif amt == "10k": amt = 10000 else: amt = int(amt) def create_index(): log.info("creating index") createindex_start = time.time() test_collection.ensure_index("indexed_id", unique=True) createindex_delta = time.time() - createindex_start ins = {"curitems": curitems, "time": createindex_delta, "action": "create_index"} return ins initialindex = bool(request.params.get("initialindex", False)) noindex = bool(request.params.get("noindex", False)) noselect = bool(request.params.get("noselect", False)) colname = request.params.get("collection", "test_collection") sleep = int(request.params.get("sleep", 0)) log.info("starting off with a count of %s" % count) records_per_iter = 100 count_of_iter = amt / records_per_iter log.info("running %s iterations of %s records each. noindex=%s" % (count_of_iter, records_per_iter, noindex)) start = time.time() log.info("getting collection") test_collection = get_collection(colname) global connection inserts = [] curitems = int(request.params.get("startfrom", test_collection.count())) def insappend(ins): ins["stamp"] = datetime.datetime.now().strftime("%s") fname = "static/%s.data.js" % colname log.info(ins) inserts.append(ins) if os.path.exists(fname): fp = open(fname, "r") dt = fp.read() fp.close() else: dt = "var data = [];" objres = re.compile("var data = (.*);").search(dt) if objres: objstr = objres.group(1) else: objstr = "[]" obj = json.loads(objstr) obj.append(ins) fp = open(fname, "w") fp.write("var data = %s;" % json.dumps(obj)) fp.close() args = { "action": "begin", "amt": amt, "count": count, "initialindex": initialindex, "noindex": noindex, "noselect": noselect, "colname": colname, "sleep": sleep, } insappend(args) if initialindex: insappend(create_index()) for cnt in range(count): log.info("index section, going %s / %s" % (cnt, count)) if not noindex: try: dropindex_start = time.time() log.info("dropping index (if exists)") test_collection.drop_index("indexed_id_1") dropindex_delta = time.time() - dropindex_start ins = {"curitems": curitems, "time": dropindex_delta, "action": "drop_index"} insappend(ins) except pymongo.errors.OperationFailure: log.info("index did not exist previously, lols") # get the current amount of items log.info("currently have %s items" % curitems) log.info("starting off insert") start_ins = time.time() log.info("starting ins %s" % cnt) for i in range(count_of_iter): ins_start = time.time() if i % 1000 == 0: log.info("batch %s, generating %s docs" % (i, records_per_iter)) # docs = [get_test_item(1000000*int(count) + i*records_per_iter + e) for e in range(records_per_iter)] docs = [] for e in range(records_per_iter): curitems += 1 ti = get_test_item(curitems) if bool(request.params.get("verbose", False)): log.info(ti) docs.append(ti) test_collection.insert(docs) if i % 1000 == 0: log.info("inserted, curitems = %s" % curitems) ins_delta = time.time() - start_ins ins = {"time": ins_delta, "curitems": curitems, "action": "insert"} #'amt':amt,'count':cnt, # log.info('insert %s done in %s: %s'%(cnt,ins_delta,ins)) insappend(ins) if "single" in colname: st, op = gso("du -s -m /var/lib/mongodb | cut -f1") print "du returned %s,%s" % (op, st) assert st == 0 datasize = int(op) elif "sharded" in colname: st, op = gso('mongo --eval "db.%s.dataSize()" --quiet localhost:10000/test_database' % colname) assert st == 0 datasize = int(op) st, op = gso('mongo --eval "db.printShardingStatus()" --quiet localhost:10000/test_database') assert st == 0 cres = re.finditer("^([ \t]+)([\w]+)([ \t]+)([\d]+)", op, re.M) tchunks = 0 shardsamt = 0 achunks = [] for r in cres: # print r.groups() shardname = r.group(2) chunks = int(r.group(4)) tchunks += chunks shardsamt += 1 achunks.append(chunks) ins = {"time": tchunks, "curitems": curitems, "action": "chunks_amt"} insappend(ins) avgdev = sum([abs(chnk - (tchunks / shardsamt)) for chnk in achunks]) ins = {"time": avgdev, "curitems": curitems, "action": "chunks_deviation"} insappend(ins) ins = {"time": shardsamt, "curitems": curitems, "action": "shards_amt"} insappend(ins) ins = {"time": datasize, "curitems": curitems, "action": "data_size"} insappend(ins) if sleep: log.info("sleeping (for sharding pacification purposes) - %s" % sleep) time.sleep(sleep) log.info("insert phase done") if not noindex: try: ins = create_index() insappend(ins) except pymongo.errors.DuplicateKeyError: log.error("FAILURE TO CREATE UNIQUE INDEX") return {"success": False, "message": "unique index could not be created", "inserts": inserts} if not noselect and curitems: seldelta = 0 selectnum = 1000 for i in range(selectnum): uid = random.randrange(curitems) rt = get_item_worker(uid, colname) assert rt["success"], "we were not succesful getting worker %s out of %s inserted (%s/%s)" % ( uid, curitems, i, selectnum, ) seldelta += rt["time"] ins = {"curitems": curitems, "time": seldelta, "action": "select%s" % selectnum} insappend(ins) delta = time.time() - start log.info("took us %s" % delta) return {"success": True, "time": delta, "count": test_collection.count(), "inserts": inserts}
#!/usr/bin/python from commands import getstatusoutput as gso import json,time colname = 'sharded_30nodes_nobalancer' mongos = open('masters.txt').read().strip().split('\n')[0] simulate=False runonce=True while True: countcmd = """cat slaves.txt | xargs -P0 -n1 -I{} ssh {} 'annotate-output +{} mongo --eval "db.%s.count()" --quiet localhost:27017/test_database' | egrep -v '(Finished|Started)' | awk '{print $3" "$1}' |sort -n"""%(colname) st,op = gso(countcmd) assert st==0 sizes = op.split("\n") targetshard = sizes[0].split(' ')[1] sourceshard = sizes[-1].split(' ')[1] print '%s vs %s'%(sizes[0],sizes[-1]) print 'going to move a chunk %s -> %s'%(sourceshard,targetshard) findchunk = """db.chunks.find({shard:'\"'%s'\"',ns:'\"'test_database.%s'\"'})"""%(sourceshard,colname) findchunkcmd ="""ssh %s 'echo "%s" | mongo --quiet localhost:10000/config'"""%(mongos,findchunk) #print findchunkcmd st,op = gso(findchunkcmd) assert st==0 chunksop = '['+','.join(op.split('\n')[1:-2])+']' #print chunksop chunks = json.loads(chunksop) print 'loaded %s chunks. moving a random one'%len(chunks) movcmd = """db.runCommand({moveChunk:'\"'test_database.%s'\"',find:{indexed_id:'\"'%s'\"'},to:'\"'%s'\"'})"""%(colname,chunks[0]['min']['indexed_id'],targetshard)
def parse(self, name): s, o = gso(self.command %(name)) if s: raise OSError(o) return o