def _leave_safe_mode(self):
"""leave safe mode"""
rcode = subprocess.cal(["hdfs", "dfsadmin", "-safemode", "leave"])
if rcode != 0:
self.msg.clear()
self.msg.code = rcode
self.msg.message = "[Error]: Failed to turn off safemode of hdfs"
raise self.msg
if re.search(pattern, data_lines):
data_lines = re.sub(pattern, r"\1" + rep_str, data_lines)
with open(conf, mode="w", encoding="cp932") as f:
f.write(data_lines)
else:
with open(conf, mode="a", encoding="cp932") as f:
f.write(add_str)
with open(conf, encoding="cp932") as f:
print(f.read())
# nftables
green("----- Start nftables -----")
nf = "nftables"
cmd = "systemctl status " + nf + " --no-pager"
if subp.run(cmd, shell=True).returncode !=0:
cmd = "systemctl start " + nf
subp.call(cmd, shell=True)
cmd = "systemctl is-enabled " + nf
if subp.run(cmd, shell=True).returncode !=0:
cmd = "systemctl enable " + nf
subp.cal(cmd, shell=True)
green("nftables backup")
cmd="nft list ruleset > /etc/nftables/org.nft"
subp.call(cmd, shell=True)
def Run_FASTX_Barcode_Splitter(files,output_dir,settings={'orientation':'bol'},search_reverse_complement=True):
parameters = copy.deepcopy(settings)
if 'orientation' not in parameters:
raise Exception('"Orientation" is required in the parameters field')
if not type(files) is list:
files = [files]
for i,each_file in enumerate(files):
if each_file.endswith('.gz'):
print "Unzipping file: {0}...".format(each_file)
files[i] = useful.gunzip_python(each_file)
files[i] = each_file[:-3]
print "Unzipping complete"
suffix = parameters.pop('suffix') if 'suffix' in parameters else ''
barcode_splitter_command = 'cat '+' '.join(files)+' | '
if output_dir[-1] == '/':
output_dir = output_dir[:-1]
if 'prefix' in parameters and parameters['prefix'] != '':
prefix = output_dir+'/'+parameters['prefix']
else:
prefix = output_dir+'/'
parameters.pop('prefix',None)
additional_folders = os.path.dirname(prefix)
if not os.path.isdir(additional_folders):
os.mkdir(additional_folders)
orientation = parameters.pop('orientation',None)
barcode_splitter_command += 'fastx_barcode_splitter.pl '
for p in parameters:
barcode_splitter_command+='--{0} {1} '.format(p,parameters[p])
barcode_splitter_command +='--prefix '+prefix+ ' --suffix '+suffix+' --'+orientation
output = useful.get_stdout(barcode_splitter_command).rstrip(' \n').split('\n')#output = subprocess.check_output(barcode_splitter_command,shell=True).rstrip(' \n').split('\n')
if output[0].lower().startswith('error'):
raise Exception("Error found in barcode split program: "+output[0])
result = {'barcodes':defaultdict(int)}
for line in output[1:-2]:
line = line.split('\t')
result['barcodes'][line[2]] = int(line[1])
result['total'] = int(output[-1].split('\t')[1])
result['unmatched'] = int(output[-2].split('\t')[1])
if search_reverse_complement:
initial_file = []
new_file = []
map_barcode_to_file = {}
with open(parameters['bcfile']) as file:
lines=file.readlines()
new_bcfile=open(settings['bcfile']+'rc','w')
for l in lines:
c = l.split('\t')
initial_file.append(c[0].strip())
new_file.append(c[0].strip()+'rev')
map_barcode_to_file[c[0].strip()+'rev'] = prefix+c[0].strip()+suffix
new_bcfile.write(c[0].strip()+'rev'+'\t'+Reverse_Complement(c[1].strip())+'\n')
new_bcfile.close()
shutil.copyfile(prefix + 'unmatched' + suffix, prefix + 'unmatched' + suffix + '.temp')
files = [prefix + 'unmatched' + suffix + '.temp']
parameters['bcfile'] +='rc'
if orientation == 'eol':
orientation='bol'
elif orientation == 'bol':
orientation= 'eol'
barcode_splitter_command = 'cat "'+' '.join(files)+'" | '
barcode_splitter_command += barcode_split_perl_script
for p in parameters:
barcode_splitter_command+='--{0} {1} '.format(p,parameters[p])
barcode_splitter_command += '--prefix ' + prefix + ' --suffix ' + suffix + ' --' + orientation
output = useful.get_stdout(barcode_splitter_command).rstrip(' \n').split('\n') # subprocess.check_output(barcode_splitter_command,shell=True).rstrip(' \n').split('\n')
for i, line in enumerate(output[1:-2]):
line = line.split('\t')
result['barcodes'][map_barcode_to_file[line[0].strip()]] += int(line[1])
result['unmatched'] = int(output[-2].split('\t')[1])
cleanup_command = ''
for i, each_bc_file in enumerate(initial_file):
cleanup_command += "mv '{0}{1}{3}' '{0}{1}{3}.temp';cat '{0}{1}{3}.temp' '{0}{2}{3}' > '{0}{1}{3}'; rm '{0}{1}{3}.temp';rm '{0}{2}{3}';".format(prefix, each_bc_file, new_file[i], suffix)
cleanup_command += "rm '{0}{1}'; ".format(prefix, 'unmatched' + suffix + '.temp')
subprocess.cal(cleanup_command, shell=True)
return result
#!/bin/bash/python
import subprocess
for i in range(1, 255):
address = "10.11.0." + str(i)
subprocess.cal(['ping', '-c', '1', address])
signeve_str, bkgneve_str, cuts_str, SRCDIR, EXENAME, SRCDIR, EXENAME),
shell=True)
subprocess.call("cd %s;make %s.x;cd -" % (SRCDIR, EXENAME), shell=True)
if rmode == 'p':
subprocess.call("%s/%s.x %s %s %d" %
(SRCDIR, EXENAME, args.inputdir, tag, renorm),
shell=True)
subprocess.call(
"cd Plots; GEINDEX; cd -; cd Plots/Plots_LamWZ_%s; GEINDEX; cd -;" %
(tag),
shell=True)
elif rmode == 'ct' or rmode == 'ca':
subprocess.call("%s/%s.x %s %s %d" %
(SRCDIR, EXENAME, args.inputdir, tag, channelID),
shell=True)
subprocess.call(
"cd Plots; GEINDEX; cd -; cd Plots/CUT_Train_LamWZ_%s; GEINDEX; cd -;"
% (tag),
shell=True)
elif rmode == 'bt' or rmode == 'ba' or rmode == 'br':
subprocess.call("%s/%s.x %s %s %d" %
(SRCDIR, EXENAME, args.inputdir, tag, channelID),
shell=True)
subprocess.call(
"cd Plots; GEINDEX; cd -; cd Plots/BDT_Train_LamWZ_%s; GEINDEX; cd -;"
% (tag),
shell=True)
subprocess.cal('echo "Tag is: %s"' % (tag), shell=True)
def main():
num_args = len(sys.argv)
if num_args != 0:
print "USAGE: python populate.py"
return
#trial_num = sys.argv[1]
with open(SERV_INFO_ATTACK_US, 'rU') as csvfile:
reader = csv.DictReader(csvfile)
nodes = list(reader)
#- send scripts to machines
for row in nodes:
ip = row["privateip"]
#python populate_nodes <mode>
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../../experiment_scripts', 'ubuntu@', ip, ':'])
# machine 1
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/1', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/2', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/3', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/4', 'ubuntu@', , ':'])
# machine 2
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/5', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/6', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/7', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/8', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/3', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/4', 'ubuntu@', , ':'])
#machine 3
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/9', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/10', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/11', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/12', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/5', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/6', 'ubuntu@', , ':'])
#machine 4
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/13', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/14', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/15', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/16', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/7', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/8', 'ubuntu@', , ':'])
#machine 5
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/9', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/10', 'ubuntu@', , ':'])
# machine 6
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/11', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/12', 'ubuntu@', , ':'])
# machine 7
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/13', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/14', 'ubuntu@', , ':'])
# machine 8
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/15', 'ubuntu@', , ':'])
subprocess.cal(['ssh', '-i', key, 'ubuntu@', ip, 'sudo', 'screen', '-d', '-m', 'scp', '-r', '-i', '../containers/16', 'ubuntu@', , ':'])
def Run_FASTX_Barcode_Splitter(files,
output_dir,
settings={'orientation': 'bol'},
search_reverse_complement=True):
parameters = copy.deepcopy(settings)
if 'orientation' not in parameters:
raise Exception('"Orientation" is required in the parameters field')
if not type(files) is list:
files = [files]
for i, each_file in enumerate(files):
if each_file.endswith('.gz'):
print "Unzipping file: {0}...".format(each_file)
files[i] = useful.gunzip_python(each_file)
files[i] = each_file[:-3]
print "Unzipping complete"
suffix = parameters.pop('suffix') if 'suffix' in parameters else ''
barcode_splitter_command = 'cat ' + ' '.join(files) + ' | '
if output_dir[-1] == '/':
output_dir = output_dir[:-1]
if 'prefix' in parameters and parameters['prefix'] != '':
prefix = output_dir + '/' + parameters['prefix']
else:
prefix = output_dir + '/'
parameters.pop('prefix', None)
additional_folders = os.path.dirname(prefix)
if not os.path.isdir(additional_folders):
os.mkdir(additional_folders)
orientation = parameters.pop('orientation', None)
barcode_splitter_command += 'fastx_barcode_splitter.pl '
for p in parameters:
barcode_splitter_command += '--{0} {1} '.format(p, parameters[p])
barcode_splitter_command += '--prefix ' + prefix + ' --suffix ' + suffix + ' --' + orientation
output = useful.get_stdout(barcode_splitter_command).rstrip(' \n').split(
'\n'
) #output = subprocess.check_output(barcode_splitter_command,shell=True).rstrip(' \n').split('\n')
if output[0].lower().startswith('error'):
raise Exception("Error found in barcode split program: " + output[0])
result = {'barcodes': defaultdict(int)}
for line in output[1:-2]:
line = line.split('\t')
result['barcodes'][line[2]] = int(line[1])
result['total'] = int(output[-1].split('\t')[1])
result['unmatched'] = int(output[-2].split('\t')[1])
if search_reverse_complement:
initial_file = []
new_file = []
map_barcode_to_file = {}
with open(parameters['bcfile']) as file:
lines = file.readlines()
new_bcfile = open(settings['bcfile'] + 'rc', 'w')
for l in lines:
c = l.split('\t')
initial_file.append(c[0].strip())
new_file.append(c[0].strip() + 'rev')
map_barcode_to_file[c[0].strip() +
'rev'] = prefix + c[0].strip() + suffix
new_bcfile.write(c[0].strip() + 'rev' + '\t' +
Reverse_Complement(c[1].strip()) + '\n')
new_bcfile.close()
shutil.copyfile(prefix + 'unmatched' + suffix,
prefix + 'unmatched' + suffix + '.temp')
files = [prefix + 'unmatched' + suffix + '.temp']
parameters['bcfile'] += 'rc'
if orientation == 'eol':
orientation = 'bol'
elif orientation == 'bol':
orientation = 'eol'
barcode_splitter_command = 'cat "' + ' '.join(files) + '" | '
barcode_splitter_command += barcode_split_perl_script
for p in parameters:
barcode_splitter_command += '--{0} {1} '.format(p, parameters[p])
barcode_splitter_command += '--prefix ' + prefix + ' --suffix ' + suffix + ' --' + orientation
output = useful.get_stdout(barcode_splitter_command).rstrip(
' \n'
).split(
'\n'
) # subprocess.check_output(barcode_splitter_command,shell=True).rstrip(' \n').split('\n')
for i, line in enumerate(output[1:-2]):
line = line.split('\t')
result['barcodes'][map_barcode_to_file[line[0].strip()]] += int(
line[1])
result['unmatched'] = int(output[-2].split('\t')[1])
cleanup_command = ''
for i, each_bc_file in enumerate(initial_file):
cleanup_command += "mv '{0}{1}{3}' '{0}{1}{3}.temp';cat '{0}{1}{3}.temp' '{0}{2}{3}' > '{0}{1}{3}'; rm '{0}{1}{3}.temp';rm '{0}{2}{3}';".format(
prefix, each_bc_file, new_file[i], suffix)
cleanup_command += "rm '{0}{1}'; ".format(
prefix, 'unmatched' + suffix + '.temp')
subprocess.cal(cleanup_command, shell=True)
return result
