def CopyLibrary(path):
new_path = os.path.join(frameworks_dir, os.path.basename(path))
args = ["ditto", "--arch=i386", "--arch=x86_64", path, new_path]
commands.append(args)
args = ["chmod", "u+w", new_path]
commands.append(args)
return new_path
def activateSamba(self):
commands = []
if fileExists('/etc/rc2.d/S20samba'):
commands.append('update-rc.d -f samba remove')
else:
commands.append('update-rc.d -f samba defaults')
self.Console.eBatch(commands, self.StartStopCallback, debug=True)
def buildCommands( logins, machines, clients ):
"""
@summary: Build all the commands that will be needed to synch the files that need to be synchronized.
@param logins: Logins to be used to build the command.
@param machines: Machines with whom you want to be synchronised.
@param clients: Specific clients/sources for wich you want to be synchronised.
@return: The list of commands that need to be run to be synchronised.
"""
commands = []
if clients[0] == "All" :
for login,machine in map( None, logins, machines) :
for i in range(3):#do 3 times in case of currently turning log files.
commands.append( "rsync -avzr -e ssh %s@%s:%s %s" %( login, machine, STATSPATHS.STATSPICKLES, STATSPATHS.STATSPICKLES ) )
else:
for client in clients :
path = STATSPATHS.STATSPICKLES + client + "/"
for login, machine in map( None, logins, machines ):
for i in range(3):#do 3 times in case of currently turning log files.
commands.append( "rsync -avzr -e ssh %s@%s:%s %s" %( login, machine, path, path ) )
return commands
def CopyLibrary(path): new_path = os.path.join(frameworks_dir, os.path.basename(path)) args = ['ditto', '--arch=i386', '--arch=x86_64', path, new_path] commands.append(args) args = ['chmod', 'u+w', new_path] commands.append(args) return new_path
def get_random_command(request):
"""S."""
commands = []
bash = {'command': 'uptime', 'args': ''}
commands.append(bash)
bash = {'command': 'host', 'args': remote_addr(request)}
commands.append(bash)
return random.choice(commands)
def CopyLibrary(path): print "CopyLibrary:", path CopyDSYM(path) new_path = os.path.join(binary_dir, os.path.basename(path)) args = ['ditto', '--arch=x86_64', path, new_path] commands.append(args) args = ['chmod', 'u+w', new_path] commands.append(args) return new_path
def output_to_command(string):
numberEntries = string.count("new")
string = string.split("-")
for i in xrange(numberEntries):
command = Command()
command.mic = int(string[5 * i + 1])
command.text = string[5 * i + 2]
command.time = float(string[5 * i + 3])
command.amplitude = float(string[5 * i + 4])
commands.append(command)
def initCommands() :
"""initCommands() -> list of python modules
return the list of avaible commands
"""
import plugins
import os, os.path
commandsDir = os.path.dirname(plugins.__file__)
commands = []
for com in os.listdir(commandsDir):
if com.endswith(".py") and com != "__init__.py" :
commands.append(__import__("uptodate.plugins."+com[:-3], globals, locals, "uptodate.plugins"))
return commands
def readconfcues(show):
global cues
cues = []
f = open("./Shows/" + show + "/" + settings["cuesfile"], "r").read()
f = removecomments(f)
sectionsanddata = re_startsection.split(f)[1:]
while sectionsanddata:
commands = []
comment = sectionsanddata[0].strip()
data_raw = sectionsanddata[1].strip()
for line in data_raw.splitlines():
commands.append(line.strip())
a = CueObject(comment, commands)
cues.append(a)
sectionsanddata = sectionsanddata[2:]
def getCommands(self,user,host="localhost"):
if(host=="localhost" or host==None):
host=socket.gethostname()
commands = []
match = False
for rule in self.rules:
if(rule.matchUser(user) and rule.matchHost(host)):
match = True
for cmnd in rule.command:
commands.append(cmnd)
#if(not match):
#commands.append("No Sudo Matches")
return commands
def version_independent_misc_cleanup():
commands = []
# Remove /export/hd?3 dirs
rm_cmd = ("echo %s | tr \\ \\\\n | fgrep -v versionmanager | xargs rm -rf" %
REMOVE_EXPORT_DIRS)
commands.append(comment('Removing %s') % REMOVE_EXPORT_DIRS )
commands.append(rm_cmd)
# Restore dirs
for dir in RESTORE_DIRS:
commands.append(comment('Creating %s') % dir )
commands.append("mkdir -p %s" % dir)
commands.append("chown nobody:nobody %s" % dir)
return commands
def SambaStartStop(self):
commands = []
if not self.my_Samba_run:
commands.append('/etc/init.d/samba start')
elif self.my_Samba_run:
commands.append('/etc/init.d/samba stop')
commands.append('killall nmbd')
commands.append('killall smbd')
self.Console.eBatch(commands, self.StartStopCallback, debug=True)
def rm_lock_uuid(mirror, username, password):
if username and password:
extra_opt = " --username %(username)s --password %(password)s " % locals()
else:
extra_opt = " "
commands = []
commands.append( SVNCMD + extra_opt + "pd svn:sync-lock --revprop -r0 %(mirror)s" % locals() )
commands.append( SVNCMD + extra_opt + "pd svn:sync-currently-copying --revprop -r0 %(mirror)s" % locals() )
for command in commands:
proc = Popen( command, stdout=PIPE, stderr=STDOUT, close_fds=True, shell=True )
output = proc.communicate()[0]
if proc.returncode != 0:
log.error( "Failed to remove sync lock from %s" % mirror )
if output:
log.error( "Command output:\n" + output )
return False
def CopyDSYM(dep):
frameworkString = '.framework'
pos = dep.rfind(frameworkString)
dsym_path = dep + ".dSYM"
if pos > -1:
frameworkPath = dep[:pos+len(frameworkString)]
dsym_path = frameworkPath + ".dSYM"
if os.path.exists(dsym_path):
new_dsym_path = os.path.join(dsyms_dir, os.path.basename(dsym_path))
print("CopyDSYM: [%s]" % dsym_path)
args = ['rm', '-rf', os.path.join(dsyms_dir, os.path.basename(dsym_path))]
commands.append(args)
args = ['cp', '-R', dsym_path, dsyms_dir]
commands.append(args)
else:
print("CopyDSYM: no .dSYM found.")
def rosExecute(apiros, command):
x = [""]
commands = command.split(" ")
commands.append(" ")
result = []
apiros.writeSentence(commands)
while True:
x = apiros.readSentence()
# print x
if x[0] == "!done":
break
result.append(x)
# s.close()
return make_dict(result)
def version_dependent_named_cleanup(version):
commands = []
# Removing version specific zone information from named. Starting from zone "ent<version> ,
# 5 lines need to be removed.
commands.append(comment('Modifying named for version: %s' %version))
commands.append("sed '/^zone \"ent%s/{N;N;N;N;d;}' /etc/named.conf > /etc/named.conf.new" % version.replace('.', '-'))
commands.append("cp /etc/named.conf /etc/named.conf.bak")
commands.append("mv /etc/named.conf.new /etc/named.conf")
return commands
def version_independent_named_cleanup():
commands = []
# Removing all the zone information from named. Starting from zone "ent ,
# 5 lines need to be removed.
commands.append(comment('Modifying named for all versions'))
commands.append("sed '/^zone \"ent/{N;N;N;N;d;}' /etc/named.conf > /etc/named.conf.new" )
commands.append("cp /etc/named.conf /etc/named.conf.bak")
commands.append("mv /etc/named.conf.new /etc/named.conf")
return commands
def test_003_can_mount_volume(self):
ip = self.data['instance'].private_dns_name
conn = self.connect_ssh(ip, TEST_KEY)
commands = []
commands.append('mkdir -p /mnt/vol')
commands.append('mkfs.ext2 %s' % self.device)
commands.append('mount %s /mnt/vol' % self.device)
commands.append('echo success')
stdin, stdout, stderr = conn.exec_command(' && '.join(commands))
out = stdout.read()
conn.close()
if not out.strip().endswith('success'):
self.fail('Unable to mount: %s %s' % (out, stderr.read()))
def RunJob(self,dataset):
# Getting the dataset name
name=InstanceName.Get(dataset.name)
# Creating Output folder is not defined
if not os.path.isdir(self.path+"/Output/"+name):
os.mkdir(self.path+"/Output/"+name)
# folder where the program is launched
folder = self.path+'/Output/'+name
# shell command
commands = ['../../Build/MadAnalysis5job']
# Weighted events
if not dataset.weighted_events:
commands.append('--no_event_weight')
# Release
commands.append('--ma5_version="'+self.main.version+';'+self.main.date+'"')
# Inputs
commands.append('../../Input/'+name+'.list')
# Running SampleAnalyzer
result = ShellCommand.Execute(commands,folder)
return result
def test_003_me_can_mount_volume(self):
conn = self.connect_ssh(data['private_ip'], test_key)
# HACK: the tiny image doesn't create the node properly
# this will make /dev/vd* if it doesn't exist
stdin, stdout, stderr = conn.exec_command('grep %s /proc/partitions | `awk \'{print "mknod /dev/"$4" b "$1" "$2}\'`' % DEVICE)
commands = []
commands.append('mkdir -p /mnt/vol')
commands.append('mkfs.ext2 /dev/%s' % DEVICE)
commands.append('mount /dev/%s /mnt/vol' % DEVICE)
commands.append('echo success')
stdin, stdout, stderr = conn.exec_command(' && '.join(commands))
out = stdout.read()
conn.close()
if not out.strip().endswith('success'):
self.fail('Unable to mount: %s %s' % (out, stderr.read()))
def CopyFramework(path):
parts = path.split(os.sep)
print "CopyFramework:", path
for i, part in enumerate(parts):
matchObj = re.match(r'(\w+\.framework)', part)
if matchObj:
full_path = os.path.join(frameworks_dir, *parts[i:-1])
framework = matchObj.group(1)
break
args = ['mkdir', '-p', full_path]
commands.append(args)
args = ['ditto', '--arch=x86_64', path, full_path]
commands.append(args)
args = ['chmod', 'u+w', os.path.join(full_path, parts[-1])]
commands.append(args)
args = ['chmod', 'u+w', os.path.join(frameworks_dir, framework, "Resources")]
commands.append(args)
info_plist = os.path.join(os.path.split(path)[0], '..', '..', 'Contents', 'Info.plist')
if os.path.exists(info_plist):
args = ['cp', '-r', info_plist, os.path.join(frameworks_dir, framework, "Resources")]
commands.append(args)
return os.path.join(full_path, parts[-1])
def CopyPlugin(path, subdir):
new_path = os.path.join(plugins_dir, subdir, os.path.basename(path))
args = ["mkdir", "-p", os.path.dirname(new_path)]
commands.append(args)
args = ["ditto", "--arch=i386", "--arch=x86_64", path, new_path]
commands.append(args)
args = ["chmod", "u+w", new_path]
commands.append(args)
return new_path
def CopyPlugin(path, subdir): new_path = os.path.join(plugins_dir, subdir, os.path.basename(path)) args = ['mkdir', '-p', os.path.dirname(new_path)] commands.append(args) args = ['ditto', '--arch=i386', '--arch=x86_64', path, new_path] commands.append(args) args = ['chmod', 'u+w', new_path] commands.append(args) return new_path
def init():
global command_names
global known_commands
#logging.debug("initCommands")
commands = []
for d in ["AVAgent", "AVCommon", "AVMaster"]:
for side in ["server", "client", "meta"]:
search = os.path.join(parent, d, "commands", side, "*.py")
dcommands = glob.glob(search)
for dc in dcommands:
name_file = os.path.split(dc)[1]
name = os.path.splitext(name_file)[0]
if name.startswith("__init__"):
continue
path = "%s.%s.%s.%s" % (d, "commands", side, name)
commands.append((name, side, path))
#logging.debug("%s" % (name))
for name, side, path in commands:
m = importlib.import_module(path)
m.side = side
known_commands[name] = m
def CopyFramework(path):
parts = path.split(os.sep)
for i, part in enumerate(parts):
if re.match(r'\w+\.framework', part):
full_path = os.path.join(frameworks_dir, *parts[i:-1])
break
args = ['mkdir', '-p', full_path]
commands.append(args)
args = ['ditto', '--arch=i386', '--arch=x86_64', path, full_path]
commands.append(args)
args = ['chmod', 'u+w', os.path.join(full_path, parts[-1])]
commands.append(args)
menu_nib = os.path.join(os.path.split(path)[0], 'Resources', 'qt_menu.nib')
if os.path.exists(menu_nib):
args = ['cp', '-r', menu_nib, resources_dir]
commands.append(args)
return os.path.join(full_path, parts[-1])
def CopyFramework(path):
parts = path.split(os.sep)
for i, part in enumerate(parts):
if re.match(r"\w+\.framework", part):
full_path = os.path.join(frameworks_dir, *parts[i:-1])
break
args = ["mkdir", "-p", full_path]
commands.append(args)
args = ["ditto", "--arch=i386", "--arch=x86_64", path, full_path]
commands.append(args)
args = ["chmod", "u+w", os.path.join(full_path, parts[-1])]
commands.append(args)
menu_nib = os.path.join(os.path.split(path)[0], "Resources", "qt_menu.nib")
if os.path.exists(menu_nib):
args = ["cp", "-r", menu_nib, resources_dir]
commands.append(args)
return os.path.join(full_path, parts[-1])
def FixVLCPlugin(abs_path, subdir):
broken_libs = GetBrokenLibraries(abs_path)
FixAllLibraries(broken_libs)
# print "Copying plugin....%s %s %s" % (plugins_dir, subdir, os.path.join(abs_path.split('/')[-2:]))
new_path = os.path.join(plugins_dir, subdir, os.path.basename(abs_path))
args = ["mkdir", "-p", os.path.dirname(new_path)]
commands.append(args)
args = ["ditto", "--arch=i386", "--arch=x86_64", abs_path, new_path]
commands.append(args)
args = ["chmod", "u+w", new_path]
commands.append(args)
for framework in broken_libs["frameworks"]:
FixFrameworkInstallPath(framework, new_path)
for library in broken_libs["libs"]:
FixLibraryInstallPath(library, new_path)
def FixVLCPlugin(abs_path, subdir):
broken_libs = GetBrokenLibraries(abs_path)
FixAllLibraries(broken_libs)
#print "Copying plugin....%s %s %s" % (plugins_dir, subdir, os.path.join(abs_path.split('/')[-2:]))
new_path = os.path.join(plugins_dir, subdir, os.path.basename(abs_path))
args = ['mkdir', '-p', os.path.dirname(new_path)]
commands.append(args)
args = ['ditto', '--arch=i386', '--arch=x86_64', abs_path, new_path]
commands.append(args)
args = ['chmod', 'u+w', new_path]
commands.append(args)
for framework in broken_libs['frameworks']:
FixFrameworkInstallPath(framework, new_path)
for library in broken_libs['libs']:
FixLibraryInstallPath(library, new_path)
def FixId(path, library_name):
id = '@executable_path/../Frameworks/%s' % library_name
args = ['install_name_tool', '-id', id, path]
commands.append(args)
def main(args):
try:
opts, pargs = getopt.getopt(args, "", ["inter=", "inter_ssh=",
"version=", "machines=",
"force" ])
except:
sys.exit(__doc__)
# default values
machines = []
inter_machine = None
inter_ssh_port = 22
version = None
force = 0
if len(opts) == 0:
# no flags were given, use old style args: <inter> <machine_list>
if len(pargs) < 2: sys.exit(__doc__)
inter_machine = pargs[0]
machines = pargs[1:]
else:
# flags were given, use new style
if len(pargs) != 0: sys.exit(__doc__)
for name,value in opts:
if name == '--inter': inter_machine = value
elif name == '--inter_ssh': inter_ssh_port = string.atoi(value)
elif name == '--version': version = value
elif name == '--machines': machines = string.split(value, ',')
elif name == '--force': force = 1
if len(machines) == 0:
sys.exit("ERROR: Must specify machines to clean")
commands = []
if version:
# Cleanup up only a specific version of RPMS
test_cmd = 'cat /export/hda3/%s/STATE 2>/dev/null' % version
# Change the STATE file to "REMOVE"
commands.append('echo REMOVE > /export/hda3/%s/STATE' % version)
commands.extend( uninstall_version_rpms(version) )
commands.append( "rm -rf /export/hd?3/%s" % version )
# Remove any lingering chubby config info and localbabysitter files
commands.append( "rm -f /etc/google/ent%s.chubby_cell "
"/etc/localbabysitter.d/*-%s.conf" %
(version.replace('.', '-'), version) )
if force:
# Kill any lingering processes with this version number
commands.append( "/usr/bin/pkill -KILL -f '/%s/'" %
version.replace('.', '\\.') )
commands.append( "/usr/bin/pkill -KILL -f '/%s/'" %
version.replace('.', '\\.') )
commands.extend( version_dependent_named_cleanup(version) )
else:
# Uninstall all enterprise RPMS
# all new versions should clean up after themselves in %preun/%postun
test_cmd = 'cat /export/hda3/*/STATE 2>/dev/null'
# Change the STATE file(s) to "REMOVE"
commands.append('echo REMOVE | tee `ls /export/hda3/*/STATE` > /dev/null')
commands.extend(uninstall_all_rpms())
# Cleanup legacy versions and third party RPMS and some other files
commands.extend(version_independent_misc_cleanup())
# Remove any lingering chubby config info and version related
# localbabysitter files. syslogd.conf and klogd.conf belong to
# OS and cannot be removed.
commands.append( "rm -f /etc/google/ent*.chubby_cell "
"/etc/localbabysitter.d/*-*.conf")
# Remove zone stuff from named.conf
commands.extend( version_independent_named_cleanup() )
if force:
# Kill any lingering processes belonging to nobody
commands.append( "/usr/bin/pkill -KILL -u nobody" )
commands.append( "/usr/bin/pkill -KILL -u nobody" )
raw_cmd = string.join(commands, '; ')
for machine in machines:
print
print "#################"
print "Cleaning machine %s" % machine
print "#################"
ssh_cmd = remote_cmd(machine, test_cmd)
if inter_machine:
cmd = remote_cmd(inter_machine, ssh_cmd, inter_ssh_port)
else:
cmd = ssh_cmd
p = os.popen(cmd, 'r')
for line in p.readlines():
if line.find('ACTIVE') == 0:
sys.stdout.write('Warning: Machine %s in active state\n' % machine)
if not force:
sys.exit('Quitting. Use --force flag to force clean.')
p.close()
ssh_cmd = remote_cmd(machine, raw_cmd)
if inter_machine:
cmd = remote_cmd(inter_machine, ssh_cmd, inter_ssh_port)
else:
cmd = ssh_cmd
os.system(cmd)
lines = (c / th) + 1
call("split -l %s %s %s " % (str(lines), dat, dat), shell=True)
onlyfiles = [f for f in listdir(".") if isfile(join(".", f))]
splits = []
for f in onlyfiles:
if f.startswith(dat + "a"):
splits.append(f)
splits.sort()
commands = []
for n in range(0, len(splits)):
com = "sat_subfam2fam_multi_support.py %s %s" % (splits[n], pat)
commands.append(com)
processes = [Popen(cmd, shell=True) for cmd in commands]
for p in processes:
p.wait()
splits_fam = []
for s in splits:
splits_fam.append(s + ".fam")
call("cat %s > %s.fam" % (" ".join(splits_fam), dat), shell=True)
print splits
call("rm %s %s" % (" ".join(splits), " ".join(splits_fam)), shell=True)
def FixId(path, library_name): id = '@executable_path/../Frameworks/%s' % library_name args = ['install_name_tool', '-id', id, path] commands.append(args)
if(bsub_command not in ["Y","N"]):print 'python Processing_sequences.py [sample file list] [commands (comma separated list)] [bsub command: Y/N] [print commands: Y/N] [run commands: Y/N]\n\tError: bsub command must be: Y or N'
if(bsub_command not in ["Y","N"]):print 'python Processing_sequences.py [sample file list] [commands (comma separated list)] [bsub command: Y/N] [print commands: Y/N] [run commands: Y/N] \n\tError: print command must be: Y or N'
if(bsub_command not in ["Y","N"]):print 'python Processing_sequences.py [sample file list] [commands (comma separated list)] [bsub command: Y/N] [print commands: Y/N] [run commands: Y/N] \n\tError: run command must be: Y or N'
idss,dirss='',''
commands = []
bsubs = []
for i in range(0,len(samples)):
info,sample, gene_types, pair,pair_final, dir,platforms, primers,other=infos[i], samples[i], gene[i], pairs[i],pairs_final[i], dirs[i], platform[i], primer[i],others[i]
bsub = ''
id,sources,species=ids[i],source[i],spec[i]
if(bsub_command=="Y"):
bsub = " | xargs -i echo qsub -P immune-rep.prjc -q "+queue+" -b y -o "+wkg_dir+"out_STANDARD_"+id+" -e "+wkg_dir+"error_log_"+id+" -N job_name \"{}\" | sh"
bsubs.append(bsub)
if( "1" in command):
command1 = python+" "+wkg_dir+"Read_processing_and_quality.py "+dir+" "+id+" "+sample+" "+gene_types+" "+pair+" "+species+" "+sources +" "+str(200)+" "+primers+" "+platforms+" 1 "+other+" "+reverse_primer_group[i]
commands.append(command1)
if( "2" in command):
command1 = python+" "+wkg_dir+"Read_processing_and_quality.py "+dir+" "+id+" "+sample+" "+gene_types+" "+pair+" "+species+" "+sources +" "+str(200)+" "+primers+" "+platforms+" 2 "+other+" "+reverse_primer_group[i]
commands.append(command1)
if( "3" in command):
command1 = python+" "+wkg_dir+"Read_processing_and_quality.py "+dir+" "+id+" "+sample+" "+gene_types+" "+pair+" "+species+" "+sources +" "+str(200)+" "+primers+" "+platforms+" 3 "+other+" "+reverse_primer_group[i]
commands.append(command1)
if( "4" in command):
command2 = python+" "+wkg_dir+"Generate_repertoire_statistics.py "+dir+"ORIENTATED_SEQUENCES/ANNOTATIONS/ "+id+" "+dir+"ORIENTATED_SEQUENCES/NETWORKS/Fully_reduced_"+id+".fasta "+dir+"ORIENTATED_SEQUENCES/Filtered_ORFs_sequences_all_"+id+".fasta "+gene_types+" "+species+" "+dir+"ORIENTATED_SEQUENCES/NETWORKS/Cluster_identities_"+id+".txt ANNOTATE,STATISTICS "+reverse_primer_group[i]
commands.append(command2)
if("ISO1" in command):
command1 = "python "+wkg_dir+"IsoTyper_1.0.py "+id+" "+id+" "+dir+" "+species+" "+reverse_primer_group[i]
commands.append(command1)
if("ISO2" in command):
command1 = "python "+wkg_dir+"Per_isotype_cluster_analyses.py "+id+" "+id+" "+dir+" "+species+" "+reverse_primer_group[i]
commands.append(command1)
def run_once(self, machines, extra_params, cycles):
AUTOTEST_DIR = self.job.clientdir
VIRT_DIR = os.path.join(AUTOTEST_DIR, 'tests', 'virt')
TEST_DIR = os.path.join(VIRT_DIR, 'tests')
SHARED_DIR = os.path.join(VIRT_DIR, 'shared')
VIRT_TYPE = 'qemu'
bootstrap.create_config_files(TEST_DIR, SHARED_DIR, interactive=False)
bootstrap.create_subtests_cfg(VIRT_TYPE)
sys.path.insert(0, VIRT_DIR)
CONTROL_MAIN_PART = """
testname = "virt"
bindir = os.path.join(job.testdir, testname)
job.install_pkg(testname, 'test', bindir)
qemu_test_dir = os.path.join(os.environ['AUTODIR'],'tests', 'virt')
sys.path.append(qemu_test_dir)
"""
logging.info("QEMU test running on hosts %s\n", machines)
_hosts = {}
for machine in machines:
_hosts[machine] = Machines(hosts.create_host(machine))
for host in _hosts.itervalues():
host.at = autotest_remote.Autotest(host.host)
cfg_file = os.path.join(VIRT_DIR, "qemu", "cfg",
"multi-host-tests.cfg")
if not os.path.exists(cfg_file):
raise error.JobError("Config file %s was not found", cfg_file)
# Get test set (dictionary list) from the configuration file
parser = cartesian_config.Parser()
parser.parse_file(cfg_file)
parser.parse_string(extra_params)
test_dicts = parser.get_dicts()
ips = []
for machine in machines:
host = _hosts[machine]
ips.append(host.host.ip)
logging.info("")
for i, params in enumerate(test_dicts):
logging.info("Test %d: %s" % (i, params.get("shortname")))
logging.info("")
test_dicts = parser.get_dicts()
test_dicts_ar = [
x for x in map(lambda x: utils_params.Params(x), test_dicts)
]
if not test_dicts_ar:
error.TestNAError("Impossible start any test with"
"this configuration.")
for params in test_dicts_ar:
params['hosts'] = ips
params['not_preprocess'] = "yes"
for vm in params.get("vms").split():
for nic in params.get('nics', "").split():
params['mac_%s_%s' % (nic, vm)] = generate_mac_address()
params['master_images_clone'] = "image1"
params['kill_vm'] = "yes"
s_host = _hosts[machines[0]]
s_host.params = params.object_params("host1")
s_host.params['clone_master'] = "yes"
s_host.params['hostid'] = machines[0]
for host_id, machine in enumerate(machines[1:]):
host = _hosts[machine]
host.params = params.object_params("host%s" % (host_id + 2))
host.params['clone_master'] = "no"
host.params['hostid'] = machine
# Report the parameters we've received
logging.debug("Test parameters:")
keys = params.keys()
keys.sort()
for key in keys:
logging.debug(" %s = %s", key, params[key])
for machine in machines:
host = _hosts[machine]
host.control = CONTROL_MAIN_PART
for machine in machines:
host = _hosts[machine]
host.control += ("job.run_test('virt', tag='%s', params=%s)" %
(host.params['shortname'], host.params))
logging.debug('Master control file:\n%s',
_hosts[machines[0]].control)
for machine in machines[1:]:
host = _hosts[machine]
logging.debug('Slave control file:\n%s', host.control)
commands = []
for machine in machines:
host = _hosts[machine]
commands.append(
subcommand.subcommand(host.at.run,
[host.control, host.host.hostname]))
try:
subcommand.parallel(commands)
except error.AutoservError, e:
logging.error(e)
def FixInstallPath(library_path, library, new_path):
args = [INSTALL_NAME_TOOL, '-change', library_path, new_path, library]
commands.append(args)
def is_exe(fpath):
return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
bundle_dir = sys.argv[1]
qmake_path = sys.argv[2]
bundle_name = os.path.basename(bundle_dir).split('.')[0]
commands = []
binary_dir = os.path.join(bundle_dir, 'Contents', 'MacOS')
frameworks_dir = os.path.join(bundle_dir, 'Contents', 'Frameworks')
commands.append(['mkdir', '-p', frameworks_dir])
resources_dir = os.path.join(bundle_dir, 'Contents', 'Resources')
commands.append(['mkdir', '-p', resources_dir])
plugins_dir = os.path.join(bundle_dir, 'Contents', 'PlugIns')
binaries = [
i for i in glob(os.path.join(bundle_dir, 'Contents', 'MacOS', "*"))
if is_exe(i)
]
qt_version = QueryQMake('QT_VERSION')
print "Using Qt", qt_version
fixed_libraries = []
fixed_frameworks = []
def CopyFramework(source_dylib):
parts = source_dylib.split(os.sep)
print "CopyFramework:", source_dylib
for i, part in enumerate(parts):
matchObj = re.match(r'(\w+\.framework)', part)
if matchObj:
framework = matchObj.group(1)
dylib_name = parts[-1]
source_path = os.path.join('/', *parts[:i + 1])
dest_path = os.path.join(frameworks_dir, framework)
dest_dylib_path = os.path.join(frameworks_dir, *parts[i:-1])
break
if os.path.exists(dest_path):
print dest_path, "already exists, skipping copy..."
return os.path.join(dest_dylib_path, dylib_name)
args = ['mkdir', '-p', dest_dylib_path]
commands.append(args)
args = ['ditto', '--arch=x86_64', source_dylib, dest_dylib_path]
commands.append(args)
args = ['chmod', 'u+w', os.path.join(dest_dylib_path, parts[-1])]
commands.append(args)
args = ['ln', '-s', '5', os.path.join(dest_path, 'Versions', 'Current')]
commands.append(args)
args = [
'ln', '-s',
os.path.join('Versions', 'Current', dylib_name),
os.path.join(dest_path, dylib_name)
]
commands.append(args)
args = [
'ln', '-s',
os.path.join('Versions', 'Current', 'Resources'),
os.path.join(dest_path, 'Resources')
]
commands.append(args)
args = [
'cp', '-r',
os.path.join(source_path, 'Versions', '5', 'Resources'),
os.path.join(dest_path, 'Versions', '5')
]
commands.append(args)
return os.path.join(dest_dylib_path, dylib_name)
def CopyFramework(path):
parts = path.split(os.sep)
print "CopyFramework:", path
for i, part in enumerate(parts):
matchObj = re.match(r'(\w+\.framework)', part)
if matchObj:
full_path = os.path.join(frameworks_dir, *parts[i:-1])
framework = matchObj.group(1)
break
args = ['mkdir', '-p', full_path]
commands.append(args)
args = ['ditto', '--arch=x86_64', path, full_path]
commands.append(args)
args = ['chmod', 'u+w', os.path.join(full_path, parts[-1])]
commands.append(args)
resources_dir = os.path.join(frameworks_dir, framework, "Resources")
args = ['mkdir', resources_dir]
commands.append(args)
args = ['chmod', 'u+w', resources_dir]
commands.append(args)
info_plist = os.path.join(
os.path.split(path)[0], '..', '..', 'Contents', 'Info.plist')
if not os.path.exists(info_plist):
info_plist = os.path.join(
os.path.split(path)[0], 'Resources', 'Info.plist')
if os.path.exists(info_plist):
args = ['cp', '-r', info_plist, resources_dir]
commands.append(args)
return os.path.join(full_path, parts[-1])
def FixInstallPath(library_path, library, new_path):
args = ['install_name_tool', '-change', library_path, new_path, library]
commands.append(args)
class CouldNotFindScriptPluginError(Error):
pass
if len(sys.argv) < 2:
print 'Usage: %s <bundle.app>' % sys.argv[0]
bundle_dir = sys.argv[1]
bundle_name = os.path.basename(bundle_dir).split('.')[0]
commands = []
framework_paths = []
binary_dir = os.path.join(bundle_dir, 'Contents', 'MacOS')
frameworks_dir = os.path.join(bundle_dir, 'Contents', 'Frameworks')
commands.append(['mkdir', '-p', frameworks_dir])
vlcplugins_dir = os.path.join(frameworks_dir, 'vlc', 'plugins')
commands.append(['mkdir', '-p', vlcplugins_dir])
snoreplugins_dir = os.path.join(binary_dir, 'libsnore')
commands.append(['mkdir', '-p', snoreplugins_dir])
resources_dir = os.path.join(bundle_dir, 'Contents', 'Resources')
commands.append(['mkdir', '-p', resources_dir])
plugins_dir = os.path.join(bundle_dir, 'Contents', 'qt-plugins')
binary = os.path.join(bundle_dir, 'Contents', 'MacOS', bundle_name)
fixed_libraries = []
fixed_frameworks = []
def GetBrokenLibraries(binary):
#print "Checking libs for binary: %s" % binary
def SambaStartStop(self):
commands = []
if not self.my_Samba_run:
commands.append('/etc/init.d/samba start')
commands.append('nmbd -D')
commands.append('smbd -D')
elif self.my_Samba_run:
commands.append('/etc/init.d/samba stop')
commands.append('killall nmbd')
commands.append('killall smbd')
self.Console.eBatch(commands, self.StartStopCallback, debug=True)
def CopyFramework(src_binary):
while os.path.islink(src_binary):
src_binary = os.path.realpath(src_binary)
m = re.match(r'(.*/([^/]+)\.framework)/Versions/([^/]+)/.*', src_binary)
if not m:
raise CouldNotParseFrameworkNameError(src_binary)
src_base = m.group(1)
name = m.group(2)
version = m.group(3)
LOGGER.info('Copying framework %s version %s', name, version)
dest_base = os.path.join(frameworks_dir, '%s.framework' % name)
dest_dir = os.path.join(dest_base, 'Versions', version)
dest_binary = os.path.join(dest_dir, name)
commands.append(['mkdir', '-p', dest_dir])
commands.append(['cp', src_binary, dest_binary])
# Copy special files from various places:
# QtCore has Resources/qt_menu.nib (copy to app's Resources)
# Sparkle has Resources/*
# Qt* have Resources/Info.plist
resources_src = os.path.join(src_base, 'Resources')
menu_nib = os.path.join(resources_src, 'qt_menu.nib')
if os.path.exists(menu_nib):
LOGGER.info("Copying qt_menu.nib '%s'", menu_nib)
commands.append(['cp', '-r', menu_nib, resources_dir])
elif os.path.exists(resources_src):
LOGGER.info("Copying resources dir '%s'", resources_src)
commands.append(['cp', '-r', resources_src, dest_dir])
info_plist = os.path.join(src_base, 'Contents', 'Info.plist')
if os.path.exists(info_plist):
LOGGER.info("Copying special file '%s'", info_plist)
resources_dest = os.path.join(dest_dir, 'Resources')
commands.append(['mkdir', resources_dest])
commands.append(['cp', '-r', info_plist, resources_dest])
# Create symlinks in the Framework to make it look like
# https://developer.apple.com/library/mac/documentation/MacOSX/Conceptual/BPFrameworks/Concepts/FrameworkAnatomy.html
commands.append([
'ln', '-sf',
'Versions/Current/%s' % name,
os.path.join(dest_base, name)
])
commands.append([
'ln', '-sf', 'Versions/Current/Resources',
os.path.join(dest_base, 'Resources')
])
commands.append(
['ln', '-sf', version,
os.path.join(dest_base, 'Versions/Current')])
return dest_binary
def FixId(path, library_name):
id = '@executable_path/../Frameworks/%s' % library_name
args = [INSTALL_NAME_TOOL, '-id', id, path]
commands.append(args)
def FixInstallPath(library_path, library, new_path): args = ['install_name_tool', '-change', library_path, new_path, library] commands.append(args)
def CopyFramework(path):
parts = path.split(os.sep)
for i, part in enumerate(parts):
if re.match(r'\w+\.framework', part):
full_path = os.path.join(frameworks_dir, *parts[i:-1])
framework_name = part.split(".framework")[0]
break
if full_path in framework_paths:
return os.path.join(full_path, parts[-1])
framework_paths.append(full_path)
args = ['mkdir', '-p', full_path]
commands.append(args)
args = ['ditto', '--arch=i386', '--arch=x86_64', path, full_path]
commands.append(args)
args = ['chmod', 'u+w', os.path.join(full_path, parts[-1])]
commands.append(args)
menu_nib = os.path.join(os.path.split(path)[0], 'Resources', 'qt_menu.nib')
if os.path.exists(menu_nib):
args = ['cp', '-rf', menu_nib, resources_dir]
commands.append(args)
# Fix framework structure for signing
path_base_dir = os.path.join(os.path.split(path)[0], '..', '..')
path_versions_dir = os.path.join(path_base_dir, 'Versions')
if not os.path.exists(os.path.join(full_path, 'Versions', 'Current')):
framework_base_dir = os.path.join(full_path, '..', '..')
framework_versions_dir = os.path.join(framework_base_dir, 'Versions')
versionParts = glob.glob(path_versions_dir + '/*')[0].split(os.sep)
args = [
'ln', '-s', versionParts[-1], framework_versions_dir + '/Current'
]
commands.append(args)
args = [
'ln', '-s', 'Versions/Current/' + framework_name,
framework_base_dir + '/' + framework_name
]
commands.append(args)
args = [
'ln', '-s', 'Versions/Current/Resources',
framework_base_dir + '/Resources'
]
commands.append(args)
# Copy Contents/Info.plist to Resources/Info.plist if Resources/Info.plist does not exist
# If Contents/Info.plist doesn't exist either, error out. If we actually see this, we can copy QtCore's Info.plist
info_plist_in_resources = os.path.join(
os.path.split(path)[0], '..', '..', 'Resources', 'Info.plist')
info_plist_in_contents = os.path.join(
os.path.split(path)[0], '..', '..', 'Contents', 'Info.plist')
framework_resources_dir = os.path.join(framework_versions_dir,
versionParts[-1], 'Resources')
args = ['mkdir', '-p', framework_resources_dir]
commands.append(args)
if os.path.exists(info_plist_in_contents):
args = ['cp', '-rf', info_plist_in_contents, framework_resources_dir]
commands.append(args)
args = [
'chmod', '+rw',
os.path.join(framework_resources_dir, 'Info.plist')
]
commands.append(args)
elif not os.path.exists(info_plist_in_resources):
print "%s: Framework does not contain an Info.plist file in Contents/ folder." % (
path)
sys.exit(-1)
return os.path.join(full_path, parts[-1])
