-
Notifications
You must be signed in to change notification settings - Fork 1
/
export-mob-project.py
186 lines (177 loc) · 7.6 KB
/
export-mob-project.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
"""
Put a project onto a Mobyle server.
This was originally written to create XMLs and install
onto a local installation of Mobyle 0.96.
Since then, support has been added for the creation of XMLs
on a different machine than the target machine.
This was added because there was some problem installing the lxml module
on the machine which was serving Mobyle.
This script is further evolving to support the installation of tools
onto the development version of Mobyle 0.98 on LSF without using DRMAA.
This new target platform is guaranteed to have lxml installed
because it is required by Mobyle 0.98,
but on the other hand the scripts need to be modified
so that they use a custom written script which in turn calls bsub.
Example usages:
python export-mob-project.py
--mobyle_version=0.96
--make_installer=ztools --universal_category=ztools
--mobyle_core=/home/mobyleuser/Mobyle96
--python_path=/home/mobyleuser/install/bin/python
--manifest=manifests/carbone.manifest --target=/home/mobyleuser/ztools
python export-mob-project.py
--deploy --create_all --mobyle_version=1.04
--mobyle_core=/home/username/mymobyle/core
--target=/home/username/mymobyle/ztools
"""
# The following is needed for the bsub hack inside the xmls
"""
<parameter ishidden="1">
<name>bsubinit</name>
<prompt lang="en">initiation</prompt>
<type>
<datatype>
<class>String</class>
</datatype>
</type>
<format>
<code proglang="python">" -n 1 -W 720 -N -q dean -o genetree.out -e genetree.err __bsubArgsEnd__ genetreerunner "</code>
</format>
<argpos>0</argpos>
<interface type="job_input"><xhtml:div xmlns:xhtml="http://www.w3.org/1999/xhtml" data-parametername="bsubinit"><xhtml:span class="prompt">initiation</xhtml:span></xhtml:div></interface></parameter>
"""
import re
import os
import sys
import shutil
import subprocess
import argparse
import meta
import mobyle
import mobenv
import Util
def create_installer(args, cat_info, env_info, module_names):
"""
Overwrite a staging subtree of the current directory.
The idea it to tgz this directory and send it to the target server.
On the target server it should be unzipped and extracted,
and the installer script should put everything in the correct
prespecified directories.
"""
# define the local staging directory
stage = args.make_installer
# remove the old staging directory if it exists
if os.path.isdir(stage):
shutil.rmtree(stage)
# create some directories
os.makedirs(os.path.join(stage, 'python-files'))
os.makedirs(os.path.join(stage, 'xml-files'))
# add the appropriate python files and the const data
meta.add_python_files(module_names, os.path.join(stage, 'python-files'))
# create the xml files
identifiers, import_errors = mobyle.add_xml_files(
cat_info, env_info,
module_names, args.short_length,
os.path.join(stage, 'xml-files'),
args.runbsub)
for e in import_errors:
print e
# copy the installer script
shutil.copy('install-mob-tools.py',
os.path.join(stage, 'install-mob-tools.py'))
# copy the installer script dependency
shutil.copy('mobenv.py',
os.path.join(stage, 'mobenv.py'))
# create the installer configuration
with open(os.path.join(stage, 'install-mob-tools.conf'), 'w') as fout:
print >> fout, '#', ' '.join(sys.argv)
print >> fout, '\t'.join(['auto_path', env_info.auto_path])
print >> fout, '\t'.join(['python_path', env_info.python_path])
print >> fout, '\t'.join(['mob_core', env_info.mob_core])
print >> fout, '\t'.join(['mob_version', args.mobyle_version])
# create a file with a fragment that could be added to the config
with open(os.path.join(stage, 'sys.config.fragment'), 'w') as fout:
print >> fout, '# begin autogenerated code'
print >> fout, 'my_pca_names = ['
for ident in sorted(identifiers):
print >> fout, "\t\t'%s'," % ident
print >> fout, "\t\t]"
dstring = "dict((x, 'Sys') for x in my_pca_names)"
print >> fout, 'PARTICULAR_BATCH.update(%s)' % dstring
print >> fout, '# end autogenerated code'
# use subprocess instead of tarfile to create the tgz
cmd = ['tar', 'czvf', stage + '.tgz', stage]
subprocess.call(cmd)
def main(args):
# check for flag conflicts
if args.deploy and args.make_installer:
raise ValueError(
'the "deploy" and "make_installer" flags are incompatible')
# initialize the mobyle category information
cat_info = mobyle.CategoryInfo(
args.show_io_types, args.show_tags, args.universal_category)
# get the module names
module_names = meta.get_module_names(
args.manifest, args.create_all, args.create_tagged)
# define the environment on the target server
auto_path = os.path.join(args.target, 'auto.py')
env_info = mobenv.create_environment_info(
auto_path, args.python_path,
args.mobyle_core, args.mobyle_version)
if args.make_installer:
create_installer(args, cat_info, env_info, module_names)
else:
# create the python subtree
meta.add_python_files(module_names, args.target)
# create the mobyle xml interface files
import_errors = mobyle.add_xml_files(
cat_info, env_info,
module_names, args.short_length, env_info.get_xml_dir(),
args.runbsub)
for e in import_errors:
print e
if args.clean:
cmd = env_info.get_clean_command()
subprocess.call(cmd)
if args.index:
cmd = env_info.get_index_command()
subprocess.call(cmd)
if args.deploy:
cmd = env_info.get_deploy_command()
subprocess.call(cmd)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--manifest',
help='create xmls for snippets listed in this file')
parser.add_argument('--create_all', action='store_true',
help='create xmls for all snippets')
parser.add_argument('--create_tagged',
help='create xmls for snippets with this tag')
parser.add_argument('--mobyle_version', metavar='VER',
choices=('0.96', '0.98', '1.04'), default='1.04',
help='the target Mobyle version')
parser.add_argument('--runbsub',
help='use this command for submission to the lsf queue')
parser.add_argument('--target', required=True,
help='python files will be created in this directory')
parser.add_argument('--python_path', default='python',
help='path to the python executable')
parser.add_argument('--mobyle_core', required=True,
help='path to the Mobyle core directory')
parser.add_argument('--short_length', type=int, default=20,
help='max length of shortened snippet names')
parser.add_argument('--show_io_types', action='store_true',
help='add mobyle categories for input and output types')
parser.add_argument('--show_tags', action='store_true',
help='add mobyle categories for module tags')
parser.add_argument('--universal_category',
help='add this mobyle category for all modules')
parser.add_argument('--make_installer', metavar='DIR',
help='create a local directory to be used for a remote install')
parser.add_argument('--deploy', action='store_true',
help='deploy the xml files via mobdeploy')
parser.add_argument('--clean', action='store_true',
help='clean the xml files via mobdeploy')
parser.add_argument('--index', action='store_true',
help='index the xml files via mobdeploy')
main(parser.parse_args())