forked from JCVI-Cloud/galaxy-tools-vcr
/
__init__.py
executable file
·2740 lines (2658 loc) · 140 KB
/
__init__.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
Classes encapsulating galaxy tools and tool configuration.
"""
import pkg_resources
pkg_resources.require( "simplejson" )
import logging, os, string, sys, tempfile, glob, shutil, types, urllib, subprocess, random
import simplejson
import binascii
from UserDict import DictMixin
from galaxy.util.odict import odict
from galaxy.util.bunch import Bunch
from galaxy.util.template import fill_template
from galaxy import util, jobs, model
from elementtree import ElementTree
from parameters import *
from parameters.grouping import *
from parameters.output import ToolOutputActionGroup
from parameters.validation import LateValidationError
from parameters.input_translation import ToolInputTranslator
from galaxy.util.expressions import ExpressionContext
from galaxy.tools.test import ToolTestBuilder
from galaxy.tools.actions import DefaultToolAction
from galaxy.tools.deps import DependencyManager
from galaxy.model import directory_hash_id
from galaxy.model.orm import *
from galaxy.util.none_like import NoneDataset
from galaxy.datatypes import sniff
from cgi import FieldStorage
from galaxy.util.hash_util import *
from galaxy.util import listify
from galaxy.util.shed_util import *
from galaxy.web import url_for
from galaxy.visualization.tracks.visual_analytics import TracksterConfig
log = logging.getLogger( __name__ )
class ToolNotFoundException( Exception ):
pass
class ToolBox( object ):
"""Container for a collection of tools"""
def __init__( self, config_filenames, tool_root_dir, app ):
"""
Create a toolbox from the config files named by `config_filenames`, using
`tool_root_dir` as the base directory for finding individual tool config files.
"""
# The shed_tool_confs list contains dictionaries storing information about the tools defined in each
# shed-related shed_tool_conf.xml file.
self.shed_tool_confs = []
self.tools_by_id = {}
self.workflows_by_id = {}
# In-memory dictionary that defines the layout of the tool panel.
self.tool_panel = odict()
# File that contains the XML section and tool tags from all tool panel config files integrated into a
# single file that defines the tool panel layout. This file can be changed by the Galaxy administrator
# (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel.
self.integrated_tool_panel_config = os.path.join( app.config.root, 'integrated_tool_panel.xml' )
# In-memory dictionary that defines the layout of the tool_panel.xml file on disk.
self.integrated_tool_panel = odict()
self.integrated_tool_panel_config_has_contents = os.path.exists( self.integrated_tool_panel_config ) and os.stat( self.integrated_tool_panel_config ).st_size > 0
if self.integrated_tool_panel_config_has_contents:
self.load_integrated_tool_panel_keys()
# The following refers to the tool_path config setting for backward compatibility. The shed-related
# (e.g., shed_tool_conf.xml) files include the tool_path attribute within the <toolbox> tag.
self.tool_root_dir = tool_root_dir
self.app = app
self.init_dependency_manager()
for config_filename in listify( config_filenames ):
try:
self.init_tools( config_filename )
except:
log.exception( "Error loading tools defined in config %s", config_filename )
if self.integrated_tool_panel_config_has_contents:
# Load self.tool_panel based on the order in self.integrated_tool_panel.
self.load_tool_panel()
if app.config.update_integrated_tool_panel:
# Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file.
# This will cover cases where the Galaxy administrator manually edited one or more of the tool panel
# config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel
# will be False when things like functional tests are the caller.
self.write_integrated_tool_panel_config_file()
def init_tools( self, config_filename ):
"""
Read the configuration file and load each tool. The following tags are currently supported:
<toolbox>
<tool file="data_source/upload.xml"/> # tools outside sections
<label text="Basic Tools" id="basic_tools" /> # labels outside sections
<workflow id="529fd61ab1c6cc36" /> # workflows outside sections
<section name="Get Data" id="getext"> # sections
<tool file="data_source/biomart.xml" /> # tools inside sections
<label text="In Section" id="in_section" /> # labels inside sections
<workflow id="adb5f5c93f827949" /> # workflows inside sections
</section>
</toolbox>
"""
if self.app.config.get_bool( 'enable_tool_tags', False ):
log.info("removing all tool tag associations (" + str( self.sa_session.query( self.app.model.ToolTagAssociation ).count() ) + ")" )
self.sa_session.query( self.app.model.ToolTagAssociation ).delete()
self.sa_session.flush()
log.info( "Parsing the tool configuration %s" % config_filename )
tree = util.parse_xml( config_filename )
root = tree.getroot()
tool_path = root.get( 'tool_path' )
if tool_path:
# We're parsing a shed_tool_conf file since we have a tool_path attribute.
parsing_shed_tool_conf = True
# Keep an in-memory list of xml elements to enable persistence of the changing tool config.
config_elems = []
else:
parsing_shed_tool_conf = False
# Default to backward compatible config setting.
tool_path = self.tool_root_dir
# Only load the panel_dict under certain conditions.
load_panel_dict = not self.integrated_tool_panel_config_has_contents
for index, elem in enumerate( root ):
if parsing_shed_tool_conf:
config_elems.append( elem )
if elem.tag == 'tool':
self.load_tool_tag_set( elem, self.tool_panel, self.integrated_tool_panel, tool_path, load_panel_dict, guid=elem.get( 'guid' ), index=index )
elif elem.tag == 'workflow':
self.load_workflow_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index )
elif elem.tag == 'section':
self.load_section_tag_set( elem, tool_path, load_panel_dict, index=index )
elif elem.tag == 'label':
self.load_label_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index )
if parsing_shed_tool_conf:
shed_tool_conf_dict = dict( config_filename=config_filename,
tool_path=tool_path,
config_elems=config_elems )
self.shed_tool_confs.append( shed_tool_conf_dict )
def load_tool_panel( self ):
for key, val in self.integrated_tool_panel.items():
if key.startswith( 'tool_' ):
tool_id = key.replace( 'tool_', '', 1 )
if tool_id in self.tools_by_id:
tool = self.tools_by_id[ tool_id ]
self.tool_panel[ key ] = tool
log.debug( "Loaded tool id: %s, version: %s." % ( tool.id, tool.version ) )
elif key.startswith( 'workflow_' ):
workflow_id = key.replace( 'workflow_', '', 1 )
if workflow_id in self.workflows_by_id:
workflow = self.workflows_by_id[ workflow_id ]
self.tool_panel[ key ] = workflow
log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) )
elif key.startswith( 'label_' ):
self.tool_panel[ key ] = val
elif key.startswith( 'section_' ):
elem = Element( 'section' )
elem.attrib[ 'id' ] = val.id or ''
elem.attrib[ 'name' ] = val.name or ''
elem.attrib[ 'version' ] = val.version or ''
section = ToolSection( elem )
log.debug( "Loading section: %s" % elem.get( 'name' ) )
for section_key, section_val in val.elems.items():
if section_key.startswith( 'tool_' ):
tool_id = section_key.replace( 'tool_', '', 1 )
if tool_id in self.tools_by_id:
tool = self.tools_by_id[ tool_id ]
section.elems[ section_key ] = tool
log.debug( "Loaded tool id: %s, version: %s." % ( tool.id, tool.version ) )
elif section_key.startswith( 'workflow_' ):
workflow_id = section_key.replace( 'workflow_', '', 1 )
if workflow_id in self.workflows_by_id:
workflow = self.workflows_by_id[ workflow_id ]
section.elems[ section_key ] = workflow
log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) )
elif section_key.startswith( 'label_' ):
section.elems[ section_key ] = section_val
self.tool_panel[ key ] = section
def load_integrated_tool_panel_keys( self ):
"""
Load the integrated tool panel keys, setting values for tools and workflows to None. The values will
be reset when the various tool panel config files are parsed, at which time the tools and workflows are
loaded.
"""
tree = util.parse_xml( self.integrated_tool_panel_config )
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
key = 'tool_%s' % elem.get( 'id' )
self.integrated_tool_panel[ key ] = None
elif elem.tag == 'workflow':
key = 'workflow_%s' % elem.get( 'id' )
self.integrated_tool_panel[ key ] = None
elif elem.tag == 'section':
section = ToolSection( elem )
for section_elem in elem:
if section_elem.tag == 'tool':
key = 'tool_%s' % section_elem.get( 'id' )
section.elems[ key ] = None
elif section_elem.tag == 'workflow':
key = 'workflow_%s' % section_elem.get( 'id' )
section.elems[ key ] = None
elif section_elem.tag == 'label':
key = 'label_%s' % section_elem.get( 'id' )
section.elems[ key ] = ToolSectionLabel( section_elem )
key = 'section_%s' % elem.get( 'id' )
self.integrated_tool_panel[ key ] = section
elif elem.tag == 'label':
key = 'label_%s' % elem.get( 'id' )
self.integrated_tool_panel[ key ] = ToolSectionLabel( elem )
def write_integrated_tool_panel_config_file( self ):
"""
Write the current in-memory version of the integrated_tool_panel.xml file to disk. Since Galaxy administrators
use this file to manage the tool panel, we'll not use util.xml_to_string() since it doesn't write XML quite right.
"""
fd, filename = tempfile.mkstemp()
os.write( fd, '<?xml version="1.0"?>\n' )
os.write( fd, '<toolbox>\n' )
for key, item in self.integrated_tool_panel.items():
if key.startswith( 'tool_' ):
if item:
os.write( fd, ' <tool id="%s" />\n' % item.id )
elif key.startswith( 'workflow_' ):
if item:
os.write( fd, ' <workflow id="%s" />\n' % item.id )
elif key.startswith( 'label_' ):
label_id = item.id or ''
label_text = item.text or ''
label_version = item.version or ''
os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) )
elif key.startswith( 'section_' ):
section_id = item.id or ''
section_name = item.name or ''
section_version = item.version or ''
os.write( fd, ' <section id="%s" name="%s" version="%s">\n' % ( section_id, section_name, section_version ) )
for section_key, section_item in item.elems.items():
if section_key.startswith( 'tool_' ):
if section_item:
os.write( fd, ' <tool id="%s" />\n' % section_item.id )
elif section_key.startswith( 'workflow_' ):
if section_item:
os.write( fd, ' <workflow id="%s" />\n' % section_item.id )
elif section_key.startswith( 'label_' ):
label_id = section_item.id or ''
label_text = section_item.text or ''
label_version = section_item.version or ''
os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) )
os.write( fd, ' </section>\n' )
os.write( fd, '</toolbox>\n' )
os.close( fd )
shutil.move( filename, os.path.abspath( self.integrated_tool_panel_config ) )
os.chmod( self.integrated_tool_panel_config, 0644 )
def get_tool( self, tool_id, tool_version=None, get_all_versions=False ):
"""Attempt to locate a tool in the tool box."""
if tool_id in self.tools_by_id:
tool = self.tools_by_id[ tool_id ]
if tool_version and tool.version == tool_version:
if get_all_versions:
return [ tool ]
else:
return tool
else:
if get_all_versions:
return [ tool ]
else:
return tool
tv = self.__get_tool_version( tool_id )
if tv:
tool_version_ids = tv.get_version_ids( self.app )
if get_all_versions:
available_tool_versions = []
for tool_version_id in tool_version_ids:
if tool_version_id in self.tools_by_id:
available_tool_versions.append( self.tools_by_id[ tool_version_id ] )
return available_tool_versions
for tool_version_id in tool_version_ids:
if tool_version_id in self.tools_by_id:
tool = self.tools_by_id[ tool_version_id ]
if tool_version and tool.version == tool_version:
return tool
else:
return tool
return None
def __get_tool_version( self, tool_id ):
"""Return a ToolVersion if one exists for the tool_id"""
return self.sa_session.query( self.app.model.ToolVersion ) \
.filter( self.app.model.ToolVersion.table.c.tool_id == tool_id ) \
.first()
def __get_tool_shed_repository( self, tool_shed, name, owner, installed_changeset_revision ):
return self.sa_session.query( self.app.model.ToolShedRepository ) \
.filter( and_( self.app.model.ToolShedRepository.table.c.tool_shed == tool_shed,
self.app.model.ToolShedRepository.table.c.name == name,
self.app.model.ToolShedRepository.table.c.owner == owner,
self.app.model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \
.first()
def load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ):
try:
path = elem.get( "file" )
if guid is None:
tool_shed_repository = None
can_load_into_panel_dict = True
else:
# The tool is contained in an installed tool shed repository, so load
# the tool only if the repository has not been marked deleted.
tool_shed = elem.find( "tool_shed" ).text
repository_name = elem.find( "repository_name" ).text
repository_owner = elem.find( "repository_owner" ).text
installed_changeset_revision_elem = elem.find( "installed_changeset_revision" )
if installed_changeset_revision_elem is None:
# Backward compatibility issue - the tag used to be named 'changeset_revision'.
installed_changeset_revision_elem = elem.find( "changeset_revision" )
installed_changeset_revision = installed_changeset_revision_elem.text
tool_shed_repository = self.__get_tool_shed_repository( tool_shed, repository_name, repository_owner, installed_changeset_revision )
if tool_shed_repository:
# Only load tools if the repository is not deactivated or uninstalled.
can_load_into_panel_dict = not tool_shed_repository.deleted
else:
# If there is not yet a tool_shed_repository record, we're in the process of installing
# a new repository, so any included tools can be loaded into the tool panel.
can_load_into_panel_dict = True
tool = self.load_tool( os.path.join( tool_path, path ), guid=guid )
key = 'tool_%s' % str( tool.id )
if can_load_into_panel_dict:
if guid is not None:
tool.tool_shed = tool_shed
tool.repository_name = repository_name
tool.repository_owner = repository_owner
tool.installed_changeset_revision = installed_changeset_revision
tool.guid = guid
tool.old_id = elem.find( "id" ).text
tool.version = elem.find( "version" ).text
# Make sure the tool has a tool_version.
if not self.__get_tool_version( tool.id ):
tool_version = self.app.model.ToolVersion( tool_id=tool.id, tool_shed_repository=tool_shed_repository )
self.sa_session.add( tool_version )
self.sa_session.flush()
if self.app.config.get_bool( 'enable_tool_tags', False ):
tag_names = elem.get( "tags", "" ).split( "," )
for tag_name in tag_names:
if tag_name == '':
continue
tag = self.sa_session.query( self.app.model.Tag ).filter_by( name=tag_name ).first()
if not tag:
tag = self.app.model.Tag( name=tag_name )
self.sa_session.add( tag )
self.sa_session.flush()
tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id )
self.sa_session.add( tta )
self.sa_session.flush()
else:
for tagged_tool in tag.tagged_tools:
if tagged_tool.tool_id == tool.id:
break
else:
tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id )
self.sa_session.add( tta )
self.sa_session.flush()
if tool.id not in self.tools_by_id:
# Allow for the same tool to be loaded into multiple places in the tool panel.
self.tools_by_id[ tool.id ] = tool
if load_panel_dict:
panel_dict[ key ] = tool
# Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file.
if key in integrated_panel_dict or index is None:
integrated_panel_dict[ key ] = tool
else:
integrated_panel_dict.insert( index, key, tool )
except:
log.exception( "Error reading tool from path: %s" % path )
def load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ):
try:
# TODO: should id be encoded?
workflow_id = elem.get( 'id' )
workflow = self.load_workflow( workflow_id )
self.workflows_by_id[ workflow_id ] = workflow
key = 'workflow_' + workflow_id
if load_panel_dict:
panel_dict[ key ] = workflow
# Always load workflows into the integrated_panel_dict.
if key in integrated_panel_dict or index is None:
integrated_panel_dict[ key ] = workflow
else:
integrated_panel_dict.insert( index, key, workflow )
except:
log.exception( "Error loading workflow: %s" % workflow_id )
def load_label_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ):
label = ToolSectionLabel( elem )
key = 'label_' + label.id
if load_panel_dict:
panel_dict[ key ] = label
if key in integrated_panel_dict or index is None:
integrated_panel_dict[ key ] = label
else:
integrated_panel_dict.insert( index, key, label )
def load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ):
key = 'section_' + elem.get( "id" )
if key in self.tool_panel:
section = self.tool_panel[ key ]
elems = section.elems
else:
section = ToolSection( elem )
elems = section.elems
if key in self.integrated_tool_panel:
integrated_section = self.integrated_tool_panel[ key ]
integrated_elems = integrated_section.elems
else:
integrated_section = ToolSection( elem )
integrated_elems = integrated_section.elems
for sub_index, sub_elem in enumerate( elem ):
if sub_elem.tag == 'tool':
self.load_tool_tag_set( sub_elem, elems, integrated_elems, tool_path, load_panel_dict, guid=sub_elem.get( 'guid' ), index=sub_index )
elif sub_elem.tag == 'workflow':
self.load_workflow_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index )
elif sub_elem.tag == 'label':
self.load_label_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index )
if load_panel_dict:
self.tool_panel[ key ] = section
# Always load sections into the integrated_tool_panel.
if key in self.integrated_tool_panel or index is None:
self.integrated_tool_panel[ key ] = integrated_section
else:
self.integrated_tool_panel.insert( index, key, integrated_section )
def load_tool( self, config_file, guid=None ):
"""Load a single tool from the file named by `config_file` and return an instance of `Tool`."""
# Parse XML configuration file and get the root element
tree = util.parse_xml( config_file )
root = tree.getroot()
# Allow specifying a different tool subclass to instantiate
if root.find( "type" ) is not None:
type_elem = root.find( "type" )
module = type_elem.get( 'module', 'galaxy.tools' )
cls = type_elem.get( 'class' )
mod = __import__( module, globals(), locals(), [cls] )
ToolClass = getattr( mod, cls )
elif root.get( 'tool_type', None ) is not None:
ToolClass = tool_types.get( root.get( 'tool_type' ) )
else:
ToolClass = Tool
return ToolClass( config_file, root, self.app, guid=guid )
def reload_tool_by_id( self, tool_id ):
"""
Attempt to reload the tool identified by 'tool_id', if successful
replace the old tool.
"""
if tool_id not in self.tools_by_id:
message = "No tool with id %s" % tool_id
status = 'error'
else:
old_tool = self.tools_by_id[ tool_id ]
new_tool = self.load_tool( old_tool.config_file )
# The tool may have been installed from a tool shed, so set the tool shed attributes.
# Since the tool version may have changed, we don't override it here.
new_tool.id = old_tool.id
new_tool.guid = old_tool.guid
new_tool.tool_shed = old_tool.tool_shed
new_tool.repository_name = old_tool.repository_name
new_tool.repository_owner = old_tool.repository_owner
new_tool.installed_changeset_revision = old_tool.installed_changeset_revision
new_tool.old_id = old_tool.old_id
# Replace old_tool with new_tool in self.tool_panel
tool_key = 'tool_' + tool_id
for key, val in self.tool_panel.items():
if key == tool_key:
self.tool_panel[ key ] = new_tool
break
elif key.startswith( 'section' ):
if tool_key in val.elems:
self.tool_panel[ key ].elems[ tool_key ] = new_tool
break
self.tools_by_id[ tool_id ] = new_tool
message = "Reloaded the tool:<br/>"
message += "<b>name:</b> %s<br/>" % old_tool.name
message += "<b>id:</b> %s<br/>" % old_tool.id
message += "<b>version:</b> %s" % old_tool.version
status = 'done'
return message, status
def load_workflow( self, workflow_id ):
"""
Return an instance of 'Workflow' identified by `id`,
which is encoded in the tool panel.
"""
id = self.app.security.decode_id( workflow_id )
stored = self.app.model.context.query( self.app.model.StoredWorkflow ).get( id )
return stored.latest_workflow
def init_dependency_manager( self ):
if self.app.config.use_tool_dependencies:
self.dependency_manager = DependencyManager( [ self.app.config.tool_dependency_dir ] )
else:
self.dependency_manager = None
@property
def sa_session( self ):
"""
Returns a SQLAlchemy session
"""
return self.app.model.context
def to_dict( self, trans, in_panel=True, trackster=False ):
def filter_for_panel( item, filters ):
"""
Filters tool panel elements so that only those that are compatible
with provided filters are kept.
"""
def _apply_filter( filter_item, filter_list ):
for filter_method in filter_list:
if not filter_method( filter_item ):
return False
return True
if isinstance( item, Tool ):
if _apply_filter( item, filters[ 'tool' ] ):
return item
elif isinstance( item, ToolSectionLabel ):
if _apply_filter( item, filters[ 'label' ] ):
return item
elif isinstance( item, ToolSection ):
# Filter section item-by-item. Only show a label if there are
# non-filtered tools below it.
if _apply_filter( item, filters[ 'section' ] ):
cur_label_key = None
tools_under_label = False
filtered_elems = item.elems.copy()
for key, section_item in item.elems.items():
if isinstance( section_item, Tool ):
# Filter tool.
if _apply_filter( section_item, filters[ 'tool' ] ):
tools_under_label = True
else:
del filtered_elems[ key ]
elif isinstance( section_item, ToolSectionLabel ):
# If there is a label and it does not have tools,
# remove it.
if ( cur_label_key and not tools_under_label ) or not _apply_filter( section_item, filters[ 'label' ] ):
del filtered_elems[ cur_label_key ]
# Reset attributes for new label.
cur_label_key = key
tools_under_label = False
# Handle last label.
if cur_label_key and not tools_under_label:
del filtered_elems[ cur_label_key ]
# Only return section if there are elements.
if len( filtered_elems ) != 0:
copy = item.copy()
copy.elems = filtered_elems
return copy
return None
#
# Dictify toolbox.
#
if in_panel:
panel_elts = [ val for val in self.tool_panel.itervalues() ]
# Filter if necessary.
filters = dict( tool=[ lambda x: not x._is_hidden_for_user( trans.user ) ], section=[], label=[] ) #hidden tools filter
if trackster:
filters[ 'tool' ].append( lambda x: x.trackster_conf ) # If tool has a trackster config, it can be used in Trackster.
filtered_panel_elts = []
for index, elt in enumerate( panel_elts ):
elt = filter_for_panel( elt, filters )
if elt:
filtered_panel_elts.append( elt )
panel_elts = filtered_panel_elts
# Produce panel.
rval = []
for elt in panel_elts:
rval.append( elt.to_dict( trans, for_link=True ) )
else:
tools = []
for id, tool in self.app.toolbox.tools_by_id.items():
tools.append( tool.to_dict( trans ) )
rval = tools
return rval
class ToolSection( object ):
"""
A group of tools with similar type/purpose that will be displayed as a
group in the user interface.
"""
def __init__( self, elem=None ):
f = lambda elem, val: elem is not None and elem.get( val ) or ''
self.name = f( elem, 'name' )
self.id = f( elem, 'id' )
self.version = f( elem, 'version' )
self.elems = odict()
def copy( self ):
copy = ToolSection()
copy.name = self.name
copy.id = self.id
copy.version = self.version
copy.elems = self.elems.copy()
return copy
def to_dict( self, trans, for_link=False ):
""" Return a dict that includes section's attributes. """
section_elts = []
for key, val in self.elems.items():
section_elts.append( val.to_dict( trans, for_link=for_link ) )
return { 'type': 'section', 'id': self.id, 'name': self.name, 'version': self.version, 'elems': section_elts }
class ToolSectionLabel( object ):
"""
A label for a set of tools that can be displayed above groups of tools
and sections in the user interface
"""
def __init__( self, elem ):
self.text = elem.get( "text" )
self.id = elem.get( "id" )
self.version = elem.get( "version" ) or ''
def to_dict( self, trans, **kwargs ):
""" Return a dict that includes label's attributes. """
return { 'type': 'label', 'id': self.id, 'name': self.text, 'version': self.version }
class DefaultToolState( object ):
"""
Keeps track of the state of a users interaction with a tool between
requests. The default tool state keeps track of the current page (for
multipage "wizard" tools) and the values of all parameters.
"""
def __init__( self ):
self.page = 0
self.inputs = None
def encode( self, tool, app, secure=True ):
"""
Convert the data to a string
"""
# Convert parameters to a dictionary of strings, and save curent
# page in that dict
value = params_to_strings( tool.inputs, self.inputs, app )
value["__page__"] = self.page
value = simplejson.dumps( value )
# Make it secure
if secure:
a = hmac_new( app.config.tool_secret, value )
b = binascii.hexlify( value )
return "%s:%s" % ( a, b )
else:
return value
def decode( self, value, tool, app, secure=True ):
"""
Restore the state from a string
"""
if secure:
# Extract and verify hash
a, b = value.split( ":" )
value = binascii.unhexlify( b )
test = hmac_new( app.config.tool_secret, value )
assert a == test
# Restore from string
values = json_fix( simplejson.loads( value ) )
self.page = values.pop( "__page__" )
self.inputs = params_from_strings( tool.inputs, values, app, ignore_errors=True )
class ToolOutput( object ):
"""
Represents an output datasets produced by a tool. For backward
compatibility this behaves as if it were the tuple:
(format, metadata_source, parent)
"""
def __init__( self, name, format=None, format_source=None, metadata_source=None,
parent=None, label=None, filters = None, actions = None, hidden=False ):
self.name = name
self.format = format
self.format_source = format_source
self.metadata_source = metadata_source
self.parent = parent
self.label = label
self.filters = filters or []
self.actions = actions
self.hidden = hidden
# Tuple emulation
def __len__( self ):
return 3
def __getitem__( self, index ):
if index == 0:
return self.format
elif index == 1:
return self.metadata_source
elif index == 2:
return self.parent
else:
raise IndexError( index )
def __iter__( self ):
return iter( ( self.format, self.metadata_source, self.parent ) )
def to_dict( self ):
return {
'name': self.name,
'format': self.format,
'label': self.label,
'hidden': self.hidden
}
class ToolRequirement( object ):
"""
Represents an external requirement that must be available for the tool to run (for example, a program, package, or library).
Requirements can optionally assert a specific version.
"""
def __init__( self, name=None, type=None, version=None ):
self.name = name
self.type = type
self.version = version
class ToolParallelismInfo(object):
"""
Stores the information (if any) for running multiple instances of the tool in parallel
on the same set of inputs.
"""
def __init__(self, tag):
self.method = tag.get('method')
self.attributes = dict([item for item in tag.attrib.items() if item[0] != 'method' ])
if len(self.attributes) == 0:
# legacy basic mode - provide compatible defaults
self.attributes['split_size'] = 20
self.attributes['split_mode'] = 'number_of_parts'
class Tool:
"""
Represents a computational tool that can be executed through Galaxy.
"""
tool_type = 'default'
def __init__( self, config_file, root, app, guid=None ):
"""Load a tool from the config named by `config_file`"""
# Determine the full path of the directory where the tool config is
self.config_file = config_file
self.tool_dir = os.path.dirname( config_file )
self.app = app
#setup initial attribute values
self.inputs = odict()
self.inputs_by_page = list()
self.display_by_page = list()
self.action = '/tool_runner/index'
self.target = 'galaxy_main'
self.method = 'post'
self.check_values = True
self.nginx_upload = False
self.input_required = False
self.display_interface = True
self.require_login = False
# Define a place to keep track of all input parameters. These
# differ from the inputs dictionary in that inputs can be page
# elements like conditionals, but input_params are basic form
# parameters like SelectField objects. This enables us to more
# easily ensure that parameter dependencies like index files or
# tool_data_table_conf.xml entries exist.
self.input_params = []
# Attributes of tools installed from Galaxy tool sheds.
self.tool_shed = None
self.repository_name = None
self.repository_owner = None
self.installed_changeset_revision = None
# The tool.id value will be the value of guid, but we'll keep the
# guid attribute since it is useful to have.
self.guid = guid
self.old_id = None
self.version = None
# Parse XML element containing configuration
self.parse( root, guid=guid )
self.external_runJob_script = app.config.drmaa_external_runjob_script
@property
def sa_session( self ):
"""Returns a SQLAlchemy session"""
return self.app.model.context
@property
def tool_version( self ):
"""Return a ToolVersion if one exists for our id"""
return self.sa_session.query( self.app.model.ToolVersion ) \
.filter( self.app.model.ToolVersion.table.c.tool_id == self.id ) \
.first()
@property
def tool_versions( self ):
# If we have versions, return them.
tool_version = self.tool_version
if tool_version:
return tool_version.get_versions( self.app )
return []
@property
def tool_version_ids( self ):
# If we have versions, return a list of their tool_ids.
tool_version = self.tool_version
if tool_version:
return tool_version.get_version_ids( self.app )
return []
def __get_job_run_config( self, run_configs, key, job_params=None ):
# Look through runners/handlers to find one with matching parameters.
available_configs = []
if len( run_configs ) == 1:
# Most tools have a single config.
return run_configs[0][ key ] # return to avoid random when this will be the case most of the time
elif job_params is None:
# Use job config with no params
for config in run_configs:
if "params" not in config:
available_configs.append( config )
else:
# Find config with matching parameters.
for config in run_configs:
if "params" in config:
match = True
config_params = config[ "params" ]
for param, value in job_params.items():
if param not in config_params or \
config_params[ param ] != job_params[ param ]:
match = False
break
if match:
available_configs.append( config )
return random.choice( available_configs )[ key ]
def get_job_runner( self, job_params=None ):
return self.__get_job_run_config( self.job_runners, key='url', job_params=job_params )
def get_job_handler( self, job_params=None ):
return self.__get_job_run_config( self.job_handlers, key='name', job_params=job_params )
def parse( self, root, guid=None ):
"""
Read tool configuration from the element `root` and fill in `self`.
"""
# Get the (user visible) name of the tool
self.name = root.get( "name" )
if not self.name:
raise Exception, "Missing tool 'name'"
# Get the UNIQUE id for the tool
# TODO: can this be generated automatically?
if guid is None:
self.id = root.get( "id" )
self.version = root.get( "version" )
else:
self.id = guid
if not self.id:
raise Exception, "Missing tool 'id'"
self.version = root.get( "version" )
if not self.version:
# For backward compatibility, some tools may not have versions yet.
self.version = "1.0.0"
# Support multi-byte tools
self.is_multi_byte = util.string_as_bool( root.get( "is_multi_byte", False ) )
# Force history to fully refresh after job execution for this tool.
# Useful i.e. when an indeterminate number of outputs are created by
# a tool.
self.force_history_refresh = util.string_as_bool( root.get( 'force_history_refresh', 'False' ) )
self.display_interface = util.string_as_bool( root.get( 'display_interface', str( self.display_interface ) ) )
self.require_login = util.string_as_bool( root.get( 'require_login', str( self.require_login ) ) )
# Load input translator, used by datasource tools to change
# names/values of incoming parameters
self.input_translator = root.find( "request_param_translation" )
if self.input_translator:
self.input_translator = ToolInputTranslator.from_element( self.input_translator )
# Command line (template). Optional for tools that do not invoke a local program
command = root.find("command")
if command is not None and command.text is not None:
self.command = command.text.lstrip() # get rid of leading whitespace
# Must pre-pend this AFTER processing the cheetah command template
self.interpreter = command.get( "interpreter", None )
else:
self.command = ''
self.interpreter = None
# Parameters used to build URL for redirection to external app
redirect_url_params = root.find( "redirect_url_params" )
if redirect_url_params is not None and redirect_url_params.text is not None:
# get rid of leading / trailing white space
redirect_url_params = redirect_url_params.text.strip()
# Replace remaining white space with something we can safely split on later
# when we are building the params
self.redirect_url_params = redirect_url_params.replace( ' ', '**^**' )
else:
self.redirect_url_params = ''
# Short description of the tool
self.description = util.xml_text(root, "description")
# Versioning for tools
self.version_string_cmd = None
version_cmd = root.find("version_command")
if version_cmd is not None:
self.version_string_cmd = version_cmd.text
# Parallelism for tasks, read from tool config.
parallelism = root.find("parallelism")
if parallelism is not None and parallelism.get("method"):
self.parallelism = ToolParallelismInfo(parallelism)
else:
self.parallelism = None
# Set job handler(s). Each handler is a dict with 'url' and, optionally, 'params'.
self_id = self.id.lower()
self.job_handlers = [ { "name" : name } for name in self.app.config.default_job_handlers ]
# Set custom handler(s) if they're defined.
if self_id in self.app.config.tool_handlers:
self.job_handlers = self.app.config.tool_handlers[ self_id ]
# Set job runner(s). Each runner is a dict with 'url' and, optionally, 'params'.
if self.app.config.start_job_runners is None:
# Jobs are always local regardless of tool config if no additional
# runners are started
self.job_runners = [ { "url" : "local:///" } ]
else:
# Set job runner to the cluster default
self.job_runners = [ { "url" : self.app.config.default_cluster_job_runner } ]
# Set custom runner(s) if they're defined.
if self_id in self.app.config.tool_runners:
self.job_runners = self.app.config.tool_runners[ self_id ]
# Is this a 'hidden' tool (hidden in tool menu)
self.hidden = util.xml_text(root, "hidden")
if self.hidden: self.hidden = util.string_as_bool(self.hidden)
# Load any tool specific code (optional) Edit: INS 5/29/2007,
# allow code files to have access to the individual tool's
# "module" if it has one. Allows us to reuse code files, etc.
self.code_namespace = dict()
self.hook_map = {}
for code_elem in root.findall("code"):
for hook_elem in code_elem.findall("hook"):
for key, value in hook_elem.items():
# map hook to function
self.hook_map[key]=value
file_name = code_elem.get("file")
code_path = os.path.join( self.tool_dir, file_name )
execfile( code_path, self.code_namespace )
# Load any tool specific options (optional)
self.options = dict( sanitize=True, refresh=False )
for option_elem in root.findall("options"):
for option, value in self.options.copy().items():
if isinstance(value, type(False)):
self.options[option] = util.string_as_bool(option_elem.get(option, str(value)))
else:
self.options[option] = option_elem.get(option, str(value))
self.options = Bunch(** self.options)
# Parse tool inputs (if there are any required)
self.parse_inputs( root )
# Parse tool help
self.parse_help( root )
# Description of outputs produced by an invocation of the tool
self.parse_outputs( root )
# Any extra generated config files for the tool
self.config_files = []
conf_parent_elem = root.find("configfiles")
if conf_parent_elem:
for conf_elem in conf_parent_elem.findall( "configfile" ):
name = conf_elem.get( "name" )
filename = conf_elem.get( "filename", None )
text = conf_elem.text
self.config_files.append( ( name, filename, text ) )
# Action
action_elem = root.find( "action" )
if action_elem is None:
self.tool_action = DefaultToolAction()
else:
module = action_elem.get( 'module' )
cls = action_elem.get( 'class' )
mod = __import__( module, globals(), locals(), [cls])
self.tool_action = getattr( mod, cls )()
# User interface hints
self.uihints = {}
uihints_elem = root.find( "uihints" )
if uihints_elem is not None:
for key, value in uihints_elem.attrib.iteritems():
self.uihints[ key ] = value
# Tests
tests_elem = root.find( "tests" )
if tests_elem:
try:
self.parse_tests( tests_elem )
except:
log.exception( "Failed to parse tool tests" )
else:
self.tests = None
# Requirements (dependencies)
self.requirements = []
requirements_elem = root.find( "requirements" )
if requirements_elem:
self.parse_requirements( requirements_elem )
# Determine if this tool can be used in workflows
self.is_workflow_compatible = self.check_workflow_compatible()
# Trackster configuration.
trackster_conf = root.find( "trackster_conf" )
if trackster_conf is not None:
self.trackster_conf = TracksterConfig.parse( trackster_conf )
else:
self.trackster_conf = None
def parse_inputs( self, root ):
"""
Parse the "<inputs>" element and create appropriate `ToolParameter`s.
This implementation supports multiple pages and grouping constructs.
"""
# Load parameters (optional)
input_elem = root.find("inputs")
enctypes = set()
if input_elem:
# Handle properties of the input form
self.check_values = util.string_as_bool( input_elem.get("check_values", self.check_values ) )
self.nginx_upload = util.string_as_bool( input_elem.get( "nginx_upload", self.nginx_upload ) )
self.action = input_elem.get( 'action', self.action )
# If we have an nginx upload, save the action as a tuple instead of
# a string. The actual action needs to get url_for run to add any
# prefixes, and we want to avoid adding the prefix to the
# nginx_upload_path. This logic is handled in the tool_form.mako