def check_for_tool_dependencies( self, trans, migration_stage ):
# Get the 000x_tools.xml file associated with migration_stage.
tools_xml_file_path = os.path.abspath( os.path.join( trans.app.config.root, 'scripts', 'migrate_tools', '%04d_tools.xml' % migration_stage ) )
tree = util.parse_xml( tools_xml_file_path )
root = tree.getroot()
tool_shed = root.get( 'name' )
tool_shed_url = self.get_tool_shed_url_from_tools_xml_file_path( trans, tool_shed )
repo_name_dependency_tups = []
if tool_shed_url:
for elem in root:
if elem.tag == 'repository':
tool_dependencies = []
tool_dependencies_dict = {}
repository_name = elem.get( 'name' )
changeset_revision = elem.get( 'changeset_revision' )
url = '%s/repository/get_tool_dependencies?name=%s&owner=devteam&changeset_revision=%s&from_install_manager=True' % \
( tool_shed_url, repository_name, changeset_revision )
response = urllib2.urlopen( url )
text = response.read()
response.close()
if text:
tool_dependencies_dict = encoding_util.tool_shed_decode( text )
for dependency_key, requirements_dict in tool_dependencies_dict.items():
tool_dependency_name = requirements_dict[ 'name' ]
tool_dependency_version = requirements_dict[ 'version' ]
tool_dependency_type = requirements_dict[ 'type' ]
tool_dependency_readme = requirements_dict.get( 'readme', '' )
tool_dependencies.append( ( tool_dependency_name, tool_dependency_version, tool_dependency_type, tool_dependency_readme ) )
repo_name_dependency_tups.append( ( repository_name, tool_dependencies ) )
return repo_name_dependency_tups
def check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_script_number ):
# Get the 000x_tools.xml file associated with the current migrate_tools version number.
tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) )
# Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config.
migrated_tool_configs_dict = odict()
tree = util.parse_xml( tools_xml_file_path )
root = tree.getroot()
tool_shed = root.get( 'name' )
tool_shed_url = get_tool_shed_url_from_tools_xml_file_path( app, tool_shed )
# The default behavior is that the tool shed is down.
tool_shed_accessible = False
if tool_shed_url:
for elem in root:
if elem.tag == 'repository':
tool_dependencies = []
tool_dependencies_dict = {}
repository_name = elem.get( 'name' )
changeset_revision = elem.get( 'changeset_revision' )
url = '%s/repository/get_tool_dependencies?name=%s&owner=%s&changeset_revision=%s&from_install_manager=True' % \
( tool_shed_url, repository_name, REPOSITORY_OWNER, changeset_revision )
try:
response = urllib2.urlopen( url )
text = response.read()
response.close()
tool_shed_accessible = True
except Exception, e:
# Tool shed may be unavailable - we have to set tool_shed_accessible since we're looping.
tool_shed_accessible = False
print "The URL\n%s\nraised the exception:\n%s\n" % ( url, str( e ) )
if tool_shed_accessible:
if text:
tool_dependencies_dict = encoding_util.tool_shed_decode( text )
for dependency_key, requirements_dict in tool_dependencies_dict.items():
tool_dependency_name = requirements_dict[ 'name' ]
tool_dependency_version = requirements_dict[ 'version' ]
tool_dependency_type = requirements_dict[ 'type' ]
tool_dependency_readme = requirements_dict.get( 'readme', '' )
tool_dependencies.append( ( tool_dependency_name, tool_dependency_version, tool_dependency_type, tool_dependency_readme ) )
for tool_elem in elem.findall( 'tool' ):
migrated_tool_configs_dict[ tool_elem.get( 'file' ) ] = tool_dependencies
if tool_shed_accessible:
# Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names.
missing_tool_configs_dict = odict()
for tool_panel_config in tool_panel_configs:
tree = util.parse_xml( tool_panel_config )
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
missing_tool_configs_dict = check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict )
elif elem.tag == 'section':
for section_elem in elem:
if section_elem.tag == 'tool':
missing_tool_configs_dict = check_tool_tag_set( section_elem, migrated_tool_configs_dict, missing_tool_configs_dict )
def check_for_tool_dependencies(self, trans, migration_stage):
# Get the 000x_tools.xml file associated with migration_stage.
tools_xml_file_path = os.path.abspath(
os.path.join(trans.app.config.root, 'scripts', 'migrate_tools',
'%04d_tools.xml' % migration_stage))
tree = util.parse_xml(tools_xml_file_path)
root = tree.getroot()
tool_shed = root.get('name')
tool_shed_url = self.get_tool_shed_url_from_tools_xml_file_path(
trans, tool_shed)
repo_name_dependency_tups = []
if tool_shed_url:
for elem in root:
if elem.tag == 'repository':
tool_dependencies = []
tool_dependencies_dict = {}
repository_name = elem.get('name')
changeset_revision = elem.get('changeset_revision')
url = '%s/repository/get_tool_dependencies?name=%s&owner=devteam&changeset_revision=%s&from_install_manager=True' % \
( tool_shed_url, repository_name, changeset_revision )
response = urllib2.urlopen(url)
text = response.read()
response.close()
if text:
tool_dependencies_dict = encoding_util.tool_shed_decode(
text)
for dependency_key, requirements_dict in tool_dependencies_dict.items(
):
tool_dependency_name = requirements_dict['name']
tool_dependency_version = requirements_dict[
'version']
tool_dependency_type = requirements_dict['type']
tool_dependency_readme = requirements_dict.get(
'readme', '')
tool_dependencies.append(
(tool_dependency_name, tool_dependency_version,
tool_dependency_type, tool_dependency_readme))
repo_name_dependency_tups.append(
(repository_name, tool_dependencies))
return repo_name_dependency_tups
def generate_workflow_image( trans, workflow_name, repository_metadata_id=None, repository_id=None ):
"""
Return an svg image representation of a workflow dictionary created when the workflow was exported. This method is called
from both Galaxy and the tool shed. When called from the tool shed, repository_metadata_id will have a value and repository_id
will be None. When called from Galaxy, repository_metadata_id will be None and repository_id will have a value.
"""
workflow_name = encoding_util.tool_shed_decode( workflow_name )
if trans.webapp.name == 'community':
# We're in the tool shed.
repository_metadata = suc.get_repository_metadata_by_id( trans, repository_metadata_id )
repository_id = trans.security.encode_id( repository_metadata.repository_id )
changeset_revision = repository_metadata.changeset_revision
metadata = repository_metadata.metadata
else:
# We're in Galaxy.
repository = suc.get_tool_shed_repository_by_id( trans, repository_id )
changeset_revision = repository.changeset_revision
metadata = repository.metadata
# metadata[ 'workflows' ] is a list of tuples where each contained tuple is
# [ <relative path to the .ga file in the repository>, <exported workflow dict> ]
for workflow_tup in metadata[ 'workflows' ]:
workflow_dict = workflow_tup[1]
if workflow_dict[ 'name' ] == workflow_name:
break
if 'tools' in metadata:
tools_metadata = metadata[ 'tools' ]
else:
tools_metadata = []
workflow, missing_tool_tups = get_workflow_from_dict( trans=trans,
workflow_dict=workflow_dict,
tools_metadata=tools_metadata,
repository_id=repository_id,
changeset_revision=changeset_revision )
data = []
canvas = svgfig.canvas( style="stroke:black; fill:none; stroke-width:1px; stroke-linejoin:round; text-anchor:left" )
text = svgfig.SVG( "g" )
connectors = svgfig.SVG( "g" )
boxes = svgfig.SVG( "g" )
svgfig.Text.defaults[ "font-size" ] = "10px"
in_pos = {}
out_pos = {}
margin = 5
# Spacing between input/outputs.
line_px = 16
# Store px width for boxes of each step.
widths = {}
max_width, max_x, max_y = 0, 0, 0
for step in workflow.steps:
step.upgrade_messages = {}
module = module_factory.from_workflow_step( trans, repository_id, changeset_revision, tools_metadata, step )
tool_errors = module.type == 'tool' and not module.tool
module_data_inputs = get_workflow_data_inputs( step, module )
module_data_outputs = get_workflow_data_outputs( step, module, workflow.steps )
step_dict = {
'id' : step.order_index,
'data_inputs' : module_data_inputs,
'data_outputs' : module_data_outputs,
'position' : step.position,
'tool_errors' : tool_errors
}
input_conn_dict = {}
for conn in step.input_connections:
input_conn_dict[ conn.input_name ] = dict( id=conn.output_step.order_index, output_name=conn.output_name )
step_dict[ 'input_connections' ] = input_conn_dict
data.append( step_dict )
x, y = step.position[ 'left' ], step.position[ 'top' ]
count = 0
module_name = get_workflow_module_name( module, missing_tool_tups )
max_len = len( module_name ) * 1.5
text.append( svgfig.Text( x, y + 20, module_name, **{ "font-size": "14px" } ).SVG() )
y += 45
for di in module_data_inputs:
cur_y = y + count * line_px
if step.order_index not in in_pos:
in_pos[ step.order_index ] = {}
in_pos[ step.order_index ][ di[ 'name' ] ] = ( x, cur_y )
text.append( svgfig.Text( x, cur_y, di[ 'label' ] ).SVG() )
count += 1
max_len = max( max_len, len( di[ 'label' ] ) )
if len( module.get_data_inputs() ) > 0:
y += 15
for do in module_data_outputs:
cur_y = y + count * line_px
if step.order_index not in out_pos:
out_pos[ step.order_index ] = {}
out_pos[ step.order_index ][ do[ 'name' ] ] = ( x, cur_y )
text.append( svgfig.Text( x, cur_y, do[ 'name' ] ).SVG() )
count += 1
max_len = max( max_len, len( do['name' ] ) )
widths[ step.order_index ] = max_len * 5.5
max_x = max( max_x, step.position[ 'left' ] )
max_y = max( max_y, step.position[ 'top' ] )
max_width = max( max_width, widths[ step.order_index ] )
for step_dict in data:
tool_unavailable = step_dict[ 'tool_errors' ]
width = widths[ step_dict[ 'id' ] ]
x, y = step_dict[ 'position' ][ 'left' ], step_dict[ 'position' ][ 'top' ]
# Only highlight missing tools if displaying in the tool shed.
if trans.webapp.name == 'community' and tool_unavailable:
fill = "#EBBCB2"
else:
fill = "#EBD9B2"
boxes.append( svgfig.Rect( x - margin, y, x + width - margin, y + 30, fill=fill ).SVG() )
box_height = ( len( step_dict[ 'data_inputs' ] ) + len( step_dict[ 'data_outputs' ] ) ) * line_px + margin
# Draw separator line.
if len( step_dict[ 'data_inputs' ] ) > 0:
box_height += 15
sep_y = y + len( step_dict[ 'data_inputs' ] ) * line_px + 40
text.append( svgfig.Line( x - margin, sep_y, x + width - margin, sep_y ).SVG() )
# Define an input/output box.
boxes.append( svgfig.Rect( x - margin, y + 30, x + width - margin, y + 30 + box_height, fill="#ffffff" ).SVG() )
for conn, output_dict in step_dict[ 'input_connections' ].iteritems():
in_coords = in_pos[ step_dict[ 'id' ] ][ conn ]
# out_pos_index will be a step number like 1, 2, 3...
out_pos_index = output_dict[ 'id' ]
# out_pos_name will be a string like 'o', 'o2', etc.
out_pos_name = output_dict[ 'output_name' ]
if out_pos_index in out_pos:
# out_conn_index_dict will be something like:
# 7: {'o': (824.5, 618)}
out_conn_index_dict = out_pos[ out_pos_index ]
if out_pos_name in out_conn_index_dict:
out_conn_pos = out_pos[ out_pos_index ][ out_pos_name ]
else:
# Take any key / value pair available in out_conn_index_dict.
# A problem will result if the dictionary is empty.
if out_conn_index_dict.keys():
key = out_conn_index_dict.keys()[0]
out_conn_pos = out_pos[ out_pos_index ][ key ]
adjusted = ( out_conn_pos[ 0 ] + widths[ output_dict[ 'id' ] ], out_conn_pos[ 1 ] )
text.append( svgfig.SVG( "circle",
cx=out_conn_pos[ 0 ] + widths[ output_dict[ 'id' ] ] - margin,
cy=out_conn_pos[ 1 ] - margin,
r = 5,
fill="#ffffff" ) )
connectors.append( svgfig.Line( adjusted[ 0 ],
adjusted[ 1 ] - margin,
in_coords[ 0 ] - 10,
in_coords[ 1 ],
arrow_end = "true" ).SVG() )
canvas.append( connectors )
canvas.append( boxes )
canvas.append( text )
width, height = ( max_x + max_width + 50 ), max_y + 300
canvas[ 'width' ] = "%s px" % width
canvas[ 'height' ] = "%s px" % height
canvas[ 'viewBox' ] = "0 0 %s %s" % ( width, height )
trans.response.set_content_type( "image/svg+xml" )
return canvas.standalone_xml()
def generate_workflow_image(trans, workflow_name, repository_metadata_id=None, repository_id=None):
"""
Return an svg image representation of a workflow dictionary created when the workflow was exported. This method is called
from both Galaxy and the tool shed. When called from the tool shed, repository_metadata_id will have a value and repository_id
will be None. When called from Galaxy, repository_metadata_id will be None and repository_id will have a value.
"""
workflow_name = encoding_util.tool_shed_decode(workflow_name)
if trans.webapp.name == "community":
# We're in the tool shed.
repository_metadata = suc.get_repository_metadata_by_id(trans, repository_metadata_id)
repository_id = trans.security.encode_id(repository_metadata.repository_id)
changeset_revision = repository_metadata.changeset_revision
metadata = repository_metadata.metadata
else:
# We're in Galaxy.
repository = suc.get_tool_shed_repository_by_id(trans, repository_id)
changeset_revision = repository.changeset_revision
metadata = repository.metadata
# metadata[ 'workflows' ] is a list of tuples where each contained tuple is
# [ <relative path to the .ga file in the repository>, <exported workflow dict> ]
for workflow_tup in metadata["workflows"]:
workflow_dict = workflow_tup[1]
if workflow_dict["name"] == workflow_name:
break
if "tools" in metadata:
tools_metadata = metadata["tools"]
else:
tools_metadata = []
workflow, missing_tool_tups = get_workflow_from_dict(
trans=trans,
workflow_dict=workflow_dict,
tools_metadata=tools_metadata,
repository_id=repository_id,
changeset_revision=changeset_revision,
)
data = []
canvas = svgfig.canvas(style="stroke:black; fill:none; stroke-width:1px; stroke-linejoin:round; text-anchor:left")
text = svgfig.SVG("g")
connectors = svgfig.SVG("g")
boxes = svgfig.SVG("g")
svgfig.Text.defaults["font-size"] = "10px"
in_pos = {}
out_pos = {}
margin = 5
# Spacing between input/outputs.
line_px = 16
# Store px width for boxes of each step.
widths = {}
max_width, max_x, max_y = 0, 0, 0
for step in workflow.steps:
step.upgrade_messages = {}
module = module_factory.from_workflow_step(trans, repository_id, changeset_revision, tools_metadata, step)
tool_errors = module.type == "tool" and not module.tool
module_data_inputs = get_workflow_data_inputs(step, module)
module_data_outputs = get_workflow_data_outputs(step, module, workflow.steps)
step_dict = {
"id": step.order_index,
"data_inputs": module_data_inputs,
"data_outputs": module_data_outputs,
"position": step.position,
"tool_errors": tool_errors,
}
input_conn_dict = {}
for conn in step.input_connections:
input_conn_dict[conn.input_name] = dict(id=conn.output_step.order_index, output_name=conn.output_name)
step_dict["input_connections"] = input_conn_dict
data.append(step_dict)
x, y = step.position["left"], step.position["top"]
count = 0
module_name = get_workflow_module_name(module, missing_tool_tups)
max_len = len(module_name) * 1.5
text.append(svgfig.Text(x, y + 20, module_name, **{"font-size": "14px"}).SVG())
y += 45
for di in module_data_inputs:
cur_y = y + count * line_px
if step.order_index not in in_pos:
in_pos[step.order_index] = {}
in_pos[step.order_index][di["name"]] = (x, cur_y)
text.append(svgfig.Text(x, cur_y, di["label"]).SVG())
count += 1
max_len = max(max_len, len(di["label"]))
if len(module.get_data_inputs()) > 0:
y += 15
for do in module_data_outputs:
cur_y = y + count * line_px
if step.order_index not in out_pos:
out_pos[step.order_index] = {}
out_pos[step.order_index][do["name"]] = (x, cur_y)
text.append(svgfig.Text(x, cur_y, do["name"]).SVG())
count += 1
max_len = max(max_len, len(do["name"]))
widths[step.order_index] = max_len * 5.5
max_x = max(max_x, step.position["left"])
max_y = max(max_y, step.position["top"])
max_width = max(max_width, widths[step.order_index])
for step_dict in data:
tool_unavailable = step_dict["tool_errors"]
width = widths[step_dict["id"]]
x, y = step_dict["position"]["left"], step_dict["position"]["top"]
# Only highlight missing tools if displaying in the tool shed.
if trans.webapp.name == "community" and tool_unavailable:
fill = "#EBBCB2"
else:
fill = "#EBD9B2"
boxes.append(svgfig.Rect(x - margin, y, x + width - margin, y + 30, fill=fill).SVG())
box_height = (len(step_dict["data_inputs"]) + len(step_dict["data_outputs"])) * line_px + margin
# Draw separator line.
if len(step_dict["data_inputs"]) > 0:
box_height += 15
sep_y = y + len(step_dict["data_inputs"]) * line_px + 40
text.append(svgfig.Line(x - margin, sep_y, x + width - margin, sep_y).SVG())
# Define an input/output box.
boxes.append(svgfig.Rect(x - margin, y + 30, x + width - margin, y + 30 + box_height, fill="#ffffff").SVG())
for conn, output_dict in step_dict["input_connections"].iteritems():
in_coords = in_pos[step_dict["id"]][conn]
# out_pos_index will be a step number like 1, 2, 3...
out_pos_index = output_dict["id"]
# out_pos_name will be a string like 'o', 'o2', etc.
out_pos_name = output_dict["output_name"]
if out_pos_index in out_pos:
# out_conn_index_dict will be something like:
# 7: {'o': (824.5, 618)}
out_conn_index_dict = out_pos[out_pos_index]
if out_pos_name in out_conn_index_dict:
out_conn_pos = out_pos[out_pos_index][out_pos_name]
else:
# Take any key / value pair available in out_conn_index_dict.
# A problem will result if the dictionary is empty.
if out_conn_index_dict.keys():
key = out_conn_index_dict.keys()[0]
out_conn_pos = out_pos[out_pos_index][key]
adjusted = (out_conn_pos[0] + widths[output_dict["id"]], out_conn_pos[1])
text.append(
svgfig.SVG(
"circle",
cx=out_conn_pos[0] + widths[output_dict["id"]] - margin,
cy=out_conn_pos[1] - margin,
r=5,
fill="#ffffff",
)
)
connectors.append(
svgfig.Line(adjusted[0], adjusted[1] - margin, in_coords[0] - 10, in_coords[1], arrow_end="true").SVG()
)
canvas.append(connectors)
canvas.append(boxes)
canvas.append(text)
width, height = (max_x + max_width + 50), max_y + 300
canvas["width"] = "%s px" % width
canvas["height"] = "%s px" % height
canvas["viewBox"] = "0 0 %s %s" % (width, height)
trans.response.set_content_type("image/svg+xml")
return canvas.standalone_xml()
def check_for_missing_tools(app, tool_panel_configs,
latest_tool_migration_script_number):
# Get the 000x_tools.xml file associated with the current migrate_tools version number.
tools_xml_file_path = os.path.abspath(
os.path.join('scripts', 'migrate_tools',
'%04d_tools.xml' % latest_tool_migration_script_number))
# Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config.
migrated_tool_configs_dict = odict()
tree = util.parse_xml(tools_xml_file_path)
root = tree.getroot()
tool_shed = root.get('name')
tool_shed_url = get_tool_shed_url_from_tools_xml_file_path(app, tool_shed)
# The default behavior is that the tool shed is down.
tool_shed_accessible = False
if tool_shed_url:
for elem in root:
if elem.tag == 'repository':
tool_dependencies = []
tool_dependencies_dict = {}
repository_name = elem.get('name')
changeset_revision = elem.get('changeset_revision')
url = '%s/repository/get_tool_dependencies?name=%s&owner=%s&changeset_revision=%s&from_install_manager=True' % \
( tool_shed_url, repository_name, REPOSITORY_OWNER, changeset_revision )
try:
response = urllib2.urlopen(url)
text = response.read()
response.close()
tool_shed_accessible = True
except Exception, e:
# Tool shed may be unavailable - we have to set tool_shed_accessible since we're looping.
tool_shed_accessible = False
print "The URL\n%s\nraised the exception:\n%s\n" % (url,
str(e))
if tool_shed_accessible:
if text:
tool_dependencies_dict = encoding_util.tool_shed_decode(
text)
for dependency_key, requirements_dict in tool_dependencies_dict.items(
):
tool_dependency_name = requirements_dict['name']
tool_dependency_version = requirements_dict[
'version']
tool_dependency_type = requirements_dict['type']
tool_dependency_readme = requirements_dict.get(
'readme', '')
tool_dependencies.append(
(tool_dependency_name, tool_dependency_version,
tool_dependency_type, tool_dependency_readme))
for tool_elem in elem.findall('tool'):
migrated_tool_configs_dict[tool_elem.get(
'file')] = tool_dependencies
if tool_shed_accessible:
# Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names.
missing_tool_configs_dict = odict()
for tool_panel_config in tool_panel_configs:
tree = util.parse_xml(tool_panel_config)
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
missing_tool_configs_dict = check_tool_tag_set(
elem, migrated_tool_configs_dict,
missing_tool_configs_dict)
elif elem.tag == 'section':
for section_elem in elem:
if section_elem.tag == 'tool':
missing_tool_configs_dict = check_tool_tag_set(
section_elem, migrated_tool_configs_dict,
missing_tool_configs_dict)
