class TestAll(unittest.TestCase):
def setUp(self):
connect("mongodb://localhost:27017/unittest")
self.tool1 = MongoTool()
self.tool1.id = "tool-1"
self.tool1.name = "tool-1"
self.tool1.description = "PeptideShaker is a search engine independent platform for interpretation of proteomics identification results from multiple search engines," \
"currently supporting X!Tandem, MS-GF+, MS Amanda, OMSSA, MyriMatch, Comet, Tide, Mascot, Andromeda and mzIdentML. " \
"By combining the results from multiple search engines, while re-calculating PTM localization scores and redoing the protein inference," \
"PeptideShaker attempts to give you the best possible understanding of your proteomics data"
self.tool1.save()
self.tool2 = MongoTool()
self.tool2.id = "tool-2"
self.tool2.name = "tool-2"
self.tool2.tool_classes = ['TOOL']
self.tool2.save()
self.assertTrue(len(self.tool2.tool_classes) == 1)
def test_01_func(self):
self.tool_version = MongoToolVersion()
self.tool_version.id = "version-1"
self.tool_version.name = "tool-1"
self.tool_version.ref_tool = self.tool1
self.tool_version.save()
self.assertEqual(self.tool1.id, "tool-1")
self.assertTrue(len(self.tool1.get_tool_versions()) == 1)
def tearDown(self):
_get_db().client.drop_database('unittest')
def test_01_func(self):
self.tool_version = MongoToolVersion()
self.tool_version.id = "version-1"
self.tool_version.name = "tool-1"
self.tool_version.ref_tool = self.tool1
self.tool_version.save()
self.assertEqual(self.tool1.id, "tool-1")
self.assertTrue(len(self.tool1.get_tool_versions()) == 1)
def annotate_docker_containers(docker_recipes):
for entry in docker_recipes:
logger.info("Annotating the recipe -- " + entry['name'])
name = entry['name']
name_parts = name.split("/")
tool_version_id = name_parts[0] + "-v" + name_parts[1]
tool_id = name_parts[0]
tool_version = MongoToolVersion.get_tool_version_by_id(tool_version_id)
tool = MongoTool.get_tool_by_id(tool_id)
if tool_version is not None:
if entry["recipe"].get_description() is not None:
tool_version.description = entry["recipe"].get_description().capitalize()
if entry['recipe'].get_home_url() is not None:
tool_version.home_url = entry['recipe'].get_home_url()
if entry['recipe'].get_license() is not None:
tool_version.license = entry['recipe'].get_license()
else:
tool_version.license = NOT_AVAILABLE
tool_version.save()
logger.info("Updated tool version description of -- " + tool_version_id)
if tool is not None:
if entry["recipe"].get_description() is not None:
tool.description = entry["recipe"].get_description().capitalize()
if entry['recipe'].get_home_url() is not None:
tool.home_url = entry['recipe'].get_home_url()
if entry['recipe'].get_license() is not None:
tool.license = entry['recipe'].get_license()
else:
tool.license = NOT_AVAILABLE
if entry['recipe'].get_tags() is not None:
tool.tool_tags = entry['recipe'].get_tags()
if entry['recipe'].get_additional_ids() is not None:
tool.add_additional_identifiers(entry['recipe'].get_additional_ids())
tool.save()
logger.info("Updated tool description of -- " + tool_version_id)
def stats():
"""
This method returns a list of stats for the API
:return:
"""
tools = MongoTool.get_all_tools()
stats = []
stats.append(Stat('num_tools', str(len(tools))))
tool_versions = MongoToolVersion.get_all_tool_versions()
stats.append(Stat('num_versions', str(len(tool_versions))))
num_containers = 0
num_docker = 0
num_conda = 0
for key in tool_versions:
num_containers = num_containers + len(key.image_containers)
for container in key.image_containers:
if (container.container_type == 'DOCKER'):
num_docker = num_docker + 1
elif container.container_type == 'CONDA':
num_conda = num_conda + 1
stats.append(Stat('num_containers', str(num_containers)))
stats.append(Stat('num_conda_containers', str(num_conda)))
stats.append(Stat('num_docker_containers', str(num_docker)))
return stats
def annotate_conda_recipes():
conda_helper = CondaMetrics()
mongo_versions = MongoToolVersion.get_all_tool_versions()
tools = []
for tool_version in mongo_versions:
count = 0
tool_not_found = True
for tool in tools:
if tool['id'] == tool_version.name:
count = tool['count']
tool_not_found = False
old_images = []
for image in tool_version.image_containers:
if image.container_type == 'CONDA':
annotations = conda_helper.get_number_downloas_by_version(tool_version.name, tool_version.version)
image.downloads = annotations['downloads']
count = count + image.downloads
image.size = annotations['size']
if annotations['last_update'][0:10] is not None and bool(annotations['last_update'][0:10].strip()):
image.last_updated = annotations['last_update'][0:10]
# else:
# image.last_updated = None
print(annotations)
old_images.append(image)
tool_version.image_containers = old_images
if tool_not_found and count > 0:
tools.append({"id": tool_version.name, "count":count})
else:
for tool in tools:
if tool['id'] == tool_version.name:
tool['count'] = count
tool_version.save()
print(tools)
for stat in tools:
tool = MongoTool.get_tool_by_id(stat['id'])
tool.add_pull_provider("conda", stat['count'])
tool.save()
def annotate_quayio_containers(conda_recipes):
for entry in conda_recipes:
logger.info("Annotating the recipe -- " + entry['name'])
tool_version_id = None
if (entry['recipe'].get_name() is not None) and (entry['recipe'].get_version() is not None) \
and ("{" not in entry['recipe'].get_name()) \
and ("|" not in entry['recipe'].get_name()) and ("{" not in entry['recipe'].get_version()) \
and ("|" not in entry['recipe'].get_version()):
tool_version_id = (entry['recipe'].get_name() + "-" + entry['recipe'].get_version()).lower()
tool_id = entry['recipe'].get_name().lower()
tool_version = MongoToolVersion.get_tool_version_by_id(tool_version_id)
tool = MongoTool.get_tool_by_id(tool_id)
if tool_version is not None:
if entry["recipe"].get_description() is not None:
tool_version.description = entry["recipe"].get_description().capitalize()
if entry['recipe'].get_home_url() is not None:
tool_version.home_url = entry['recipe'].get_home_url()
if entry['recipe'].get_license() is not None and len(entry['recipe'].get_license()) > 0:
tool_version.license = entry['recipe'].get_license()
else:
tool_version.license = NOT_AVAILABLE
tool_version.save()
logger.info("Updated tool version description of -- " + tool_version_id)
if tool is not None:
if entry["recipe"].get_description() is not None:
tool.description = entry["recipe"].get_description().capitalize()
if entry['recipe'].get_home_url() is not None:
tool.home_url = entry['recipe'].get_home_url()
if entry['recipe'].get_license() is not None and bool(entry['recipe'].get_license()):
tool.license = entry['recipe'].get_license()
if entry['recipe'].get_biotool_ids() is not None:
tool.add_additional_identifiers(entry['recipe'].get_biotool_ids())
else:
tool.license = NOT_AVAILABLE
tool.save()
logger.info("Updated tool description of -- " + tool_version_id)
logger.info("The following tool has been analyzed -- " + str(tool_version_id))
def insert_singularity_containers(singularity_containers):
"""
This method provide the mechanism to insert Singularity containers into the Mongo Database
:param singularity_containers: List of Singularity containers
:return:
"""
for key in singularity_containers.keys():
tool_version = MongoToolVersion.get_tool_version_by_id(key)
if tool_version is not None:
update = False
image_containers = tool_version.image_containers
for i in singularity_containers[key]:
found = False
for j in image_containers:
if i.full_tag == j.full_tag:
found = True
break
if not found:
update = True
tool_version.image_containers.append(i)
logger.info("Added singularity image: " + i.tag + "to tool version: " + key)
if update:
tool_version.save()
def tools_container_type_container_tag_get(container_type, container_tag):
"""Get the container using container_type and container_tag.
:param container_type: container_type, for example: conda, docker, singulariry etc.,
:type container_type: str
:param container_tag: container_tag, for example: im-pipelines:1.0.0--pyh9f0ad1d_0
:type container_tag: str
:rtype: string
"""
tool_name = container_tag.split(":")[0]
tool_version = container_tag.split(":")[1]
tools_list = MongoToolVersion.get_tool_version_by_name(tool_name)
if tools_list is None:
return None
container_type = container_type.upper()
for tool in tools_list:
containers = tool.image_containers
for container in containers:
if container.container_type == container_type:
if container_type == "CONDA" and container.tag == "conda:" + tool_version:
return container.full_tag
if container.tag == tool_version: # this case works for quay images
return container.full_tag
try:
tag_replace = container.tag.replace("'", '"')
tag_replace = tag_replace.replace("None", "null")
tag_replace = tag_replace.replace("True", "true")
tag_replace = tag_replace.replace("False", "false")
tag_obj = json.loads(tag_replace) # this case works for docker images
if tag_obj['name'] == tool_version:
return container.full_tag
except ValueError as e:
continue
def insert_dockerhub_containers(dockerhub_containers):
"""
This method provide the mechanism to insert dockerhub containers into the Mongo Database
:param dockerhub_containers: List of DockerHub containers
:return:
"""
list_versions = list(MongoToolVersion.get_all_tool_versions())
tool_versions_dic = {}
for tool_version in list_versions:
tool_versions_dic[tool_version.id] = tool_version
tools_dic = {}
list_tools = list(MongoTool.get_all_tools())
for tool in list_tools:
tools_dic[tool.id] = tool
for container in dockerhub_containers:
# The version is read from the container tag.
current_tool = None
for key in container.tags:
# First insert Tool version containers. For that we need to parse first the version of the tool. Version is also handle as defined by
# the container provider Docker or Quay.io
version = key['name'].split("_", 1)[0]
tool_version_id = container.name() + TOOL_VERSION_SPLITTER + version
if tool_version_id not in tool_versions_dic:
mongo_tool_version = MongoToolVersion()
mongo_tool_version.name = container.name()
mongo_tool_version.version = version
mongo_tool_version.description = container.description()
mongo_tool_version.tool_classes = [_CONSTANT_TOOL_CLASSES['CommandLineTool']]
mongo_tool_version.id = tool_version_id
mongo_tool_version.add_author(BIOCONTAINERS_USER)
mongo_tool_version.organization = container.organization()
else:
mongo_tool_version = tool_versions_dic[tool_version_id]
## Get the tag information (Container image) and add to the ToolVersion
container_image = ContainerImage()
container_image.tag = key
container_image.full_tag = DOCKER_DOMAIN + container.name() + ":" + key['name']
container_image.container_type = 'DOCKER'
datetime_object = datetime.datetime.strptime(key['last_updated'][0:10], '%Y-%m-%d')
container_image.last_updated = datetime_object
container_image.size = int(int(key['full_size']))
mongo_tool_version.add_image_container(container_image)
tool_versions_dic[tool_version_id] = mongo_tool_version
# Insert the corresponding tool
tool_id = container.name()
if tool_id not in tools_dic:
mongo_tool = MongoTool()
mongo_tool.name = container.name()
mongo_tool.id = container.name()
mongo_tool.description = container.description()
mongo_tool.tool_classes = [_CONSTANT_TOOL_CLASSES['CommandLineTool']]
tools_dic[tool_id] = mongo_tool
mongo_tool.add_authors(mongo_tool_version.authors)
mongo_tool.organization = container.organization()
mongo_tool.checker = container.checker()
else:
mongo_tool = tools_dic[tool_id]
mongo_tool.add_registry(container.registry())
mongo_tool.add_alias(container.alias())
tools_dic[tool_id] = mongo_tool
try:
mongo_tool.save()
current_tool = mongo_tool
except DuplicateKeyError as error:
logger.error(" A tool with same name is already in the database -- " + tool_id)
mongo_tool_version.ref_tool = mongo_tool
# mongo_versions = mongo_tool.get_tool_versions()
try:
mongo_tool_version.save()
except DuplicateKeyError as error:
logger.error(
" A tool version with a same name and version is in the database -- " + tool_version_id)
if current_tool is not None:
current_tool.add_pull_provider("dockerhub", container.get_pull_count())
current_tool.save()
containers_list = list(tool_versions_dic.values())
def insert_quayio_containers(quayio_containers):
"""
This method provide the mechanism to insert quayio containers into the Mongo Database
:param quayio_containers: List of Quay.io containers
:return:
"""
list_versions = list(MongoToolVersion.get_all_tool_versions())
tool_versions_dic = {}
for tool_version in list_versions:
tool_versions_dic[tool_version.id] = tool_version
tools_dic = {}
list_tools = list(MongoTool.get_all_tools())
for tool in list_tools:
tools_dic[tool.id] = tool
for container in quayio_containers:
# The version is read from the container tag.
version_list = []
current_tool = None
for key, val in container.tags().items():
# First insert Tool version containers. For that we need to parse first the version of the tool. Version is also handle as defined by
# the container provider Docker or Quay.io
version = key.split("--", 1)[0]
tool_version_id = container.name() + TOOL_VERSION_SPLITTER + version
if tool_version_id not in tool_versions_dic:
mongo_tool_version = MongoToolVersion()
mongo_tool_version.name = container.name()
mongo_tool_version.version = version
mongo_tool_version.description = container.description()
mongo_tool_version.organization = container.organization()
if "mulled-v2" not in mongo_tool_version.name:
mongo_tool_version.tool_classes = [_CONSTANT_TOOL_CLASSES['CommandLineTool']]
else:
mongo_tool_version.tool_classes = [_CONSTANT_TOOL_CLASSES['CommandLineMultiTool']]
mongo_tool_version.id = tool_version_id
mongo_tool_version.add_author(BIOCONTAINERS_USER)
mongo_tool_version.add_author(BICONDA_USER)
else:
mongo_tool_version = tool_versions_dic[tool_version_id]
## Add only one conda package for each version
if key not in version_list:
container_image = ContainerImage()
container_image.tag = "conda:" + key
container_image.full_tag = container.name() + "==" + key
container_image.container_type = 'CONDA'
container_image.size = 0
container_image.downloads = 0
mongo_tool_version.add_image_container(container_image)
version_list.append(key)
## Add container
container_image = ContainerImage()
container_image.tag = key
container_image.full_tag = QUAYIO_DOMAIN + container.name() + ":" + key
container_image.container_type = 'DOCKER'
datetime_object = datetime.datetime.strptime(val['last_modified'][0:-15], '%a, %d %b %Y')
container_image.last_updated = datetime_object
container_image.size = int(int(val['size']))
container_image.downloads = 0
mongo_tool_version.add_image_container(container_image)
tool_versions_dic[tool_version_id] = mongo_tool_version
# Insert the corresponding tool
tool_id = container.name()
if tool_id not in tools_dic:
mongo_tool = MongoTool()
mongo_tool.name = container.name()
if "mulled-v2" not in mongo_tool_version.name:
mongo_tool.tool_classes = [_CONSTANT_TOOL_CLASSES['CommandLineTool']]
else:
mongo_tool.tool_classes = [_CONSTANT_TOOL_CLASSES['CommandLineMultiTool']]
mongo_tool.id = container.name()
mongo_tool.description = container.description()
mongo_tool.add_authors(mongo_tool_version.authors)
mongo_tool.organization = container.organization()
mongo_tool.checker = container.checker()
else:
mongo_tool = tools_dic[tool_id]
mongo_tool.add_registry(container.registry())
mongo_tool.add_alias(container.alias())
tools_dic[tool_id] = mongo_tool
try:
mongo_tool.save()
current_tool = mongo_tool
except DuplicateKeyError as error:
logger.error(" A tool with same name is already in the database -- " + tool_id)
mongo_tool_version.ref_tool = mongo_tool
try:
mongo_tool_version.save()
except DuplicateKeyError as error:
logger.error(
" A tool version with a same name and version is in the database -- " + tool_version_id)
if current_tool is not None:
count = 0
for stat in container.pulls():
count = count + stat['count']
current_tool.add_pull_provider("quay.io", count)
current_tool.save()
containers_list = list(tool_versions_dic.values())