def setUp(self): self.tmpdir = os.path.join(tempfile.gettempdir(), 'schemaorg-test') if not os.path.exists(self.tmpdir): os.mkdir(self.tmpdir) self.here = os.path.abspath(os.path.dirname(__file__)) recipe_yml = os.path.join(self.here, "recipe.yml") self.recipe = RecipeParser(recipe_yml) self.dataset = Schema("Dataset") self.templates = [ 'google/dataset-table.html', # bootstrap 'google/visual-dataset.html', # default 'google/dataset.html', # json only 'google/dataset-vue-table.html' ] # vue js person = make_person(name="Dinosaur Pancakes", description='Dataset maintainer', url='https://www.github.com/vsoch', contact_type='customer support', telephone='999-999-9999') self.dataset.add_property('creator', person) self.dataset.add_property('version', "1.0.0") self.dataset.add_property('description', "This is the best dataset.") self.dataset.add_property('name', "Dinosaur Dataset") self.dataset.add_property( 'thumbnailUrl', 'https://vsoch.github.io/datasets/assets/img/avocado.png') self.dataset.add_property('about', "This is a dataset") download = Schema('DataDownload') download.add_property( 'contentUrl', 'https://vsoch.github.io/datasets/assets/img/avocado.png') download.add_property('encodingFormat', 'CSV') self.dataset.add_property('distribution', [download]) self.recipe.validate(self.dataset)
here = os.path.abspath(os.path.dirname(__file__)) recipe_yml = os.path.join(here, "recipe.yml") index_html = os.path.join(here, "index.html") dockerfile = os.path.join(os.path.dirname(here), "Dockerfile") # Step 1: Show required and recommended fields from recipe recipe = RecipeParser(recipe_yml) print(recipe.loaded) # Step 2: Generate a Person (these are Google Helper functions) from schemaorg.templates.google import (make_person, make_dataset) # make_person(name, description, url="", telephone="", email="") person = make_person(name="@vsoch", description='research software engineer, dinosaur') # Step 3: Create SoftwareSourceCode from spython.main.parse.parsers import DockerParser parser = DockerParser(dockerfile).parse() sourceCode = Schema("SoftwareSourceCode") # sourceCode.properties sourceCode.add_property('creator', person) sourceCode.add_property('version', sourceCode.version) sourceCode.add_property('description', 'A Dockerfile build recipe') sourceCode.add_property('name', parser.fromHeader)
def extract(dockerfile, contact, container_name=None, output_html=True): '''extract a dataset from a given dockerfile, write to html output file. Use container-diff and spython to get information about the container. ''' # Step 0. Define absolute paths to our Dockerfile, recipe, output here = os.path.abspath(os.path.dirname(__file__)) recipe_yml = os.path.join(here, "recipe.yml") spec_yml = os.path.join(here, "specification.yml") # Step 1: Show required and recommended fields from recipe recipe = RecipeParser(recipe_yml) # Step 2: Create Dataset parser = DockerRecipe(dockerfile) image = Schema(spec_yml) # We can obtain these from the environment, or use reasonable defaults thumbnail = os.environ.get( 'IMAGE_THUMBNAIL', 'https://vsoch.github.io/datasets/assets/img/avocado.png') about = os.environ.get( 'IMAGE_ABOUT', 'This is a Dockerfile parsed by the openschemas/extractors container.') repository = os.environ.get('GITHUB_REPOSITORY', 'openschemas/extractors') description = os.environ.get('IMAGE_DESCRIPTION', 'A Dockerfile build recipe') # Step 3: Generate a Person (these are Google Helper functions) contact = os.environ.get('GITHUB_ACTOR', contact) contact_url = os.environ.get('CONTACT_URL', repository) contact_description = os.environ.get('CONTACT_DESCRIPTION', 'Dockerfile maintainer') contact_type = os.environ.get('CONTACT_TYPE', 'customer support') contact_telephone = os.environ.get('CONTACT_TELEPHONE') # Get the repository full url for contact if not contact_url.startswith('http'): contact_url = "https://www.github.com/%s" % contact_url if contact is not None: person = make_person(name=contact, description=contact_description, url=contact_url, contact_type=contact_type, telephone=contact_telephone) image.properties['creator'] = person image.properties['author'] = person # image.properties if len(parser.environ) > 0: image.properties['environment'] = parser.environ image.properties['entrypoint'] = parser.entrypoint image.properties['version'] = image.version image.properties['description'] = description image.properties['ContainerImage'] = parser.fromHeader image.properties['name'] = container_name # Fun properties :) image.properties['thumbnailUrl'] = thumbnail image.properties['sameAs'] = 'ImageDefinition' image.properties['about'] = about image.properties[ 'codeRepository'] = 'https://www.github.com/%s' % repository image.properties['runtime'] = 'Docker' # Generate temporary filename output_file = "%s.json" % get_tmpfile("image-definition") # Try using container name, if not available default to ContainerImage (FROM) layers = run_container_diff(container_name, parser.fromHeader, output_file) if len(layers) > 0: # softwareRequirements requires = [] # APT and PIP # note that the top level key here can be history, files, pip, apt, etc. for layer in layers: ## Pip and Apt will go into softwareRequirements if layer['AnalyzeType'] in ["Pip", "Apt"]: for pkg in layer['Analysis']: requires.append( '%s > %s==%s' % (layer['AnalyzeType'], pkg['Name'], pkg['Version'])) image.properties["softwareRequirements"] = requires if output_html: return make_dataset(image) return image.dump_json(pretty_print=True)
def extract(name, version=None, contact=None, output_html=True, description=None, thumbnail=None, sameAs=None, about=None, repository=None): ''' extract a Dataset to describe some Github repository. To add more properties, just add them via additional keyword args (kwargs) Parameters ========== url: the url to get the catalog name: the name of the DataCatalog contact: name of a person that is in charge of the dataset description: a description of the DataCatalog thumbnail: an image thumbnail (web url) about: text about the data catalog (optional). ''' # Step 0. Define absolute paths to our Dockerfile, recipe, output here = os.path.abspath(os.path.dirname(__file__)) recipe_yml = os.path.join(here, "recipe.yml") # Step 1: Show required and recommended fields from recipe recipe = RecipeParser(recipe_yml) # Step 2: Create Dataset dataset = Schema("Dataset") # We can obtain these from the environment, or use reasonable defaults thumbnail = os.environ.get( 'DATASET_THUMBNAIL', thumbnail or 'https://vsoch.github.io/datasets/assets/img/avocado.png') about = os.environ.get( 'DATASET_ABOUT', about or 'This is a Dataset parsed by the openschemas/extractors container.') repository = os.environ.get('GITHUB_REPOSITORY', repository or 'openschemas/extractors') description = os.environ.get('DATASET_DESCRIPTION', 'A Dataset') email = os.environ.get('DATASET_EMAIL') template = os.environ.get('DATASET_TEMPLATE', "google/dataset-table.html") # Can be one of: # google/dataset-table.html (bootstrap) # google/visual-dataset.html (see vsoch.github.io/zenodo-ml) # google/dataset.html (just blank page, json metadata) # google/dataset-vue-table.html # see https://openschemas.github.io/schemaorg#7-embed-in-html-with-json-ld # Contact metadata contact = os.environ.get('GITHUB_ACTOR', contact) contact_url = os.environ.get('CONTACT_URL', repository) contact_description = os.environ.get('CONTACT_DESCRIPTION', 'Dataset maintainer') contact_type = os.environ.get('CONTACT_TYPE', 'customer support') contact_telephone = os.environ.get('CONTACT_TELEPHONE') contact = add_kwargs(contact, 'DATASET_DOWNLOAD_KWARGS') # Download Link download_link = os.environ.get('DATASET_DOWNLOAD_LINK') encoding = os.environ.get('DATASET_ENCODING_FORMAT') if download != None: download = Schema('DataDownload') download.add_property('encodingFormat', encoding) download.add_property('contentUrl', download_link) download = add_kwargs(download, 'DATASET_DOWNLOAD_KWARGS') dataset.add_property('distribution', [download]) # Get the repository full url for contact if not contact_url.startswith('http'): contact_url = "https://www.github.com/%s" % contact_url if contact is not None: person = make_person(name=contact, description=contact_description, url=contact_url, contact_type=contact_type, telephone=contact_telephone) person = add_kwargs(person, 'CONTACT_KWARGS') dataset.add_property('creator', person) # dataset.properties dataset.add_property('version', version) dataset.add_property('description', description) dataset.add_property('name', name) dataset.add_property('thumbnailUrl', thumbnail) dataset.add_property('about', about) dataset = add_kwargs(dataset, 'DATASET_KWARGS') # Step 5: Validate Data Structure recipe.validate(dataset) if output_html: return make_dataset(dataset, template=template) return dataset.dump_json(pretty_print=True)