def _transform_scraped_resource(target_dept, resource): distribution = Resource() downloadURL = str() if urlparse(resource.get('url')).scheme: downloadURL = resource.get('url') else: downloadURL = urljoin(resource.get('source_url'), resource.get('url')) #remove spaces in links downloadURL = downloadURL.replace(' ', '%20') distribution.downloadURL = downloadURL resource_name = str(resource.get('name')) if resource_name and resource_name.lower() not in resources_common_names: distribution.title = resource.get('name') else: distribution.title = h.extract_resource_name_from_url( distribution.downloadURL) if resource.get('description'): distribution.description = resource.get('description') if resource.get('format'): distribution.resource_format = resource.get('format') distribution.mediaType = h.get_media_type(resource.get('format')) else: extension = h.extract_resource_format_from_url( distribution.downloadURL) if extension: distribution.resource_format = extension distribution.mediaType = h.get_media_type(extension) return distribution
def __init__(self): self.resource_type = "dcat:Distribution" self.description = "n/a" self.resource_format = "txt" self.mediaType = h.get_media_type(self.resource_format) self.headerMetadata = dict()