def handle(self, *args, **options): interactive = True if len(args) and args[0] == '--no-input': interactive = False # Ask the user if interactive if interactive: correct = False print "This process will delete the indexes directory. Continue: [y/n]" while not correct: opt = read_from_cmd() if opt != 'y' and opt != 'n': print "Please include 'y' or 'n'" else: correct = True if opt == 'n': return # Remove the index directory index_path = settings.DATADIR index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') rmtree(index_path, True) # Generate new search indexes se = SearchEngine(index_path) for o in Offering.objects.all(): se.create_index(o)
def handle(self, *args, **options): interactive = True if len(args) and args[0] == '--no-input': interactive = False # Ask the user if interactive if interactive: correct = False print "This process will delete the indexes directory. Continue: [y/n]" while not correct: opt = read_from_cmd() if opt != 'y' and opt != 'n': print "Please include 'y' or 'n'" else: correct = True if opt == 'n': return # Remove the index directory index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') rmtree(index_path, True) # Generate new search indexes se = SearchEngine(index_path) for o in Offering.objects.all(): se.create_index(o)
def _create_indexes(): index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') search_engine = SearchEngine(index_path) for off in Offering.objects.all(): search_engine.create_index(off)
def test_basic_index_creaton(self): offering = Offering.objects.get(name='test_offering') se = SearchEngine(settings.BASEDIR + '/wstore/test/test_index') se.create_index(offering) # Get the index reader index = open_dir(settings.BASEDIR + '/wstore/test/test_index') with index.searcher() as searcher: query = QueryParser('content', index.schema).parse(unicode('widget')) total_hits = searcher.search(query) self.assertEqual(len(total_hits), 1) doc = total_hits[0] self.assertEqual(offering.pk, doc['id'])
def create_offering(provider, json_data): profile = provider.userprofile data = {} if not 'name' in json_data or not 'version' in json_data: raise ValueError('Missing required fields') data['name'] = json_data['name'] data['version'] = json_data['version'] if not re.match(re.compile(r'^(?:[1-9]\d*\.|0\.)*(?:[1-9]\d*|0)$'), data['version']): raise ValueError('Invalid version format') if not is_valid_id(data['name']): raise ValueError('Invalid name format') # Get organization organization = profile.current_organization # Check if the offering already exists existing = True try: Offering.objects.get(name=data['name'], owner_organization=organization, version=data['version']) except: existing = False if existing: raise Exception('The offering already exists') # Check if the version of the offering is lower than an existing one offerings = Offering.objects.filter(owner_organization=organization, name=data['name']) for off in offerings: if is_lower_version(data['version'], off.version): raise ValueError('A bigger version of the current offering exists') is_open = json_data.get('open', False) # If using the idm, get the applications from the request if settings.OILAUTH: # Validate application structure data['applications'] = [] for app in json_data['applications']: data['applications'].append({ 'name': app['name'], 'url': app['url'], 'id': app['id'], 'description': app['description'] }) data['related_images'] = [] # Check the URL to notify the provider notification_url = '' if 'notification_url' in json_data: if json_data['notification_url'] == 'default': notification_url = organization.notification_url if not notification_url: raise ValueError('There is not a default notification URL defined for the organization ' + organization.name + '. To configure a default notification URL provide it in the settings menu') else: # Check the notification URL if not is_valid_url(json_data['notification_url']): raise ValueError("Invalid notification URL format: It doesn't seem to be an URL") notification_url = json_data['notification_url'] # Create the directory for app media dir_name = organization.name + '__' + data['name'] + '__' + data['version'] path = os.path.join(settings.MEDIA_ROOT, dir_name) os.makedirs(path) if not 'image' in json_data: raise ValueError('Missing required field: Logo') if not isinstance(json_data['image'], dict): raise TypeError('Invalid image type') image = json_data['image'] if not 'name' in image or not 'data' in image: raise ValueError('Missing required field in image') # Save the application image or logo f = open(os.path.join(path, image['name']), "wb") dec = base64.b64decode(image['data']) f.write(dec) f.close() data['image_url'] = settings.MEDIA_URL + dir_name + '/' + image['name'] # Save screen shots if 'related_images' in json_data: for image in json_data['related_images']: # images must be encoded in base64 format f = open(os.path.join(path, image['name']), "wb") dec = base64.b64decode(image['data']) f.write(dec) f.close() data['related_images'].append(settings.MEDIA_URL + dir_name + '/' + image['name']) # Save USDL document # If the USDL itself is provided if 'offering_description' in json_data: usdl_info = json_data['offering_description'] repository = Repository.objects.get(name=json_data['repository']) repository_adaptor = RepositoryAdaptor(repository.host, 'storeOfferingCollection') offering_id = organization.name + '__' + data['name'] + '__' + data['version'] usdl = usdl_info['data'] data['description_url'] = repository_adaptor.upload(usdl_info['content_type'], usdl_info['data'], name=offering_id) # If the USDL is already uploaded in the repository elif 'description_url' in json_data: # Check that the link to USDL is unique since could be used to # purchase offerings from Marketplace usdl_info = {} usdl_url = json_data['description_url'] off = Offering.objects.filter(description_url=usdl_url) if len(off) != 0: raise ValueError('The provided USDL description is already registered') # Download the USDL from the repository repository_adaptor = RepositoryAdaptor(usdl_url) accept = "text/plain; application/rdf+xml; text/turtle; text/n3" usdl = repository_adaptor.download(content_type=accept) usdl_info['content_type'] = usdl['content_type'] usdl = usdl['data'] data['description_url'] = usdl_url # If the USDL is going to be created elif 'offering_info' in json_data: _validate_offering_info(json_data['offering_info']) offering_info = json_data['offering_info'] offering_info['image_url'] = data['image_url'] offering_info['name'] = data['name'] repository = Repository.objects.get(name=json_data['repository']) offering_info['base_uri'] = repository.host usdl = _create_basic_usdl(offering_info) usdl_info = { 'content_type': 'application/rdf+xml' } repository_adaptor = RepositoryAdaptor(repository.host, 'storeOfferingCollection') offering_id = organization.name + '__' + data['name'] + '__' + data['version'] data['description_url'] = repository_adaptor.upload(usdl_info['content_type'], usdl, name=offering_id) else: raise Exception('No USDL description provided') # Validate the USDL data['open'] = is_open data['organization'] = organization valid = validate_usdl(usdl, usdl_info['content_type'], data) if not valid[0]: raise Exception(valid[1]) # Check new currencies used if len(valid) > 2: new_curr = valid[2] # Set the currency as used cont = Context.objects.all()[0] currency = None # Search the currency for c in cont.allowed_currencies['allowed']: if c['currency'].lower() == new_curr.lower(): currency = c break cont.allowed_currencies['allowed'].remove(currency) currency['in_use'] = True cont.allowed_currencies['allowed'].append(currency) cont.save() # Serialize and store USDL info in json-ld format graph = rdflib.Graph() rdf_format = usdl_info['content_type'] if rdf_format == 'text/turtle' or rdf_format == 'text/plain': rdf_format = 'n3' elif rdf_format == 'application/json': rdf_format = 'json-ld' graph.parse(data=usdl, format=rdf_format) data['offering_description'] = graph.serialize(format='json-ld', auto_compact=True) # Create the offering offering = Offering.objects.create( name=data['name'], owner_organization=organization, owner_admin_user=provider, version=data['version'], state='uploaded', description_url=data['description_url'], resources=[], comments=[], tags=[], image_url=data['image_url'], related_images=data['related_images'], offering_description=json.loads(data['offering_description']), notification_url=notification_url, creation_date=datetime.now(), open=is_open ) if settings.OILAUTH: offering.applications = data['applications'] offering.save() if 'resources' in json_data and len(json_data['resources']) > 0: bind_resources(offering, json_data['resources'], profile.user) # Load offering document to the search index index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') search_engine = SearchEngine(index_path) search_engine.create_index(offering)
def create_offering(provider, data): """ Creates a new offering including the media files and the repository uploads """ profile = provider.userprofile # Validate basic fields if 'name' not in data or 'version' not in data or 'offering_description' not in data \ or 'image' not in data: missing_fields = '' if 'name' not in data: missing_fields += ' name' if 'version' not in data: missing_fields += ' version' if 'offering_description' not in data: missing_fields += ' offering_description' if 'image' not in data: missing_fields += ' image' raise ValueError('Missing required fields:' + missing_fields) if not re.match(re.compile(r'^(?:[1-9]\d*\.|0\.)*(?:[1-9]\d*|0)$'), data['version']): raise ValueError('Invalid version format') if not is_valid_id(data['name']): raise ValueError('Invalid name format') # Get organization organization = profile.current_organization # Check if the offering already exists if len(Offering.objects.filter(name=data['name'], owner_organization=organization, version=data['version'])) > 0: raise ConflictError('The offering ' + data['name'] + ' version ' + data['version'] + ' already exists') # Check if the version of the offering is lower than an existing one offerings = Offering.objects.filter(owner_organization=organization, name=data['name']) for off in offerings: if is_lower_version(data['version'], off.version): raise ValueError('A bigger version of the current offering exists') is_open = data.get('open', False) # If using the idm, get the applications from the request if settings.OILAUTH and 'applications' in data: # Validate application structure for app in data['applications']: if 'name' not in app or 'url' not in app or 'id' not in app or 'description' not in app: raise ValueError('Missing a required field in application definition') # Check the URL to notify the provider notification_url = '' if 'notification_url' in data: if data['notification_url'] == 'default': notification_url = organization.notification_url if not notification_url: raise ValueError('There is not a default notification URL defined for the organization ' + organization.name + '. To configure a default notification URL provide it in the settings menu') else: # Check the notification URL if not is_valid_url(data['notification_url']): raise ValueError("Invalid notification URL format: It doesn't seem to be an URL") notification_url = data['notification_url'] # Create the directory for app media dir_name = organization.name + '__' + data['name'] + '__' + data['version'] path = os.path.join(settings.MEDIA_ROOT, dir_name) os.makedirs(path) # Validate image format if not isinstance(data['image'], dict): raise TypeError('Invalid image type') data['image_url'] = _create_image(dir_name, path, data['image']) # Save screen shots screenshots = [] if 'related_images' in data: for image in data['related_images']: screenshots.append(_create_image(dir_name, path, image)) else: data['related_images'] = [] # Validate the USDL data['open'] = is_open data['organization'] = organization # Create offering USDL offering_info = deepcopy(data['offering_description']) offering_info['image_url'] = data['image_url'] offering_info['name'] = data['name'] offering_info['version'] = data['version'] offering_info['organization'] = organization.name offering_info['base_id'] = 'pk' created = datetime.now() offering_info['created'] = unicode(created) offering_info['modified'] = unicode(created) data['offering_description']['modified'] = unicode(created) usdl_generator = USDLGenerator() usdl_generator.validate_info(offering_info, organization, open_=is_open) # Create the offering offering = Offering( name=data['name'], owner_organization=organization, owner_admin_user=provider, version=data['version'], state='uploaded', resources=[], comments=[], tags=[], image_url=data['image_url'], related_images=screenshots, notification_url=notification_url, creation_date=created, open=is_open, offering_description=data['offering_description'] ) if settings.OILAUTH and 'applications' in data: offering.applications = data['applications'] offering.save() offering.offering_description['base_id'] = offering.pk offering.save() if 'resources' in data and len(data['resources']) > 0: bind_resources(offering, data['resources'], profile.user) # Load offering document to the search index index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') search_engine = SearchEngine(index_path) search_engine.create_index(offering)