def handle(self, *args, **options): interactive = True if len(args) and args[0] == '--no-input': interactive = False # Ask the user if interactive if interactive: correct = False print "This process will delete the indexes directory. Continue: [y/n]" while not correct: opt = read_from_cmd() if opt != 'y' and opt != 'n': print "Please include 'y' or 'n'" else: correct = True if opt == 'n': return # Remove the index directory index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') rmtree(index_path, True) # Generate new search indexes se = SearchEngine(index_path) for o in Offering.objects.all(): se.create_index(o)
def handle(self, *args, **options): interactive = True if len(args) and args[0] == '--no-input': interactive = False # Ask the user if interactive if interactive: correct = False print "This process will delete the indexes directory. Continue: [y/n]" while not correct: opt = read_from_cmd() if opt != 'y' and opt != 'n': print "Please include 'y' or 'n'" else: correct = True if opt == 'n': return # Remove the index directory index_path = settings.DATADIR index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') rmtree(index_path, True) # Generate new search indexes se = SearchEngine(index_path) for o in Offering.objects.all(): se.create_index(o)
def test_search_for_purchased_offerings(self): user = User.objects.get(username='******') user_profile = UserProfile.objects.get(user=user) org = Organization.objects.get(name='test_organization') user_profile.current_organization = org user_profile.organizations.append({ 'organization': org.pk, 'roles': ['provider', 'customer'] }) user_profile.save() org.offerings_purchased = ["61000a0a8905ac2115f022f0"] org.save() purchase = Purchase.objects.all()[0] purchase.organization_owned = True purchase.owner_organization = org purchase.save() se = SearchEngine(settings.BASEDIR + '/wstore/test/test_index') result = se.full_text_search(user, 'purchased', state='purchased') self.assertEqual(len(result), 1) self.assertEqual(result[0]['name'], 'purchased_offering') self.assertEqual(result[0]['version'], '1.0') self.assertEqual(result[0]['state'], 'purchased')
def read(self, request, text): index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') search_engine = SearchEngine(index_path) filter_ = request.GET.get('filter', None) action = request.GET.get('action', None) start = request.GET.get('start', None) limit = request.GET.get('limit', None) sort = request.GET.get('sort', None) # Check the filter value if filter_ and filter_ != 'published' and filter_ != 'provided' and filter_ != 'purchased': return build_response(request, 400, 'Invalid filter') count = False pagination = None # Check if the action is count if action != None: if action == 'count': count = True else: return build_response(request, 400, 'Invalid action') else: # Check pagination params (Only when action is none) if start != None and limit != None: pagination = { 'start': int(start), 'limit': int(limit) } elif (start != None and limit == None) or (start == None and limit != None): return build_response(request, 400, 'Missing pagination param') # Check sorting values if sort != None: if sort != 'date' and sort != 'popularity' and sort != 'name': return build_response(request, 400, 'Invalid sorting') if not filter_: response = search_engine.full_text_search(request.user, text, count=count, pagination=pagination, sort=sort) elif filter_ == 'provided': state = request.GET.get('state', 'all') # Check the state value if state != 'all' and state != 'uploaded'\ and state != 'published' and state != 'deleted': return build_response(request, 400, 'Invalid state') response = search_engine.full_text_search(request.user, text, state=state, count=count, pagination=pagination, sort=sort) elif filter_ == 'purchased': response = search_engine.full_text_search(request.user, text, state='purchased', count=count, pagination=pagination, sort=sort) return HttpResponse(json.dumps(response), status=200, mimetype='application/json')
def _create_indexes(): index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') search_engine = SearchEngine(index_path) for off in Offering.objects.all(): search_engine.create_index(off)
def get_named_entities(self): # Get USDL text aggregator se = SearchEngine('') # Get usdl text text = se._aggregate_text(self._offering) # Get stemmed tokens analyzer = StemmingAnalyzer() named_entities = set([token.text for token in analyzer(unicode(text))]) return named_entities
def test_basic_index_creaton(self): offering = Offering.objects.get(name='test_offering') se = SearchEngine(settings.BASEDIR + '/wstore/test/test_index') se.create_index(offering) # Get the index reader index = open_dir(settings.BASEDIR + '/wstore/test/test_index') with index.searcher() as searcher: query = QueryParser('content', index.schema).parse(unicode('widget')) total_hits = searcher.search(query) self.assertEqual(len(total_hits), 1) doc = total_hits[0] self.assertEqual(offering.pk, doc['id'])
def test_search_no_results(self): user = User.objects.get(username='******') user_profile = UserProfile.objects.get(user=user) org = Organization.objects.get(name='test_organization') user_profile.current_organization = org user_profile.organizations.append({ 'organization': org.pk, 'roles': ['provider', 'customer'] }) user_profile.save() se = SearchEngine(settings.BASEDIR + '/wstore/test/test_index') result = se.full_text_search(user, 'no result') self.assertEqual(len(result), 0)
def test_basic_search(self): user = User.objects.get(username='******') user_profile = UserProfile.objects.get(user=user) org = Organization.objects.get(name='test_organization') user_profile.current_organization = org user_profile.organizations.append({ 'organization': org.pk, 'roles': ['provider', 'customer'] }) user_profile.save() se = SearchEngine(settings.BASEDIR + '/wstore/test/test_index') result = se.full_text_search(user, 'first') self.assertEqual(len(result), 1) self.assertEqual(result[0]['name'], 'test_offering1') self.assertEqual(result[0]['version'], '1.0')
def remove_review(self, user, review): """ Removes a given review """ rev = self._get_and_validate_review(user, review) # Remove review from offering rev.offering.comments.remove(review) # Update offering rating if len(rev.offering.comments): rev.offering.rating = ((rev.offering.rating * (len(rev.offering.comments) + 1)) - rev.rating) / len(rev.offering.comments) else: rev.offering.rating = 0 rev.offering.save() # Update offering indexes index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') se = SearchEngine(index_path) se.update_index(rev.offering) # Update top rated offerings self._update_top_rated() # Update user info to allow her to create a new review if rev.user == user: # Check if is a private review or an organization review if user.userprofile.is_user_org(): user.userprofile.rated_offerings.remove(rev.offering.pk) user.userprofile.save() else: self._remove_review_from_org( user, rev.offering, user.userprofile.current_organization) else: self._remove_review_from_org(rev.user, rev.offering, rev.organization) rev.delete()
def remove_review(self, user, review): """ Removes a given review """ rev = self._get_and_validate_review(user, review) # Remove review from offering rev.offering.comments.remove(review) # Update offering rating if len(rev.offering.comments): rev.offering.rating = ((rev.offering.rating * (len(rev.offering.comments) + 1)) - rev.rating) / len( rev.offering.comments ) else: rev.offering.rating = 0 rev.offering.save() # Update offering indexes index_path = settings.DATADIR index_path = os.path.join(index_path, "search") index_path = os.path.join(index_path, "indexes") se = SearchEngine(index_path) se.update_index(rev.offering) # Update top rated offerings self._update_top_rated() # Update user info to allow her to create a new review if rev.user == user: # Check if is a private review or an organization review if user.userprofile.is_user_org(): user.userprofile.rated_offerings.remove(rev.offering.pk) user.userprofile.save() else: self._remove_review_from_org(user, rev.offering, user.userprofile.current_organization) else: self._remove_review_from_org(rev.user, rev.offering, rev.organization) rev.delete()
def publish_offering(offering, data): # Validate data if not 'marketplaces' in data: raise ValueError('Publication error: missing required field, marketplaces') # Validate the state of the offering if not offering.state == 'uploaded': raise PermissionDenied('Publication error: The offering ' + offering.name + ' ' + offering.version +' cannot be published') # Validate the offering has enough content to be published # Open offerings cannot be published in they do not contain # digital assets (applications or resources) if offering.open and not len(offering.resources) and not len(offering.applications): raise PermissionDenied('Publication error: Open offerings cannot be published if they do not contain at least a digital asset (resource or application)') # Publish the offering in the selected marketplaces for market in data['marketplaces']: try: m = Marketplace.objects.get(name=market) except: raise ValueError('Publication error: The marketplace ' + market + ' does not exist') market_adaptor = MarketAdaptor(m.host) info = { 'name': offering.name, 'url': offering.description_url } market_adaptor.add_service(settings.STORE_NAME, info) offering.marketplaces.append(m.pk) offering.state = 'published' offering.publication_date = datetime.now() offering.save() # Update offering indexes index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') se = SearchEngine(index_path) se.update_index(offering)
def test_search_not_existing_index(self): user = User.objects.get(username='******') user_profile = UserProfile.objects.get(user=user) org = Organization.objects.get(name='test_organization') user_profile.current_organization = org user_profile.organizations.append({ 'organization': org.pk, 'roles': ['provider', 'customer'] }) user_profile.save() se = SearchEngine(settings.BASEDIR + '/wstore/test/no_index') error = False msg = None try: se.full_text_search(user, 'index offering') except Exception, e: error = True msg = e.message
def update_review(self, user, review, review_data): """ Updates a given review """ # Check data validation = self._validate_content(review_data) if validation: raise validation rev = self._get_and_validate_review(user, review) # Calculate new rating rate = ( (rev.offering.rating * len(rev.offering.comments)) - rev.rating + review_data['rating']) / len(rev.offering.comments) # update review rev.title = review_data['title'] rev.comment = review_data['comment'] rev.rating = review_data['rating'] rev.timestamp = datetime.now() rev.save() # Update offering rating rev.offering.rating = rate rev.offering.save() # Update offering indexes index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') se = SearchEngine(index_path) se.update_index(rev.offering) # Update top rated offerings self._update_top_rated()
def update_review(self, user, review, review_data): """ Updates a given review """ # Check data validation = self._validate_content(review_data) if validation: raise validation rev = self._get_and_validate_review(user, review) # Calculate new rating rate = ((rev.offering.rating * len(rev.offering.comments)) - rev.rating + review_data["rating"]) / len( rev.offering.comments ) # update review rev.title = review_data["title"] rev.comment = review_data["comment"] rev.rating = review_data["rating"] rev.timestamp = datetime.now() rev.save() # Update offering rating rev.offering.rating = rate rev.offering.save() # Update offering indexes index_path = settings.DATADIR index_path = os.path.join(index_path, "search") index_path = os.path.join(index_path, "indexes") se = SearchEngine(index_path) se.update_index(rev.offering) # Update top rated offerings self._update_top_rated()
def update_review(self, user, review, review_data): """ Updates a given review """ # Check data validation = self._validate_content(review_data) if validation: raise validation rev = self._get_and_validate_review(user, review) # Calculate new rating rate = ((rev.offering.rating * len(rev.offering.comments)) - rev.rating + review_data['rating']) / len(rev.offering.comments) # update review rev.title = review_data['title'] rev.comment = review_data['comment'] rev.rating = review_data['rating'] rev.timestamp = datetime.now() rev.save() # Update offering rating rev.offering.rating = rate rev.offering.save() # Update offering indexes index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') se = SearchEngine(index_path) se.update_index(rev.offering) # Update top rated offerings self._update_top_rated()
def test_search_offerings(self, expected_result, side_effect=None, state=None, count=False, pagination=None, sort=None, err_type=None, err_msg=None): # Create user data user = User.objects.get(username='******') if side_effect: side_effect(self) # Create the search engine se = SearchEngine(settings.BASEDIR + '/wstore/test/test_index') # Call full text search error = None try: result = se.full_text_search(user, 'offering', state=state, count=count, pagination=pagination, sort=sort) except Exception as e: error = e # Check results if not err_type: self.assertEquals(error, None) if count: self.assertEquals(result, expected_result) else: # If a sorting has been defined the result must be strict self.assertEquals(len(result), len(expected_result)) i = 0 for res in result: if sort: self.assertEquals(res['name'], expected_result[i]) else: self.assertTrue(res['name'] in expected_result) i += 1 else: self.assertTrue(isinstance(error, err_type)) self.assertEquals(unicode(e), err_msg)
def test_update_index(self, update_method, offering, query_, owned=False, err_type=None, err_msg=None): # Get the offering off = None if not err_type: off = Offering.objects.get(pk=offering) # Create the search engine se = SearchEngine(settings.BASEDIR + '/wstore/test/test_index') # Call the update method update_method(self, off, sa=se) # Call full text search error = None try: se.update_index(off) except Exception as e: error = e if not err_type: self.assertEquals(error, None) index = open_dir(settings.BASEDIR + '/wstore/test/test_index') with index.searcher() as searcher: if owned: user = User.objects.get(pk="51000aba8e05ac2115f022f9") pk = user.userprofile.current_organization.pk query_ = query_ & query.Term('purchaser', pk) search_result = searcher.search(query_) self.assertEquals(len(search_result), 1) else: self.assertTrue(isinstance(error, err_type)) self.assertEquals(unicode(error), err_msg)
def create_offering(provider, data): """ Creates a new offering including the media files and the repository uploads """ profile = provider.userprofile # Validate basic fields if 'name' not in data or 'version' not in data or 'offering_description' not in data \ or 'image' not in data: missing_fields = '' if 'name' not in data: missing_fields += ' name' if 'version' not in data: missing_fields += ' version' if 'offering_description' not in data: missing_fields += ' offering_description' if 'image' not in data: missing_fields += ' image' raise ValueError('Missing required fields:' + missing_fields) if not re.match(re.compile(r'^(?:[1-9]\d*\.|0\.)*(?:[1-9]\d*|0)$'), data['version']): raise ValueError('Invalid version format') if not is_valid_id(data['name']): raise ValueError('Invalid name format') # Get organization organization = profile.current_organization # Check if the offering already exists if len(Offering.objects.filter(name=data['name'], owner_organization=organization, version=data['version'])) > 0: raise ConflictError('The offering ' + data['name'] + ' version ' + data['version'] + ' already exists') # Check if the version of the offering is lower than an existing one offerings = Offering.objects.filter(owner_organization=organization, name=data['name']) for off in offerings: if is_lower_version(data['version'], off.version): raise ValueError('A bigger version of the current offering exists') is_open = data.get('open', False) # If using the idm, get the applications from the request if settings.OILAUTH and 'applications' in data: # Validate application structure for app in data['applications']: if 'name' not in app or 'url' not in app or 'id' not in app or 'description' not in app: raise ValueError('Missing a required field in application definition') # Check the URL to notify the provider notification_url = '' if 'notification_url' in data: if data['notification_url'] == 'default': notification_url = organization.notification_url if not notification_url: raise ValueError('There is not a default notification URL defined for the organization ' + organization.name + '. To configure a default notification URL provide it in the settings menu') else: # Check the notification URL if not is_valid_url(data['notification_url']): raise ValueError("Invalid notification URL format: It doesn't seem to be an URL") notification_url = data['notification_url'] # Create the directory for app media dir_name = organization.name + '__' + data['name'] + '__' + data['version'] path = os.path.join(settings.MEDIA_ROOT, dir_name) os.makedirs(path) # Validate image format if not isinstance(data['image'], dict): raise TypeError('Invalid image type') data['image_url'] = _create_image(dir_name, path, data['image']) # Save screen shots screenshots = [] if 'related_images' in data: for image in data['related_images']: screenshots.append(_create_image(dir_name, path, image)) else: data['related_images'] = [] # Validate the USDL data['open'] = is_open data['organization'] = organization # Create offering USDL offering_info = deepcopy(data['offering_description']) offering_info['image_url'] = data['image_url'] offering_info['name'] = data['name'] offering_info['version'] = data['version'] offering_info['organization'] = organization.name offering_info['base_id'] = 'pk' created = datetime.now() offering_info['created'] = unicode(created) offering_info['modified'] = unicode(created) data['offering_description']['modified'] = unicode(created) usdl_generator = USDLGenerator() usdl_generator.validate_info(offering_info, organization, open_=is_open) # Create the offering offering = Offering( name=data['name'], owner_organization=organization, owner_admin_user=provider, version=data['version'], state='uploaded', resources=[], comments=[], tags=[], image_url=data['image_url'], related_images=screenshots, notification_url=notification_url, creation_date=created, open=is_open, offering_description=data['offering_description'] ) if settings.OILAUTH and 'applications' in data: offering.applications = data['applications'] offering.save() offering.offering_description['base_id'] = offering.pk offering.save() if 'resources' in data and len(data['resources']) > 0: bind_resources(offering, data['resources'], profile.user) # Load offering document to the search index index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') search_engine = SearchEngine(index_path) search_engine.create_index(offering)
def create_review(self, user, offering, review): """ Creates a new review for a given offering """ # Check if the user has purchased the offering (Not if the offering is open) if not offering.open: try: purchase = Purchase.objects.get( offering=offering, owner_organization=user.userprofile.current_organization ) except: raise PermissionDenied("You cannot review this offering since you has not acquire it") # Check if the user has already review the offering. if (user.userprofile.is_user_org() and offering.pk in user.userprofile.rated_offerings) or ( not user.userprofile.is_user_org() and user.userprofile.current_organization.has_rated_offering(user, offering) ): raise PermissionDenied( "You cannot review this offering again. Please update your review to provide new comments" ) # Validate review data validation = self._validate_content(review) if validation: raise validation # Create the review rev = Review.objects.create( user=user, organization=user.userprofile.current_organization, offering=offering, timestamp=datetime.now(), title=review["title"], comment=review["comment"], rating=review["rating"], ) offering.comments.insert(0, rev.pk) # Calculate new offering rate old_rate = offering.rating if old_rate == 0: offering.rating = review["rating"] else: offering.rating = ((old_rate * (len(offering.comments) - 1)) + review["rating"]) / len(offering.comments) offering.save() # Update offering indexes index_path = settings.DATADIR index_path = os.path.join(index_path, "search") index_path = os.path.join(index_path, "indexes") se = SearchEngine(index_path) se.update_index(offering) # Save the offering as rated if user.userprofile.is_user_org(): user.userprofile.rated_offerings.append(offering.pk) user.userprofile.save() else: user.userprofile.current_organization.rated_offerings.append({"user": user.pk, "offering": offering.pk}) user.userprofile.current_organization.save() # Update top rated list self._update_top_rated()
def read(self, request, text): index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') search_engine = SearchEngine(index_path) filter_ = request.GET.get('filter', None) action = request.GET.get('action', None) start = request.GET.get('start', None) limit = request.GET.get('limit', None) sort = request.GET.get('sort', None) state = request.GET.get('state', None) if state: if state == 'ALL': state = ['uploaded', 'published', 'deleted'] else: state = state.split(',') # Check the filter value if filter_ and filter_ != 'published' and filter_ != 'provided' and filter_ != 'purchased': return build_response(request, 400, 'Invalid filter') if state and filter_ != 'provided': return build_response(request, 400, 'Invalid filters') if filter_ == 'provider' and not state: return build_response(request, 400, 'Invalid filters') count = False pagination = None # Check if the action is count if action != None: if action == 'count': count = True else: return build_response(request, 400, 'Invalid action') else: # Check pagination params (Only when action is none) if start != None and limit != None: pagination = { 'start': int(start), 'limit': int(limit) } elif (start != None and limit == None) or (start == None and limit != None): return build_response(request, 400, 'Missing pagination param') # Check sorting values if sort != None: if sort != 'date' and sort != 'popularity' and sort != 'name': return build_response(request, 400, 'Invalid sorting') if not filter_ or filter_ == 'published': response = search_engine.full_text_search(request.user, text, count=count, pagination=pagination, sort=sort) elif filter_ == 'provided': response = search_engine.full_text_search(request.user, text, state=state, count=count, pagination=pagination, sort=sort) elif filter_ == 'purchased': response = search_engine.full_text_search(request.user, text, state=['purchased'], count=count, pagination=pagination, sort=sort) return HttpResponse(json.dumps(response), status=200, mimetype='application/json')
def delete_offering(offering): # If the offering has been purchased it is not deleted # it is marked as deleted in order to allow customers that # have purchased the offering to install it if needed #delete the usdl description from the repository if offering.state == 'deleted': raise PermissionDenied('The offering is already deleted') parsed_url = urlparse(offering.description_url) path = parsed_url.path host = parsed_url.scheme + '://' + parsed_url.netloc path = path.split('/') host += '/' + path[1] + '/' + path[2] collection = path[3] repository_adaptor = RepositoryAdaptor(host, collection) repository_adaptor.delete(path[4]) index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') se = SearchEngine(index_path) if offering.state == 'uploaded': _remove_offering(offering, se) else: offering.state = 'deleted' offering.save() # Delete the offering from marketplaces for market in offering.marketplaces: m = Marketplace.objects.get(pk=market) market_adaptor = MarketAdaptor(m.host) market_adaptor.delete_service(settings.STORE_NAME, offering.name) # Update offering indexes if not offering.open: se.update_index(offering) context = Context.objects.all()[0] # Check if the offering is in the newest list if offering.pk in context.newest: # Remove the offering from the newest list newest = context.newest if len(newest) < 8: newest.remove(offering.pk) else: # Get the 8 newest offerings using the publication date for sorting connection = MongoClient() db = connection[settings.DATABASES['default']['NAME']] offerings = db.wstore_offering newest_off = offerings.find({'state': 'published'}).sort('publication_date', -1).limit(8) newest = [] for n in newest_off: newest.append(str(n['_id'])) context.newest = newest context.save() # Check if the offering is in the top rated list if offering.pk in context.top_rated: # Remove the offering from the top rated list top_rated = context.top_rated if len(top_rated) < 8: top_rated.remove(offering.pk) else: # Get the 4 top rated offerings connection = MongoClient() db = connection[settings.DATABASES['default']['NAME']] offerings = db.wstore_offering top_off = offerings.find({'state': 'published', 'rating': {'$gt': 0}}).sort('rating', -1).limit(8) top_rated = [] for t in top_off: top_rated.append(str(t['_id'])) context.top_rated = top_rated context.save() if offering.open: _remove_offering(offering, se)
def create_review(self, user, offering, review): """ Creates a new review for a given offering """ # Check if the user has purchased the offering (Not if the offering is open) if not offering.open: try: purchase = Purchase.objects.get( offering=offering, owner_organization=user.userprofile.current_organization) except: raise PermissionDenied( 'You cannot review this offering since you has not acquire it' ) elif offering.is_owner(user): # If the offering is open, check that the user is not owner of the offering raise PermissionDenied('You cannot review your own offering') # Check if the user has already review the offering. if (user.userprofile.is_user_org() and offering.pk in user.userprofile.rated_offerings)\ or (not user.userprofile.is_user_org() and user.userprofile.current_organization.has_rated_offering(user, offering)): raise PermissionDenied( 'You cannot review this offering again. Please update your review to provide new comments' ) # Validate review data validation = self._validate_content(review) if validation: raise validation # Create the review rev = Review.objects.create( user=user, organization=user.userprofile.current_organization, offering=offering, timestamp=datetime.now(), title=review['title'], comment=review['comment'], rating=review['rating']) offering.comments.insert(0, rev.pk) # Calculate new offering rate old_rate = offering.rating if old_rate == 0: offering.rating = review['rating'] else: offering.rating = ((old_rate * (len(offering.comments) - 1)) + review['rating']) / len(offering.comments) offering.save() # Update offering indexes index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') se = SearchEngine(index_path) se.update_index(offering) # Save the offering as rated if user.userprofile.is_user_org(): user.userprofile.rated_offerings.append(offering.pk) user.userprofile.save() else: user.userprofile.current_organization.rated_offerings.append({ 'user': user.pk, 'offering': offering.pk }) user.userprofile.current_organization.save() # Update top rated list self._update_top_rated()
def create_purchase(user, offering, org_owned=False, payment_info=None): if offering.state != 'published': raise PermissionDenied("This offering can't be purchased") if offering.open: raise PermissionDenied('Open offerings cannot be purchased') if accepted_needed(offering) and not payment_info['accepted']: raise PermissionDenied('You must accept the terms and conditions of the offering to acquire it') profile = UserProfile.objects.get(user=user) # Check if the offering is already purchased if (org_owned and offering.pk in profile.current_organization.offerings_purchased) \ or (not org_owned and offering.pk in profile.offerings_purchased): raise PermissionDenied('The offering has been already purchased') organization = profile.current_organization plan = None # Check the selected plan if payment_info and 'plan' in payment_info: plan = payment_info['plan'] # Get the effective tax address if not 'tax_address' in payment_info: if org_owned: tax = organization.tax_address else: tax = profile.tax_address # Check that the customer has a tax address if not 'street' in tax: raise ValueError('The customer does not have a tax address') else: tax = payment_info['tax_address'] # Check tax_address fields if (not 'street' in tax) or (not 'postal' in tax) or (not 'city' in tax) or (not 'country' in tax): raise ValueError('The tax address is not valid') # Check the payment method before purchase creation in order to avoid # an inconsistent state in the database credit_card_info = None if payment_info['payment_method'] == 'credit_card': if 'credit_card' in payment_info: # Check credit card info if (not ('number' in payment_info['credit_card'])) or (not ('type' in payment_info['credit_card']))\ or (not ('expire_year' in payment_info['credit_card'])) or (not ('expire_month' in payment_info['credit_card']))\ or (not ('cvv2' in payment_info['credit_card'])): raise ValueError('Invalid credit card info') credit_card_info = payment_info['credit_card'] else: if org_owned: credit_card_info = organization.payment_info else: credit_card_info = profile.payment_info # Check the credit card info if not 'number' in credit_card_info: raise Exception('The customer does not have payment info') elif payment_info['payment_method'] != 'paypal': raise ValueError('Invalid payment method') # Create the purchase purchase = Purchase.objects.create( customer=user, date=datetime.now(), offering=offering, organization_owned=org_owned, state='pending', tax_address=tax, owner_organization = organization ) # Load ref purchase.ref = purchase.pk purchase.save() if credit_card_info != None: charging_engine = ChargingEngine(purchase, payment_method='credit_card', credit_card=credit_card_info, plan=plan) else: charging_engine = ChargingEngine(purchase, payment_method='paypal', plan=plan) redirect_url = charging_engine.resolve_charging(new_purchase=True) if redirect_url == None: result = purchase # If no redirect URL is provided the purchase has ended so the user profile # info is updated if org_owned: organization.offerings_purchased.append(offering.pk) organization.save() else: profile.offerings_purchased.append(offering.pk) profile.save() notify_provider(purchase) else: result = redirect_url # Update offering indexes index_path = settings.DATADIR index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') se = SearchEngine(index_path) se.update_index(offering) return result
def update_offering(offering, data): # Check if the offering has been published, # if published the offering cannot be updated if offering.state != 'uploaded' and not offering.open: raise PermissionDenied('The offering cannot be edited') dir_name = offering.owner_organization.name + '__' + offering.name + '__' + offering.version path = os.path.join(settings.MEDIA_ROOT, dir_name) # Update the logo if 'image' in data: logo_path = offering.image_url logo_path = os.path.join(settings.BASEDIR, logo_path[1:]) # Remove the old logo os.remove(logo_path) # Save the new logo f = open(os.path.join(path, data['image']['name']), "wb") dec = base64.b64decode(data['image']['data']) f.write(dec) f.close() offering.image_url = settings.MEDIA_URL + dir_name + '/' + data['image']['name'] # Update the related images if 'related_images' in data: # Delete old related images for img in offering.related_images: old_image = os.path.join(settings.BASEDIR, img[1:]) os.remove(old_image) offering.related_images = [] # Create new images for img in data['related_images']: f = open(os.path.join(path, img['name']), "wb") dec = base64.b64decode(img['data']) f.write(dec) f.close() offering.related_images.append(settings.MEDIA_URL + dir_name + '/' + img['name']) new_usdl = False # Update the USDL description if 'offering_description' in data: usdl_info = data['offering_description'] repository_adaptor = RepositoryAdaptor(offering.description_url) usdl = usdl_info['data'] repository_adaptor.upload(usdl_info['content_type'], usdl) new_usdl = True # The USDL document has changed in the repository elif 'description_url' in data: usdl_info = {} usdl_url = data['description_url'] # Check the link if usdl_url != offering.description_url: raise ValueError('The provided USDL URL is not valid') # Download new content repository_adaptor = RepositoryAdaptor(usdl_url) accept = "text/plain; application/rdf+xml; text/turtle; text/n3" usdl = repository_adaptor.download(content_type=accept) usdl_info['content_type'] = usdl['content_type'] usdl = usdl['data'] new_usdl = True elif 'offering_info' in data: usdl_info = { 'content_type': 'application/rdf+xml' } # Validate USDL info if not 'description' in data['offering_info'] or not 'pricing' in data['offering_info']: raise ValueError('Invalid USDL info') offering_info = data['offering_info'] offering_info['image_url'] = offering.image_url offering_info['name'] = offering.name splited_desc_url = offering.description_url.split('/') base_uri = splited_desc_url[0] + '//' splited_desc_url.remove(splited_desc_url[0]) splited_desc_url.remove(splited_desc_url[0]) splited_desc_url.remove(splited_desc_url[-1]) splited_desc_url.remove(splited_desc_url[-1]) for p in splited_desc_url: base_uri += (p + '/') offering_info['base_uri'] = base_uri usdl = _create_basic_usdl(offering_info) usdl_info = { 'content_type': 'application/rdf+xml' } repository_adaptor = RepositoryAdaptor(offering.description_url) repository_adaptor.upload(usdl_info['content_type'], usdl) new_usdl = True # If the USDL has changed store the new description # in the offering model if new_usdl: # Validate the USDL valid = validate_usdl(usdl, usdl_info['content_type'], { 'name': offering.name, 'organization': offering.owner_organization }) if not valid[0]: raise ValueError(valid[1]) # Serialize and store USDL info in json-ld format graph = rdflib.Graph() rdf_format = usdl_info['content_type'] if usdl_info['content_type'] == 'text/turtle' or usdl_info['content_type'] == 'text/plain': rdf_format = 'n3' elif usdl_info['content_type'] == 'application/json': rdf_format = 'json-ld' off_description = usdl if rdf_format != 'json-ld': graph.parse(data=usdl, format=rdf_format) off_description = graph.serialize(format='json-ld', auto_compact=True) offering.offering_description = json.loads(off_description) offering.save() # Update offering indexes index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') se = SearchEngine(index_path) se.update_index(offering)
def create_offering(provider, json_data): profile = provider.userprofile data = {} if not 'name' in json_data or not 'version' in json_data: raise ValueError('Missing required fields') data['name'] = json_data['name'] data['version'] = json_data['version'] if not re.match(re.compile(r'^(?:[1-9]\d*\.|0\.)*(?:[1-9]\d*|0)$'), data['version']): raise ValueError('Invalid version format') if not is_valid_id(data['name']): raise ValueError('Invalid name format') # Get organization organization = profile.current_organization # Check if the offering already exists existing = True try: Offering.objects.get(name=data['name'], owner_organization=organization, version=data['version']) except: existing = False if existing: raise Exception('The offering already exists') # Check if the version of the offering is lower than an existing one offerings = Offering.objects.filter(owner_organization=organization, name=data['name']) for off in offerings: if is_lower_version(data['version'], off.version): raise ValueError('A bigger version of the current offering exists') is_open = json_data.get('open', False) # If using the idm, get the applications from the request if settings.OILAUTH: # Validate application structure data['applications'] = [] for app in json_data['applications']: data['applications'].append({ 'name': app['name'], 'url': app['url'], 'id': app['id'], 'description': app['description'] }) data['related_images'] = [] # Check the URL to notify the provider notification_url = '' if 'notification_url' in json_data: if json_data['notification_url'] == 'default': notification_url = organization.notification_url if not notification_url: raise ValueError('There is not a default notification URL defined for the organization ' + organization.name + '. To configure a default notification URL provide it in the settings menu') else: # Check the notification URL if not is_valid_url(json_data['notification_url']): raise ValueError("Invalid notification URL format: It doesn't seem to be an URL") notification_url = json_data['notification_url'] # Create the directory for app media dir_name = organization.name + '__' + data['name'] + '__' + data['version'] path = os.path.join(settings.MEDIA_ROOT, dir_name) os.makedirs(path) if not 'image' in json_data: raise ValueError('Missing required field: Logo') if not isinstance(json_data['image'], dict): raise TypeError('Invalid image type') image = json_data['image'] if not 'name' in image or not 'data' in image: raise ValueError('Missing required field in image') # Save the application image or logo f = open(os.path.join(path, image['name']), "wb") dec = base64.b64decode(image['data']) f.write(dec) f.close() data['image_url'] = settings.MEDIA_URL + dir_name + '/' + image['name'] # Save screen shots if 'related_images' in json_data: for image in json_data['related_images']: # images must be encoded in base64 format f = open(os.path.join(path, image['name']), "wb") dec = base64.b64decode(image['data']) f.write(dec) f.close() data['related_images'].append(settings.MEDIA_URL + dir_name + '/' + image['name']) # Save USDL document # If the USDL itself is provided if 'offering_description' in json_data: usdl_info = json_data['offering_description'] repository = Repository.objects.get(name=json_data['repository']) repository_adaptor = RepositoryAdaptor(repository.host, 'storeOfferingCollection') offering_id = organization.name + '__' + data['name'] + '__' + data['version'] usdl = usdl_info['data'] data['description_url'] = repository_adaptor.upload(usdl_info['content_type'], usdl_info['data'], name=offering_id) # If the USDL is already uploaded in the repository elif 'description_url' in json_data: # Check that the link to USDL is unique since could be used to # purchase offerings from Marketplace usdl_info = {} usdl_url = json_data['description_url'] off = Offering.objects.filter(description_url=usdl_url) if len(off) != 0: raise ValueError('The provided USDL description is already registered') # Download the USDL from the repository repository_adaptor = RepositoryAdaptor(usdl_url) accept = "text/plain; application/rdf+xml; text/turtle; text/n3" usdl = repository_adaptor.download(content_type=accept) usdl_info['content_type'] = usdl['content_type'] usdl = usdl['data'] data['description_url'] = usdl_url # If the USDL is going to be created elif 'offering_info' in json_data: _validate_offering_info(json_data['offering_info']) offering_info = json_data['offering_info'] offering_info['image_url'] = data['image_url'] offering_info['name'] = data['name'] repository = Repository.objects.get(name=json_data['repository']) offering_info['base_uri'] = repository.host usdl = _create_basic_usdl(offering_info) usdl_info = { 'content_type': 'application/rdf+xml' } repository_adaptor = RepositoryAdaptor(repository.host, 'storeOfferingCollection') offering_id = organization.name + '__' + data['name'] + '__' + data['version'] data['description_url'] = repository_adaptor.upload(usdl_info['content_type'], usdl, name=offering_id) else: raise Exception('No USDL description provided') # Validate the USDL data['open'] = is_open data['organization'] = organization valid = validate_usdl(usdl, usdl_info['content_type'], data) if not valid[0]: raise Exception(valid[1]) # Check new currencies used if len(valid) > 2: new_curr = valid[2] # Set the currency as used cont = Context.objects.all()[0] currency = None # Search the currency for c in cont.allowed_currencies['allowed']: if c['currency'].lower() == new_curr.lower(): currency = c break cont.allowed_currencies['allowed'].remove(currency) currency['in_use'] = True cont.allowed_currencies['allowed'].append(currency) cont.save() # Serialize and store USDL info in json-ld format graph = rdflib.Graph() rdf_format = usdl_info['content_type'] if rdf_format == 'text/turtle' or rdf_format == 'text/plain': rdf_format = 'n3' elif rdf_format == 'application/json': rdf_format = 'json-ld' graph.parse(data=usdl, format=rdf_format) data['offering_description'] = graph.serialize(format='json-ld', auto_compact=True) # Create the offering offering = Offering.objects.create( name=data['name'], owner_organization=organization, owner_admin_user=provider, version=data['version'], state='uploaded', description_url=data['description_url'], resources=[], comments=[], tags=[], image_url=data['image_url'], related_images=data['related_images'], offering_description=json.loads(data['offering_description']), notification_url=notification_url, creation_date=datetime.now(), open=is_open ) if settings.OILAUTH: offering.applications = data['applications'] offering.save() if 'resources' in json_data and len(json_data['resources']) > 0: bind_resources(offering, json_data['resources'], profile.user) # Load offering document to the search index index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') search_engine = SearchEngine(index_path) search_engine.create_index(offering)
def delete_offering(user, offering): # If the offering has been purchased it is not deleted # it is marked as deleted in order to allow customers that # have purchased the offering to install it if needed # delete the usdl description from the repository if offering.state == 'deleted': raise PermissionDenied('The offering is already deleted') if offering.state == 'published' and len(offering.description_url): repository_adaptor = unreg_repository_adaptor_factory(offering.description_url) if settings.OILAUTH: repository_adaptor.set_credentials(user.userprofile.access_token) repository_adaptor.delete() index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') se = SearchEngine(index_path) if offering.state == 'uploaded': _remove_offering(offering, se) else: offering.state = 'deleted' offering.save() # Delete the offering from marketplaces for market in offering.marketplaces: market_adaptor = marketadaptor_factory(market.marketplace, user) market_adaptor.delete_service(market.offering_name) # Update offering indexes if not offering.open: se.update_index(offering) context = Context.objects.all()[0] # Check if the offering is in the newest list if offering.pk in context.newest: # Remove the offering from the newest list newest = context.newest if len(newest) < 8: newest.remove(offering.pk) else: # Get the 8 newest offerings using the publication date for sorting db = get_database_connection() offerings = db.wstore_offering newest_off = offerings.find({'state': 'published'}).sort('publication_date', -1).limit(8) newest = [] for n in newest_off: newest.append(str(n['_id'])) context.newest = newest context.save() # Check if the offering is in the top rated list if offering.pk in context.top_rated: # Remove the offering from the top rated list top_rated = context.top_rated if len(top_rated) < 8: top_rated.remove(offering.pk) else: # Get the 4 top rated offerings db = get_database_connection() offerings = db.wstore_offering top_off = offerings.find({'state': 'published', 'rating': {'$gt': 0}}).sort('rating', -1).limit(8) top_rated = [] for t in top_off: top_rated.append(str(t['_id'])) context.top_rated = top_rated context.save() if offering.open: _remove_offering(offering, se)
def publish_offering(user, offering, data): # Validate data if 'marketplaces' not in data: raise ValueError('Publication error: missing required field, marketplaces') # Validate the state of the offering if not offering.state == 'uploaded': raise PermissionDenied('Publication error: The offering ' + offering.name + ' ' + offering.version + ' cannot be published') # Validate the offering has enough content to be published # Open offerings cannot be published in they do not contain # digital assets (applications or resources) if offering.open and not len(offering.resources) and not len(offering.applications): raise PermissionDenied('Publication error: Open offerings cannot be published if they do not contain at least a digital asset (resource or application)') # Check if it possible to publish the offering in a Marketplace if not len(Repository.objects.all()) > 0 and len(data['marketplaces']) > 0: raise PermissionDenied('Publication error: It is not possible to publish an offering in a Markteplace if no Repositories has been registered') # Upload the USDL description of the offering to the repository if len(Repository.objects.all()) > 0: repository = Repository.objects.get(is_default=True) # Generate the USDL of the offering generator = USDLGenerator() usdl, offering_uri = generator.generate_offering_usdl(offering) repository_adaptor = repository_adaptor_factory(repository) offering_id = offering.owner_organization.name + '__' + offering.name + '__' + offering.version repository_adaptor.set_uri(offering_uri) if settings.OILAUTH: repository_adaptor.set_credentials(user.userprofile.access_token) offering.description_url = repository_adaptor.upload('application/rdf+xml', usdl, name=offering_id) # Publish the offering in the selected marketplaces for market in data['marketplaces']: try: m = Marketplace.objects.get(name=market) except: raise ValueError('Publication error: The marketplace ' + market + ' does not exist') market_adaptor = marketadaptor_factory(m, user) off_market_name = market_adaptor.add_service(offering) offering.marketplaces.append(MarketOffering( marketplace=m, offering_name=off_market_name )) # Create revenue sharing models if needed build_rs_model(offering) offering.state = 'published' offering.publication_date = datetime.now() offering.save() # Update offering indexes index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') se = SearchEngine(index_path) se.update_index(offering)
def rollback(purchase): # If the purchase state is paid means that the purchase has been made # so the models must not be deleted offering = purchase.offering if purchase.state != 'paid': # Check that the payment has been made contract = True try: contr = purchase.contract except: contract = False to_del = True if contract: # If the charges field contains any charge means that it is not # the first charge so the models cannot be deleted if len(contr.charges) > 0: purchase.state = 'paid' purchase.save() to_del = False if to_del: # Check organization owned if purchase.organization_owned: org = purchase.owner_organization if purchase.offering.pk in org.offerings_purchased: org.offerings_purchased.remove(purchase.offering.pk) org.save() # Delete the offering from the user profile user_profile = UserProfile.objects.get(user=purchase.customer) if purchase.offering.pk in user_profile.offerings_purchased: user_profile.offerings_purchased.remove(purchase.offering.pk) user_profile.save() # Delete the contract if contract: purchase.contract.delete() # Delete the Purchase purchase.delete() # If the purchase is paid the offering must be included in the customer # offerings purchased list else: if purchase.organization_owned: org = purchase.owner_organization if not purchase.offering.pk in org.offerings_purchased: org.offerings_purchased.append(purchase.offering.pk) org.save() else: profile = purchase.customer.userprofile if not purchase.offering.pk in profile.offerings_purchased: profile.offerings_purchased.append(purchase.offering.pk) profile.save() # Update offering indexes: Offering index must be updated in any case index_path = settings.DATADIR index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') se = SearchEngine(index_path) se.update_index(offering)
def update_offering(user, offering, data): # Check if the offering has been published, # if published the offering cannot be updated if offering.state != 'uploaded' and not offering.open: raise PermissionDenied('The offering cannot be edited') dir_name = offering.owner_organization.name + '__' + offering.name + '__' + offering.version path = os.path.join(settings.MEDIA_ROOT, dir_name) # Update the logo if 'image' in data: logo_path = offering.image_url logo_path = os.path.join(settings.BASEDIR, logo_path[1:]) # Remove the old logo os.remove(logo_path) # Save the new logo _save_encoded_image(path, data['image']['name'], data['image']['data']) offering.image_url = settings.MEDIA_URL + dir_name + '/' + data['image']['name'] # Update the related images if 'related_images' in data: # Delete old related images for img in offering.related_images: old_image = os.path.join(settings.BASEDIR, img[1:]) os.remove(old_image) offering.related_images = [] # Create new images for img in data['related_images']: _save_encoded_image(path, img['name'], img['data']) offering.related_images.append(settings.MEDIA_URL + dir_name + '/' + img['name']) if 'offering_description' in data: # Create offering USDL offering_info = deepcopy(data['offering_description']) offering_info['image_url'] = offering.image_url offering_info['name'] = offering.name offering_info['version'] = offering.version offering_info['organization'] = offering.owner_organization.name offering_info['base_id'] = offering.pk offering_info['created'] = unicode(offering.creation_date) mod = unicode(datetime.now()) offering_info['modified'] = mod usdl_generator = USDLGenerator() usdl_generator.validate_info(offering_info, offering.owner_organization, open_=offering.open) data['offering_description']['modified'] = mod offering.offering_description = data['offering_description'] if offering.open and offering.state == 'published' and len(offering.description_url): repository_adaptor = unreg_repository_adaptor_factory(offering.description_url) if settings.OILAUTH: repository_adaptor.set_credentials(user.userprofile.access_token) repository_adaptor.upload( 'application/rdf+xml', usdl_generator.generate_offering_usdl(offering)[0] ) offering.save() # Update offering indexes index_path = os.path.join(settings.BASEDIR, 'wstore') index_path = os.path.join(index_path, 'search') index_path = os.path.join(index_path, 'indexes') se = SearchEngine(index_path) se.update_index(offering)