def handle(self, *args, **options):
interactive = True
if len(args) and args[0] == '--no-input':
interactive = False
# Ask the user if interactive
if interactive:
correct = False
print "This process will delete the tag indexes directory. Continue: [y/n]"
while not correct:
opt = read_from_cmd()
if opt != 'y' and opt != 'n':
print "Please include 'y' or 'n'"
else:
correct = True
if opt == 'n':
return
# Remove the index directory
index_path = os.path.join(settings.BASEDIR, 'wstore')
index_path = os.path.join(index_path, 'social')
index_path = os.path.join(index_path, 'indexes')
rmtree(index_path, True)
# Generate new search indexes
tag_manager = TagManager(index_path)
for o in Offering.objects.all():
aux_tags = list(o.tags)
o.tags = []
o.save()
tag_manager.update_tags(o, aux_tags)
def handle(self, *args, **options):
interactive = True
if len(args) and args[0] == '--no-input':
interactive = False
# Ask the user if interactive
if interactive:
correct = False
print "This process will delete the tag indexes directory. Continue: [y/n]"
while not correct:
opt = read_from_cmd()
if opt != 'y' and opt != 'n':
print "Please include 'y' or 'n'"
else:
correct = True
if opt == 'n':
return
# Remove the index directory
index_path = os.path.join(settings.BASEDIR, 'wstore')
index_path = os.path.join(index_path, 'social')
index_path = os.path.join(index_path, 'indexes')
rmtree(index_path, True)
# Generate new search indexes
tag_manager = TagManager(index_path)
for o in Offering.objects.all():
aux_tags = list(o.tags)
o.tags = []
o.save()
tag_manager.update_tags(o, aux_tags)
def _remove_offering(offering, se):
# If the offering has media files delete them
dir_name = offering.owner_organization.name + '__' + offering.name + '__' + offering.version
path = os.path.join(settings.MEDIA_ROOT, dir_name)
try:
files = os.listdir(path)
for f in files:
file_path = os.path.join(path, f)
os.remove(file_path)
os.rmdir(path)
except OSError as e:
# An OS error means that offering files
# does not exist so continue with the deletion
pass
# Remove the search index
se.remove_index(offering)
# Remove the offering ID from the tag indexes
if len(offering.tags):
tm = TagManager()
tm.delete_tag(offering)
# Remove the offering pk from the bound resources
for res in offering.resources:
resource = Resource.objects.get(pk=unicode(res))
resource.offerings.remove(offering.pk)
resource.save()
offering.delete()
def _create_tags():
tm = TagManager()
offering1 = Offering.objects.get(name='test_offering1')
offering2 = Offering.objects.get(name='test_offering2')
offering3 = Offering.objects.get(name='test_offering3')
tm.update_tags(offering1, ['service', 'tag'])
tm.update_tags(offering2, ['dataset', 'tag'])
tm.update_tags(offering3, ['widget'])
def run(self):
tag_manager = TagManager()
frequency_map = {
'user_tags_freq': {}, # Absolute frequencies of user tags
'co-tags':{} # tags and user tags intersection frequency
}
# Iterate over user tags
for tag in self._user_tags:
# Stem the tag
st_tag = stem(tag)
# Get indexed documents where the tag is included
docs = tag_manager.get_index_doc_by_tag(st_tag)
# Include user tags frequency
if len(docs) > 0:
frequency_map['user_tags_freq'][st_tag] = len(docs)
for doc in docs:
# Get tags
tags = doc['named_tags'].split(' ')
# Populate the frequencies map: for user tags it contains the absolute
# frequency of the tag, for co-occurrence tags it contains the frequency
# of the intersection with the corresponding user tag
for not_stem_tag in tags:
t = stem(not_stem_tag)
if t and (self._include_user_tags or (not self._include_user_tags and t != st_tag)):
if not t in frequency_map['co-tags']:
frequency_map['co-tags'][t] = {
'named_tag': not_stem_tag,
'user_tags': {
st_tag: 1
}
}
else:
if st_tag in frequency_map['co-tags'][t]['user_tags']:
frequency_map['co-tags'][t]['user_tags'][st_tag] += 1
else:
frequency_map['co-tags'][t]['user_tags'][st_tag] = 1
# Calculate tag rank and populate tag container
self._rank_tags(frequency_map)
def update(self, request, organization, name, version):
# Get offering
try:
org = Organization.objects.get(name=organization)
offering = Offering.objects.get(owner_organization=org, name=name, version=version)
except:
return build_response(request, 404, 'Not found')
# Check that the user has enough rights
if request.user.userprofile.current_organization != org:
return build_response(request, 403, 'Forbidden')
# Build tag manager
try:
data = json.loads(request.raw_post_data)
manager = TagManager()
manager.update_tags(offering, data['tags'])
except Exception, e:
return build_response(request, 400, e.message)
def update(self, request, organization, name, version):
# Get offering
try:
org = Organization.objects.get(name=organization)
offering = Offering.objects.get(owner_organization=org,
name=name,
version=version)
except:
return build_response(request, 404, 'Not found')
# Check that the user has enough rights
if request.user.userprofile.current_organization != org\
or (not offering.is_owner(request.user) and not request.user.pk in org.managers):
return build_response(request, 403, 'Forbidden')
# Build tag manager
try:
data = json.loads(request.raw_post_data)
manager = TagManager()
manager.update_tags(offering, data['tags'])
except Exception, e:
return build_response(request, 400, e.message)
def _create_tags():
tm = TagManager()
offering1 = Offering.objects.get(name='test_offering1')
offering2 = Offering.objects.get(name='test_offering2')
offering3 = Offering.objects.get(name='test_offering3')
tm.update_tags(offering1, ['service', 'tag'])
tm.update_tags(offering2, ['dataset', 'tag'])
tm.update_tags(offering3, ['widget'])
def read(self, request, tag):
# Get query params
action = request.GET.get('action', None)
start = request.GET.get('start', None)
limit = request.GET.get('limit', None)
sort = request.GET.get('sort', None)
state = request.GET.get('filter', None)
# Check action format
if action and not (isinstance(action, str)
or isinstance(action, unicode)):
return build_response(request, 400, 'Invalid action format')
# Validate action
if action and action != 'count':
return build_response(request, 400, 'Invalid action')
if action and (start or limit or sort):
return build_response(
request, 400, 'Actions cannot be combined with pagination')
# Validate pagination
if (start and not limit) or (not start and limit):
return build_response(request, 400,
'Both pagination params are required')
if start and limit and (not start.isnumeric()
or not limit.isnumeric()):
return build_response(request, 400,
'Invalid format in pagination parameters')
elif start and limit:
start = int(start)
limit = int(limit)
if start and limit and (start < 1 or limit < 1):
return build_response(
request, 400,
'Pagination params must be equal or greater that 1')
# Validate sorting
allowed_sorting = ['name', 'date', 'popularity']
if sort and (not isinstance(sort, str)
and not isinstance(sort, unicode)):
return build_response(request, 400, 'Invalid sorting format')
if sort and not sort in allowed_sorting:
return build_response(request, 400, 'Invalid sorting value')
# Validate state
if state and state != 'published' and state != 'purchased':
return build_response(request, 400, 'Invalid filter')
try:
# Build tag manager
tm = TagManager()
# Select action
if action == 'count':
response = {'number': tm.count_offerings(tag)}
else:
offerings = tm.search_by_tag(tag)
response = []
# Get offering info
for off in offerings:
offering_info = get_offering_info(off, request.user)
if not state and offering_info['state'] != 'published'\
and offering_info['state'] != 'purchased' and offering_info['state'] != 'rated':
continue
response.append(offering_info)
# Sort offerings if needed
if sort:
rev = True
if sort == 'name':
rev = False
elif sort == 'date':
sort = 'publication_date'
elif sort == 'popularity':
sort = 'rating'
response = sorted(response,
key=lambda off: off[sort],
reverse=rev)
# If sort was needed pagination must be done after sorting
if start and limit:
response = response[start - 1:(limit + (start - 1))]
except Exception as e:
return build_response(request, 400, unicode(e))
# Create response
return HttpResponse(json.dumps(response),
status=200,
mimetype='application/json; charset=utf-8')
def read(self, request, tag):
# Get query params
action = request.GET.get('action', None)
start = request.GET.get('start', None)
limit = request.GET.get('limit', None)
sort = request.GET.get('sort', None)
state = request.GET.get('filter', None)
# Check action format
if action and not (isinstance(action,str) or isinstance(action,unicode)):
return build_response(request, 400, 'Invalid action format')
# Validate action
if action and action != 'count':
return build_response(request, 400, 'Invalid action')
if action and (start or limit or sort):
return build_response(request, 400, 'Actions cannot be combined with pagination')
# Validate pagination
if (start and not limit) or (not start and limit):
return build_response(request, 400, 'Both pagination params are required')
if start and limit and (not start.isnumeric() or not limit.isnumeric()):
return build_response(request, 400, 'Invalid format in pagination parameters')
elif start and limit:
start = int(start)
limit = int(limit)
if start and limit and (start < 1 or limit < 1):
return build_response(request, 400, 'Pagination params must be equal or greater that 1')
# Validate sorting
allowed_sorting = ['name', 'date', 'popularity']
if sort and (not isinstance(sort, str) and not isinstance(sort, unicode)):
return build_response(request, 400, 'Invalid sorting format')
if sort and not sort in allowed_sorting:
return build_response(request, 400, 'Invalid sorting value')
# Validate state
if state and state != 'published' and state != 'purchased':
return build_response(request, 400, 'Invalid filter')
try:
# Build tag manager
tm = TagManager()
# Select action
if action == 'count':
response = {
'number': tm.count_offerings(tag)
}
else:
offerings = tm.search_by_tag(tag)
response = []
# Get offering info
for off in offerings:
offering_info = get_offering_info(off, request.user)
if not state and offering_info['state'] != 'published'\
and offering_info['state'] != 'purchased' and offering_info['state'] != 'rated':
continue
response.append(offering_info)
# Sort offerings if needed
if sort:
rev = True
if sort == 'name':
rev = False
elif sort == 'date':
sort = 'publication_date'
elif sort == 'popularity':
sort = 'rating'
response = sorted(response, key=lambda off: off[sort], reverse=rev)
# If sort was needed pagination must be done after sorting
if start and limit:
response = response[start - 1: (limit + (start - 1))]
except Exception as e:
return build_response(request, 400, unicode(e))
# Create response
return HttpResponse(json.dumps(response), status=200, mimetype='application/json; charset=utf-8')