def filter(css_selector, lxmltree, include_references = True): """ Returns an iterable over the elements from @lxmltree matching the css selector @css_selector. If @include_references is True, the iterable will contain an additional <references> element which will contain all referenced elements not already present. """ try: selector = CSSSelector(css_selector) if include_references: refs_selector = CSSSelector('ref') ref_ids = set([]) ret = [] # Find all references in the selected elements # and store the referenced urls (should be of the form #id) # in ref_ids for ch in selector(lxmltree): ret.append(ch) for ref_node in refs_selector(ch): a_nodes = ref_node.findall('a') if len(a_nodes) > 0: ref_ids.add(a_nodes[-1].get('href',None)) if len(ref_ids) > 0: # Create the <references> element and an appropriate heading for it ref_parent = lxml.etree.Element('references') ref_h = lxml.etree.SubElement(ref_parent,'h1') ref_h.set('class','do_not_number') ref_h.text = 'References' references_found = False # Iterate over the reference ids and check that they exist # and are not already present in the selected elements for id in sorted(ref_ids): if id: logger.debug("Checking reference "+id) # Only include the reference if it is not already present in the selected elements if get_by_id(selector(lxmltree),id[1:]) is None: logger.debug("Adding reference "+id) ref = get_by_id(lxmltree,id[1:]) if ref is None: logger.warn("Reference "+id+" not found") else: references_found = True ref_parent.append(ref) # Do not include the <references> element if there were no references if references_found: ret.append(ref_parent) return ret return selector(lxmltree) except Exception as e: logger.critical("Error parsing filter: "+css_selector+" ("+str(e)+")") return []
def order(qs, sort, prefix='sort.'): order_by = [] if len(sort) == 1: additional_sort = settings.config['user']['ui']['listSort'] key = utils.get_by_id(models.Item.item_keys, sort[0]['key']) for s in key.get('additionalSort', additional_sort): if s['key'] not in [e['key'] for e in sort]: sort.append(s) for e in sort: operator = e['operator'] if operator != '-': operator = '' else: operator = ' DESC' key = {}.get(e['key'], e['key']) if key not in ('fixme', ): key = "%s%s" % (prefix, key) order = '%s%s' % (key, operator) order_by.append(order) if order_by: #nulllast not supported in sqlite, use IS NULL hack instead #order_by = map(nullslast, order_by) _order_by = [] for order in order_by: nulls = "%s IS NULL" % order.split(' ')[0] _order_by.append(nulls) _order_by.append(order) order_by = _order_by qs = qs.join(models.Sort).order_by(*order_by) return qs
def test_get_by_valid_id(self): id = f"{osu_id}-{term_id}" res = utils.get_by_id(id) self.assertIn('data', res.json()) self.assertIsInstance(res.json()['data'], dict) self.assert_response_time(res, 3) self.assertEqual(res.status_code, 200)
def test_get(self, type, res_type, has_id): # Test GET /type valid_types = utils.get(type) validate_response(self, valid_types, 200) check_null_fields(self, valid_types.json()) # Test GET /type/{id} if has_id: # Valid ID try: valid_id = valid_types.json()["data"][0]["id"] valid_type = utils.get_by_id(type, valid_id) validate_response(self, valid_type, 200, res_type) except IndexError: warnings.append("Could not test GET /{0}/{{id}} with valid ID. " "No {0} found".format(type)) # Invalid ID invalid_id = utils.get_by_id(type, utils.invalid_id) validate_response(self, invalid_id, 404, message="Not Found") return valid_types
def autocomplete(data): ''' takes { key: string, value: string, operator: string // '=', '==', '^', '$' query: object // item query to limit results range: [int, int] } returns { items: [string, ...] //array of matching values } ''' response = {} response['items'] = [] if not 'range' in data: data['range'] = [0, 10] op = data.get('operator', '=') key = utils.get_by_id(settings.config['itemKeys'], data['key']) order_by = key.get('autocompleteSort', False) add_itemsort = False if order_by: for o in order_by: if o['operator'] != '-': o['operator'] = '' order_by = ['%(operator)ssort.%(key)s' % o for o in order_by] add_itemsort = True else: order_by = ['-items'] items = query.parse({'query': data.get('query', {})})['qs'].options(load_only('id')) qs = state.db.session.query(models.Find.value, func.count(models.Find.value).label('items')) qs = qs.filter(models.Find.item_id.in_(items)) if data['value']: value = data['value'].lower() qs = qs.filter(models.Find.key.is_(data['key'])) if op == '=': qs = qs.filter(models.Find.findvalue.contains(value)) elif op == '==': qs = qs.filter(models.Find.findvalue.is_(value)) elif op == '^': qs = qs.filter(models.Find.findvalue.startswith(value)) elif op == '$': qs = qs.filter(models.Find.findvalue.endswith(value)) qs = qs.group_by(models.Find.value) if add_itemsort: qs = qs.join(models.Item).join(models.Sort) qs = qs.order_by(*order_by) response['items'] = [ r.value for r in qs[data['range'][0]:data['range'][1]] ] return response
def autocomplete(data): ''' takes { key: string, value: string, operator: string // '=', '==', '^', '$' query: object // item query to limit results range: [int, int] } returns { items: [string, ...] //array of matching values } ''' response = {} response['items'] = [] if not 'range' in data: data['range'] = [0, 10] op = data.get('operator', '=') key = utils.get_by_id(settings.config['itemKeys'], data['key']) order_by = key.get('autocompleteSort', False) add_itemsort = False if order_by: for o in order_by: if o['operator'] != '-': o['operator'] = '' order_by = ['%(operator)ssort.%(key)s' % o for o in order_by] add_itemsort = True else: order_by = ['-items'] items = query.parse({'query': data.get('query', {})})['qs'].options(load_only('id')) qs = state.db.session.query(models.Find.value, func.count(models.Find.value).label('items')) qs = qs.filter(models.Find.item_id.in_(items)) if data['value']: value = data['value'].lower() qs = qs.filter(models.Find.key.is_(data['key'])) if op == '=': qs = qs.filter(models.Find.findvalue.contains(value)) elif op == '==': qs = qs.filter(models.Find.findvalue.is_(value)) elif op == '^': qs = qs.filter(models.Find.findvalue.startswith(value)) elif op == '$': qs = qs.filter(models.Find.findvalue.endswith(value)) qs = qs.group_by(models.Find.value) if add_itemsort: qs = qs.join(models.Item).join(models.Sort) qs = qs.order_by(*order_by) response['items'] = [r.value for r in qs[data['range'][0]:data['range'][1]]] return response
def _order_query(qs, sort, prefix='sort__'): order_by = [] if len(sort) == 1: key = utils.get_by_id(settings.CONFIG['itemKeys'], sort[0]['key']) for s in key.get('additionalSort', settings.CONFIG.get('additionalSort', [])): sort.append(s) for e in sort: operator = e['operator'] if operator != '-': operator = '' key = { 'id': 'itemId', 'index': 'listitem__index' }.get(e['key'], e['key']) if key not in ('listitem__index', ): key = "%s%s" % (prefix, key) order = '%s%s' % (operator, key) order_by.append(order) if order_by: qs = qs.order_by(*order_by, nulls_last=True) return qs
def details(id): dt = utils.get_by_id(id) introduction = utils.cleanhtml(dt["introduction"]) s_note = utils.cleanhtml(dt["special_note"]) u_fts = utils.cleanhtml(dt["unique_features"]) f_t = dt["featured_photo"] m_f = dt["Phi_tang_them"] min_ = dt["So_dem_toi_thieu"] max_ = dt["So_dem_toi_da"] c_in = dt["checkin_time"] c_out = dt["checkout_time"] add = dt["full_address"] m_t = dt['Mon_to_thurs'] f_s = dt["Fri_to_sun"] pt_1 = dt["photos_1"] pt_2 = dt["photos_2"] pt_3 = dt["photos_3"] pt_4 = dt["photos_4"] return render_template("detail.html", introduction=introduction, name=dt["name"], pt_1=pt_1, pt_2=pt_2, pt_3=pt_3, pt_4=pt_4, s_note=s_note, u_f=u_fts, m_t=m_t, f_s=f_s, m_f=m_f, min=min_, max=max_, c_in=c_in, c_out=c_out, add=add, f_t=f_t)
def parse_condition(self, condition): ''' condition: { value: "war" } or condition: { key: "year", value: [1970, 1980], operator: "=" } ... ''' #logger.debug('parse_condition %s', condition) if not 'value' in condition: return None k = condition.get('key', '*') if not k: k = '*' v = condition['value'] op = condition.get('operator') if not op: op = '=' if op.startswith('!'): op = op[1:] exclude = True else: exclude = False key_type = (utils.get_by_id(self.item_keys, k) or {'type': 'string'}).get('type') if isinstance(key_type, list): key_type = key_type[0] if k == 'list': key_type = '' if (not exclude and op == '=' or op in ('$', '^')) and v == '': return None elif k == 'resolution': q = self.parse_condition({'key': 'width', 'value': v[0], 'operator': op}) \ & self.parse_condition({'key': 'height', 'value': v[1], 'operator': op}) if exclude: q = ~q return q elif isinstance(v, list) and len(v) == 2 and op == '=': q = self.parse_condition({'key': k, 'value': v[0], 'operator': '>='}) \ & self.parse_condition({'key': k, 'value': v[1], 'operator': '<'}) if exclude: q = ~q return q elif key_type == 'boolean': q = getattr(self._model, 'find_%s' % k) == v if exclude: q = ~q return q elif key_type in ("string", "text"): if isinstance(v, str): v = unicodedata.normalize('NFKD', v).lower() else: v = v.lower() q = get_operator(op)(self._find.findvalue, v) if k != '*': q &= (self._find.key == k) ids = self._model.query.join(self._find).filter(q).options(load_only('id')) in_op = operators.notin_op if exclude else operators.in_op q = in_op(self._model.id, ids) return q elif k == 'list': nickname, name = v.split(':', 1) if nickname: u = self._user.query.filter_by(nickname=nickname).one() else: u = self._user.query.filter_by(id=settings.USER_ID).one() l = self._list.query.filter_by(user_id=u.id, name=name).one() ids = l.get_items().options(load_only('id')) in_op = operators.notin_op if exclude else operators.in_op q = in_op(self._model.id, ids) return q elif key_type == 'date': def parse_date(d): while len(d) < 3: d.append(1) return datetime(*[int(i) for i in d]) #using sort here since find only contains strings v = parse_date(v.split('-')) vk = getattr(self._sort, k) q = get_operator(op, 'int')(vk, v) ids = self._model.query.join(self._find).filter(q).options(load_only('id')) in_op = operators.notin_op if exclude else operators.in_op q = in_op(self._model.id, ids) return q else: #integer, float, time q = get_operator(op, 'int')(getattr(self._sort, k), v) ids = self._model.query.join(self._find).filter(q).options(load_only('id')) in_op = operators.notin_op if exclude else operators.in_op q = in_op(self._model.id, ids) return q
def find(data): ''' takes { query { conditions [{}] operator string } group string keys [string] sort [{}] range [int, int] } ''' response = {} q = query.parse(data) if 'group' in q: names = {} groups = {} key = 'group:' + hashlib.sha1(json.dumps(data).encode('utf-8')).hexdigest() g = state.cache.get(key) if g is None: items = q['qs'].options(load_only('id')) qs = models.Find.query.filter_by(key=q['group']) if items.first(): qs = qs.filter(models.Find.item_id.in_(items)) for f in qs.values('value', 'findvalue'): value = f[0] findvalue = f[1] if findvalue not in groups: groups[findvalue] = 0 groups[findvalue] += 1 names[findvalue] = value g = [{'name': names[k], 'items': groups[k]} for k in groups] else: g = [] if 'sort' in q: sort_type = utils.get_by_id(settings.config['itemKeys'], q['group']).get('sortType') def _sort_key(k): if sort_type == 'person' and q['sort'][0]['key'] == 'name': v = get_sort_name(k[q['sort'][0]['key']]) else: v = k[q['sort'][0]['key']] if isinstance(v, str): v = unicodedata.normalize('NFKD', v).lower() return v g.sort(key=_sort_key, reverse=q['sort'][0]['operator'] == '-') state.cache.set(key, g) if 'positions' in data: response['positions'] = {} ids = [k['name'] for k in g] response['positions'] = utils.get_positions(ids, data['positions']) elif 'range' in data: response['items'] = g[q['range'][0]:q['range'][1]] else: response['items'] = len(g) elif 'position' in data: ids = [i.id for i in q['qs'].options(load_only('id'))] response['position'] = utils.get_positions(ids, [data['qs'][0].id])[0] elif 'positions' in data: ids = [i.id for i in q['qs'].options(load_only('id'))] response['positions'] = utils.get_positions(ids, data['positions']) elif 'keys' in data: response['items'] = [ i.json(data['keys']) for i in q['qs'][q['range'][0]:q['range'][1]] ] else: size = [i.info.get('size', 0) for i in q['qs'].options(load_only('id', 'info'))] response['items'] = len(size) response['size'] = sum(size) return response
def autocomplete(request): ''' takes { key: string, value: string, operator: string // '=', '==', '^', '$' query: object // item query to limit results range: [int, int] } returns { items: [string, ...] //array of matching values } ''' data = json.loads(request.POST['data']) if not 'range' in data: data['range'] = [0, 10] op = data.get('operator', '=') key = utils.get_by_id(settings.CONFIG['itemKeys'], data['key']) order_by = key.get('autocompleteSort', False) if order_by: for o in order_by: if o['operator'] != '-': o['operator'] = '' order_by = ','.join( ['%(operator)ssort__%(key)s' % o for o in order_by]) else: order_by = '-items' sort_type = key.get('sortType', key.get('type', 'string')) if sort_type == 'title': qs = parse_query({'query': data.get('query', {})}, request.user)['qs'] if data['value']: if op == '=': qs = qs.filter(find__key=data['key'], find__value__icontains=data['value']) elif op == '==': qs = qs.filter(find__key=data['key'], find__value__iexact=data['value']) elif op == '^': qs = qs.filter(find__key=data['key'], find__value__istartswith=data['value']) elif op == '$': qs = qs.filter(find__key=data['key'], find__value__iendswith=data['value']) qs = qs.order_by(order_by, nulls_last=True) qs = qs[data['range'][0]:data['range'][1]] response = json_response({}) response['data']['items'] = list(set([i.get(data['key']) for i in qs])) else: qs = models.Facet.objects.filter(key=data['key']) if data['value']: if op == '=': qs = qs.filter(value__icontains=data['value']) elif op == '==': qs = qs.filter(value__iexact=data['value']) elif op == '^': qs = qs.filter(value__istartswith=data['value']) elif op == '$': qs = qs.filter(value__iendswith=data['value']) if 'query' in data: item_query = parse_query({'query': data.get('query', {})}, request.user)['qs'] qs = qs.filter(item__in=item_query) qs = qs.values('value').annotate(items=Count('id')) qs = qs.order_by(order_by) qs = qs[data['range'][0]:data['range'][1]] response = json_response({}) response['data']['items'] = [i['value'] for i in qs] return render_to_json_response(response)
def find(data): ''' takes { query { conditions [{}] operator string } group string keys [string] sort [{}] range [int, int] } ''' response = {} q = query.parse(data) if 'group' in q: names = {} groups = {} key = 'group:' + hashlib.sha1( json.dumps(data).encode('utf-8')).hexdigest() g = state.cache.get(key) if g is None: items = q['qs'].options(load_only('id')) qs = models.Find.query.filter_by(key=q['group']) if items.first(): qs = qs.filter(models.Find.item_id.in_(items)) for f in qs.values('value', 'findvalue'): value = f[0] findvalue = f[1] if findvalue not in groups: groups[findvalue] = 0 groups[findvalue] += 1 names[findvalue] = value g = [{'name': names[k], 'items': groups[k]} for k in groups] else: g = [] if 'sort' in q: sort_type = utils.get_by_id(settings.config['itemKeys'], q['group']).get('sortType') def _sort_key(k): if sort_type == 'person' and q['sort'][0]['key'] == 'name': v = get_sort_name(k[q['sort'][0]['key']]) else: v = k[q['sort'][0]['key']] if isinstance(v, str): v = unicodedata.normalize('NFKD', v).lower() return v g.sort(key=_sort_key, reverse=q['sort'][0]['operator'] == '-') state.cache.set(key, g) if 'positions' in data: response['positions'] = {} ids = [k['name'] for k in g] response['positions'] = utils.get_positions(ids, data['positions']) elif 'range' in data: response['items'] = g[q['range'][0]:q['range'][1]] else: response['items'] = len(g) elif 'position' in data: ids = [i.id for i in q['qs'].options(load_only('id'))] response['position'] = utils.get_positions(ids, [data['qs'][0].id])[0] elif 'positions' in data: ids = [i.id for i in q['qs'].options(load_only('id'))] response['positions'] = utils.get_positions(ids, data['positions']) elif 'keys' in data: response['items'] = [ i.json(data['keys']) for i in q['qs'][q['range'][0]:q['range'][1]] ] else: size = [ i.info.get('size', 0) for i in q['qs'].options(load_only('id', 'info')) ] response['items'] = len(size) response['size'] = sum(size) return response
def parse_condition(self, condition): ''' condition: { value: "war" } or condition: { key: "year", value: [1970, 1980], operator: "=" } ... ''' #logger.debug('parse_condition %s', condition) if not 'value' in condition: return None k = condition.get('key', '*') if not k: k = '*' v = condition['value'] op = condition.get('operator') if not op: op = '=' if op.startswith('!'): op = op[1:] exclude = True else: exclude = False key_type = (utils.get_by_id(self.item_keys, k) or { 'type': 'string' }).get('type') if isinstance(key_type, list): key_type = key_type[0] if k == 'list': key_type = '' if (not exclude and op == '=' or op in ('$', '^')) and v == '': return None elif k == 'resolution': q = self.parse_condition({'key': 'width', 'value': v[0], 'operator': op}) \ & self.parse_condition({'key': 'height', 'value': v[1], 'operator': op}) if exclude: q = ~q return q elif isinstance(v, list) and len(v) == 2 and op == '=': q = self.parse_condition({'key': k, 'value': v[0], 'operator': '>='}) \ & self.parse_condition({'key': k, 'value': v[1], 'operator': '<'}) if exclude: q = ~q return q elif key_type == 'boolean': q = getattr(self._model, 'find_%s' % k) == v if exclude: q = ~q return q elif key_type in ("string", "text"): if isinstance(v, str): v = unicodedata.normalize('NFKD', v).lower() else: v = v.lower() q = get_operator(op)(self._find.findvalue, v) if k != '*': q &= (self._find.key == k) ids = self._model.query.join(self._find).filter(q).options( load_only('id')) in_op = operators.notin_op if exclude else operators.in_op q = in_op(self._model.id, ids) return q elif k == 'list': nickname, name = v.split(':', 1) if nickname: u = self._user.query.filter_by(nickname=nickname).one() else: u = self._user.query.filter_by(id=settings.USER_ID).one() l = self._list.query.filter_by(user_id=u.id, name=name).one() ids = l.get_items().options(load_only('id')) in_op = operators.notin_op if exclude else operators.in_op q = in_op(self._model.id, ids) return q elif key_type == 'date': def parse_date(d): while len(d) < 3: d.append(1) return datetime(*[int(i) for i in d]) #using sort here since find only contains strings v = parse_date(v.split('-')) vk = getattr(self._sort, k) q = get_operator(op, 'int')(vk, v) ids = self._model.query.join(self._find).filter(q).options( load_only('id')) in_op = operators.notin_op if exclude else operators.in_op q = in_op(self._model.id, ids) return q else: #integer, float, time q = get_operator(op, 'int')(getattr(self._sort, k), v) ids = self._model.query.join(self._find).filter(q).options( load_only('id')) in_op = operators.notin_op if exclude else operators.in_op q = in_op(self._model.id, ids) return q
def test_get_by_invalid_id(self): id = 'invalid_id' res = utils.get_by_id(id) self.assert_response_time(res, 2) self.assertEqual(res.status_code, 404)
def parseCondition(condition, user, owner=None): ''' condition: { value: "war" } or condition: { key: "year", value: [1970, 1980], operator: "=" } ... ''' k = condition.get('key', '*') k = {'id': 'itemId'}.get(k, k) if not k: k = '*' v = condition['value'] op = condition.get('operator') if not op: op = '=' if op.startswith('!'): op = op[1:] exclude = True else: exclude = False facet_keys = models.Item.facet_keys + ['title'] key_type = (utils.get_by_id(settings.CONFIG['itemKeys'], k) or { 'type': 'string' }).get('type') if isinstance(key_type, list): key_type = key_type[0] key_type = { 'title': 'string', 'person': 'string', 'text': 'string', 'year': 'string', 'length': 'string', 'layer': 'string', 'list': 'list', }.get(key_type, key_type) if k == 'list': key_type = '' if v == '{me}' and op == '==': if not owner: owner = user if k == 'user': v = owner.username elif k == 'groups': q = Q(groups__in=owner.groups.all()) if exclude: q = ~q return q if (not exclude and op == '=' or op in ('$', '^')) and v == '': return Q() elif k == 'filename' and (user.is_anonymous() or \ not user.get_profile().capability('canSeeMedia')): return Q(id=0) elif k == 'oshash': return Q(files__oshash=v) elif k == 'rendered': return Q(rendered=v) elif k == 'resolution': q = parseCondition({'key': 'width', 'value': v[0], 'operator': op}, user) \ & parseCondition({'key': 'height', 'value': v[1], 'operator': op}, user) if exclude: q = ~q return q elif isinstance(v, list) and len(v) == 2 and op == '=': q = parseCondition({'key': k, 'value': v[0], 'operator': '>='}, user) \ & parseCondition({'key': k, 'value': v[1], 'operator': '<'}, user) if exclude: q = ~q return q elif k in ('canplayvideo', 'canplayclips'): level = user.is_anonymous() and 'guest' or user.get_profile( ).get_level() allowed_level = settings.CONFIG['capabilities'][{ 'canplayvideo': 'canPlayVideo', 'canplayclips': 'canPlayClips' }[k]][level] if v: q = Q(level__lte=allowed_level) else: q = Q(level__gt=allowed_level) if exclude: q = ~q return q elif key_type == 'boolean': q = Q(**{'find__key': k, 'find__value': v}) if exclude: q = ~q return q elif key_type == "string": in_find = not k.startswith('itemId') if in_find: value_key = 'find__value' else: value_key = k if k in facet_keys: in_find = False facet_value = 'facets__value%s' % { '==': '__iexact', '>': '__gt', '>=': '__gte', '<': '__lt', '<=': '__lte', '^': '__istartswith', '$': '__iendswith', }.get(op, '__icontains') v = models.Item.objects.filter(**{ 'facets__key': k, facet_value: v }) value_key = 'id__in' else: value_key = '%s%s' % (value_key, { '==': '__iexact', '>': '__gt', '>=': '__gte', '<': '__lt', '<=': '__lte', '^': '__istartswith', '$': '__iendswith', }.get(op, '__icontains')) k = str(k) value_key = str(value_key) if isinstance(v, unicode): v = unicodedata.normalize('NFKD', v).lower() if k == '*': q = Q(**{value_key: v}) elif in_find: q = Q(**{'find__key': k, value_key: v}) else: q = Q(**{value_key: v}) if exclude: q = ~q return q elif k == 'list': q = Q(id=0) l = v.split(":") if len(l) == 1: vqs = Volume.objects.filter(name=v, user=user) if vqs.count() == 1: v = vqs[0] q = Q(files__instances__volume__id=v.id) elif len(l) >= 2: l = (l[0], ":".join(l[1:])) lqs = list(List.objects.filter(name=l[1], user__username=l[0])) if len(lqs) == 1 and lqs[0].accessible(user): l = lqs[0] if l.query.get('static', False) == False: data = l.query q = parseConditions(data.get('conditions', []), data.get('operator', '&'), user, l.user) else: q = Q(id__in=l.items.all()) if exclude: q = ~q else: q = Q(id=0) return q elif key_type == 'date': def parse_date(d): while len(d) < 3: d.append(1) return datetime(*[int(i) for i in d]) #using sort here since find only contains strings v = parse_date(v.split('-')) vk = 'sort__%s%s' % (k, { '==': '__exact', '>': '__gt', '>=': '__gte', '<': '__lt', '<=': '__lte', }.get(op, '')) vk = str(vk) q = Q(**{vk: v}) if exclude: q = ~q return q else: #integer, float, list, time #use sort table here if key_type == 'time': v = int(utils.parse_time(v)) vk = 'sort__%s%s' % (k, { '==': '__exact', '>': '__gt', '>=': '__gte', '<': '__lt', '<=': '__lte', }.get(op, '')) vk = str(vk) q = Q(**{vk: v}) if exclude: q = ~q return q