Exemple #1
0
    def get(uid: int = None,
            name: str = None,
            limit: int = None,
            offset: int = None,
            search: str = None):
        """
        Fetches some resourcegroups
        :param uid:
        :param name:
        :param limit:
        :param offset:
        :param search: performs a fulltext search
        :return:
        """
        q = db.session.query(ResourceGroup)

        if isinstance(uid, int):
            return [q.filter(ResourceGroup.id == uid).first()]

        if isinstance(name, str) and name:
            return q.filter(ResourceGroup.name == name).first()

        if isinstance(search, str) and search:
            q = q.filter(
                ResourceGroup.name.ilike("%" + escape_like(search) + "%"))

        if offset and isinstance(offset, int):
            q = q.offset(offset)

        if limit and isinstance(limit, int):
            q = q.limit(limit)

        return q.all()
Exemple #2
0
def TestInput(data):
    if len(data) < 10:
        pass

    fdp = atheris.FuzzedDataProvider(data)

    cast_if(FuzzTable.id, Integer)
    cast_if(FuzzTable.name, Integer)
    cast_if(FuzzTable.id, String)
    cast_if(FuzzTable.name, String)

    cast_if(fdp.ConsumeInt(10), Integer)
    cast_if(fdp.ConsumeString(10), Integer)
    cast_if(fdp.ConsumeInt(10), String)
    cast_if(fdp.ConsumeString(10), String)

    db_str = 'sqlite:///fuzz.db'

    engine = create_engine(db_str)
    Base.metadata.create_all(engine)

    try:
        with Session(engine) as session:
            name_str = fdp.ConsumeString(20)
            session.query(FuzzTable).filter(
                FuzzTable.name.ilike(escape_like(name_str))).all()
    except SQLAlchemyError as e:
        pass
Exemple #3
0
 def search(queryset, user_input):
     if user_input is None or user_input == u'':
         return None
     # TODO joins and type checks for the queries where possible
     queryset = queryset.filter(
         or_(*(getattr(model, f.name).ilike(u'%{}%'.format(
             escape_like(str(user_input)))) for f in fields
               if f.searchable)))
     return queryset
Exemple #4
0
def get_filter(l):
    if type(l) is list:
        if len(l) == 1:
            return get_filter(l[0])
        else:
            op_str = l[1]
            if op_str == "AND":
                op = and_
            else:  #OR
                op = or_

            args = []
            for e in l[0::2]:
                args += [get_filter(e)]
            return op(*args)
    else:
        return OcrMetaData.text.like("%" + escape_like(l) + "%")
Exemple #5
0
 def create_tags_clause(self, tag):
     """Creates a clause for a tag"""
     tag = tag.lower()
     return Task.tags.any(Tag.name.ilike(u'%{}%'.format(escape_like(tag))))
Exemple #6
0
 def create_text_clause(self, column, value):
     """Creates a clause for text"""
     return column.ilike(u'%{}%'.format(escape_like(value.strip())))
 def test_escapes_wildcards(self):
     assert escape_like('_*%') == '*_***%'
Exemple #8
0
    def search(**kwargs):
        kwargs['key'] = DatabaseSearchController.make_valid_key(kwargs['key'])

        # @TODO: filter by protocols / hosts
        q = Files.query

        # only find files that are not in 'temp' mode
        q = q.filter(Files.resource_id >= 1)

        # ignores certain filters
        ignore_filters = []

        # filter only files/dirs, or both
        if 'folders' in kwargs['file_type'] and 'files' in kwargs['file_type']:
            pass
        elif 'folders' in kwargs['file_type']:
            q = q.filter(Files.file_isdir == True)

            # When searching only for directories, ignore filters that are not relevant
            ignore_filters.extend(('file_size', 'file_categories', 'file_extensions'))
        elif 'files' in kwargs['file_type']:
            q = q.filter(Files.file_isdir == False)

        # size
        if kwargs['file_size'] and not 'file_size' in ignore_filters:
            try:
                file_size = kwargs['file_size'].split('-')

                if not len(file_size) == 2:
                    raise Exception()

                if file_size[0] == '*':
                    q = q.filter(Files.file_size <= int(file_size[1]))
                elif file_size[1] == '*':
                    q = q.filter(Files.file_size >= int(file_size[0]))
                else:
                    q = q.filter(Files.file_size.between(*[int(x) for x in file_size]))
            except:
                pass

        # filter categories
        filecategories = FileCategories()

        cat_ids = []
        for cat in kwargs['file_categories']:
            cat_id = filecategories.id_by_name(cat)

            if cat_id is None:
                continue

            cat_ids.append(FileCategories().id_by_name(cat))

        if cat_ids and not 'file_categories' in ignore_filters:
            q = q.filter(Files.file_format.in_(cat_ids))

        if not kwargs['file_categories']:
            file_categories = filecategories.get_names()

        # filter extensions
        if kwargs['file_extensions'] and not 'file_extensions' in ignore_filters:
            exts = []

            for ext in kwargs['file_extensions']:
                if ext.startswith('.'):
                    ext = ext[1:]

                exts.append(ext)

            q = q.filter(Files.file_ext.in_(exts))

        # Search
        if kwargs['autocomplete'] or kwargs['lazy_search'] or app.config['db_file_count'] > 5000000:
            q = q.filter(Files.searchable.like(escape_like(kwargs['key']) + '%'))
        else:
            q = q.filter(Files.searchable.like('%' + escape_like(kwargs['key']) + '%'))

        # pagination
        q = q.offset(kwargs['page'])

        if kwargs['autocomplete']:
            q = q.limit(5)
            q = q.distinct(func.lower(Files.file_name))
        else:
            q = q.limit(kwargs['per_page'])

        # fetch
        results = q.all()

        resource_ids = set([z.resource_id for z in results])
        resource_obs = {z.id: z for z in Resources.query.filter(Resources.id.in_(resource_ids)).all()}

        for result in results:
            setattr(result, 'resource', resource_obs[result.resource_id])

        results = [result.fancify() for result in results]

        return results
Exemple #9
0
    def _search(**kwargs):
        kwargs["key"] = CrawlController.make_valid_key(kwargs["key"])
        if not kwargs["key"]:
            raise Exception("Invalid search. Too short?")

        q = ZdbQuery(Files, session=db.session) if config(
            "findex:elasticsearch:enabled") else Files.query

        # @TODO: filter by protocols / hosts
        # only find files that are not in "temp" mode
        # q = q.filter(Files.resource_id >= 1)

        # ignores certain filters
        ignore_filters = []

        # filter only files/dirs
        if kwargs.get("file_type"):
            if "both" in kwargs["file_type"]:
                pass
            if "folders" in kwargs["file_type"]:
                q = q.filter(Files.file_isdir == True)
                ignore_filters.extend(
                    ("file_size", "file_categories", "file_extensions"))
            elif "files" in kwargs["file_type"]:
                q = q.filter(Files.file_isdir == False)

        # size
        if kwargs["file_size"] and "file_size" not in ignore_filters:
            try:
                file_size = kwargs["file_size"].split("-")

                if not len(file_size) == 2:
                    raise Exception()

                if file_size[0] == "*":
                    q = q.filter(Files.file_size <= int(file_size[1]))
                elif file_size[1] == "*":
                    q = q.filter(Files.file_size >= int(file_size[0]))
                else:
                    q = q.filter(
                        Files.file_size.between(*[int(x) for x in file_size]))
            except:
                pass

        # filter categories
        filecategories = FileCategories()

        cat_ids = []
        cats = kwargs.get("file_categories", [])
        cats = [] if cats is None else cats
        for cat in cats:
            cat_id = filecategories.id_by_name(cat)

            if cat_id is None:
                continue
            cat_ids.append(FileCategories().id_by_name(cat))

        if cat_ids and "file_categories" not in ignore_filters:
            q = q.filter(Files.file_format.in_(cat_ids))

        if not kwargs["file_categories"]:
            file_categories = filecategories.get_names()

        # filter extensions
        if kwargs[
                "file_extensions"] and "file_extensions" not in ignore_filters:
            exts = []

            for ext in kwargs["file_extensions"]:
                if ext.startswith("."):
                    ext = ext[1:]

                exts.append(ext)

            q = q.filter(Files.file_ext.in_(exts))

        if isinstance(kwargs["meta_movie_id"], int):
            q = q.filter(Files.meta_movie_id == kwargs["meta_movie_id"])

        # Search
        if config("findex:elasticsearch:enabled"):
            val = kwargs["key"]
        else:
            if kwargs["autocomplete"] or app.config["db_file_count"] > 5000000:
                print("warning: too many rows, enable ElasticSearch")
                val = "%s%%" % escape_like(kwargs["key"])
            else:
                val = "%%%s%%" % escape_like(kwargs["key"])

        if val != "*":
            q = q.filter(Files.searchable.like(val))

        q = q.order_by(Files.file_size.desc())

        # pagination
        q = q.offset(kwargs["page"])

        if kwargs["autocomplete"]:
            q = q.limit(5)
            # q = q.distinct(func.lower(Files.file_name))
            q = q.distinct(Files.file_size)
        else:
            q = q.limit(kwargs["per_page"])

        # fetch
        try:
            results = q.all()
        except Exception as ex:
            raise Exception(ex)

        results = SearchController.assign_resource_objects(results)
        return results
OPERATOR_FILTER_MAP = {
    'eq':
    lambda f, v: f == v,
    'neq':
    lambda f, v: f != v,
    'gt':
    lambda f, v: f > v,
    'gte':
    lambda f, v: f >= v,
    'lt':
    lambda f, v: f < v,
    'lte':
    lambda f, v: f <= v,
    'contains':
    lambda f, v: f.ilike('%{}%'.format(escape_like(v))),
    'contains_cs':
    lambda f, v: f.like('%{}%'.format(escape_like(v))),
    'eq_cs':
    lambda f, v: f.ilike('{}'.format(escape_like(v))),
    'neq_cs':
    lambda f, v: ~f.ilike('{}'.format(escape_like(v))),
    'near':
    lambda f, v: f.ST_DWithin(ga_point_from_dict(v), v.get('radius', '20000')),
    'in':
    lambda f, v: sa.or_(*(f.any(vv)
                          for vv in v)),  # field: array; value: array
    'all':
    lambda f, v: sa.and_(*(f.any(vv)
                           for vv in v)),  # field: array; value: array
    'any':