def get_resources(uid: int = None,
name: str = None,
address: str = None,
port: int = None,
limit: int = None,
offset: int = None,
by_owner: int = None,
search: str = None,
protocol: int = None,
scheduled: bool = None,
order_by: str = None):
"""
Fetches some resources
:param uid:
:param name:
:param address:
:param port:
:param limit:
:param offset:
:param by_owner:
:param protocol:
:param order_by: order_by a column
:param search: performs a fulltext search on column 'search' which
:param scheduled: filter on scheduled
includes: IP/DOMAIN, NAME, DISPLAY_URL, PROTOCOL
:return:
"""
# normal sqla or zdb?
if search and config("findex:elasticsearch:enabled"):
q = ZdbQuery(Resource, session=db.session)
else:
q = db.session.query(Resource)
if isinstance(by_owner, int):
q = q.filter(Resource.created_by_id == by_owner)
if isinstance(uid, int):
q = q.filter(Resource.id == uid)
if isinstance(protocol, int):
q = q.filter(Resource.protocol == protocol)
if isinstance(scheduled, bool):
if scheduled:
q = q.filter(Resource.date_crawl_next <= datetime.now())
else:
q = q.filter(Resource.date_crawl_end.isnot(None))
if isinstance(address, str) and address:
qs = Server.query
server = qs.filter(Server.address == address).first()
if not server:
raise Exception("Could not find server")
q = q.filter(Resource.server_id == server.id)
if isinstance(port, int):
q = q.filter(Resource.port == port)
if isinstance(search, str) and search:
q = q.filter(Resource.search.like(search))
if isinstance(name, str) and name:
qs = Server.query
server = qs.filter(Server.name == name).first()
if not server:
raise Exception("Could not find server")
q = q.filter(Resource.server_id == server.id)
if isinstance(order_by, str):
c = getattr(Resource, order_by)
q = q.order_by(desc(c))
if offset and isinstance(offset, int):
q = q.offset(offset)
if limit and isinstance(limit, int):
q = q.limit(limit)
return q.all()
def _search(**kwargs):
kwargs["key"] = CrawlController.make_valid_key(kwargs["key"])
if not kwargs["key"]:
raise Exception("Invalid search. Too short?")
q = ZdbQuery(Files, session=db.session) if config(
"findex:elasticsearch:enabled") else Files.query
# @TODO: filter by protocols / hosts
# only find files that are not in "temp" mode
# q = q.filter(Files.resource_id >= 1)
# ignores certain filters
ignore_filters = []
# filter only files/dirs
if kwargs.get("file_type"):
if "both" in kwargs["file_type"]:
pass
if "folders" in kwargs["file_type"]:
q = q.filter(Files.file_isdir == True)
ignore_filters.extend(
("file_size", "file_categories", "file_extensions"))
elif "files" in kwargs["file_type"]:
q = q.filter(Files.file_isdir == False)
# size
if kwargs["file_size"] and "file_size" not in ignore_filters:
try:
file_size = kwargs["file_size"].split("-")
if not len(file_size) == 2:
raise Exception()
if file_size[0] == "*":
q = q.filter(Files.file_size <= int(file_size[1]))
elif file_size[1] == "*":
q = q.filter(Files.file_size >= int(file_size[0]))
else:
q = q.filter(
Files.file_size.between(*[int(x) for x in file_size]))
except:
pass
# filter categories
filecategories = FileCategories()
cat_ids = []
cats = kwargs.get("file_categories", [])
cats = [] if cats is None else cats
for cat in cats:
cat_id = filecategories.id_by_name(cat)
if cat_id is None:
continue
cat_ids.append(FileCategories().id_by_name(cat))
if cat_ids and "file_categories" not in ignore_filters:
q = q.filter(Files.file_format.in_(cat_ids))
if not kwargs["file_categories"]:
file_categories = filecategories.get_names()
# filter extensions
if kwargs[
"file_extensions"] and "file_extensions" not in ignore_filters:
exts = []
for ext in kwargs["file_extensions"]:
if ext.startswith("."):
ext = ext[1:]
exts.append(ext)
q = q.filter(Files.file_ext.in_(exts))
if isinstance(kwargs["meta_movie_id"], int):
q = q.filter(Files.meta_movie_id == kwargs["meta_movie_id"])
# Search
if config("findex:elasticsearch:enabled"):
val = kwargs["key"]
else:
if kwargs["autocomplete"] or app.config["db_file_count"] > 5000000:
print("warning: too many rows, enable ElasticSearch")
val = "%s%%" % escape_like(kwargs["key"])
else:
val = "%%%s%%" % escape_like(kwargs["key"])
if val != "*":
q = q.filter(Files.searchable.like(val))
q = q.order_by(Files.file_size.desc())
# pagination
q = q.offset(kwargs["page"])
if kwargs["autocomplete"]:
q = q.limit(5)
# q = q.distinct(func.lower(Files.file_name))
q = q.distinct(Files.file_size)
else:
q = q.limit(kwargs["per_page"])
# fetch
try:
results = q.all()
except Exception as ex:
raise Exception(ex)
results = SearchController.assign_resource_objects(results)
return results