def addTags(scopeitem, tags: Iterable):
from app.models import Tag
for tag in tags:
if tag.strip(
) == "": # If the tag is an empty string then don't use it
continue
tag_obj = Tag.create_if_none(tag)
scopeitem.addTag(tag_obj)
def import_scope_list(address_list: Iterable, blacklist: bool) -> dict:
result = {"fail": [], "success": 0, "exist": 0}
prefixes = {"sqlite": " OR IGNORE", "mysql": " IGNORE"}
selected_prefix = prefixes.get(db.session.bind.dialect.name)
scope_import = {}
scope_tag_import = {}
address_list = [line.strip() for line in address_list]
tags = ScopeItem.extract_import_tags(address_list)
tag_dict = {}
from app.models import Tag
for tag in tags:
tag_dict[tag] = Tag.create_if_none(tag)
db.session.commit()
for line in address_list:
ip, tags = ScopeItem.parse_import_line(line)
if not ip:
result["fail"].append(line)
continue
tags = [tag_dict[tag] for tag in tags]
item = ScopeItem(target=str(ip), blacklist=blacklist).as_dict()
scope_tag_import[item["target"]] = tags
scope_import[item["target"]] = item
import_list = [v for _, v in scope_import.items()]
chunk_size = 10000
import_chunks = [
import_list[i:i + chunk_size]
for i in range(0, len(import_list), chunk_size)
]
for chunk in import_chunks:
ins_stmt = (ScopeItem.__table__.insert().prefix_with(
selected_prefix).values(chunk))
ins_result = db.session.execute(ins_stmt)
result["success"] += ins_result.rowcount
result["exist"] = len(address_list) - len(
result["fail"]) - result["success"]
all_scope = {item.target: item.id for item in ScopeItem.query.all()}
tags_to_import = []
for k, v in scope_tag_import.items():
for tag in v:
tags_to_import.append(
dict(scope_id=all_scope[k], tag_id=tag.id))
import_chunks = [
tags_to_import[i:i + chunk_size]
for i in range(0, len(tags_to_import), chunk_size)
]
for chunk in import_chunks:
tag_stmt = scopetags.insert().prefix_with(selected_prefix).values(
chunk)
db.session.execute(tag_stmt)
db.session.commit()
return result
