def addTags(scopeitem, tags: Iterable): from app.models import Tag for tag in tags: if tag.strip( ) == "": # If the tag is an empty string then don't use it continue tag_obj = Tag.create_if_none(tag) scopeitem.addTag(tag_obj)
def import_scope_list(address_list: Iterable, blacklist: bool) -> dict: result = {"fail": [], "success": 0, "exist": 0} prefixes = {"sqlite": " OR IGNORE", "mysql": " IGNORE"} selected_prefix = prefixes.get(db.session.bind.dialect.name) scope_import = {} scope_tag_import = {} address_list = [line.strip() for line in address_list] tags = ScopeItem.extract_import_tags(address_list) tag_dict = {} from app.models import Tag for tag in tags: tag_dict[tag] = Tag.create_if_none(tag) db.session.commit() for line in address_list: ip, tags = ScopeItem.parse_import_line(line) if not ip: result["fail"].append(line) continue tags = [tag_dict[tag] for tag in tags] item = ScopeItem(target=str(ip), blacklist=blacklist).as_dict() scope_tag_import[item["target"]] = tags scope_import[item["target"]] = item import_list = [v for _, v in scope_import.items()] chunk_size = 10000 import_chunks = [ import_list[i:i + chunk_size] for i in range(0, len(import_list), chunk_size) ] for chunk in import_chunks: ins_stmt = (ScopeItem.__table__.insert().prefix_with( selected_prefix).values(chunk)) ins_result = db.session.execute(ins_stmt) result["success"] += ins_result.rowcount result["exist"] = len(address_list) - len( result["fail"]) - result["success"] all_scope = {item.target: item.id for item in ScopeItem.query.all()} tags_to_import = [] for k, v in scope_tag_import.items(): for tag in v: tags_to_import.append( dict(scope_id=all_scope[k], tag_id=tag.id)) import_chunks = [ tags_to_import[i:i + chunk_size] for i in range(0, len(tags_to_import), chunk_size) ] for chunk in import_chunks: tag_stmt = scopetags.insert().prefix_with(selected_prefix).values( chunk) db.session.execute(tag_stmt) db.session.commit() return result