def main(args): setup_logger(args) # Set up model state_dict = torch.load(args.checkpoint) config = state_dict['config'] model = TemporalFusionTransformer(config).cuda() model.load_state_dict(state_dict['model']) model.eval() model.cuda() # Set up dataset test_split = TFTDataset(args.data, config) data_loader = DataLoader(test_split, batch_size=args.batch_size, num_workers=4) scalers = pickle.load(open(args.tgt_scalers, 'rb')) cat_encodings = pickle.load(open(args.cat_encodings, 'rb')) if args.visualize: # TODO: abstract away all forms of visualization. visualize_v2(args, config, model, data_loader, scalers, cat_encodings) quantiles, perf_dict = inference(args, config, model, data_loader, scalers, cat_encodings) quantiles = {'test_p10': quantiles[0].item(), 'test_p50': quantiles[1].item(), 'test_p90': quantiles[2].item(), 'sum':sum(quantiles).item()} finish_log = {**quantiles, **perf_dict} dllogger.log(step=(), data=finish_log, verbosity=1) print('Test q-risk: P10 {} | P50 {} | P90 {}'.format(*quantiles)) print('Latency:\n\tAverage {:.3f}s\n\tp90 {:.3f}s\n\tp95 {:.3f}s\n\tp99 {:.3f}s'.format( perf_dict['latency_avg'], perf_dict['latency_p90'], perf_dict['latency_p95'], perf_dict['latency_p99']))
import logging import log_helper import winreg import enum from system_utils import is_x64os logger = log_helper.setup_logger(name="registry_helper", level=logging.DEBUG, log_to_file=False) __doc__ = """File contains registry-related functions, for creating, enumerating, editing and removing Windows registry keys and values. enums and dictionaries with registry-related integer values, imported from winreg module Also is_x64os() helper function provided """ HIVES_MAP = { "HKEY_CLASSES_ROOT": winreg.HKEY_CLASSES_ROOT, "HKEY_CURRENT_USER": winreg.HKEY_CURRENT_USER, "HKEY_LOCAL_MACHINE": winreg.HKEY_LOCAL_MACHINE, "HKEY_USERS": winreg.HKEY_USERS, "HKEY_PERFORMANCE_DATA": winreg.HKEY_PERFORMANCE_DATA, "HKEY_CURRENT_CONFIG": winreg.HKEY_CURRENT_CONFIG, "HKEY_DYN_DATA": winreg.HKEY_DYN_DATA } class RegistryKeyType(enum.IntEnum): # Binary data in any form REG_BINARY = 0 # 32-bit number
import sys import argparse import logging import winreg import log_helper import system_fingerprint import hardware_fingerprint import telemetry_fingerprint import random_utils import registry_helper from registry_helper import RegistryKeyType, Wow64RegistryEntry from system_utils import is_x64os, platform_version logger = log_helper.setup_logger(name="antidetect", level=logging.INFO, log_to_file=False) def generate_telemetry_fingerprint(): """ IDs related to Windows 10 Telemetry All the telemetry is getting around the DeviceID registry value It can be found in the following kays: HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\SQMClient HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\SettingsRequests """ windows_ver = platform_version() if not windows_ver.startswith("Windows-10"): logger.warning("Telemetry ID replace available for Windows 10 only") return
# parser.add_argument("--epochs",default=EPOCHS) # args = parser.parse_args() # EPOCHS = int(args.epochs) ''' ============================================================================= save name ''' SAVE_NAME = f'test' ''' ============================================================================= init ''' ### gpu 지정 GPU_ID = set_gpu(GPU_ID) ### log now = datetime.datetime.now() save_name = SAVE_NAME + ' ' + now.strftime('%y%m%d_%p:%I:%M:%S') save_path = LOG_DIR + save_name log = setup_logger(save_path, save_name) ### code copy pylist = glob.glob('./*.py') for p in pylist: filename = os.path.split(p)[-1] shutil.copy(p, os.path.join(save_path, filename)) ### # import imgaug # imgaug.random.seed(0) # np.random.seed(0) # tf.random.set_seed(0) # random.seed(0) ### mixed precision
import sys import logging import log_helper import random import uuid import string import random_utils logger = log_helper.setup_logger(name="system_fingerpring", level=logging.INFO, log_to_file=False) class WinFingerprint: """ """ EDITIONS = { 7: [["Starter", "Starter"], ["HomeBasic", "Home Basic"], ["HomePremium", "Home Premium"], ["Professional", "Professional"], ["ProfessionalN", "Professional N"], ["ProfessionalKN", "Professional KN"], ["Enterprise", "Enterprise"], ["Ultimate", "Ultimate"]], 8: [["Core", "Core"], ["Pro", "Pro"], ["ProN", "Pro N"], ["Enterprise", "Enterprise"], ["EnterpriseN", "Enterprise N"], ["OEM", "OEM"], ["withBing", "with Bing"]], 10: [["Home", "Home"],
def main(args): # Enable CuDNN autotuner nproc_per_node = torch.cuda.device_count() if args.affinity != 'disabled': affinity = gpu_affinity.set_affinity( args.local_rank, nproc_per_node, args.affinity ) print(f'{args.local_rank}: thread affinity: {affinity}') torch.backends.cudnn.benchmark = True ### INIT DISTRIBUTED if args.distributed_world_size > 1: args.local_rank = int(os.environ.get('LOCAL_RANK', args.local_rank)) torch.cuda.set_device(args.local_rank) dist.init_process_group(backend='nccl', init_method='env://') args.distributed_world_size = int(os.environ['WORLD_SIZE']) args.distributed_rank = dist.get_rank() print_once(f'Distributed training with {args.distributed_world_size} GPUs') torch.cuda.synchronize() if args.seed: np.random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) setup_logger(args) config = CONFIGS[args.dataset]() if args.overwrite_config: config.__dict__.update(json.loads(args.overwrite_config)) dllogger.log(step='HPARAMS', data={**vars(args), **vars(config)}, verbosity=1) model = TemporalFusionTransformer(config).cuda() if args.ema_decay: model_ema = ModelEma(model, decay=args.ema_decay) print_once('Model params: {}'.format(sum(p.numel() for p in model.parameters()))) criterion = QuantileLoss(config).cuda() optimizer = FusedAdam(model.parameters(), lr=args.lr) if args.use_amp: model, optimizer = amp.initialize(model, optimizer, opt_level="O2", loss_scale="dynamic") if args.distributed_world_size > 1: #model = DDP(model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True) model = DDP(model) train_loader, valid_loader, test_loader = load_dataset(args, config) global_step = 0 perf_meter = PerformanceMeter() for epoch in range(args.epochs): start = time.time() dllogger.log(step=global_step, data={'epoch': epoch}, verbosity=1) model.train() for local_step, batch in enumerate(train_loader): perf_meter.reset_current_lap() batch = {key: tensor.cuda() if tensor.numel() else None for key, tensor in batch.items()} predictions = model(batch) targets = batch['target'][:,config.encoder_length:,:] p_losses = criterion(predictions, targets) loss = p_losses.sum() if args.use_amp: with amp.scale_loss(loss, optimizer) as scaled_loss: scaled_loss.backward() else: loss.backward() if not args.grad_accumulation or (global_step+1) % args.grad_accumulation == 0: if args.clip_grad: torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip_grad) optimizer.step() optimizer.zero_grad() if args.ema_decay: model_ema.update(model) if args.distributed_world_size > 1: dist.all_reduce(p_losses) p_losses /= args.distributed_world_size loss = p_losses.sum() torch.cuda.synchronize() ips = perf_meter.update(args.batch_size * args.distributed_world_size, exclude_from_total=local_step in [0, len(train_loader)-1]) log_dict = {'P10':p_losses[0].item(), 'P50':p_losses[1].item(), 'P90':p_losses[2].item(), 'loss': loss.item(), 'items/s':ips} dllogger.log(step=global_step, data=log_dict, verbosity=1) global_step += 1 validate(args, config, model_ema if args.ema_decay else model, criterion, valid_loader, global_step) if validate.early_stop_c >= args.early_stopping: print_once('Early stopping') break ### TEST PHASE ### state_dict = torch.load(os.path.join(args.results, 'checkpoint.pt'), map_location='cpu') if isinstance(model, DDP): model.module.load_state_dict(state_dict['model']) else: model.load_state_dict(state_dict['model']) model.cuda().eval() tgt_scalers = pickle.load(open(os.path.join(args.data_path, 'tgt_scalers.bin'), 'rb')) cat_encodings = pickle.load(open(os.path.join(args.data_path,'cat_encodings.bin'), 'rb')) unscaled_predictions, unscaled_targets, _, _ = predict(args, config, model, test_loader, tgt_scalers, cat_encodings) losses = QuantileLoss(config)(unscaled_predictions, unscaled_targets) normalizer = unscaled_targets.abs().mean() quantiles = 2 * losses / normalizer if args.distributed_world_size > 1: quantiles = quantiles.cuda() dist.all_reduce(quantiles) quantiles /= args.distributed_world_size quantiles = {'test_p10': quantiles[0].item(), 'test_p50': quantiles[1].item(), 'test_p90': quantiles[2].item(), 'sum':sum(quantiles).item()} finish_log = {**quantiles, 'average_ips':perf_meter.avg, 'convergence_step':validate.conv_step} dllogger.log(step=(), data=finish_log, verbosity=1)
import os import sys import argparse import logging import enum import log_helper import validatable_record import generate_fingerprint as gen from soft_assert import SoftAssert from test_type import TestType from validation_type import ValidationType logger = log_helper.setup_logger(name="fingerprint_test", level=logging.DEBUG, log_to_file=False) class ApplicationType(enum.IntEnum): APP_PROTOTYPE = 0 APP_PRODUCTION = 1 APP_PROOF = 2 TEST_TYPE_MAP = { "telemetry": TestType.telemetry_fingerprint, "network": TestType.network_fingerprint, "system": TestType.windows_fingerprint, "hardware": TestType.hardware_fingerprint, "font": TestType.font_fingerprint }
import os import sys import argparse import logging import log_helper import subprocess import psutil import getpass logger = log_helper.setup_logger(name="win10_cleaner", level=logging.DEBUG, log_to_file=True) """ We use PowerShell for most of operations. Python provides convenient wrapper for its output """ POWERSHELL_COMMAND = r'C:\WINDOWS\system32\WindowsPowerShell\v1.0\powershell.exe' """ Disable Cortana by killing the process and renaming Cortana directory in SystemApps so that RintimeBroker did not start it again """ def find_cortana_directory(name, path): """ :param name: Name of Cortana executable :param path: Path to Windows Store applications directory :return: Path to the directory contains Cortana """ for root, dirs, files in os.walk(path): if name in files: return root
import sys import logging import log_helper import random import registry_helper from registry_helper import Wow64RegistryEntry logger = log_helper.setup_logger(name="font_fp", level=logging.INFO, log_to_file=False) __doc__ = "The script deletes N random fonts from the system" def delete_random_font(fonts_delete): """ Delete several random fonts from the system :param fonts_delete: Fonts to delete """ hive = "HKEY_LOCAL_MACHINE" fonts64 = registry_helper.enumerate_key( hive, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Fonts") for _ in range(0, fonts_delete): delete_font = random.choice(fonts64) logger.info("Delete font {0}".format(delete_font)) registry_helper.delete_value( hive, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Fonts", delete_font)
import os import sys import stat import subprocess import argparse import logging import zipfile import log_helper logger = log_helper.setup_logger(name="qt_builder", level=logging.DEBUG, log_to_file=False) QT_CONFIG_PARAMS = "-mp -opensource -confirm-license -nomake tests -nomake examples -no-compile-examples" \ " -release -shared -pch -no-ltcg -accessibility -no-sql-sqlite -opengl desktop -no-openvg" \ " -no-nis -no-iconv -no-evdev -no-mtdev -no-inotify -no-eventfd -largefile -no-system-proxies" \ " -qt-zlib -qt-pcre -no-icu -qt-libpng -qt-libjpeg -qt-freetype -no-fontconfig -qt-harfbuzz" \ " -no-angle -incredibuild-xge -no-plugin-manifests -qmake -qreal double -rtti -strip -no-ssl" \ " -no-openssl -no-libproxy -no-dbus -no-audio-backend -no-wmf-backend -no-qml-debug -no-direct2d" \ " -directwrite -no-style-fusion -native-gestures -skip qt3d -skip qtactiveqt -skip qtandroidextras" \ " -skip qtcanvas3d -skip qtconnectivity -skip qtdeclarative -skip qtdoc -skip qtenginio" \ " -skip qtgraphicaleffects -skip qtlocation -skip qtmacextras -skip qtmultimedia" \ " -skip qtquickcontrols -skip qtquickcontrols2 -skip qtscript -skip qtsensors -skip qtserialbus" \ " -skip qtserialport -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebsockets" \ " -skip qtwebview -skip qtx11extras -skip qtxmlpatterns" ########################################################################### def on_rm_error(*args): """
import os import sys import stat import argparse import logging import log_helper logger = log_helper.setup_logger(name="simple_archive", level=logging.DEBUG, log_to_file=False) ########################################################################### def on_rm_error(*args): """ In case the file or directory is read-only and we need to delete it this function will help to remove 'read-only' attribute :param args: (func, path, exc_info) tuple """ # path contains the path of the file that couldn't be removed # let's just assume that it's read-only and unlink it. _, path, _ = args logger.warning("OnRmError: {0}".format(path)) os.chmod(path, stat.S_IWRITE) os.unlink(path) def environment_value(environment_name): """ :param environment_name: Name of the environment variable :return: Value of the environment variable or the empty string if not exists
import logging import log_helper import random import uuid import string import random_utils logger = log_helper.setup_logger(name="hardware_fingerpring", level=logging.DEBUG, log_to_file=False) class HardwareFingerprint: """ """ def __init__(self): self.hw_profile_guid = ("{%s}" % str(uuid.uuid4())) self.performance_guid = ("{%s}" % str(uuid.uuid4())) self.machine_guid = str(uuid.uuid4()) self.win_update_guid = str(uuid.uuid4()) self.system_client_id = self.__random_system_client_id() def random_hw_profile_guid(self): """ :return: Hardware profile GUID """ return self.hw_profile_guid def random_performance_guid(self): """ :return: Performance\BootCKCLSettings and Performance\BShutdownCKCLSettings GUID
import os import sys import argparse import logging import log_helper import registry_helper logger = log_helper.setup_logger(name="windows_uninstall", level=logging.DEBUG, log_to_file=False) def environment_value(environment_name): """ :param environment_name: Name of the environment variable :return: Value of the environment variable or the empty string if not exists """ try: return os.environ[environment_name] except KeyError: return '' class WinUninstallApplication: """ Application for uninstalling Windows applications based on the applications list """ # Necessary commands WMIC_ENUMERATE_COMMAND = 'wmic product get name' WMIC_UNINSTALL_COMMAND = 'wmic product where name="{0}" call uninstall'
import os import sys import re import argparse import logging import log_helper __dec__ = """Bulk files processor. Does not intended to be universal, change the code every time you need different conditions. Allows rename or remove, non-recursive or recursive """ logger = log_helper.setup_logger(name="bulk_rename", level=logging.DEBUG, log_to_file=False) class FileProcessor: def __init__(self, input_dir): """ Counter is for mass renaming with a new index """ self.counter = 0 self.input_dir = input_dir def rename(self): """ Rename file with leading 2 digits to leading 3 digits """ for item in os.listdir(self.input_dir): if re.match(u"[0-9]{2}\\D.+\.jpg", item):
import os import sys import argparse import logging import log_helper import xml.etree.ElementTree as XmlTree import xml.dom.minidom as minidom logger = log_helper.setup_logger(name="dictionary_merge", level=logging.DEBUG, log_to_file=False) class BaseDictionaryProvider: """ Base class for all dictionary providers """ def __init__(self, file_path): self.file_path = file_path if not os.path.isfile(self.file_path): raise RuntimeError("File does not exist: %s" % self.file_path) self.dictionary_name = os.path.splitext( os.path.basename(self.file_path))[0] def name(self): """ :return: Dictionary name """ return self.dictionary_name
import logging import string import log_helper import identity_data import random import time import datetime import itertools import platform logger = log_helper.setup_logger(name="random_utils", level=logging.DEBUG, log_to_file=False) __doc__ = """Service functions for generation random values and sequences with given format. Hostname, user name and MAC address, randomly selected from lists imported from identity_data module, random unix time, random string sequences. Helper functions for writing special values to Windows registry Also is_x64os() helper function provided """ def is_x64os(): """ :return: True if system is 64-bit, False otherwise """ return platform.machine().endswith('64') def random_hostname(): """ :return: random host name from the list """