def generate_dataset(dry_dpath, fx_fpath, output_dpath=None, func=None):
"""Generate dataset of wet samples.
Fill a JSON file wich matches dry and wet samples.
"""
if not output_dpath:
output_dpath = mkrdir()
elif not pth.__exists(output_dpath):
pth.__make_dir(output_dpath)
fx = _read(fx_fpath)
jsn.init()
info, save_steps = dict(), tml.value('json',
section='data',
subkey='save_steps')
for idx, dryfpath in enumerate(__list_audio_files(dry_dpath)):
wet_signal = apply_fx(_read(dryfpath), fx, func)
dpath = pth.__with_extension(
rfname(path=output_dpath, prefix='{0}_'.format(idx)), '.wav')
_save(wet_signal, dpath)
info[dryfpath] = dpath
if (idx + 1) % save_steps == 0:
log.debug("{0} samples processed".format(idx + 1))
jsn.dump(info)
jsn.dump(info)
def _filter(dpath):
"""Filtering <dpath> by removing a certain amount of files.
Keep files corresponding to specific instruments and sources, given in configuration file.
For now, filtering removes files from the tail.
"""
log.debug("Filtering \"{0}\"".format(dpath))
instruments = tml.value('instruments', section='data')
sources = tml.value('sources', section='data')
instr_conds = list(
map(lambda instr: eval('ns.is_{0}'.format(instr)), instruments))
src_conds = list(map(lambda src: eval('ns.is_{0}'.format(src)), sources))
ns.filter_elements(dpath, [instr_conds, src_conds, [ns.is_pitch_63]])
max_samples = tml.value('max_samples', section='data')
audio_files = __list_audio_files(dpath)
if len(audio_files) <= max_samples:
log.debug("Keeping all remaining files in {0}".format(dpath))
return
for afile in audio_files[max_samples:]:
pth.__remove_file(afile)
def train(self, data, labels):
log.debug("Training model")
train_data, valid_data = split_valid(data, labels)
self.model.fit(*train_data,
batch_size=self.bsiz,
epochs=self.epoc,
callbacks=self.cbac,
validation_data=valid_data)
def write_binary_file(path: Union[str, Path], content, mode=0o0755, symlink: Union[str, Path] = None):
p = Path(path)
logger.debug(f"Dump to file [{p}]")
with open(str(p.absolute()), 'wb') as f:
f.write(content)
os.chmod(p, mode)
if symlink:
os.symlink(p, symlink)
def _read(fpath):
"""Read content of numpy archive file at <fpath>."""
if not pth.__is_file(fpath):
log.critical(
"Numpy file \"{0}\" not found, cannot read data".format(fpath))
log.debug("Reading data from \"{0}\"".format(fpath))
return np.load(fpath)
def get_vpn_ip(self, nic: str, lenient=True):
try:
logger.log(self.log_lvl, f'Query VPN IPv4 on {nic}...')
return netifaces.ifaddresses(nic)[netifaces.AF_INET]
except Exception as err:
if lenient:
logger.debug(f'Not found VPN IP {nic}. Error: {err}')
return None
raise err
def _dump(fpath, _dict, mode='w', n_indent=4):
"""Dump <_dict> into json file at <fpath>.
By default, mode is set to overwrite previous data.
As well as an indentation is set for visualization purpose.
"""
log.debug("Dumping data into \"{0}\"".format(fpath))
fjson = pth.__open_file(fpath, _mode=mode)
json.dump(_dict, fjson, indent=n_indent)
pth.__close_file(fjson)
def extract(fpath, dpath):
"""Extract archive at <fpath> into <dpath>."""
log.debug("Extracting \"{0}\" into \"{1}\"".format(fpath, dpath))
try:
unpack_archive(fpath, dpath)
except ValueError:
log.critical("\"{0}\" not a valid archive".format(fpath))
pth.__remove_file(fpath)
def decode_base64(value: str, url_safe=False, without_padding=False, lenient=False) -> str:
v = value + ("=" * (4 - (len(value) % 4))) if without_padding else value
try:
v = base64.urlsafe_b64decode(v) if url_safe else base64.b64decode(v)
return v.decode(DEFAULT_ENCODING)
except (TypeError, ValueError, UnicodeError) as err:
if lenient:
logger.debug(f'Failed when decoding base64. Value[{value}]. Error[{err}]')
return value
raise
def _common_adapt_dnsmasq(self, vpn_service: str):
identity = self.config.identity
logger.debug(
f'Adapt [{identity}] DNS resolver service to compatible with [dnsmasq] and [{vpn_service}]...'
)
FileHelper.mkdirs(self.config.config_dir)
FileHelper.copy(self.resource_dir.joinpath(f'dnsmasq-{identity}.conf'),
self.config.to_fqn_cfg(self.DNSMASQ_TUNED_CFG), True)
FileHelper.chmod(self.config.to_fqn_cfg(self.DNSMASQ_TUNED_CFG),
mode=0o0644)
return self.config.runtime_resolv
def restore_config(self, vpn_service: str, keep_dnsmasq=True):
if not keep_dnsmasq:
logger.debug(
f'Remove dnsmasq vpn hook config [{self._dnsmasq_vpn_hook_cfg}]'
)
FileHelper.rm(self._dnsmasq_vpn_hook_cfg)
logger.debug(
f'Remove dnsmasq vpn config [{self._dnsmasq_vpn_cfg(vpn_service)}]'
)
FileHelper.rm(self._dnsmasq_vpn_cfg(vpn_service))
if self._resolver:
self._resolver.restore_config(vpn_service, keep_dnsmasq)
def parseYamlFile(absoluteFilePath):
with open(absoluteFilePath) as f:
try:
inputConfig = yaml.load(f)
except yaml.YAMLError as e:
logger.error(str(e))
sys.exit(1)
logger.debug("Parsed config:\n{}".format(json.dumps(inputConfig)))
return inputConfig
def testMysqlSeeder(self):
logger.info("Initializing mysql integration testing components..")
ctx = Context(self.conn, self.inputConfig)
orderInfo, schemaForDatagen = SchemaBuilder(ctx).getSchemaForDataGen()
logger.debug("Schema for data generation:\n{}".format(json.dumps(schemaForDatagen)))
logger.debug("Will be worked in order:\n{}".format(json.dumps(orderInfo)))
writer = Writer(ctx)
dataGen = DataGen(ctx)
for results in dataGen.generate(schemaForDatagen, orderInfo):
logger.info("Writing {} documents into {}..".format(len(results["docs"]), results["table"]))
writer.doBulkWrite(results["table"], results["docs"])
logger.info("Finally, Done with it!")
def write_data():
_dict = jsn.load()
if not _dict:
log.error("{0} is empty".format(tml.value('json', section='data', subkey='fname')))
data = []
for key in _dict:
data.append(list(map(__convert, map(_read, (_dict[key], key)), repeat(DTYPE))))
log.debug("{0} couples data/label have been retrieved".format(len(data)))
npz.write(np.asarray(data))
def _extract(archive, to, suffix, compression, force_reload, extract_func):
if isinstance(to, str):
to = Path(to)
to.mkdir(parents=True, exist_ok=True)
filename = archive.name.rstrip(suffix)
if (to / filename).exists() and not force_reload:
logger.debug("Skipping file extraction. Already exist")
else:
logger.debug("Extracting %s to %s", filename, to)
extract_func(archive, to, compression)
return to / filename
def _load(fpath):
"""Get content of json file at <fpath>."""
if not pth.__is_file(fpath):
log.critical(
"\"{0}\" doesn\'t exist, can\'t load data from it".format(fpath))
log.debug("Loading data from \"{0}\"".format(fpath))
fjson = pth.__open_file(fpath)
ret = json.load(fjson)
pth.__close_file(fjson)
return ret
def generate(self, target_label: int):
logger.info(self.gather_parameters())
labels: Tensor = next(iter(self._dataloader))[1]
target_label_vector = torch.tensor(
[target_label] * labels.shape[0],
dtype=labels.dtype,
device=self._device
)
# store former classified label
former_classified_label_vector = torch.tensor(
[0] * labels.shape[0],
dtype=labels.dtype,
device=self._device
)
self._model.eval()
for ep in range(self._epoch):
# number of images with triggers that inferred as targeted label by models
attack_success_number = 0
for inputs, _ in self._dataloader:
# skip the last batch, owing to inconsistent shape
if inputs.shape != self._mask.shape:
logger.debug("skip last batch")
continue
inputs = inputs.to(self._device)
self._add_trigger(inputs)
inputs.requires_grad = True
self._model.zero_grad()
outputs = self._model(inputs)
loss = self._loss_function(outputs, target_label_vector) - \
self._loss_function(outputs, former_classified_label_vector)
loss.backward()
self._mask = self._mask - self._lr * inputs.grad.sign()
self._mask = torch.clamp(self._mask, 0, 1)
_, former_classified_label_vector = torch.max(outputs, 1)
attack_success_number += (former_classified_label_vector == target_label_vector).sum().item()
logger.info(f"epoch: {ep}, attack success number: {attack_success_number}, "
f"attack success rate: {attack_success_number / len(self._dataloader.dataset)}")
if ep % 9 == 0:
torch.save(self._mask.data, settings.trigger_dir / f"triggers-epoch{ep}")
def shape(data):
"""Shape <data> to fit input of neural network."""
log.debug("Shaping data")
if data.ndim != 2:
log.error("\'shape\' expects a two-dimensional array : (n_samples, sample_len)")
return data
_dtype = 'int{0}'.format(tml.value('bit_depth', section='audio'))
if data.dtype != _dtype:
log.warning("\'shape\' expects an {0} array".format(_dtype))
data = data.astype('float64')
for i in range(data.shape[0]):
data[i] = __pcm2float(data[i].astype(_dtype))
return data.reshape(*data.shape, 1)
def unshape(data):
"""Unshape <data> output of neural network."""
log.debug("Unshaping data")
if data.ndim != 3:
log.error("\'unshape\' expects a three-dimensional array : (n_samples, sample_len, n_channels)")
return data
data = data.reshape(*data.shape[:-1])
if data.dtype.kind != 'f':
log.warning("\'unshape\' expects a float array")
data = data.astype('float64')
for i in range(data.shape[0]):
data[i] = __float2pcm(data[i])
return data.astype('int{0}'.format(tml.value('bit_depth', section='audio')))
def seedFromTableRef(self, table, field, inSerial=True, offset=0, limit=10000):
cacheKey = "seedFromTableRef___{}___{}".format(table, field)
hit = cache.getCacheKey(cacheKey)
if not hit:
logger.debug("Queries into table for {}".format(cacheKey))
self.cursor.execute("SELECT {} FROM {} LIMIT {}, {}".format(field, table, offset, limit))
hit = [result[field] for result in self.cursor.fetchall()]
cache.setCacheKey(cacheKey, hit)
# TODO (3): Following should go in some utility file
# Used in seeders/j.py also
if not inSerial:
return random.choice(hit)
k = "MySQL__seedFromTableRef__{}__{}".format(table, field)
i = cache.getCacheKey(k, 0)
cache.setCacheKey(k, i + 1)
hitLen = len(hit)
return hit[i%hitLen]
def download(furl, dpath='.'):
"""Download file at <furl> and write it at <dpath>.
Return path to extracted file.
"""
log.debug("Downloading file from \"{0}\" into \"{1}\"".format(furl, dpath))
response = req.get(furl)
if response.status_code != req.codes.ok:
if response.status_code == HTTP.NOT_FOUND:
log.critical(''.join(['File not found at: ', furl]))
else:
log.critical(''.join(
['Error code: ', response.status_code, ', getting: ', furl]))
fpath = pth.__join_path(dpath, pth.__file_name(furl))
pth.__write_file(fpath, response.content)
return fpath
def setup(self, vpn_service: str, origin_resolv_conf: Path,
vpn_resolv_conf: Path, vpn_nameserver_hook_conf: Path):
if not self._available:
logger.error('[dnsmasq] is not yet installed or is corrupted')
sys.exit(ErrorCode.MISSING_REQUIREMENT)
logger.info('Setup DNS resolver[dnsmasq]...')
dnsmasq_vpn_cfg = self._dnsmasq_vpn_cfg(vpn_service)
runtime_resolv_cfg = self.adapt_dnsmasq(origin_resolv_conf,
vpn_service)
dnsmasq_opts = {
'{{DNS_RESOLVED_FILE}}':
self.__build_dnsmasq_conf('resolv-file', runtime_resolv_cfg),
'{{PORT}}':
self.__build_dnsmasq_conf('port',
self.dnsmasq_options().get('port',
None)),
'{{CACHE_SIZE}}':
self.__build_dnsmasq_conf(
'cache-size',
self.dnsmasq_options().get('cache_size', None))
}
logger.debug(
f'Add [dnsmasq] config for {vpn_service}[{dnsmasq_vpn_cfg}]...')
FileHelper.copy(self.resource_dir.joinpath(self.DNSMASQ_CONFIG_TMPL),
dnsmasq_vpn_cfg,
force=True)
FileHelper.replace_in_file(dnsmasq_vpn_cfg, dnsmasq_opts, backup='')
FileHelper.chmod(dnsmasq_vpn_cfg, mode=0o0644)
logger.debug(
f'Symlink [dnsmasq] VPN nameserver runtime configuration [{vpn_nameserver_hook_conf}]...'
)
FileHelper.create_symlink(vpn_nameserver_hook_conf,
self._dnsmasq_vpn_hook_cfg,
force=True)
logger.info(f'Generate System DNS config file from VPN service...')
FileHelper.write_file(vpn_resolv_conf,
self.__dnsmasq_resolv(vpn_service),
mode=0o0644)
FileHelper.create_symlink(vpn_resolv_conf,
DNSResolver.DNS_SYSTEM_FILE,
force=True)
self.service.enable(self.config.identity)
def probe(self) -> 'DNSResolver':
self.kind = next(
(t for t in DNSResolverType.as_services()
if self.service.status(t.config.identity).is_enabled()),
self.kind)
if self.kind.might_be_command():
self.kind = next(t for t in DNSResolverType.as_command()
if SystemHelper.verify_command(t.config.identity))
if self.kind.is_unknown():
logger.warn(
'Unknown DNS resolver. DNS VPN IP might be not resolved correctly'
)
if self.kind not in [DNSResolverType.DNSMASQ, DNSResolverType.UNKNOWN]:
dnsmasq_name = DNSResolverType.DNSMASQ.config.identity
self._is_dnsmasq = self.service.status(dnsmasq_name).is_enabled(
) or shutil.which(dnsmasq_name) is not None
logger.debug(
f'Current DNS resolver [{self.kind.name}], is dnsmasq available [{self._is_dnsmasq}]'
)
return self
def copy(file_or_folder: Union[str, Path], dest: Union[str, Path], force=False, skip_if_no_source=True):
p = Path(file_or_folder)
t = Path(dest)
logger.debug(f'Copy [{p}] to [{t}]...')
if not FileHelper.is_exists(p):
if skip_if_no_source:
return
raise RuntimeError(f'Given path[{file_or_folder}] is not existed')
if FileHelper.is_dir(t):
FileHelper.mkdirs(t)
else:
if t.exists():
if not force:
raise RuntimeError(f'Destination[{dest}] is existed')
FileHelper.rm(t)
FileHelper.mkdirs(t.parent)
if p.is_dir():
copy_tree(str(p.absolute()), str(t.absolute()))
else:
shutil.copy(p, t)
def get_vpn_status(self, vpn_acc: str) -> dict:
if not vpn_acc:
return {'connected': False}
try:
ss = self.exec_command('AccountStatusGet',
params=vpn_acc,
silent=True,
log_lvl=logger.DEBUG)
ss_msg = TextHelper.awk(next(
iter(TextHelper.grep(ss, r'Session Status.+')), None),
sep='|',
pos=1).strip()
return {
'connected':
ss_msg == 'Connection Completed (Session Established)',
'msg': ss_msg
}
except Exception as err:
logger.debug(
f'Something wrong when getting VPN status. Error[{err}]')
return {'connected': False}
def parseOptions(argv):
# Intialize defaults
inputFile = None
try:
options, args = getopt.getopt(argv, "hi:", ["inputFile="])
except getopt.GetoptError as e:
logger.error(str(e))
sys.exit(1)
for option, arg in options:
if option == "-h":
print(getCliHelpText())
sys.exit()
elif option in ("-i", "--inputFile"):
inputFile = arg
if not inputFile:
raise Exception('-i/--inputFile is required')
logger.debug("Input file: {}".format(inputFile))
return (inputFile)
def run_neuralnet(data, labels):
"""Run tool to train network on <data> and <labels>.
Predict on <data> and export them as wave files.
"""
# Shaping data and splitting them into train and test parts
log.info("Shaping data")
data, labels = map(utls.shape, (data, labels))
train_data, test_data = utls.split_test(data, labels)
log.debug("Computing initial MSE")
train_mses, test_mses = utls.mse(*train_data), utls.mse(*test_data)
avg_train_mse = sum(train_mses) / train_mses.shape[0]
avg_test_mse = sum(test_mses) / test_mses.shape[0]
log.debug("Average MSE of train dataset: {0}".format(avg_train_mse))
log.debug("Average MSE of test dataset: {0}".format(avg_test_mse))
# Building and training model
mdl_dname = tml.value('dnames', section='neuralnet', subkey='saved_models')
if not pth.__is_empty(mdl_dname):
log.warning(
"Model has already been trained in a previous session, picking up best model from \'{0}\' directory"
.format(mdl_dname))
NN = NeuralNetwork(model=utls.load_best_model())
else:
log.info("Training model")
pth.__make_dir(mdl_dname)
NN = NeuralNetwork()
NN.compile()
NN.train(*train_data)
# Making predictions using model
log.info("Predicting with model")
predictions = NN.predict(test_data[0])
# Exporting predicted data along with expected data
log.info("Exporting data")
dnames = tml.value('dnames', section='neuralnet')
_export(utls.unshape(predictions), dnames['predicted_labels'])
_export(utls.unshape(test_data[1]), dnames['expected_labels'])
_export(utls.unshape(test_data[0]), dnames['original_data'])
def _reconnect_vpn(_executor: VPNClientExecutor, _default_acc: str,
_current_acc: str):
logger.debug(
f'UPGRADE::Reconnect VPN previous state: default[{_default_acc}] - current[{_current_acc}]'
)
if not _current_acc and not _default_acc:
return
if not _current_acc and _default_acc:
logger.debug('UPGRADE::Enable VPN service but no connect...')
return _executor.lease_vpn_service(is_enable=True,
is_restart=False,
is_lease_ip=False)
if _current_acc == _default_acc:
logger.debug('UPGRADE::Enable then restart VPN service...')
return _executor.lease_vpn_service(is_enable=True,
is_restart=True,
is_lease_ip=False)
logger.debug(
f'UPGRADE::Start VPN service then connect to previous current acc [{_current_acc}]...'
)
_executor.device.unix_service.restart(_executor.vpn_service, delay=0)
_executor.do_disconnect_current(log_lvl=logger.DEBUG)
_executor.do_connect(_current_acc)
def setUpClass(self):
logger.debug("Setting up class..")
self.inputConfig = {
"engine": "mysql",
"host": "localhost",
"user": "******",
"database": "jseeder",
"password": "******",
"port": 3306,
"includeTables": {
"users": {
"seedSize": 10,
"excludeFields": ["middle_name"],
"inclusionPolicy": "all", # "all"/"none" - Include all/ none fields, default - "none"
"includeFields": {
"first_name": {
"seeder": "j.fromList",
"seederArgs": {
"l": ["jitendra", "kumar", "ojha"],
"inSerial": True
}
},
"last_name": {
"seeder": "j.fromList",
"seederArgs": {
"l": ["jitendra", "kumar", "ojha"],
"inSerial": True
}
},
"fav_num": {
"seeder": "j.fromBetween",
"seederArgs": {
"i": 0,
"j": 3,
"inSerial": False
}
},
"city_id": {
"seederArgs": {
"inSerial": True,
"offset": 3,
"limit": 5
}
}
}
},
"cities": {
"seedSize": 10,
"inclusionPolicy": "all",
"includeFields": {
"name": {
"seeder": "j.fromList",
"seederArgs": {
"l": ["Bangalore", "Patna"],
"inSerial": True
}
}
}
}
}
}
logger.info("Using following input config:\n{}".format(json.dumps(self.inputConfig)))
self.conn = MySQLdb.connect(
self.inputConfig["host"],
self.inputConfig["user"],
self.inputConfig["password"],
self.inputConfig["database"],
self.inputConfig["port"]
)
logger.info("Creating required test tables..")
self.cursor = self.conn.cursor()
sql = """CREATE TABLE cities (
id INT PRIMARY KEY AUTO_INCREMENT NOT NULL,
name VARCHAR(20) NOT NULL)"""
self.cursor.execute(sql)
sql = """CREATE TABLE users (
id INT PRIMARY KEY AUTO_INCREMENT NOT NULL,
first_name VARCHAR(20) NOT NULL,
middle_name VARCHAR(20),
last_name VARCHAR(20),
fav_num INT,
city_id INT,
CONSTRAINT fk_users_cities_city_id_id FOREIGN KEY (city_id) REFERENCES cities(id))"""
self.cursor.execute(sql)
def tearDownClass(self):
logger.debug("Tearing down class..")
logger.info("Droping all tests tables..")
self.cursor.execute("DROP TABLE users")
self.cursor.execute("DROP TABLE cities")
logger.error(str(e))
sys.exit(1)
inputConfig = parseYamlFile(inputFile)
if inputConfig["engine"] == "mysql":
from src.schema_builders.mysql import MysqlSchemaBuilder as SchemaBuilder
from src.writers.mysql import MysqlWriter as Writer
from src.contexts.mysql import MysqlContext as Context
conn = MySQLdb.connect(
inputConfig["host"],
inputConfig["user"],
inputConfig["password"],
inputConfig["database"],
inputConfig["port"]
)
ctx = Context(conn, inputConfig)
else:
logger.error("Engine - {} not supported".format(inputConfig["engine"]))
sys.exit(1)
orderInfo, schemaForDatagen = SchemaBuilder(ctx).getSchemaForDataGen()
logger.debug("Schema for data generation:\n{}".format(json.dumps(schemaForDatagen)))
logger.debug("Will be worked in order:\n{}".format(json.dumps(orderInfo)))
writer = Writer(ctx)
dataGen = DataGen(ctx)
for results in dataGen.generate(schemaForDatagen, orderInfo):
logger.info("Writing {} documents into {}..".format(len(results["docs"]), results["table"]))
writer.doBulkWrite(results["table"], results["docs"])
logger.info("Finally, Done with it!")
def _write(data, fpath):
"""Write <data> into numpy archive file at <fpath>."""
log.debug("Writing data in \"{0}\"".format(fpath))
np.savez_compressed(fpath, *data)
def evaluate(self, data, labels):
log.debug("Evaluating model")
return self.model.evaluate(data,
labels,
batch_size=self.bsiz,
callbacks=self.cbac)
def predict(self, data):
log.debug("Generating predictions")
return self.model.predict(data, batch_size=self.bsiz)
def _error(_path: Union[str, Path], _strict: bool, _err=None):
if strict:
if _err:
logger.debug(f'Unable read json file [{path}]. Error:{err}')
raise FileNotFoundError(f'Not found or unreadable file[{path}]')
return {}
def compile(self):
log.debug("Compiling model")
self.model.compile(self.opti, loss=self.loss, metrics=self.metr)
def _common_remove_dnsmasq(self, vpn_service: str, keep_dnsmasq: bool):
if not keep_dnsmasq:
cfg = self.config.to_fqn_cfg(self.DNSMASQ_TUNED_CFG)
logger.debug(
f'Remove [dnsmasq] and [{vpn_service}] plugin[{cfg}]...')
FileHelper.rm(cfg)
