def check_arg_input(parser, args):
if not args.input:
parser.error("-i, --input INPUT is required.")
loader = Loader.get_loader(args.input)
if loader == FSLoader:
if os.path.isfile(
args.input) and not is_a_supported_image_file_extension(
args.input):
parser.error("Input {} file not supported format.".format(
args.input))
if os.path.isfile(args.input):
check_image_file_validity(args.input)
elif loader == HTTPLoader:
if not check_url(args.input):
parser.error(
"Url {} of the http ressource doesn't exist or is not accesible."
.format(args.input))
if not is_a_supported_image_file_extension(args.input):
parser.error(
"Url {} is not file with a supported extension format.".format(
args.input))
else:
parser.error(
"Input {} is not a valid file or directory or url.".format(
args.input))
return args.input
def load(self, data, predictor):
f = self.check_keys(data, "file", "")
amount = self.check_keys(data, "amount", -1)
print "Start file: " + str(data['file']) + "."
from loader import Loader, BinaryLoader
loader = Loader.get_loader(self.check_keys(data, "loader", "BinaryLoader"), BinaryLoader())
print "Using Loader \"" + data["loader"] + "\" to load the data."
if not loader.load(f, amount, predictor):
print "Dataset \"" + data['file'] + "\" could not be loaded."
return None
return None
def load(self, data, predictor):
self.check_keys(data, "from", 0)
self.check_keys(data, "file", "")
self.check_keys(data, "to", -1)
print "Start file: " + str(data['file']) + "."
from loader import Loader, CTULoader
loader = Loader.get_loader(self.check_keys(data, "loader", "CTULoader"), CTULoader())
print "Using Loader \"" + data["loader"] + "\" to load the data."
if not loader.load(data['file'], data['from'], data['to'], predictor.get_good_labels(), False):
print "Dataset \"" + data['file'] + "\" could not be loaded."
return None
return loader.get_netflow()
def train(self, algorithm, data, feature, good_labels, config):
fr = self.check_keys(data, "from", 0)
file = self.check_keys(data, "file", "")
to = self.check_keys(data, "to", -1)
from loader import Loader, CTULoader
loader = Loader.get_loader(
self.check_keys(data, "loader", "CTULoader"), CTULoader())
print "Using Loader \"" + data["loader"] + "\" to load the data."
if not loader.load(file, fr, to):
print "Training set \"" + file + "\" could not be loaded."
return None
return self.default(algorithm, loader, feature, file)
def load(self, data, predictor):
host = self.check_keys(data, "host", "")
user = self.check_keys(data, "user", "")
password = self.check_keys(data, "password", "")
db = self.check_keys(data, "db", "")
amount = self.check_keys(data, "amount", 2000)
print "Start file: " + host+":"+db + "."
from loader import Loader, PickleLoader
loader = Loader.get_loader(self.check_keys(data, "loader", "PickleLoader"), PickleLoader())
print "Using Loader \"" + data["loader"] + "\" to load the data."
if not loader.load(host, user, password, db, amount, classify=True, db_file=predictor.get_db_file()):
print "Dataset \"" + host+":"+db + "\" could not be loaded."
return None
return loader.get_netflow()
def _setup(self, *args):
"""
Process Image Constructor.
:param args: <dict> args parameter to run the image transformation (default use Conf.args)
"""
self.__phases = select_phases(self._args)
self.__input_path = self._args['input']
self.__output_path = self._args['output']
self.__altered_path = self._args.get('altered')
self.__starting_step = self._args['steps'][0] if self._args.get(
'steps') else 0
self.__ending_step = self._args['steps'][1] if self._args.get(
'steps') else None
Conf.log.debug("All Phases : {}".format(self.__phases))
Conf.log.debug("To Be Executed Phases : {}".format(
self.__phases[self.__starting_step:self.__ending_step]))
path = self.__altered_path if os.path.isfile(self.__input_path) or not self._args.get('folder_altered') \
else os.path.join(self._args['folder_altered'], os.path.basename(self.__output_path))
self.__image_steps = [self.__input_path] + [
os.path.join(path, "{}.png".format(p().__class__.__name__))
for p in self.__phases[:self.__starting_step]
]
Conf.log.info("Processing on {}".format(str(self.__image_steps)[2:-2]))
Conf.log.debug(self.__image_steps)
try:
self.__image_steps = [
(Loader.get_loader(x)).load(x) if isinstance(x, str) else x
for x in self.__image_steps
]
except (FileNotFoundError, AttributeError) as e:
Conf.log.error(e)
Conf.log.error(
"{} is not able to resume because it not able to load required images. "
.format(camel_case_to_str(self.__class__.__name__)))
Conf.log.error(
"Possible source of this error is that --altered argument is not a correct "
"directory path that contains valid images.")
sys.exit(1)
def train(self, algorithm, data, feature, good_labels, config):
host = self.check_keys(data, "host", "")
user = self.check_keys(data, "user", "")
password = self.check_keys(data, "password", "")
db = self.check_keys(data, "db", "")
amount = self.check_keys(data, "amount", 10000)
from loader import Loader, PickleLoader
loader = Loader.get_loader(
self.check_keys(data, "loader", "PickleLoader"), PickleLoader())
print "Using Loader \"" + data["loader"] + "\" to load the data."
if not loader.load(host,
user,
password,
db,
amount,
classify=True,
db_file=config.get_db_file()):
print "Training set \"" + host + ":" + db + "\" could not be loaded."
return None
return self.default(algorithm, loader, feature, host + ":" + db)
def _setup(self, *args):
"""
Process Image Constructor.
:param args: <dict> args parameter to run the image transformation (default use Conf.args)
"""
self.__phases = select_phases(self._args)
self.__input_path = self._args['input']
self.__output_path = self._args['output']
self.__altered_path = self._args.get('altered')
self.__starting_step = self._args['steps'][0] if self._args.get(
'steps') else 0
self.__ending_step = self._args['steps'][1] if self._args.get(
'steps') else None
Conf.log.debug("All Phases : {}".format(self.__phases))
Conf.log.debug("To Be Executed Phases : {}".format(
self.__phases[self.__starting_step:self.__ending_step]))
imagename_no_ext = os.path.splitext(os.path.basename(
self.__input_path))[0]
if (self._args.get('folder_altered')):
folder_name = imagename_no_ext + '_' + str(
hashlib.md5(open(self.__input_path, 'rb').read()).hexdigest())
folder_path = os.path.join(self._args['folder_altered'],
folder_name)
if (not os.path.isdir(folder_path)):
os.makedirs(folder_path, exist_ok=True)
self._args['folder_altered'] = folder_path
path = self._args['folder_altered']
self.__image_steps = [self.__input_path] + [
os.path.join(path, "{}.png".format(p().__class__.__name__))
for p in self.__phases[:self.__starting_step]
]
elif (self.__altered_path):
folder_name = imagename_no_ext + '_' + str(
hashlib.md5(open(self.__input_path, 'rb').read()).hexdigest())
folder_path = os.path.join(self.__altered_path, folder_name)
if (not os.path.isdir(folder_path)):
os.makedirs(folder_path, exist_ok=True)
self.__altered_path = folder_path
path = self.__altered_path
self.__image_steps = [self.__input_path] + [
os.path.join(path, "{}.png".format(p().__class__.__name__))
for p in self.__phases[:self.__starting_step]
]
else:
# TODO: refactor me, please!
self.__image_steps = [self.__input_path] + [
self.__input_path for p in self.__phases[:self.__starting_step]
]
Conf.log.info("Processing on {}".format(str(self.__image_steps)))
try:
self.__image_steps = [
(Loader.get_loader(x)).load(x) if isinstance(x, str) else x
for x in self.__image_steps
]
except (FileNotFoundError, AttributeError) as e:
Conf.log.error(e)
Conf.log.error(
"{} is not able to resume because it not able to load required images. "
.format(camel_case_to_str(self.__class__.__name__)))
Conf.log.error(
"Possible source of this error is that --altered argument is not a correct "
"directory path that contains valid images.")
sys.exit(1)
