def add_params(name: str, *obj: object):
log = ChangeLog.query.filter_by(name=name).all()
successful = False
if log is not None and len(log) > 0:
return
else:
try:
changelog = ChangeLog()
for o in obj:
try:
db.session.add(o)
successful = True
except Exception as e:
Logger.info(e)
db.session.rollback()
successful = False
if successful:
changelog.name = name
changelog.created = str(int(dt.now().timestamp()))
db.session.add(changelog)
db.session.commit()
except Exception as e:
Logger.info(e)
db.session.rollback()
raise
def load_data():
if config.Server.Settings.load_gameobjects:
WorldLoader.load_gameobjects()
else:
Logger.info('Skipped game object loading.')
if config.Server.Settings.load_creatures:
WorldLoader.load_creature_loot_templates()
WorldLoader.load_creatures()
WorldLoader.load_creature_quests()
WorldLoader.load_creature_involved_quests()
else:
Logger.info('Skipped creature loading.')
WorldLoader.load_item_templates()
WorldLoader.load_quests()
WorldLoader.load_spells()
WorldLoader.load_skills()
WorldLoader.load_skill_line_abilities()
WorldLoader.load_taxi_nodes()
WorldLoader.load_taxi_path_nodes()
# Character related data
WorldLoader.load_groups()
WorldLoader.load_guilds()
def load_data():
# Map tiles
MapManager.initialize_maps()
MapManager.initialize_area_tables()
# Gameobject spawns
if config.Server.Settings.load_gameobjects:
WorldLoader.load_gameobjects()
WorldLoader.load_gameobject_loot_templates()
else:
Logger.info('Skipped game object loading.')
# Creature spawns
if config.Server.Settings.load_creatures:
WorldLoader.load_creature_loot_templates()
WorldLoader.load_creatures()
WorldLoader.load_creature_quests()
WorldLoader.load_creature_involved_quests()
else:
Logger.info('Skipped creature loading.')
WorldLoader.load_item_templates()
WorldLoader.load_quests()
WorldLoader.load_spells()
WorldLoader.load_skills()
WorldLoader.load_skill_line_abilities()
WorldLoader.load_char_base_infos()
WorldLoader.load_taxi_nodes()
WorldLoader.load_taxi_path_nodes()
WorldLoader.load_factions()
WorldLoader.load_faction_templates()
# Character related data
WorldLoader.load_groups()
WorldLoader.load_guilds()
def parseStringDtToDate(string_date):
try:
date_formate = datetime.datetime.strptime(string_date, '%Y-%m-%d')
parsedDate = date_formate.strftime("%Y-%m-%d")
return parsedDate
except Exception as e:
Logger.info(e)
def load_data():
# Map tiles
MapManager.initialize_maps()
MapManager.initialize_area_tables()
# Below order matters.
# Loot related, even if not loading creatures or gameobjects, loot might be referenced.
WorldLoader.load_gameobject_loot_templates()
WorldLoader.load_fishing_loot_templates()
WorldLoader.load_creature_loot_templates()
WorldLoader.load_item_templates()
WorldLoader.load_reference_loot_templates()
WorldLoader.load_pickpocketing_loot_templates()
WorldLoader.load_item_loot_templates()
# Spells.
WorldLoader.load_spells()
WorldLoader.load_creature_spells()
# Gameobject spawns
if config.Server.Settings.load_gameobjects:
WorldLoader.load_gameobject_quest_starters()
WorldLoader.load_gameobject_quest_finishers()
WorldLoader.load_gameobjects()
else:
Logger.info('Skipped game object loading.')
# Creature spawns
if config.Server.Settings.load_creatures:
WorldLoader.load_creature_equip_templates()
WorldLoader.load_creatures()
WorldLoader.load_creature_on_kill_reputation()
WorldLoader.load_creature_quest_starters()
WorldLoader.load_creature_quest_finishers()
WorldLoader.load_creature_display_info()
WorldLoader.load_creature_model_info()
WorldLoader.load_creature_families()
WorldLoader.load_npc_gossip()
WorldLoader.load_npc_text()
else:
Logger.info('Skipped creature loading.')
WorldLoader.load_area_trigger_quest_relations()
WorldLoader.load_quests()
WorldLoader.load_spell_chains()
WorldLoader.load_trainer_spells()
WorldLoader.load_skills()
WorldLoader.load_skill_line_abilities()
WorldLoader.load_char_base_infos()
WorldLoader.load_taxi_nodes()
WorldLoader.load_taxi_path_nodes()
WorldLoader.load_factions()
WorldLoader.load_faction_templates()
WorldLoader.load_locks()
# Character related data
WorldLoader.load_groups()
WorldLoader.load_guilds()
def start():
Logger.info('Proxy server started.')
ThreadedProxyServer.allow_reuse_address = True
with ThreadedProxyServer((config.Server.Connection.RealmServer.host,
config.Server.Connection.RealmProxy.port), ProxyServerSessionHandler) \
as proxy_instance:
proxy_session_thread = threading.Thread(
target=proxy_instance.serve_forever())
proxy_session_thread.daemon = True
proxy_session_thread.start()
def start():
Logger.info('Login server started.')
ThreadedLoginServer.allow_reuse_address = True
with ThreadedLoginServer((config.Server.Connection.RealmServer.host,
config.Server.Connection.RealmServer.port), LoginServerSessionHandler) \
as login_instance:
login_session_thread = threading.Thread(
target=login_instance.serve_forever())
login_session_thread.daemon = True
login_session_thread.start()
def start():
ThreadedProxyServer.allow_reuse_address = True
with ThreadedProxyServer((config.Server.Connection.RealmProxy.host,
config.Server.Connection.RealmProxy.port), ProxyServerSessionHandler) \
as proxy_instance:
Logger.success(f'Proxy server started, listening on {proxy_instance.server_address[0]}:{proxy_instance.server_address[1]}')
try:
proxy_session_thread = threading.Thread(target=proxy_instance.serve_forever())
proxy_session_thread.daemon = True
proxy_session_thread.start()
except KeyboardInterrupt:
Logger.info("Proxy server turned off.")
def _load_data():
# TODO: Use threads to load the data more efficiently
if config.Server.Settings.load_gameobjects:
WorldServerSessionHandler._load_gameobjects()
else:
Logger.info('Skipped game object loading.')
if config.Server.Settings.load_creatures:
WorldServerSessionHandler._load_creatures()
else:
Logger.info('Skipped creature loading.')
WorldServerSessionHandler._load_spells()
def start():
Logger.success('Login server started.')
ThreadedLoginServer.allow_reuse_address = True
with ThreadedLoginServer((config.Server.Connection.RealmServer.host,
config.Server.Connection.RealmServer.port), LoginServerSessionHandler) \
as login_instance:
try:
login_session_thread = threading.Thread(target=login_instance.serve_forever())
login_session_thread.daemon = True
login_session_thread.start()
except KeyboardInterrupt:
Logger.info("Login server turned off.")
async def launch(self):
try:
self.ws = await websockets.connect(f"ws://{self.host}:{self.port}",
extra_headers=self._headers())
if self.ws.open:
Logger.info(
f"Node connected with host: {self.host} and port: {self.port}."
)
self._manager.bot.loop.create_task(self._wait_for_ws_message())
self.ready = True
except OSError as error:
Logger.error(
f"Failed to connect to LavaLink with host: {self.host} reason: {error}"
)
def start():
ThreadedLoginServer.allow_reuse_address = True
with ThreadedLoginServer((config.Server.Connection.RealmServer.host,
config.Server.Connection.RealmServer.port), LoginServerSessionHandler) \
as login_instance:
Logger.success(f'Login server started, listening on {login_instance.server_address[0]}:{login_instance.server_address[1]}')
# Make sure all characters have online = 0 on realm start.
RealmDatabaseManager.character_set_all_offline()
try:
login_session_thread = threading.Thread(target=login_instance.serve_forever())
login_session_thread.daemon = True
login_session_thread.start()
except KeyboardInterrupt:
Logger.info("Login server turned off.")
def load_data():
if config.Server.Settings.load_gameobjects:
WorldLoader.load_gameobjects()
else:
Logger.info('Skipped game object loading.')
if config.Server.Settings.load_creatures:
WorldLoader.load_creatures()
else:
Logger.info('Skipped creature loading.')
WorldLoader.load_spells()
WorldLoader.load_skills()
WorldLoader.load_skill_line_abilities()
WorldLoader.load_taxi_nodes()
WorldLoader.load_taxi_path_nodes()
def load_data():
# Map tiles
MapManager.initialize_maps()
MapManager.initialize_area_tables()
# Gameobject spawns
if config.Server.Settings.load_gameobjects:
WorldLoader.load_gameobjects()
WorldLoader.load_gameobject_loot_templates()
WorldLoader.load_gameobject_quest_starters()
WorldLoader.load_gameobject_quest_finishers()
else:
Logger.info('Skipped game object loading.')
# Creature spawns
if config.Server.Settings.load_creatures:
WorldLoader.load_creature_loot_templates()
WorldLoader.load_creature_equip_templates()
WorldLoader.load_creatures()
WorldLoader.load_creature_quest_starters()
WorldLoader.load_creature_quest_finishers()
WorldLoader.load_creature_display_info()
WorldLoader.load_creature_model_info()
WorldLoader.load_npc_gossip()
WorldLoader.load_npc_text()
else:
Logger.info('Skipped creature loading.')
WorldLoader.load_item_templates()
WorldLoader.load_item_loot_templates()
WorldLoader.load_quests()
WorldLoader.load_spells()
WorldLoader.load_spell_chains()
WorldLoader.load_trainer_spells()
WorldLoader.load_skills()
WorldLoader.load_skill_line_abilities()
WorldLoader.load_char_base_infos()
WorldLoader.load_taxi_nodes()
WorldLoader.load_taxi_path_nodes()
WorldLoader.load_factions()
WorldLoader.load_faction_templates()
# Character related data
WorldLoader.load_groups()
WorldLoader.load_guilds()
def start():
WorldLoader.load_data()
Logger.success('World server started.')
WorldServerSessionHandler.schedule_updates()
ThreadedWorldServer.allow_reuse_address = True
ThreadedWorldServer.timeout = 10
with ThreadedWorldServer((config.Server.Connection.RealmServer.host,
config.Server.Connection.WorldServer.port),
WorldServerSessionHandler) as world_instance:
try:
world_session_thread = threading.Thread(
target=world_instance.serve_forever())
world_session_thread.daemon = True
world_session_thread.start()
except KeyboardInterrupt:
Logger.info("World server turned off.")
def main():
logger = Logger(name = "runner_test")
service = FDTService(logger)
report(service, logger)
# number of FDT Java servers to create
numServers = 3
ids = ["request-%03d" % i for i in range(numServers)]
threads = []
for id in ids:
runner = ThreadCaller(id, "FDT caller", service.service, logger)
runner.start()
threads.append(runner)
# wait until all threads finish ... no matter how but must finish
logger.info("Waiting for FDT caller threads to terminate ...")
for t in threads:
t.join()
report(service, logger)
threads = []
# commenting out this section - cleaning up and restarting the
# script should show reliable failure reporting since the ports
# occupied from the previous script run
for id in ids:
wiper = ThreadCaller(id, "FDT wiper", service.cleanUp, logger)
wiper.start()
threads.append(wiper)
# wait until all threads finish ... no matter how but must finish
logger.info("Waiting for FDT wiper threads to terminate ...")
for t in threads:
t.join()
report(service, logger)
async def on_ready():
bot.started_at = time.time()
Logger.info(
f"Bot ready as: {bot.user} serving in {len(bot.guilds)} guilds with {len(bot.users)} users."
)
Logger.info("Loading cogs...")
for cog in bot.cog_files:
bot.load_extension(f"cogs.{cog}")
Logger.info(f"Loaded cog: {cog}")
Logger.info("All cogs loaded.")
bot.loop.create_task(status_change())
Logger.task("Status change has started.")
bot.loop.create_task(bot.music_manager.audio_task())
Logger.task("Audio task has started.")
def main():
params = ParameterParser().get_params()
urls_manager = URLsManager(
params['search_terms'],
params['n_pics_in_batch'],
params['number_pictures']
)
browser = BrowserWrapper(params['search_terms'], params['target_directory'])
logger = Logger(params['log_file_directory'])
try:
logger.info('Started new session for search terms {}.'.format(
' '.join(params['search_terms'])
))
if params['links_only']:
logger.info('Using URL search-only mode.')
elif params['pictures_only']:
logger.info('Using picture download-only mode.')
while not urls_manager.total_pic_number_collected():
collect_urls(params, urls_manager, browser, logger)
logger.info('Collected full batch of URLs. {} in queue.'.format(urls_manager.in_queue()))
try:
download_pics(params, urls_manager, browser, logger)
except ElementDoesNotExistException:
time.sleep(30)
try:
download_pics(params, urls_manager, browser, logger)
except ElementDoesNotExistException:
continue
logger.info('Downloaded all images from current batch, collecting new URLs.')
except KeyboardInterrupt:
logger.warn('Keyboard interrupt, trying to make URL lists persistent...')
urls_manager.print_url_list_sizes()
urls_manager.write_urls()
def main():
logger = Logger(name="runner_test")
service = FDTService(logger)
report(service, logger)
# number of FDT Java servers to create
numServers = 3
ids = ["request-%03d" % i for i in range(numServers)]
threads = []
for id in ids:
runner = ThreadCaller(id, "FDT caller", service.service, logger)
runner.start()
threads.append(runner)
# wait until all threads finish ... no matter how but must finish
logger.info("Waiting for FDT caller threads to terminate ...")
for t in threads:
t.join()
report(service, logger)
threads = []
# commenting out this section - cleaning up and restarting the
# script should show reliable failure reporting since the ports
# occupied from the previous script run
for id in ids:
wiper = ThreadCaller(id, "FDT wiper", service.cleanUp, logger)
wiper.start()
threads.append(wiper)
# wait until all threads finish ... no matter how but must finish
logger.info("Waiting for FDT wiper threads to terminate ...")
for t in threads:
t.join()
report(service, logger)
def _load_data():
# TODO: Use threads to load the data more efficiently
if config.Server.Settings.load_gameobjects:
Logger.info('Loading game objects...')
gobject_number = WorldServerSessionHandler._load_gameobjects()
Logger.success('%u game objects successfully loaded.' %
gobject_number)
else:
Logger.info('Skipped game object loading.')
if config.Server.Settings.load_creatures:
Logger.info('Loading creature spawns...')
creature_number = WorldServerSessionHandler._load_creatures()
Logger.success('%u creature spawns successfully loaded.' %
creature_number)
else:
Logger.info('Skipped creature loading.')
def start():
Logger.info('World server started.')
Logger.info('Loading realm tables...')
RealmDatabaseManager.load_tables()
Logger.info('Realm tables loaded.')
ThreadedWorldServer.allow_reuse_address = True
with ThreadedWorldServer((config.Server.Connection.RealmServer.host, config.Server.Connection.WorldServer.port),
WorldServerSessionHandler) as world_instance:
world_session_thread = threading.Thread(target=world_instance.serve_forever())
world_session_thread.daemon = True
world_session_thread.start()
torch.random.manual_seed(conf.seed)
device = torch.device('cuda') if torch.cuda.is_available() else torch.device(
'cpu')
dataset = Dataset(data_dir=conf.data_dir,
data_name=model_conf.data_name,
train_ratio=model_conf.train_ratio,
device=device)
log_dir = os.path.join('saves', conf.model)
logger = Logger(log_dir)
model_conf.save(os.path.join(logger.log_dir, 'config.json'))
eval_pos, eval_target = dataset.eval_data()
item_popularity = dataset.item_popularity
evaluator = Evaluator(eval_pos, eval_target, item_popularity, model_conf.top_k)
model_base = getattr(models, conf.model)
model = model_base(model_conf, dataset.num_users, dataset.num_items, device)
logger.info(model_conf)
logger.info(dataset)
trainer = Trainer(dataset=dataset,
model=model,
evaluator=evaluator,
logger=logger,
conf=model_conf)
trainer.train()
os.makedirs(exp_visual_dir)
exp_ckpt_dir = os.path.join(exp_dir, "checkpoints")
if not os.path.exists(exp_ckpt_dir):
os.makedirs(exp_ckpt_dir)
now_str = datetime.datetime.now().__str__().replace(' ', '_')
writer_path = os.path.join(exp_visual_dir, now_str)
writer = SummaryWriter(writer_path)
logger_path = os.path.join(exp_log_dir, now_str + ".log")
logger = Logger(logger_path).get_logger()
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
logger.info("argument parser settings: {}".format(args))
logger.info("basic configuration settings: {}".format(basic_configs))
# Part 2-4. configurations for loss function, network, and optimizer
loss_function = nn.CrossEntropyLoss()
max_val_acc = 0.0
max_val_acc_epoch = -1
network_configs = collections.OrderedDict()
network_configs['output_dim'] = 345
network_configs['n_heads'] = 8
def __init__():
# Set up the global logger
log = Logger('debug', 180)
# Parse arguments
parser = ArgumentParser(
"ImageCaptionLearn_py: Neural Network for Nonvisual "
"Prediction; Bidirectional LSTM to hidden layer "
"to softmax over (v)isual and (n)onvisual labels")
parser.add_argument(
"--epochs",
type=int,
default=20,
help="train opt; number of times to iterate over the dataset")
parser.add_argument(
"--batch_size",
type=int,
default=512,
help="train opt; number of random mention pairs per batch")
parser.add_argument("--lstm_hidden_width",
type=int,
default=200,
help="train opt; number of hidden units within "
"the LSTM cells")
parser.add_argument("--start_hidden_width",
type=int,
default=512,
help="train opt; number of hidden units in the "
"layer after the LSTM")
parser.add_argument("--hidden_depth",
type=int,
default=2,
help="train opt; number of hidden layers after the "
"lstm, where each is last_width/2 units wide, "
"starting with start_hidden_width")
parser.add_argument("--weighted_classes",
action="store_true",
help="Whether to inversely weight the classes "
"in the loss")
parser.add_argument("--learn_rate",
type=float,
default=0.001,
help="train opt; optimizer learning rate")
parser.add_argument("--adam_epsilon",
type=float,
default=1e-08,
help="train opt; Adam optimizer epsilon value")
parser.add_argument("--clip_norm",
type=float,
default=5.0,
help='train opt; global clip norm value')
parser.add_argument(
"--data_norm",
action='store_true',
help="train opt; Whether to L2-normalize the w2v word vectors")
parser.add_argument("--lstm_input_dropout",
type=float,
default=0.5,
help="train opt; probability to keep lstm input nodes")
parser.add_argument("--dropout",
type=float,
default=0.5,
help="train opt; probability to keep all other nodes")
parser.add_argument(
"--data_dir",
required=True,
type=lambda f: util.arg_path_exists(parser, f),
help="Directory containing raw/, feats/, and scores/ directories")
parser.add_argument(
"--data",
choices=["flickr30k", "mscoco", "coco30k", "flickr30k_v1"],
required=True,
help="Dataset to use")
parser.add_argument("--split",
choices=["train", "dev", "test", "trainDev"],
required=True,
help="Dataset split")
parser.add_argument("--eval_data",
choices=["flickr30k", "mscoco", "coco30k"],
help="Evaluation dataset to use")
parser.add_argument("--eval_split",
choices=["train", "dev", "test", "trainDev"],
help="Evaluation dataset split")
parser.add_argument(
"--encoding_scheme",
choices=["first_last_sentence", 'first_last_mention'],
default="first_last_mention",
help="train opt; specifies how lstm outputs are transformed")
parser.add_argument("--train", action='store_true', help='Trains a model')
parser.add_argument(
"--activation",
choices=['sigmoid', 'tanh', 'relu', 'leaky_relu'],
default='relu',
help='train opt; which nonlinear activation function to use')
parser.add_argument("--predict",
action='store_true',
help='Predicts using pre-trained model')
parser.add_argument("--model_file",
type=str,
help="Model file to save/load")
parser.add_argument("--embedding_type",
choices=['w2v', 'glove'],
default='w2v',
help="Word embedding type to use")
parser.add_argument(
"--early_stopping",
action='store_true',
help="Whether to implement early stopping based on the "
"evaluation performance")
args = parser.parse_args()
arg_dict = vars(args)
if arg_dict['train'] and arg_dict['model_file'] is None:
arg_dict['model_file'] = "/home/ccervan2/models/tacl201801//" + \
nn_data.build_model_filename(arg_dict, "affinity_lstm")
model_file = arg_dict['model_file']
util.dump_args(arg_dict, log)
# Construct data files from the root directory and filename
data_dir = arg_dict['data_dir'] + "/"
data_root = arg_dict['data'] + "_" + arg_dict['split']
eval_data_root = None
if arg_dict['train']:
eval_data_root = arg_dict['eval_data'] + "_" + arg_dict['eval_split']
sentence_file = data_dir + "raw/" + data_root + "_captions.txt"
mention_idx_file = data_dir + "raw/" + data_root + "_mentions_affinity.txt"
feature_file = data_dir + "feats/" + data_root + "_affinity_neural.feats"
feature_meta_file = data_dir + "feats/" + data_root + "_affinity_neural_meta.json"
box_dir = data_dir + "feats/" + arg_dict['data'] + "_boxes/" + arg_dict[
"split"] + "/"
mention_box_label_file = data_dir + "raw/" + data_root + "_affinity_labels.txt"
box_category_file = None
#if "coco" in data_root:
# box_category_file = data_dir + "raw/" + data_root + "_box_cats.txt"
if eval_data_root is not None:
eval_box_dir = data_dir + "feats/" + arg_dict[
'eval_data'] + "_boxes/" + arg_dict["eval_split"] + "/"
eval_sentence_file = data_dir + "raw/" + eval_data_root + "_captions.txt"
eval_mention_idx_file = data_dir + "raw/" + eval_data_root + "_mentions_affinity.txt"
eval_feature_file = data_dir + "feats/" + eval_data_root + "_affinity_neural.feats"
eval_feature_meta_file = data_dir + "feats/" + eval_data_root + "_affinity_neural_meta.json"
eval_mention_box_label_file = data_dir + "raw/" + eval_data_root + "_affinity_labels.txt"
eval_box_category_file = None
# if "coco" in eval_data_root:
# eval_box_category_file = data_dir + "raw/" + eval_data_root + "_box_cats.txt"
#endif
# Load the appropriate word embeddings
embedding_type = arg_dict['embedding_type']
if embedding_type == 'w2v':
log.info("Initializing word2vec")
nn_data.init_w2v()
elif embedding_type == 'glove':
log.info("Initializing glove")
nn_data.init_glove()
#endif
# Set the random seeds identically every run
nn_util.set_random_seeds()
# Set up the minimum tensorflow logging level
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
# Train, if training was specified
if arg_dict['train']:
train(encoding_scheme=arg_dict['encoding_scheme'],
embedding_type=embedding_type,
sentence_file=sentence_file,
mention_idx_file=mention_idx_file,
feature_file=feature_file,
feature_meta_file=feature_meta_file,
box_dir=box_dir,
mention_box_label_file=mention_box_label_file,
epochs=arg_dict['epochs'],
batch_size=arg_dict['batch_size'],
lstm_hidden_width=arg_dict['lstm_hidden_width'],
start_hidden_width=arg_dict['start_hidden_width'],
hidden_depth=arg_dict['hidden_depth'],
weighted_classes=arg_dict['weighted_classes'],
input_dropout=arg_dict['lstm_input_dropout'],
other_dropout=arg_dict['dropout'],
lrn_rate=arg_dict['learn_rate'],
clip_norm=arg_dict['clip_norm'],
data_norm=arg_dict['data_norm'],
adam_epsilon=arg_dict['adam_epsilon'],
activation=arg_dict['activation'],
model_file=model_file,
eval_sentence_file=eval_sentence_file,
eval_mention_idx_file=eval_mention_idx_file,
eval_feature_file=eval_feature_file,
eval_feature_meta_file=eval_feature_meta_file,
eval_box_dir=eval_box_dir,
eval_mention_box_label_file=eval_mention_box_label_file,
early_stopping=arg_dict['early_stopping'],
box_category_file=box_category_file,
eval_box_category_file=eval_box_category_file,
log=log)
elif arg_dict['predict']:
scores_file = data_dir + "scores/" + data_root + "_affinity.scores"
# Restore our variables
tf.reset_default_graph()
with tf.Session() as sess:
saver = tf.train.import_meta_graph(model_file + ".meta")
saver.restore(sess, model_file)
predict(encoding_scheme=arg_dict['encoding_scheme'],
embedding_type=embedding_type,
tf_session=sess,
batch_size=arg_dict['batch_size'],
sentence_file=sentence_file,
mention_idx_file=mention_idx_file,
feature_file=feature_file,
feature_meta_file=feature_meta_file,
box_dir=box_dir,
mention_box_label_file=mention_box_label_file,
scores_file=scores_file,
box_category_file=box_category_file,
log=log)
class SenderWindow:
def __init__(self, sequenceN, windowSize):
self.sequenceNoBit = sequenceN
# secventa de numere a primului frame
self.SeqFirst = 0
self.logger = Logger("SenderWindow")
self.maxSeq = math.pow(
2, self.sequenceNoBit
) #fereastra are dim max jumatatate din numarul de secvente
# secventa de numere a ultimului frame
self.transmitWindow = OrderedDict(
) # va contine timerul si daca a fost ack(true/false)
self.expectedAck = 0
self.nextPkt = 0 # used to iterate through packetList
self.nextSeqNo = 0
self.in_progress = 1
if 0 < windowSize <= self.maxSeq / 2:
self.windowSize = windowSize
else:
self.logger.error(
"Window size should be greater than 0 and less then 2^(sequenceN-1)!"
)
def getMaxSeq(self):
return self.maxSeq
def isEmpty(self):
if len(self.transmitWindow) == 0:
return True
return False
def insideWindow(self, pktNo):
if pktNo in self.transmitWindow:
return True
return False
def mark_ack(self, pktNo):
with lock:
self.transmitWindow[int(pktNo)][1] = True
def unacked(self, pktNo):
if pktNo in self.transmitWindow:
return not self.transmitWindow[pktNo][1]
def ackRecv(self, pktNo):
with lock:
if self.insideWindow(pktNo):
self.transmitWindow[pktNo][0] = None # stop timer for packet
if pktNo == self.expectedAck:
self.slideWindow()
with lock:
if len(self.transmitWindow) == 0:
self.expectedAck = self.nextSeqNo
else:
self.expectedAck = list(self.transmitWindow.items())[0][
0] # urm ack devine capatul inferior al ferestrei
def slideWindow(self):
to_delete = []
for k, v in self.transmitWindow.items():
if v[0] == None and v[1] == True:
to_delete.append(k)
else:
break
with lock:
for item in to_delete:
del self.transmitWindow[item]
def getSeqNo(self):
with lock:
self.transmitWindow[self.nextSeqNo] = [None, False]
self.nextSeqNo += 1
if self.nextSeqNo >= self.maxSeq:
self.nextSeqNo = int(self.nextSeqNo % self.maxSeq)
self.nextPkt += 1
def getNextPkt(self):
return self.nextPkt
def startTimer(self, pktNo): # pkt.time=timpul la momentul curent
self.logger.info(f"On thread-{pktNo} function startTimer")
if pktNo in self.transmitWindow:
with lock:
self.transmitWindow[pktNo][0] = time.time()
def getStartTime(
self, pktNo
): # va returna valoarea timpului in momentul trimiterii pachetului
return self.transmitWindow[pktNo][0]
def restartTimer(
self, pktNo
): # pkt.time=timpul la momentul curent- in caz de nevoie de retransmitere
if self.insideWindow(pktNo):
self.transmitWindow[pktNo][0] = time.time()
def stopTimer(self, pktNo):
self.logger.info(f"On thread-{pktNo} function stopTimer")
with lock:
self.transmitWindow[pktNo][0] = None
def full(
self
): # in transmiterea unui nou pachet, se are in vederea dim max a ferestrei
if len(self.transmitWindow) >= self.windowSize:
return True
return False
def stopTransm(self):
self.in_progress = 0
def isTransmissionDone(self):
return not self.in_progress
eval_type_file = abspath(
expanduser(data_root + "cca/" + eval_prefix + "_type_" + lexical_types +
".csv"))
eval_scores_file = abspath(
expanduser(data_root + "cca/" + eval_prefix + "_ccaScores.csv"))
scores_file = abspath(
expanduser(data_root + "scores/" + eval_prefix + "_" + lexical_types +
"Types_affinity.scores"))
if arg_dict['train']:
type_id_dict, type_x_dict, type_y_dict = \
load_cca_data(fit_id_file, fit_scores_file, fit_label_file, fit_type_file)
# learn a separate curve for each lexical type
for type in type_x_dict.keys():
log.info('Training ' + type)
x = np.array(type_x_dict[type]).reshape((-1, 1))
y = np.array(type_y_dict[type])
learner = LogisticRegression(max_iter=arg_dict['max_iter'], n_jobs=-1)
learner.fit(x, y)
model_file = abspath(
expanduser(model_root + "affinity_" + type + ".model"))
with open(model_file, 'wb') as pickle_file:
cPickle.dump(learner, pickle_file)
#endwith
#endfor
#endif
if arg_dict['eval']:
type_id_dict_eval, type_x_dict_eval, type_y_dict_eval = \
load_cca_data(eval_id_file, eval_scores_file, eval_label_file, eval_type_file)
proxy_process.start()
world_process = context.Process(
target=WorldManager.WorldServerSessionHandler.start)
world_process.start()
try:
if os.getenv('CONSOLE_MODE', config.Server.Settings.console_mode) in [
True, 'True', 'true'
]:
while input() != 'exit':
Logger.error('Invalid command.')
else:
world_process.join()
except:
Logger.info('Shutting down the core...')
# Send SIGTERM to processes.
world_process.terminate()
Logger.info('World process terminated.')
proxy_process.terminate()
Logger.info('Proxy process terminated.')
login_process.terminate()
Logger.info('Login process terminated.')
# Release process resources.
Logger.info('Waiting to release resources...')
release_process(world_process)
release_process(proxy_process)
release_process(login_process)
exp_ckpt_dir = os.path.join(exp_dir, "checkpoints")
if not os.path.exists(exp_ckpt_dir):
os.makedirs(exp_ckpt_dir)
now_str = datetime.datetime.now().__str__().replace(' ', '_')
writer_path = os.path.join(exp_visual_dir, now_str)
writer = SummaryWriter(writer_path)
logger_path = os.path.join(exp_log_dir, now_str + ".log")
logger = Logger(logger_path).get_logger()
# TODO
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
logger.info("basic configuration settings: {}".format(basic_configs))
#logger.info("dataloader configuration settings: {}".format(dataloader_configs))
# Part 2-4. configurations for loss function, and optimizer
# TODO
loss_function = nn.CrossEntropyLoss()
net = models.inception_v3(num_classes=345)
logger.info("withOUT ImageNet pretraining!!!")
net = net.cuda()
# net = torch.nn.DataParallel(net, device_ids=[int(x) for x in args.gpu.split(',')]).cuda()
# optimizer = torch.optim.SGD(net.parameters(), lr=basic_configs['learning_rate'], momentum=0.9, weight_decay=5e-4)
# TODO change as RMSProb
exp_ckpt_dir = os.path.join(exp_dir, "checkpoints")
if not os.path.exists(exp_ckpt_dir):
os.makedirs(exp_ckpt_dir)
now_str = datetime.datetime.now().__str__().replace(' ', '_')
writer_path = os.path.join(exp_visual_dir, now_str)
writer = SummaryWriter(writer_path)
logger_path = os.path.join(exp_log_dir, now_str + ".log")
logger = Logger(logger_path).get_logger()
# TODO
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
logger.info("basic configuration settings: {}".format(basic_configs))
#logger.info("dataloader configuration settings: {}".format(dataloader_configs))
# Part 2-4. configurations for loss function, SiamRPN++ network, and optimizer
# TODO
loss_function = nn.CrossEntropyLoss()
training_loss = AverageMeter()
net = SiamRPN()
#net = net.cuda()
net = torch.nn.DataParallel(net,
device_ids=[int(x)
for x in args.gpu.split(',')]).cuda()
optimizer = torch.optim.SGD(net.parameters(),
def __init__():
# Set up the logger
log = Logger('debug', 180)
# Parse arguments
parser = ArgumentParser("ImageCaptionLearn_py: Core Neural "
"Network classification architecture; "
"used for nonvis and cardinality prediction")
parser.add_argument("--epochs",
type=int,
default=20,
help="train opt; number of times to "
"iterate over the dataset")
parser.add_argument("--batch_size",
type=int,
default=512,
help="train opt; number of random mention "
"pairs per batch")
parser.add_argument("--lstm_hidden_width",
type=int,
default=200,
help="train opt; number of hidden units "
"within the LSTM cells")
parser.add_argument("--start_hidden_width",
type=int,
default=512,
help="train opt; number of hidden units "
"in the layer after the LSTM")
parser.add_argument("--hidden_depth",
type=int,
default=2,
help="train opt; number of hidden layers "
"after the lstm, where each is "
"last_width/2 units wide, starting "
"with start_hidden_width")
parser.add_argument("--weighted_classes",
action="store_true",
help="Whether to inversely weight the "
"classes in the loss")
parser.add_argument("--learn_rate",
type=float,
default=0.001,
help="train opt; optimizer learning rate")
parser.add_argument("--adam_epsilon",
type=float,
default=1e-08,
help="train opt; Adam optimizer epsilon value")
parser.add_argument("--clip_norm",
type=float,
default=5.0,
help='train opt; global clip norm value')
parser.add_argument(
"--data_norm",
action='store_true',
help="train opt; Whether to L2-normalize the w2v word vectors")
parser.add_argument("--lstm_input_dropout",
type=float,
default=0.5,
help="train opt; probability to keep lstm input nodes")
parser.add_argument("--dropout",
type=float,
default=0.5,
help="train opt; probability to keep all other nodes")
parser.add_argument(
"--data_dir",
required=True,
type=lambda f: util.arg_path_exists(parser, f),
help="Directory containing raw/, feats/, and scores/ directories")
parser.add_argument("--data_root",
type=str,
required=True,
help="Data file root (eg. flickr30k_train)")
parser.add_argument(
"--eval_data_root",
type=str,
help="Data file root for eval data (eg. flickr30k_dev)")
parser.add_argument("--train", action='store_true', help='Trains a model')
parser.add_argument(
"--activation",
choices=['sigmoid', 'tanh', 'relu', 'leaky_relu'],
default='relu',
help='train opt; which nonlinear activation function to use')
parser.add_argument("--predict",
action='store_true',
help='Predicts using pre-trained model')
parser.add_argument("--model_file",
type=str,
help="Model file to save/load")
parser.add_argument("--embedding_type",
choices=['w2v', 'glove'],
default='w2v',
help="Word embedding type to use")
parser.add_argument(
"--early_stopping",
action='store_true',
help="Whether to implement early stopping based on the " +
"evaluation performance")
parser.add_argument("--skip_epoch_eval",
action='store_true',
help='Skips evaluation each epoch during training')
parser.add_argument("--encoding_scheme",
choices=['first_last_sentence', 'first_last_mention'],
default='first_last_mention')
parser.add_argument("--task", required=True, choices=['nonvis', 'card'])
args = parser.parse_args()
arg_dict = vars(args)
task = arg_dict['task']
if arg_dict['train']:
arg_dict['model_file'] = "/home/ccervan2/models/tacl201801/" + \
nn_data.build_model_filename(arg_dict, task + "_lstm")
model_file = arg_dict['model_file']
util.dump_args(arg_dict, log)
# Construct data files from the root directory and filename
data_dir = arg_dict['data_dir'] + "/"
data_root = arg_dict['data_root']
eval_data_root = arg_dict['eval_data_root']
sentence_file = data_dir + "raw/" + data_root + "_captions.txt"
mention_idx_file = data_dir + "raw/" + data_root + "_mentions_" + task + ".txt"
feature_file = data_dir + "feats/" + data_root + "_" + task + "_neural.feats"
feature_meta_file = data_dir + "feats/" + data_root + "_" + task + "_neural_meta.json"
if eval_data_root is not None:
eval_sentence_file = data_dir + "raw/" + eval_data_root + "_captions.txt"
eval_mention_idx_file = data_dir + "raw/" + eval_data_root + "_mentions_" + task + ".txt"
eval_feature_file = data_dir + "feats/" + eval_data_root + "_" + task + "_neural.feats"
eval_feature_meta_file = data_dir + "feats/" + eval_data_root + "_" + task + "_neural_meta.json"
#endif
# Load the appropriate word embeddings
embedding_type = arg_dict['embedding_type']
if embedding_type == 'w2v':
log.info("Initializing word2vec")
nn_data.init_w2v()
elif embedding_type == 'glove':
log.info("Initializing glove")
nn_data.init_glove()
#endif
# Set the random seeds identically every run
nn_util.set_random_seeds()
# Set up the minimum tensorflow logging level
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
# Train, if training was specified
if arg_dict['train']:
train(task=task,
encoding_scheme=arg_dict['encoding_scheme'],
embedding_type=embedding_type,
sentence_file=sentence_file,
mention_idx_file=mention_idx_file,
feature_file=feature_file,
feature_meta_file=feature_meta_file,
epochs=arg_dict['epochs'],
batch_size=arg_dict['batch_size'],
lstm_hidden_width=arg_dict['lstm_hidden_width'],
start_hidden_width=arg_dict['start_hidden_width'],
hidden_depth=arg_dict['hidden_depth'],
weighted_classes=arg_dict['weighted_classes'],
lstm_input_dropout=arg_dict['lstm_input_dropout'],
dropout=arg_dict['dropout'],
lrn_rate=arg_dict['learn_rate'],
clip_norm=arg_dict['clip_norm'],
data_norm=arg_dict['data_norm'],
adam_epsilon=arg_dict['adam_epsilon'],
activation=arg_dict['activation'],
model_file=model_file,
eval_sentence_file=eval_sentence_file,
eval_mention_idx_file=eval_mention_idx_file,
eval_feature_file=eval_feature_file,
eval_feature_meta_file=eval_feature_meta_file,
early_stopping=arg_dict['early_stopping'],
log=log)
elif arg_dict['predict']:
scores_file = data_dir + "scores/" + data_root + "_" + task + ".scores"
# Restore our variables
tf.reset_default_graph()
with tf.Session() as sess:
saver = tf.train.import_meta_graph(model_file + ".meta")
saver.restore(sess, model_file)
predict(task=task,
encoding_scheme=arg_dict['encoding_scheme'],
embedding_type=embedding_type,
tf_session=sess,
batch_size=arg_dict['batch_size'],
sentence_file=sentence_file,
mention_idx_file=mention_idx_file,
feature_file=feature_file,
feature_meta_file=feature_meta_file,
scores_file=scores_file,
log=log)
class Pipeline(object):
def __init__(self, hostname="scanme.nmap.org", config=None):
self.config = Config.from_file(config) if config else Config()
self.target = Target(hostname)
self.tools = []
self.outdir = self.make_outdir()
self.assetdir = self.get_assetdir()
self.logger = Logger(self.config, f"{self.outdir}/logs")
self.available = self.check_dependencies()
self.load_tools()
self.logger.info(f"Pipeline initialized for target: {hostname}")
# load from config
def load_tools(self):
missing = []
for k, v in self.config.tools():
tool, *options = v.split(";")
options = ";".join(options)
Tool = [
T for T in TOOLS if T.__name__ == tool and T in self.available
]
if len(Tool) == 0:
missing.append(tool)
else:
self.add_tool(Tool.pop()(options), from_config=True)
if len(missing) > 0:
os.system("clear")
[
self.logger.error(
f"Skipping tool {tool} in configuration (Missing)")
for tool in missing
]
acknowledge()
def add_tool(self, tool, from_config=False):
self.logger.info(f"Adding {tool}")
for t in self.tools:
if tool == t: # same tool with same options...
self.logger.info(f"Duplicate")
return
tool.set_logger(self.logger)
tool.set_target(self.target)
tool.set_outdir(f"{self.outdir}/commands")
tool.set_reportdir(f"{self.outdir}/reports")
tool.set_assetdir(self.assetdir)
self.tools.append(tool)
if from_config == False:
if not hasattr(self, "tool_iota"):
# self.tool_iota = len(self.config.tools())
tool_indexes = [
int(k.split("_")[-1]) for k, v in self.config.tools()
]
self.tool_iota = len(tool_indexes) and max(tool_indexes)
self.tool_iota += 1
setattr(self.config, f"TOOL_{self.tool_iota}", f"{repr(tool)}")
self.logger.info(f"Success")
return -1 # "só" ficar uniforme com o comportamento do remove_tool
def remove_tool(self, tool):
self.logger.info(f"Removing {tool}")
for t in self.tools:
if tool == t: # same tool with same options...
self.tools.remove(t)
configentry = [v for k, v in self.config.tools()
].index(repr(tool))
delattr(self.config, f"{self.config.tools()[configentry][0]}")
self.logger.info(f"Success")
return -1 # só para sair do menu... (forçar a atualizar)
self.logger.info(f"Not found")
# def update_target(self, hostname):
# self.target.set_target(hostname)
# need to update for each tool? maybe refactor?
def run(self):
if len(self.tools) == 0:
self.logger.error("Must add tools...")
return
missing = missing_tool_dependencies(self.tools)
if missing:
self.logger.error(missing)
self.logger.error(
"Did not execute. Please fullfil requirements...")
return
sorted_tools = sortby_dependencies(self.tools)
if not sorted_tools: return
self.logger.debug(
f"Tool order: {','.join([t.__class__.__name__ for t in sorted_tools])}"
)
outputs = {}
reports = {}
report_obj = None
for tool in sorted_tools:
out, err = tool.run(outputs)
for p in tool.PROVIDES:
outputs[p] = out
if err and not tool.IGNORE_STDERR:
self.logger.error(f"{err.decode('ascii')}\nExiting...")
return
else:
self.logger.info("Output saved")
report_obj = tool.report(reports)
for p in tool.PROVIDES:
reports[p] = report_obj
self.create_report(reports, sorted_tools)
def create_report(self, reports, sorted_tools):
outfile = f"{self.outdir}/reports/Report.md"
title = f"PENSEC - Report of {self.target.hostname}"
reportfile = MdUtils(file_name=outfile, title=title)
# "Execute Summary"
reportfile.new_header(level=3, title="Common Statistics")
for tool in sorted_tools:
tool.write_report_summary(reportfile, reports)
# "Technical Details"
for tool in sorted_tools:
tool.write_report(reportfile, reports)
reportfile.create_md_file()
self.logger.info("Report saved in " + outfile)
def check_dependencies(self):
self.logger.info("Checking dependencies...")
dependencies = TOOLS
available = check_dependencies(dependencies, self.logger)
if len(dependencies) != len(available):
acknowledge()
return available
def make_outdir(self):
hostname = self.target.hostname
timestamp = re.sub(r'[/:]', '-', datetime.now().strftime('%x_%X'))
outdir = f"{self.config.OUTPUT_DIRECTORY}/{hostname}/{timestamp}"
os.makedirs(outdir)
subdirs = ["logs", "commands", "reports"]
for sd in subdirs:
os.mkdir(f"{outdir}/{sd}")
return outdir
def get_assetdir(self):
path = Path().absolute()
return str(path) + "/assets/"
def viewconfig(self):
self.logger.info(f"Showing current config:")
self.logger.debug("\n" + str(self.config))
print("\n")
acknowledge()
def saveconfig(self):
outfile = input("Config name\n>> ")
with open(f"config/{outfile}", "w+") as f:
f.write(str(self.config))
self.logger.info(f"Config saved to {outfile}")
# called on exit from main menu
def cleanup(self):
self.logger.end()
from utils.Menu import Menu
return Menu.EXIT
