def gimmeTrame(self,daNewState):
"""
Return the update trame to be sent
"""
if (str(daNewState)=="toggle"):
if self.current_state=="off":
#on met à on
rawTrame="A55A6B0570000000"+ "FF9F1E0"+"7" +"30"
self.current_state="on"
elif(self.current_state=="on"):
#on met à off
rawTrame="A55A6B0550000000"+ "FF9F1E0"+"7" +"30"
self.current_state="off"
else:
rawTrame="A55A6B0550000000"+ "FF9F1E0"+"7" +"30"
self.current_state="off"
else:
LOGGER.warn("Strange new state : {}. Trram not send".format(daNewState))
return ""
LOGGER.info("State after : {}".format(self.current_state))
myTrame=Trame.trame(rawTrame)
myTrame.calculateChecksum()
LOGGER.info("Actuator trame generated, to be send : {}".format(myTrame.lessRawView()))
self.save()
return myTrame.rawView()
def log_request(record):
global hpclient
req = json.dumps(record)
LOGGER.info(req)
if hpclient and record['is_shellshock']:
hpclient.publish(app.config['hpfeeds.channel'], req)
def checkTrame(self):
if self.trameUsed:
LOGGER.debug("Trame received : {}".format(self.trameUsed.lessRawView()))
if ("A55A" not in self.trameUsed.sep):
LOGGER.warn("Wrong separator, rejected")
if (self.doChecksum(self.trameUsed) not in self.trameUsed.checkSum):
#Mauvais checkSum
LOGGER.warn("Wrong checksum, expected : {}, rejected".format(self.doChecksum(self.trameUsed)))
with self.lock:
if (self.trameUsed.ident in self.identSet):
#Recuperer le capteur en bdd
sensorUsed = sensor.Sensor.objects(physic_id=self.trameUsed.ident)[0]
newData = '' #la nouvelle data a entrer en base, type dynamique
if (sensorUsed.__class__.__name__=="Switch"):
newData=sensorUsed.translateTrame(self.trameUsed)
elif (sensorUsed.__class__.__name__=="Temperature"):
newData = sensorUsed.translateTrame(self.trameUsed)
elif (sensorUsed.__class__.__name__=="Position"):
newData = sensorUsed.translateTrame(self.trameUsed)
else :
LOGGER.warn("Other Captor (not handle (YET !) )")
# Update de la trame au niveau de la base
if newData :
sensorUsed.update(newData)
LOGGER.info(" Sensor {} ||New data {}".format(sensorUsed.physic_id, sensorUsed.current_state))
self.trameUsed=''
def log_request(record):
global hpclient
req = json.dumps(record)
LOGGER.info(req)
if hpclient and (record['is_shellshock'] or app.config['hpfeeds.only_exploits'].lower() == 'false'):
hpclient.publish(app.config['hpfeeds.channel'], req)
def translateTrame(self,inTrame):
"""
return the temperature (range 0-40 c) from data byte 2
"""
rowTemp=int(inTrame.data1,16)
temperature = round((rowTemp*40/255.0),3)
LOGGER.info("Temperature sensor {} with temp {}".format(inTrame.ident, temperature))
return temperature
def sendTrame(self,ident,newState):
with self.lock:
sensorUsed=sensor.Device.objects(physic_id=ident)[0]
daTrame=sensorUsed.gimmeTrame(newState)
if daTrame:
self.soc.send(daTrame)
LOGGER.info("Trame sended : {}".format(daTrame))
return
def sendTrame(self,ident,newState):
"""
Ask the traductor to send a trame with the new state of a captor
"""
LOGGER.info("Lazily updating {} with {}".format(ident,newState))
self.idToUpdate=ident
self.newState=newState
self.save()
def updateOne(self,ident):
"""
Ask for update the sensor with this id
"""
LOGGER.info("lazily updating {}".format(ident))
self.idToUpdate=ident
self.newState=''
self.save()
def _import_listener(self, name, args):
listener, source = utils.import_(name, 'listener')
if not inspect.ismodule(listener):
listener = listener(*args)
elif args:
raise DataError("Listeners implemented as modules do not take arguments")
LOGGER.info("Imported listener '%s' with arguments %s (source %s)"
% (name, utils.seq2str2(args), source))
return listener
def web_request(program, url):
LOGGER.info('Performing {} request on {}'.format(program, url))
data = ''
try:
resp = requests.get(url, headers={'User-Agent': USER_AGENTS[program]})
data = resp.text
except Exception as e:
LOGGER.error(e)
return '{} {}'.format(program, url), data
def _email_config(self):
try:
self.mail_server = smtplib.SMTP('smtp.gmail.com', 587)
self.mail_server.ehlo()
self.mail_server.starttls()
self.mail_server.login(self.gmail_user, self.gmail_password)
except Exception as e:
LOGGER.info("Failed to connnect. Error: {}".format(e))
exit()
def call_method(self, method, *args):
if self.is_java:
args = [self._to_map(a) if isinstance(a, dict) else a for a in args]
try:
method(*args)
except:
message, details = utils.get_error_details()
LOGGER.error("Calling listener method '%s' of listener '%s' failed: %s"
% (method.__name__, self.name, message))
LOGGER.info("Details:\n%s" % details)
def get_page(self):
if self.page is None:
LOGGER.info('Fetching page contents from Confluence')
data = self.server.getPage(
self.get_token(),
self.settings.namespace,
self.settings.pagename
)
self.page = ConfluencePage(data)
return self.page
def DebugFile(path):
if path == 'NONE':
LOGGER.info('No debug file')
return None
try:
LOGGER.info('Debug file: %s' % path)
return _DebugFileWriter(path)
except:
LOGGER.error("Opening debug file '%s' failed and writing to debug file "
"is disabled. Error: %s" % (path, utils.get_error_message()))
return None
def return_os_version_name(self):
"""
Returns an os version name. Uses versions.py to verify os_name
"""
os_names = []
# Gathers a list of osx version keys
for key in OSX_VERSIONS:
os_names.append(key)
# Takes those keys and loops over the dictionary
for os_name in os_names:
if self.return_os_version() in OSX_VERSIONS[os_name]:
LOGGER.info('Returning OS Name')
return os_name
def fetch_cookies(cls):
LOGGER.info('get cookies from reids')
r = redis.Redis(connection_pool=cls.redis_pool)
while True:
user = r.spop('users')
r.sadd('users', user)
c = r.hget('account', user)
if c:
user_cookies = c.decode('utf-8')
cookies_json = json.loads(user_cookies)
LOGGER.info('cookies got-------')
return cookies_json
LOGGER.warn('cookies not get')
def _import_listeners(self, listener_data):
listeners = []
for name, args in listener_data:
try:
listeners.append(_ListenerProxy(name, args))
except:
message, details = utils.get_error_details()
if args:
name += ':' + ':'.join(args)
LOGGER.error("Taking listener '%s' into use failed: %s"
% (name, message))
LOGGER.info("Details:\n%s" % details)
return listeners
def log_request(record):
global hpclient
global dbh
req = json.dumps(record)
LOGGER.info(req)
if hpclient and record['is_shellshock']:
hpclient.publish(app.config['hpfeeds.channel'], req)
if dbh and record['is_shellshock']:
cursor = dbh.cursor()
cursor.execute("INSERT INTO connections (method, url, path, query_string, headers, source_ip, source_port, dest_host, dest_port, is_shellshock, command, command_data, timestamp) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
(record['method'], record['url'], record['path'], record['query_string'], str(record['headers']), record['source_ip'],request.environ.get('REMOTE_PORT'), record['dest_host'], record['dest_port'], record['is_shellshock'], record['command'], record['command_data'], record['timestamp']) )
dbh.commit()
def stop(self):
"""Stop the example by closing the channel and connection. We
set a flag here so that we stop scheduling new messages to be
published. The IOLoop is started because this method is
invoked by the Try/Catch below when KeyboardInterrupt is caught.
Starting the IOLoop again will allow the publisher to cleanly
disconnect from RabbitMQ.
"""
LOGGER.info('Stopping')
self._stopping = True
self.close_channel()
self.close_connection()
def enable_delivery_confirmations(self):
"""Send the Confirm.Select RPC method to RabbitMQ to enable delivery
confirmations on the channel. The only way to turn this off is to close
the channel and create a new one.
When the message is confirmed from RabbitMQ, the
on_delivery_confirmation method will be invoked passing in a Basic.Ack
or Basic.Nack method from RabbitMQ that will indicate which messages it
is confirming or rejecting.
"""
LOGGER.info('Issuing Confirm.Select RPC command')
self._channel.confirm_delivery(self.on_delivery_confirmation)
def setup_exchange(self, exchange_name):
"""
:param str|unicode exchange_name: The name of the exchange to declare
"""
LOGGER.info('Declaring exchange %s', exchange_name)
# Note: using functools.partial is not required, it is demonstrating
# how arbitrary data can be passed to the callback when it is called
cb = functools.partial(self.on_exchange_declareok,
userdata=exchange_name)
self._channel.exchange_declare(exchange=exchange_name,
exchange_type=self.EXCHANGE_TYPE,
callback=cb)
def get_stats_for_all_matches(self):
SQL = ("SELECT ID1_G, ID2_G, ID_T_G, DATE_G FROM (games_atp "
"INNER JOIN players_atp ON games_atp.ID1_G=players_atp.ID_P) "
"WHERE players_atp.COUNTRY_P<>'N/A' AND DATE_G IS NOT NULL "
"ORDER BY DATE_G DESC;")
matches = self.execute_sql(SQL)
dataset = []
for m in matches:
match = m
match[3] = match[3].strftime(self.d_format)
dataset.append(self.get_stats_for_match(*match))
LOGGER.info("Collected data for %s matches.", len(dataset))
return np.asarray(dataset)
def _import_listeners(self, listener_data):
listeners = []
for name, args in listener_data:
try:
listeners.append(_ListenerProxy(name, args))
except:
message, details = utils.get_error_details()
if args:
name += ':' + ':'.join(args)
LOGGER.error("Taking listener '%s' into use failed: %s" %
(name, message))
LOGGER.info("Details:\n%s" % details)
return listeners
def upload_items(self):
"""Uploads each item of the instances items list"""
for item in self.transfer_files:
r = item.upload()
if not r:
log = "Failed to upload item {0}".format(item)
LOGGER.error(log)
return False
log = "Successfully uploaded item {0}".format(item)
LOGGER.info(log)
return True
def execute_commands(self, commands: List[str]):
"""Попытка выполнить команды из списка commands.
* commands - > это список строк, каждая строка,
представляет из себя UNIX команду
"""
for command in commands:
_, stdout, _ = self.client.exec_command(command)
stdout.channel.recv_exit_status(
) # Помещаем только итоговоый результат
response = stdout.readlines()
for line in response:
LOGGER.info(f"INPUT: {command} | OUTPUT: {line}")
def bulk_upload(self, files: List[str]):
"""Upload multiple files to a remote directory.
:param files: List of local files to be uploaded.
:type files: List[str]
"""
try:
self.scp.put(files, remote_path=self.remote_path)
LOGGER.info(
f"Finished uploading {len(files)} files to {self.remote_path} on {self.host}"
)
except SCPException as e:
raise e
def compete(game, p1, p2, compete_round=100):
compete_game = game()
compete_game.add_player(p1)
compete_game.add_player(p2)
player_board = collections.defaultdict(int)
for rd in range(compete_round):
LOGGER.info("compete round[{}]".format(rd))
compete_game.init_board()
winner = compete_game.start()
if winner is None:
winner = "draw"
player_board[winner] += 1
LOGGER.info("compete result, player_board:{}".format(dict(player_board)))
def get_hpfeeds_client(config):
hpc = None
if config["hpfeeds.enabled"].lower() == "true":
LOGGER.info(
"hpfeeds enabled, creating connection to {}:{}".format(config["hpfeeds.host"], config["hpfeeds.port"])
)
hpc = hpfeeds.new(
config["hpfeeds.host"], int(config["hpfeeds.port"]), config["hpfeeds.identity"], config["hpfeeds.secret"]
)
hpc.s.settimeout(0.01)
else:
LOGGER.info("hpfeeds is disabled")
return hpc
def train(self, game, fast_move_func):
self.node_stack = []
leaf_node = self.selection(self.root_node)
is_finish, win_piece = game.judge(leaf_node.status)
if not is_finish:
action_list = game.get_action_list(leaf_node.status)
self.expansion(node=leaf_node,
action_list=action_list,
transform_func=game.transform)
win_piece = game.fast_finish(status=leaf_node.status,
move_func=game.fast_move)
LOGGER.info("{} win".format(win_piece))
self.back_progression(win_piece)
def main():
with timer('load data'):
df = pd.read_csv(TRAIN_PATH)[:10]
df = df[df.Image != "ID_6431af929"].reset_index(drop=True)
df.loc[df.pre_SOPInstanceUID == "ID_6431af929", "pre1_SOPInstanceUID"] = df.loc[
df.pre_SOPInstanceUID == "ID_6431af929", "Image"]
df.loc[df.post_SOPInstanceUID == "ID_6431af929", "post1_SOPInstanceUID"] = df.loc[
df.post_SOPInstanceUID == "ID_6431af929", "Image"]
df.loc[df.prepre_SOPInstanceUID == "ID_6431af929", "pre2_SOPInstanceUID"] = df.loc[
df.prepre_SOPInstanceUID == "ID_6431af929", "pre1_SOPInstanceUID"]
df.loc[df.postpost_SOPInstanceUID == "ID_6431af929", "post2_SOPInstanceUID"] = df.loc[
df.postpost_SOPInstanceUID == "ID_6431af929", "post1_SOPInstanceUID"]
y = df[TARGET_COLUMNS].values
df = df[
["Image", "pre1_SOPInstanceUID", "post1_SOPInstanceUID", "pre2_SOPInstanceUID", "post2_SOPInstanceUID"]]
ids = df["Image"].values
gc.collect()
with timer('preprocessing'):
test_augmentation = Compose([
CenterCrop(512 - 50, 512 - 50, p=1.0),
Resize(img_size, img_size, p=1)
])
test_dataset = RSNADatasetTest(df, img_size, IMAGE_PATH, id_colname=ID_COLUMNS,
transforms=test_augmentation, black_crop=False, subdural_window=True,
pick_type="pre_pre", n_tta=N_TTA)
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=16, pin_memory=True)
del df, test_dataset
gc.collect()
with timer('create model'):
model = CnnModel(num_classes=N_CLASSES, encoder="se_resnext50_32x4d", pretrained="imagenet", pool_type="avg")
model.load_state_dict(torch.load(model_path))
model.to(device)
model = torch.nn.DataParallel(model)
with timer('predict'):
pred = predict(model, test_loader, device, n_tta=N_TTA)
pred = np.clip(pred, 1e-6, 1-1e-6)
with timer('sub'):
sub = pd.DataFrame(pred, columns=TARGET_COLUMNS)
sub["ID"] = ids
sub = sub.set_index("ID")
sub = sub.unstack().reset_index()
sub["ID"] = sub["ID"] + "_" + sub["level_0"]
sub = sub.rename(columns={0: "Label"})
sub = sub.drop("level_0", axis=1)
LOGGER.info(sub.head())
sub.to_csv("../output/{}_train.csv".format(EXP_ID), index=False)
def main():
args = argument_parser(TOOLS).parse_args()
log.setLevel(LOG_LEVELS[args.loglevel])
cdb = None
ret = 0
processcdb_config = ConfigParser()
if args.dumpconfigs:
for tool_name in TOOLS:
tool = TOOLS[tool_name](tool_name)
processcdb_config[tool_name] = tool.default_config()
config_file = args.config.absolute()
config_file.parent.mkdir(parents=True, exist_ok=True)
if config_file.exists():
log.warn(
f"Configuration file {config_file} already exists, overwriting"
)
with config_file.open("w") as output:
processcdb_config.write(output)
log.info(f"Configuration file written to {config_file}")
sys.exit(0)
processcdb_config.read(args.config)
try:
tool = TOOLS[args.tool](args.tool, processcdb_config)
except KeyError:
log.error(f"Unknown tool '{args.tool}' - cant initilize")
return 127
if args.cdb.is_file():
cdb = json.loads(args.cdb.read_text())
if cdb:
#if args.commit_a is not None:
#cdb = filterByChangelist(cdb, (args.commit_a, args.commit_b))
if not args.allow_dupes:
cdb = remove_dupes(cdb)
try:
ret = tool.execute(cdb, args)
log.info(f"Return value from tool process: {ret}")
except EnvironmentError as e:
log.error(f"Cant process: {e}")
except Exception as e:
log.error(f"{e}")
traceback.print_exc()
else:
log.error(f"File '{args.cdb}' is empty")
else:
log.error(f"File '{args.cdb}' does not exist")
return ret # TODO: Add proper return codes
def add_downloaded_image(num, link, pdf):
"""
Download image, add it to PDF file and delete it
"""
LOGGER.info(f'Download image № {num} from {link} from received URL.')
filename, headers = urllib.request.urlretrieve(link)
image_format = headers['content-type'].replace('image/', '')
if image_format not in ('jpeg', 'png'):
LOGGER.info(
f"Image № {num} from {link} is not in an appropriate format.")
add_image_link(num, link, pdf)
else:
LOGGER.info(f"Format of image № {num} from {link} is appropriate.")
pdf.image(filename,
x=70,
y=pdf.get_y(),
h=40,
type=image_format,
link=link)
pdf.ln(40)
LOGGER.info(f'Delete downloaded image № {num} from {link}.')
os.remove(filename)
def main():
with timer('load data'):
df = pd.read_csv(TRAIN_PATH)
df = df[df.Image != "ID_6431af929"].reset_index(drop=True)
y = df[TARGET_COLUMNS].values
df = df[["Image"]]
gc.collect()
with timer('preprocessing'):
train_augmentation = Compose([
CenterCrop(512 - 50, 512 - 50, p=1.0),
HorizontalFlip(p=0.5),
OneOf([
ElasticTransform(p=0.5, alpha=120, sigma=120 * 0.05, alpha_affine=120 * 0.03),
GridDistortion(p=0.5),
OpticalDistortion(p=1, distort_limit=2, shift_limit=0.5)
], p=0.5),
Rotate(limit=30, border_mode=0, p=0.7),
Resize(img_size, img_size, p=1)
])
train_dataset = RSNADataset(df, y, img_size, IMAGE_PATH, id_colname=ID_COLUMNS,
transforms=train_augmentation, black_crop=False, subdural_window=True,
conc_type="concat_all", conc_type2="concat_prepost")
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=8, pin_memory=True)
del df, train_dataset
gc.collect()
with timer('create model'):
model = CnnModel(num_classes=N_CLASSES, encoder="se_resnext50_32x4d", pretrained="imagenet", pool_type="avg")
if model_path is not None:
model.load_state_dict(torch.load(model_path))
model.to(device)
criterion = torch.nn.BCEWithLogitsLoss(weight=torch.FloatTensor([2, 1, 1, 1, 1, 1]).cuda())
optimizer = torch.optim.Adam(model.parameters(), lr=1e-4, eps=1e-4)
model = torch.nn.DataParallel(model)
with timer('train'):
for epoch in range(1, epochs + 1):
if epoch == 5:
for param_group in optimizer.param_groups:
param_group['lr'] = param_group['lr'] * 0.1
seed_torch(SEED + epoch)
LOGGER.info("Starting {} epoch...".format(epoch))
tr_loss = train_one_epoch(model, train_loader, criterion, optimizer, device)
LOGGER.info('Mean train loss: {}'.format(round(tr_loss, 5)))
torch.save(model.module.state_dict(), 'models/{}_ep{}.pth'.format(EXP_ID, epoch))
def tap_read_cb(self, data):
LOGGER.debug("MainControl tap_read_cb")
# dns filter
if is_dns_packet(data):
LOGGER.debug("MainControl read dns packet")
qnames = get_dns_qnames(data)
for qname in qnames:
if self.filter.match_domain(qname.decode()):
LOGGER.info("DNSServer domain matched: %s" % qname)
self.dns_server.resolve(data)
return
self.client.send(data)
def get_hpfeeds_client(config):
hpc = None
if config['hpfeeds.enabled'].lower() == 'true':
LOGGER.info('hpfeeds enabled, creating connection to {}:{}'.format(config['hpfeeds.host'], config['hpfeeds.port']))
hpc = hpfeeds.new(
config['hpfeeds.host'],
int(config['hpfeeds.port']),
config['hpfeeds.identity'],
config['hpfeeds.secret']
)
hpc.s.settimeout(0.01)
else:
LOGGER.info( 'hpfeeds is disabled')
return hpc
def return_home_dirs(self):
"""
Returns a list of home directories.
"""
# This will gather a list of home directories.
directories = []
path = '/Users'
os.chdir(path)
home_dirs = os.listdir(path)
for directory in home_dirs:
if not directory.startswith('.'):
directories.append(directory)
LOGGER.info('Returning home directories')
return directories
def return_bash_history(self):
"""
Returns a list of bash history
"""
bash_history = []
history = self.shell_cmd('history')
try:
if history:
for line in history:
bash_history.append(line)
LOGGER.info('Returning bash history')
return bash_history
except OSError:
LOGGER.error('OSError')
def main(seed):
with timer('load data'):
df = pd.read_csv(FOLD_PATH)
with timer('preprocessing'):
val_df = df[df.fold_id == FOLD_ID]
val_augmentation = None
val_dataset = SeverDatasetTest(val_df,
IMG_DIR,
IMG_SIZE,
N_CLASSES,
id_colname=ID_COLUMNS,
transforms=val_augmentation)
val_loader = DataLoader(val_dataset,
batch_size=BATCH_SIZE,
shuffle=False,
num_workers=8)
del val_df, df, val_dataset
gc.collect()
with timer('create model'):
models = []
model = smp_old.Unet('resnet34',
encoder_weights="imagenet",
classes=N_CLASSES,
encoder_se_module=True,
decoder_semodule=True,
h_columns=False,
skip=True,
act="swish",
freeze_bn=True,
classification=CLASSIFICATION)
model = convert_model(model)
if base_model is not None:
model.load_state_dict(torch.load(base_model))
model.to(device)
models.append(model)
with timer('predict'):
rles, sub_ids = predict(models, val_loader, device)
sub_df = pd.DataFrame({
'ImageId_ClassId': sub_ids,
'EncodedPixels': rles
})
LOGGER.info(sub_df.head())
sub_df.to_csv('{}_{}.csv'.format(EXP_ID, FOLD_ID), index=False)
def perform_commands(headers):
for name, value in headers:
mat = ping_check_re.search(value)
if mat:
# do ping
ping = mat.groupdict()
# don't do more than 20 pings
count = min(20, int(ping.get('count', 1)))
host = ping['host']
LOGGER.info('Performing {} pings against {}'.format(count, host))
# host must match an IP regex and count must be a number, prevents command injection here
command = ['ping', '-n', '-c', str(count), host]
try:
subprocess.call(command)
except Exception as e:
LOGGER.error(e)
return ' '.join(command), ''
mat = wget_check_re.search(value)
if mat:
return web_request('wget', mat.groupdict()['url'])
mat = wget_check_re2.search(value)
if mat:
return web_request('wget', 'http://'+mat.groupdict()['url'])
mat = curl_check_re.search(value)
if mat:
return web_request('curl', mat.groupdict()['url'])
mat = curl_check_re2.search(value)
if mat:
return web_request('curl', 'http://'+mat.groupdict()['url'])
mat = telnet_check_re.search(value)
if mat:
telnet = mat.groupdict()
try:
host = telnet['host']
port = telnet['port']
LOGGER.info('Openning socket to {}:{}'.format(host, port))
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, int(port)))
s.close()
except Exception as e:
LOGGER.error(e)
return 'telnet {}'.format(host, port), ''
return None, None
def perform_commands(headers):
for name, value in headers:
mat = ping_check_re.search(value)
if mat:
# do ping
ping = mat.groupdict()
# don't do more than 20 pings
count = min(20, int(ping.get('count', 1)))
host = ping['host']
LOGGER.info('Performing {} pings against {}'.format(count, host))
# host must match an IP regex and count must be a number, prevents command injection here
command = ['ping', '-n', '-c', str(count), host]
try:
subprocess.call(command)
except Exception as e:
LOGGER.error(e)
return ' '.join(command), ''
mat = wget_check_re.search(value)
if mat:
return web_request('wget', mat.groupdict()['url'])
mat = wget_check_re2.search(value)
if mat:
return web_request('wget', 'http://' + mat.groupdict()['url'])
mat = curl_check_re.search(value)
if mat:
return web_request('curl', mat.groupdict()['url'])
mat = curl_check_re2.search(value)
if mat:
return web_request('curl', 'http://' + mat.groupdict()['url'])
mat = telnet_check_re.search(value)
if mat:
telnet = mat.groupdict()
try:
host = telnet['host']
port = telnet['port']
LOGGER.info('Openning socket to {}:{}'.format(host, port))
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, int(port)))
s.close()
except Exception as e:
LOGGER.error(e)
return 'telnet {}'.format(host, port), ''
return None, None
def train_one_epoch(model,
train_loader,
criterion,
optimizer,
device,
accumulation_steps,
total_step,
n_labels,
base_lr,
steps_upd_logging=500,
gamma=None):
model.train()
optimizer.zero_grad()
total_loss = 0.0
train_losses = []
for step, (features, targets) in enumerate(train_loader):
features = trim_tensors(features)
features, targets = features.to(device), targets.to(device)
logits = model(features)
if n_labels == 1:
loss = criterion(logits.view(-1, 1), targets.view(-1, 1))
else:
loss = criterion(logits, targets)
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
if (step + 1
) % accumulation_steps == 0: # Wait for several backward steps
optimizer.step() # Now we can do an optimizer step
optimizer.zero_grad()
if gamma is not None and step == int(total_step / 2):
for param_group in optimizer.param_groups:
param_group['lr'] = base_lr * gamma
if step == 80000:
return total_loss / (step + 1), train_losses
total_loss += loss.item()
if (step + 1) % steps_upd_logging == 0:
train_losses.append(total_loss / (step + 1))
LOGGER.info(
f'Train loss on step {step + 1} was {round(total_loss / (step + 1), 5)}'
)
return total_loss / (step + 1), train_losses
def find_ip(self):
arg = 'ip route list'
# time to get IP varies on each reboot, so try to find IP continuously
while (True):
try:
p = subprocess.Popen(arg, shell=True, stdout=subprocess.PIPE)
data = p.communicate()
split_data = data[0].split()
self.ip_addr = split_data[split_data.index('src') + 1]
self.ext_ipaddr = urllib2.urlopen("http://icanhazip.com").read().strip()
break
except:
continue
LOGGER.info('Local IP:,{}'.format(self.ip_addr))
def log_email(msg, error=False):
if error:
sys.stderr.write(f"{msg}\n")
# send_email(
# config.err_email_list,
# body,
# subject="Archiver Error",
# to="*****@*****.**",
# )
LOGGER.error(msg)
else:
LOGGER.info(msg)
print(msg)
return
def stop(self):
LOGGER.info("Controller stop")
for _, value in self.id_to_server.items():
value.stop()
self.running = False
if self.recv_thread is not None:
while self.recv_thread.is_alive():
time.sleep(1)
self.sock.close()
if self.timeout_thread is not None:
while self.timeout_thread.is_alive():
time.sleep(1)
if self.handle_traffic_thread is not None:
while self.handle_traffic_thread.is_alive():
time.sleep(1)
def __init__ (self) :
"""
Create things for synchro, get the sensors from the DB
"""
self.lock=threading.Lock()#Lock use for DB updates
self.soc = socket.socket()
self.stoppedAnalyze=False
self.trameUsed = ''
self.running=True
self.identSet = set([])
lazzyUpdate.drop_collection()
LOGGER.info("initializing sensors set : ")
for lsensor in sensor.Sensor.objects:
self.identSet.add(lsensor.physic_id)
LOGGER.info(lsensor.physic_id)
def get_last_model(model_name, from_step=False):
LOGGER.debug(f'Model checkpoint directory: {os.path.join(constants.CHECKPOINT_DIR, model_name)}')
if from_step:
model_paths = glob(os.path.join(constants.CHECKPOINT_DIR, model_name) + '/train_step-*.model')
LOGGER.debug(f'Model paths: {model_paths}')
last_step = max([int(os.path.basename(mp)[len('train_step-'):-len('.model')]) for mp in model_paths])
last_model_path = os.path.join(constants.CHECKPOINT_DIR, model_name, f'train_step-{last_step}.model')
else:
model_paths = glob(os.path.join(constants.CHECKPOINT_DIR, model_name) + '/[!train_step-]*.model')
LOGGER.debug(f'Model paths: {model_paths}')
last_iter = max([int(os.path.basename(mp)[: -len('.model')]) for mp in model_paths])
last_model_path = os.path.join(constants.CHECKPOINT_DIR, model_name, f'{str(last_iter).zfill(6)}.model')
LOGGER.info(f'Loading {last_model_path} (last)')
model = torch.load(last_model_path)
return model
def get_stories_data(self):
result = []
for row in self.soup.findAll('tr')[1:]:
if len(row.findAll('td')) != 12:
fail('column count is incorrect')
cols = row.findAll('td')
if cols[2].find('a') is None:
fail('story key is not a hyperlink')
else:
result.append({
'key': str(cols[2].find("a").text),
'status': str(cols[6].text.encode('ascii', 'ignore'))
})
LOGGER.info('Table contains %s stories' % len(result))
return result
def __init__(self, train_df, demand_non_empty_proba):
assert demand_non_empty_proba > 0, 'frequensy of non-empty images must be greater then zero'
self.positive_proba = demand_non_empty_proba
self.train_df = train_df.reset_index(drop=True)
self.positive_idxs = self.train_df[
self.train_df.sum_target != 0].index.values
self.negative_idxs = self.train_df[self.train_df.sum_target ==
0].index.values
self.n_positive = self.positive_idxs.shape[0]
self.n_negative = int(self.n_positive * (1 - self.positive_proba) /
self.positive_proba)
LOGGER.info("len data = {}".format(self.n_positive + self.n_negative))
def publish_message(self, message):
if self._channel is None or not self._channel.is_open:
LOGGER.warning('Channel is not open, could not publish event',
json.dumps(message, cls=UUIDEncoder))
return
properties = pika.BasicProperties(content_type='application/json')
self._channel.basic_publish(
self.EXCHANGE, self.ROUTING_KEY,
json.dumps(message, ensure_ascii=False, cls=UUIDEncoder),
properties)
self._message_number += 1
self._deliveries.append(self._message_number)
LOGGER.info('Published message # %i', self._message_number)
def execute(self, cdb, args):
result = 1
if not self.tool_exists():
raise EnvironmentError(f"tool: {self.tool_name} not in path, cannot execute.")
command_queue = self._generate_cmd_queue(cdb, args)
try:
tmp_dir = None
if args.output is not None:
tmp_dir = Path(tempfile.mkdtemp())
tasks = self.max_tasks(args, len(command_queue))
global list_of_futures
signal.signal(signal.SIGINT, inthandler)
log.info("Starting scanning ...")
with concurrent.futures.ProcessPoolExecutor(tasks) as executor:
for cmd in command_queue:
future = executor.submit(self.process_queue, cmd, tmp_dir)
future.add_done_callback(partial(process_cb, cmd))
future = list_of_futures.append(future)
log.info("Waiting for scanning processes to finish ..")
concurrent.futures.wait(list_of_futures)
log.info("Scanning done ...")
if args.output is not None:
log.debug(f"User requested output to {args.output}, scanning log files and saving ...")
with open(args.output, "w") as dst:
for name in tmp_dir.glob("*.log"):
dst.write(name.read_text())
if args.xml:
log.debug("User requested output as xml, converting ...")
self.format_output_to_xml(args.output, args.allow_dupes)
shutil.rmtree(tmp_dir)
log.info("All done")
result = 0
except KeyboardInterrupt:
log.debug("Keyboard interrupt!")
if tmp_dir is not None:
shutil.rmtree(tmp_dir)
os.kill(0, 9)
except Exception as e: # TODO: Add proper exception handling
log.error(f"Exception: {e} .-- ")
result = 1
log.info("All done")
return result
def anonymize_csv_wrapper(input_csv, output_folder, anon_csv_name, hash_df,
dataset):
if not os.path.exists(output_folder):
try:
os.makedirs(output_folder)
LOGGER.info('Output directory %s is created' % output_folder)
except OSError:
LOGGER.warning('Creation of the output directory %s failed' %
output_folder)
if hash_df == 'sha224':
hash_method = 'sha3'
elif hash_df == 'md5':
hash_method = 'md5'
output_path = os.path.join(output_folder, anon_csv_name)
anonymize_csv(input_csv, output_path, columns=[0], method=hash_method)
LOGGER.info('Anonymized csv is saved in %s' % output_path)
def translateTrame(self,inTrame):
"""
return close if data0=09,
open if data0=08
else nothing
"""
if (inTrame.data0=='09'):
LOGGER.info("Door sensor {} with state [close]".format(inTrame.ident))
dataToRet = "close"
elif(inTrame.data0=='08'):
LOGGER.info("Door sensor {} with state [open]".format(inTrame.ident))
dataToRet = "open"
else:
LOGGER.warn("Door sensor {}Strange state : {}".format(inTrame.ident, inTrame.data2))
dataToRet=''
return dataToRet
def launch(self,addr,port):
self.connect(addr,port)
dacount=0
self.updateIdentSet()
while self.running:
try:
if dacount >200 : #Fréquence de mise à jour de la base
self.updateIdentSet()
dacount=0
try:
self.receive()
except IOError:
# timeout !
pass
if self.trameUsed:
self.checkTrame()
dacount+=1
# LOGGER.debug("tic")
except KeyboardInterrupt:
sys.exit(0)
LOGGER.info("Le traducteur est terminé")
def updateIdentSet(self):
"""
Safely update the identifier set of the traductor
"""
for anUpdate in lazzyUpdate.objects:
LOGGER.warn("id : {} || state : {}".format(anUpdate.idToUpdate,anUpdate.newState))
if(anUpdate.idToUpdate==""):
with self.lock:
self.identSet=set([])
for lsensor in sensor.Sensor.objects:
self.identSet.add(lsensor.physic_id)
LOGGER.info(lsensor.physic_id)
LOGGER.info("Traductor's set of captors updated")
elif(anUpdate.newState==""):
with self.lock:
if (anUpdate.idToUpdate in things.physic_id for things in sensor.Sensor.objects):
self.identSet.add(anUpdate.idToUpdate)
LOGGER.info("{} added".format(anUpdate.idToUpdate))
else:
#send a trame from a captor with a newState
LOGGER.error("Sensor to update : {} ||new state : {}".format(anUpdate.idToUpdate,anUpdate.newState))
self.sendTrame(anUpdate.idToUpdate,anUpdate.newState)
anUpdate.delete()
LOGGER.warn(" {} update GROS delete de : {} || {}".format(lazzyUpdate.objects.count(),anUpdate.idToUpdate,anUpdate.newState))
return
LOGGER.debug("nothing to update")
def get_postgresql_handler(config):
dbh = None
if config["postgresql.enabled"].lower() == "true":
LOGGER.info(
"postgresql enabled, creating connection to {}:{}".format(
config["postgresql.host"], config["postgresql.port"]
)
)
dbh = psycopg2.connect(
database=config["postgresql.database"],
user=config["postgresql.user"],
password=config["postgresql.password"],
host=config["postgresql.host"],
port=config["postgresql.port"],
)
cursor = dbh.cursor()
cursor.execute(
"""CREATE TABLE IF NOT EXISTS
connections (
connection SERIAL PRIMARY KEY,
method TEXT,
url TEXT,
path TEXT,
query_string TEXT,
headers TEXT,
source_ip TEXT,
source_port INTEGER,
dest_host TEXT,
dest_port INTEGER,
is_shellshock TEXT,
command TEXT,
command_data TEXT,
timestamp INTEGER
);"""
)
dbh.commit()
else:
LOGGER.info("postgresql is disabled")
return dbh
def check_ip():
Prev_IP = None
with open('IP_Logger.csv', 'r+') as csv_file:
# check if prev ip exists
lines = csv_file.readlines()
for i in range(-1, -len(lines) - 1, -1):
if "Local IP" in lines[i]:
Prev_IP = [x.strip() for x in lines[i].split(',')][-1]
break
else:
continue
#print "OldIP: {}".format(Prev_IP)
Send_IP = Find_IP(user, password, to, args.pi_name)
Send_IP.find_ip()
New_IP = Send_IP.ip_addr
#print "NewIP: {}".format(New_IP)
if New_IP != Prev_IP:
LOGGER.info('Email sent with new ip: {}'.format(New_IP))
Send_IP.send_mail()
else:
LOGGER.info("IP Address hasn't changed")
def connect (self, addr, port) :
self.soc.connect((addr,port))
self.soc.setblocking(0)
LOGGER.info("Connected to {} : {}".format(addr,port))
def fsync(self, path, isfsyncfile = 0):
LOGGER.info("Fsync %s %s" % (path, isfsyncfile))
return self.flush (path, 0)
self.identSet.add(lsensor.physic_id)
LOGGER.info(lsensor.physic_id)
LOGGER.info("Traductor's set of captors updated")
elif(anUpdate.newState==""):
with self.lock:
if (anUpdate.idToUpdate in things.physic_id for things in sensor.Sensor.objects):
self.identSet.add(anUpdate.idToUpdate)
LOGGER.info("{} added".format(anUpdate.idToUpdate))
else:
#send a trame from a captor with a newState
LOGGER.error("Sensor to update : {} ||new state : {}".format(anUpdate.idToUpdate,anUpdate.newState))
self.sendTrame(anUpdate.idToUpdate,anUpdate.newState)
anUpdate.delete()
LOGGER.warn(" {} update GROS delete de : {} || {}".format(lazzyUpdate.objects.count(),anUpdate.idToUpdate,anUpdate.newState))
return
LOGGER.debug("nothing to update")
if __name__ == '__main__':
connect('test')
try:
myTrad=traductor()
myTrad.launch('',1515)
except socket.error:
LOGGER.info("Déconnection du serveur")
mytrad.soc.close()
finally:
pass
