def generate_label_dict_and_save_to_pkl(config):
logger = app_logger.get_logger('preprocess')
# get the basic dataset and setting info
setting = SJPathSetting()
setting.parse(config)
with open(setting.split_file) as f:
split_data = json.load(f)
train_data = filter(
lambda item: item['split'] == 'Train_tumor' or item['split'] ==
'Train_normal', split_data)
val_data = filter(
lambda item: item['split'] == 'Val_tumor' or item['split'] ==
'Val_normal', split_data)
test_data = filter(
lambda item: item['split'] == 'Test_tumor' or item['split'] ==
'Test_normal', split_data)
logger.info('# of training WSIs: {}'.format(len(train_data)))
logger.info('# of validation WSIs: {}'.format(len(val_data)))
logger.info('# of testing WSIs: {}'.format(len(test_data)))
all_labels = _prepare_all_patch(split_data, setting)
pickle_file = os.path.join(setting.pkl_folder, 'label_dict.pkl')
with open(pickle_file, 'wb') as f:
pickle.dump(all_labels, f)
logger.info('label dict file is saved to {}'.format(pickle_file))
def _prepare_all_patch(split_data, setting):
logger = app_logger.get_logger('preprocess')
processor = single_WSI_processor.WSIProcessor(setting)
logger.info('Processing Training WSIs')
preproces_data = filter(
lambda item: item['split'] != 'Test_tumor' and item['split'] !=
'Test_normal', split_data)
all_train_tiles = []
for idx, item in enumerate(tqdm(preproces_data)):
label_dict = None
if 'tumor' in item['split']:
label_dict = processor.process(item, 'pos')
label_dict['neg'] = []
if 'normal' in item['split']:
label_dict = processor.process(item, 'neg')
label_dict['pos'] = []
count_pos = len(label_dict['pos'])
count_neg = len(label_dict['neg'])
all_train_tiles.append({
'WSI': item['WSI'][0],
'split': item['split'],
'tiles': label_dict
})
logger.info(
'Initial patch # for {}, # of pos tile: {}, # of neg tile: {}'.
format(os.path.basename(item['WSI'][0]), count_pos, count_neg))
return all_train_tiles
"""
def generate_training_patches(config):
logger = app_logger.get_logger('preprocess')
# get the basic dataset and setting info
setting = SJPathSetting()
setting.parse(config)
pickle_file = os.path.join(setting.pkl_folder, 'label_dict.pkl')
if not os.path.exists(pickle_file):
logger.error('{} does not exists, please regenerate the file'.format(
pickle_file))
logger.info('Load label dict file drom {}'.format(pickle_file))
with open(pickle_file, 'rb') as f:
all_tiles = pickle.load(f)
logger.info('Tiles for {} of WSI are loaded'.format(len(all_tiles)))
logger.info('Randomly sampling training tiles')
sampled_tiles = _random_sample_tiles(all_tiles, setting.sample_upper_bound)
logger.info('Done sampling training tiles')
for item in sampled_tiles:
svs_file = item['WSI']
label_dict = item['tiles']
logger.info(
'Number of tiles sampled for {}, # of pos: {}, # of neg: {}'.
format(os.path.basename(svs_file), len(label_dict['pos']),
len(label_dict['neg'])))
# visualize the sampled training patches in the mask file
if setting.save_img:
mask_file = os.path.join(
setting.img_vis,
os.path.basename(svs_file)[:-4] + '_mask' + setting.img_ext)
lvl_mask = Image.open(mask_file)
lvl_mask = np.asarray(lvl_mask)
lvl_mask.flags.writeable = True
for key, label_tiles in label_dict.iteritems():
for (col, row) in label_tiles:
col = col / setting.patch_factor
row = row / setting.patch_factor
# label selected tumor as yellow and normal as white
if key == 'pos':
lvl_mask[row, col, :] = [255, 255, 0]
else:
lvl_mask[row, col, :] = [255, 255, 255]
lvl_mask = Image.fromarray(lvl_mask)
lvl_mask.save(
os.path.join(
setting.img_vis,
os.path.basename(svs_file)[:-4] + '_mask_selected' +
setting.img_ext))
pickle_file = os.path.join(setting.pkl_folder, 'sampled_label_dict.pkl')
with open(pickle_file, 'wb') as f:
pickle.dump(sampled_tiles, f)
logger.info('Sampled label dict file is saved to {}'.format(pickle_file))
class RomanNumerals:
logger = app_logger.get_logger(__name__)
to_from = {
'I': 1,
'V': 5,
'X': 10,
'L': 50,
'C': 100,
'D': 500,
'M': 1000
}
from_to = {
1: 'I',
5: 'V',
10: 'X',
50: 'L',
100: 'C',
500: 'D',
1000: 'M'
}
@classmethod
def to_roman(cls, input_int):
val = [
1000, 900, 500, 400,
100, 90, 50, 40,
10, 9, 5, 4,
1
]
syb = [
"M", "CM", "D", "CD",
"C", "XC", "L", "XL",
"X", "IX", "V", "IV",
"I"
]
roman_num = ''
i = 0
while input_int > 0:
for _ in range(input_int // val[i]):
roman_num += syb[i]
input_int -= val[i]
i += 1
return roman_num
@classmethod
def from_roman(cls, input_symbal):
cls.logger.info(f"log from classmethod {input_symbal} ")
result_int = 0
for iter in input_symbal:
result_int += cls.to_from[iter]
cls.logger.warning(f"log from classmethod result {result_int} ")
return result_int
def prepare_model(config):
logger = app_logger.get_logger('preprocess')
# get the basic dataset and setting info
setting = SJPathSetting()
setting.parse(config)
logger.info('Downloading pretrained model, this may take a while')
#urllib.urlretrieve(setting.pretrained_url,
# filename=setting.pretrained_model)
global rem_file # global variable to be used in dlProgress
rem_file = setting.pretrained_url
def dlProgress(count, blockSize, totalSize):
percent = int(count * blockSize * 100 / totalSize)
sys.stdout.write("\r" + rem_file + "...%d%%" % percent)
sys.stdout.flush()
if os.path.exists(setting.pretrained_model):
print('find model!')
else:
urllib.urlretrieve(rem_file,
filename=setting.pretrained_model,
reporthook=dlProgress)
net = caffe.Net(setting.original_prototxt, setting.pretrained_model,
caffe.TEST)
net_full_conv = caffe.Net(setting.full_conv_prototxt,
setting.pretrained_model, caffe.TEST)
params = ['fc6', 'fc7']
fc_params = {
pr: (net.params[pr][0].data, net.params[pr][1].data)
for pr in params
}
params_full_conv = ['fc6_conv', 'fc7_conv']
conv_params = {
pr:
(net_full_conv.params[pr][0].data, net_full_conv.params[pr][1].data)
for pr in params_full_conv
}
for pr, pr_conv in zip(params, params_full_conv):
conv_params[pr_conv][0].flat = fc_params[pr][
0].flat # flat unrolls the arrays
conv_params[pr_conv][1][...] = fc_params[pr][1]
logger.info(
'The model is transformed to FCN, it will be written to {}'.format(
setting.full_conv_model))
net_full_conv.save(setting.full_conv_model)
def generate_train_txt(config):
logger = app_logger.get_logger('preprocess')
# get the basic dataset and setting info
setting = SJPathSetting()
setting.parse(config)
# find all the training and validation patches on the disk
base = setting.train_patch_dir
logger.info('Grabbing image files from {}'.format(base))
train_pos_patches = glob.glob(os.path.join(base, 'TRAIN/pos/*.tif'))
train_neg_patches = glob.glob(os.path.join(base, 'TRAIN/neg/*.tif'))
val_pos_patches = glob.glob(os.path.join(base, 'VAL/pos/*.tif'))
val_neg_patches = glob.glob(os.path.join(base, 'VAL/neg/*.tif'))
train_pos_files = map(lambda x: (x, '1'), train_pos_patches)
train_neg_files = map(lambda x: (x, '0'), train_neg_patches)
val_pos_files = map(lambda x: (x, '1'), val_pos_patches)
val_neg_files = map(lambda x: (x, '0'), val_neg_patches)
train_files = []
train_files.extend(train_pos_files)
train_files.extend(train_neg_files)
val_files = []
val_files.extend(val_pos_files)
val_files.extend(val_neg_files)
shuffle(train_files)
shuffle(train_files)
shuffle(val_files)
shuffle(val_files)
#print len(train_files), len(val_files)
#train_txt = '/media/usbdata/pathology/SJTU_PROJ/Experiments/train/train.txt'
#val_txt = '/media/usbdata/pathology/SJTU_PROJ/Experiments/train/val.txt'
logger.info(
'train patches and their labels will be written into {}'.format(
setting.train_txt))
with open(setting.train_txt, 'w') as f:
for item in train_files:
f.write(item[0] + ' ' + item[1] + '\n')
logger.info(
'validation patches and their labels will be written into {}'.format(
setting.val_txt))
with open(setting.val_txt, 'w') as f:
for item in val_files:
f.write(item[0] + ' ' + item[1] + '\n')
def compute_img_mean(config):
logger = app_logger.get_logger('preprocess')
# get the basic dataset and setting info
setting = SJPathSetting()
setting.parse(config)
db_dir = os.path.join(setting.db_folder, 'TRAIN_lmdb')
mean_file = os.path.join(setting.db_folder, 'mean.binaryproto')
mean_bin = 'compute_image_mean'
cmd_line = mean_bin + ' ' + db_dir + ' ' + mean_file + ' ' + '-backend ' + 'lmdb'
logger.info(cmd_line)
# calculating mean image file
try:
os.system(cmd_line)
except IOError as e:
logger.error("Compute_image_mean I/O error({0}): {1}".format(
e.errno, e.strerror))
except ValueError as e:
logger.error("Compute_image_mean ValueError error({0}): {1}".format(
e.errno, e.strerror))
except:
logger.error('Compute_image_mean unexpected error')
# calculate mean values
blob = caffe.proto.caffe_pb2.BlobProto()
try:
mean_values = np.mean(out, axis=(1, 2)).tolist()
str_means = [str(k) for k in mean_values]
str_means = ';'.join(str_means)
logger.info('mean values: ' + str_means)
data = open(mean_file, 'rb').read()
blob.ParseFromString(data)
out = np.array(caffe.io.blobproto_to_array(blob))[0]
#print out.shape
except IOError as e:
logger.error("computeMeanValueFromFile I/O error({0}): {1}".format(
e.errno, e.strerror))
except ValueError as e:
logger.error(
"computeMeanValueFromFile ValueError error({0}): {1}".format(
e.errno, e.strerror))
except:
logger.error('computeMeanValueFromFile unexpected error')
return None
def __init__(self, settings, sentiment_analyzer=TweetSentiment):
"""
init class
:param settings: settings object
"""
self.settings = settings ## read settings file
self.start_date = datetime.strptime(self.settings.start_date, '%Y-%m-%d')
self.end_date = datetime.strptime(self.settings.end_date, '%Y-%m-%d')
self.username = self.settings.username
self.auth = twitter_oauth(settings.auth_file)
self.api = self.auth
self.logger = get_logger("TweetsAPI")
self.sentiment_analyzer = sentiment_analyzer()
self.engagement_url = "https://data-api.twitter.com/insights/engagement"
def main():
"""The main rountine to generate the training patches.
"""
logger = app_logger.get_logger('preprocess')
# read the configuration file
cfg_file = 'config/SJPath.cfg'
config = ConfigParser.SafeConfigParser()
logger.info('Using the config file: ' + cfg_file)
config.read(cfg_file)
generate_pkl = config.getboolean('RESULT', 'generate_pkl')
save_train_patch = config.getboolean('RESULT', 'save_train_patch')
train_patch_dir = config.get('RESULT', 'train_patch_dir')
logger.info('Start to generate potential training tiles')
if generate_pkl:
SJPath_helpers.generate_label_dict_and_save_to_pkl(config)
else:
logger.info('label pkl file already generated')
logger.info('Done generating potential training tiles')
logger.info('Start saving patches to disk')
if save_train_patch:
SJPath_helpers.generate_training_patches(config)
SJPath_helpers.save_training_patches_to_disk(config)
logger.info('Finished saving patches to disk')
logger.info('Start to convert training patches to lmdb')
SJPath_helpers.convert_training_patches_to_lmdb(config)
else:
logger.info('Training patches are already saved to folder {}'.format(
train_patch_dir))
logger.info('Computing image mean')
SJPath_helpers.compute_img_mean(config)
logger.info('Grabbing training patches and labels')
SJPath_helpers.generate_train_txt(config)
logger.info('Preparing pretrained model')
SJPath_helpers.prepare_model(config)
async def connect_iotc_device() -> IoTCClient:
log: logging.Logger = app_logger.get_logger()
device_client = None
#asyncio.sleep(0)
registration_result = await register_device()
if registration_result.status == 'assigned':
try:
device_client = IoTHubDeviceClient.create_from_symmetric_key(
symmetric_key=credentials.symmetric_key,
hostname=registration_result.registration_state.assigned_hub,
device_id=registration_result.registration_state.device_id,
)
#device_client = IoTCClient(credentials.device_id, credentials.id_scope, IOTCConnectType.IOTC_CONNECT_DEVICE_KEY, credentials.symmetric_key)
await device_client.connect()
log.info('Connected to IoTCentral')
except Exception as e:
s = e
log.error("Error Connecting to IoTCentral: {e}")
finally:
return device_client
def save_training_patches_to_disk(config):
logger = app_logger.get_logger('preprocess')
# get the basic dataset and setting info
setting = SJPathSetting()
setting.parse(config)
pickle_file = os.path.join(setting.pkl_folder, 'sampled_label_dict.pkl')
if not os.path.exists(pickle_file):
logger.error('{} does not exists, please regenerate the file'.format(
pickle_file))
logger.info('Load sampled label dict file drom {}'.format(pickle_file))
with open(pickle_file, 'rb') as f:
sampled_tiles = pickle.load(f)
logger.info('Sampled tiles for {} of WSIs are loaded'.format(
len(sampled_tiles)))
logger.info('Saving sampled training tiles to disk')
for item in tqdm(sampled_tiles):
svs_file = item['WSI']
wsi = file_api.AllSlide(svs_file)
label_dict = item['tiles']
split = item['split']
base = setting.train_patch_dir
if split in ['Train_tumor', 'Train_normal']:
base = os.path.join(base, 'TRAIN')
else:
base = os.path.join(base, 'VAL')
for key, label_tiles in label_dict.iteritems():
base_label = os.path.join(base, key)
base_label = os.path.join(base_label,
os.path.basename(svs_file)[:-4])
for origin in label_tiles:
patch = wsi.read_region(
origin, 0, (setting.patch_size, setting.patch_size))
patch = patch.convert('RGB')
patch.save(base_label +
'_{}_{}.tif'.format(origin[0], origin[1]))
patch.close()
def process(self, item, type):
logger = app_logger.get_logger('preprocess')
self._type = type
raw_file = item['WSI'][0]
wsi = file_api.AllSlide(raw_file)
level_idx = wsi.level_count - 1
mask_files = item['WSI'][1]
if mask_files is None:
lowest_mask_lvl = level_idx
else:
lowest_mask_lvl = self._read_lowest_reso_lvls(mask_files)
if level_idx < lowest_mask_lvl:
logger.error('Annotation for {} is Wrong'.format(raw_file))
level_idx = self._adjust_level(wsi, level_idx)
lvl_mask = self._generate_mask(wsi, mask_files, raw_file, level_idx)
#factor = wsi.level_dimensions[0][0] / lvl_mask.shape[1]
#print factor
return self._get_training_tiles(lvl_mask)
import iot_events.iot_commands as iot_commands
import busio, board #needed for i2c interface with proximity sensors
import adafruit_vcnl4040 #needed for the vcnl4040 proximity sensor
import adafruit_vcnl4010 #needed for the vcnl4010 proximity sensor
import time
from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient
#Define some globals
GPIO.setmode(GPIO.BCM)
camera = RCamera() #Camera connected to camera pins
credentials: Credentials = Credentials()
device_client: IoTCClient = None #IoTHubDeviceClient.create_from_connection_string(credentials.get_credentail_info(CredentialInfo.connection_string))
start_time = datetime.now()
tfclassifier = TFClassify()
tfclassifier2 = TFClassify2()
log: logging.Logger = app_logger.get_logger()
log.info(f"TensorFlow took {datetime.now() - start_time} seconds to load")
modelTier = "tier1"
_app_settings = AppSettings()
_app_settings.ensure_label_folders_exist(modelTier)
modelTier2 = "tier2"
_app_settings2 = AppSettings()
_app_settings.ensure_label_folders_exist(modelTier2)
# Set the trigger word for setting a sub tier model lookup
_subTierTrigger = _app_settings.get_SubTierTrigger(modelTier)
_subTierTrigger2 = _app_settings2.get_SubTierTrigger(modelTier2)
_USE_TEST_MODE = False
import os
import psycopg2.extras
from psycopg2 import IntegrityError
from app_logger import get_logger
logger = get_logger("database")
cursor = None
conn = None
def create_table_if_not_exists(tablename: str, create_query: str) -> bool:
"""
creates a table if not exists
:param tablename: name of the table
:param create_query: create query to execute if table does not exist
:return: True if table was created
"""
if cursor:
cursor.execute(
"select exists(select * from information_schema.tables where table_name=%s)",
(tablename, ))
if not cursor.fetchone()[0]:
cursor.execute(create_query)
return True
else:
return False
else:
class Player(Ship):
"""Class representing a player
"""
_logger = app_logger.get_logger(__name__)
def __init__(self, max_life_pt: int = 3):
"""Player class constructor
Args:
max_life_pt (int, optional): Maximum life points the player can have. Defaults to 3.
"""
#self._position = Position(0, 0) (already initialize in mother class)
self._max_life_pt = max_life_pt
self._life_pt = max_life_pt
'''@property
def position(self):
"""Attribute getter
Returns:
(Type of attribute): Value of attribute (Position, int...)
"""
if self._position is not None:
return self._position
@position.setter
def position(self, value):
"""Attribute setter
Args:
value (Type of attribute): Value of attribute (Position, int...)
"""
self._position = value
@position.deleter
def position(self):
"""Attribute "destructor"
"""
self._position = None
'''
@property
def life_pt(self):
"""Attribute getter
Returns:
(Type of attribute): Value of attribute (Position, int...)
"""
if self._life_pt is not None:
return self._life_pt
@life_pt.setter
def life_pt(self, value):
"""Attribute setter
Args:
value (Type of attribute): Value of attribute (Position, int...)
"""
self._life_pt = value
@life_pt.deleter
def life_pt(self):
"""Attribute "destructor"
"""
self._life_pt = None
@property
def max_life_pt(self):
"""Attribute getter
Returns:
(Type of attribute): Value of attribute (Position, int...)
"""
if self._max_life_pt is not None:
return self._max_life_pt
@max_life_pt.setter
def max_life_pt(self, value):
"""Attribute setter
Args:
value (Type of attribute): Value of attribute (Position, int...)
"""
self._max_life_pt = value
@max_life_pt.deleter
def max_life_pt(self):
"""Attribute "destructor"
"""
self._max_life_pt = None
# -*- coding: utf-8 -*-
import telebot
import config
import app_logger
user0 = config.admin_user
bot = telebot.TeleBot(config.token, parse_mode='HTML')
bot_info = bot.get_me()
logger = app_logger.get_logger(__name__)
logger.info('Бот начал работу!', name='Bot')
def scan_message(text):
result = None
for word in config.texts['bad_words']:
if word == text:
result = word
return result
def def_keyboard():
keyboard = telebot.types.ReplyKeyboardMarkup(True, True)
keyboard.add(config.texts['add_chat'], config.texts['help'])
keyboard.add(config.texts['add_antispam'])
return keyboard
#!/usr/bin/env python3
import os, sys
import pprint
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import settings
import app_logger
logger = app_logger.get_logger("linkedIn-preprocessing",
logfile="linkedIn.log")
import pandas as pd
from datetime import date, timedelta, datetime
import csv
## Get settings:
file = "settings_linkedIn.json"
settings_obj = settings.Settings(file=file)
settings_data = settings_obj.get_settings()
def read_excel(filepath, sheet_name):
return pd.read_excel(filepath, sheet_name=sheet_name)
"""
Visitor file path
"""
def get_visitor_dataframes(visitors_file_path, settings_data=settings_data):
def convert_training_patches_to_lmdb(config):
logger = app_logger.get_logger('preprocess')
# get the basic dataset and setting info
setting = SJPathSetting()
setting.parse(config)
# find all the training and validation patches on the disk
base = setting.train_patch_dir
logger.info('Grabbing image files from {}'.format(base))
train_pos_patches = glob.glob(os.path.join(base, 'TRAIN/pos/*.tif'))
train_neg_patches = glob.glob(os.path.join(base, 'TRAIN/neg/*.tif'))
val_pos_patches = glob.glob(os.path.join(base, 'VAL/pos/*.tif'))
val_neg_patches = glob.glob(os.path.join(base, 'VAL/neg/*.tif'))
# blend samples
train_patches = []
val_patches = []
train_patches.extend(
map(lambda x: {
'filename': x,
'label': 1
}, train_pos_patches))
train_patches.extend(
map(lambda x: {
'filename': x,
'label': 0
}, train_neg_patches))
val_patches.extend(
map(lambda x: {
'filename': x,
'label': 1
}, val_pos_patches))
val_patches.extend(
map(lambda x: {
'filename': x,
'label': 0
}, val_neg_patches))
logger.info(
'# of training patches: {} (# of pos: {}, # of neg: {})'.format(
len(train_patches), len(train_pos_patches),
len(train_neg_patches)))
logger.info(
'# of validation patches: {} (# of pos: {}, # of neg: {})'.format(
len(val_patches), len(val_pos_patches), len(val_neg_patches)))
# randomrize training
shuffle(train_patches)
shuffle(train_patches)
shuffle(val_patches)
shuffle(val_patches)
def _write_to_db(patches, file_dir, db_batch):
batch_imgs = []
# create the lmdb file
lmdb_env = lmdb.open(file_dir, map_size=int(1e12))
lmdb_txn = lmdb_env.begin(write=True)
datum = caffe.proto.caffe_pb2.Datum()
batch_idx = 0
for idx, item in enumerate(tqdm(patches)):
patch = Image.open(item['filename'])
img = np.asarray(patch)
label = item['label']
patch.close()
# map numpy array order to caffe order
img = img[:, :, (2, 1, 0)] # RGB to BGR
img = img.transpose((2, 0, 1)) # cxhxw in Caffe
# save in datum
datum = caffe.io.array_to_datum(img, label)
keystr = '{:0>8d}'.format(idx)
lmdb_txn.put(keystr, datum.SerializeToString())
if idx % db_batch == 0:
#print idx, batch_idx
lmdb_txn.commit()
lmdb_txn = lmdb_env.begin(write=True)
if idx % db_batch != 0:
lmdb_txn.commit()
# def _write_to_db(patches, file_dir, db_batch):
# batch_imgs = []
# def _write_batch(batch_imgs, db_dir, batch_idx):
# db_saver = leveldb.LevelDB(db_dir)
# batch = leveldb.WriteBatch()
# for idx, im_label in enumerate(batch_imgs):
# im_dat = caffe.io.array_to_datum(im_label[0]) # image content
# im_dat.label = label # label content
# batch.Put('{:0>10d}'.format(batch_idx) + '{:0>10d}'.format(idx),
# im_dat.SerializeToString())
# db_saver.Write(batch, sync=True)
#
# batch_idx = 0
# for idx, item in enumerate(tqdm(patches)):
# patch = Image.open(item['filename'])
# img = np.asarray(patch)
# label = item['label']
# patch.close()
#
# # map numpy array order to caffe order
# img = img[:, :, (2, 1, 0)] # RGB to BGR
# img = img.transpose((2, 0, 1)) # cxhxw in Caffe
# batch_imgs.append([img, label])
#
# if idx % db_batch == 0 or idx == len(patches) - 1:
# #print idx, batch_idx
# _write_batch(batch_imgs, file_dir, batch_idx)
# batch_imgs = []
# batch_idx += 1
# write to leveldb
file_dir = os.path.join(setting.db_folder, 'TRAIN_lmdb')
logger.info('Writing training patches to lmdb {}'.format(file_dir))
_write_to_db(train_patches, file_dir, setting.db_batch)
file_dir = os.path.join(setting.db_folder, 'VAL')
logger.info('Writing validation patches to lmdb {}'.format(file_dir))
_write_to_db(val_patches, file_dir, setting.db_batch)
class Position:
"""Class representing a position in the map of the game
"""
_logger = app_logger.get_logger(__name__)
def __init__(self, x: int = 0, y: int = 0):
"""Position class constructor
Args:
x (int): Coordinate along the x axis. Defaults to 0.
y (int): Coordinate along the y axis. Defaults to 0.
"""
self._x = int(x) #Coordinate along the x axis
self._y = int(y) #Coordinate along the y axis
@property
def x(self):
"""Attribute getter
Returns:
(Type of attribute): Value of attribute (Position, int...)
"""
if self._x is not None:
return self._x
@x.setter
def x(self, value):
"""Attribute setter
Args:
value (Type of attribute): Value of attribute (Position, int...)
"""
self._x = value
@x.deleter
def x(self):
"""Attribute "destructor"
"""
self._x = None
@property
def y(self):
"""Attribute getter
Returns:
(Type of attribute): Value of attribute (Position, int...)
"""
if self._y is not None:
return self._y
@y.setter
def y(self, value):
"""Attribute setter
Args:
value (Type of attribute): Value of attribute (Position, int...)
"""
self._y = value
@y.deleter
def y(self):
"""Attribute "destructor"
"""
self._y = None
def __str__(self):
"""User-friendly representation of Position
Returns:
string: User-friendly representation of Position
"""
try:
return f'({self._x},{self._y})'
except:
self._logger.error("Error during user-friendly Position display")
def __repr__(self):
"""Formal representation of Position
Returns:
string: Formal representation of the Position
"""
try:
return f'Position(x={self._x}, y={self._y})'
except:
self._logger.error("Error during formal Position display")
def __add__(self, other):
"""Operator overload + for Position
Args:
other (Position): The other Position to add
Returns:
Position: Sum of 2 Position
"""
if not other.__class__ is Position:
self._logger.error("Error: Argument is not a Position")
return NotImplemented
res = Position()
res.x = self._x + other.x
res.y = self._y + other.y
return res
def __sub__(self, other):
"""Operator overload - for Position
Args:
other (Position): The other Position to add
Returns:
Position: Subtraction of 2 Position
"""
if not other.__class__ is Position:
self._logger.error("Error: Argument is not a Position")
return NotImplemented
res = Position()
res.x = self._x - other.x
res.y = self._y - other.y
return res
def __eq__(self, other):
"""Operator overload == for Position
Args:
other (Position): The other Position to compare
Returns:
Position: Comparison of equality of 2 positions
"""
if not other.__class__ is Position:
self._logger.error("Error: Argument is not a Position")
return NotImplemented
if((self._x == other.x) and (self._y == other.y)):
return True
else:
return False
class Ship:
"""Class representing a vessel (mother of player and enemy)
"""
#//Peut accéder aux "private" de :
#friend class Jeu;
#friend class Terrain;
_logger = app_logger.get_logger(__name__)
def __init__(self, x: int = 0, y: int = 0):
"""Ship class constructor
Args:
x (int): Coordinate along the x axis. Defaults to 0.
y (int): Coordinate along the y axis. Defaults to 0.
"""
#Location
self._position = Positiion(x, y)
#Hit box => TO MODIFY
self._width = 0
self._height = 0
#Life point
self._max_life_pt #Maximum life points the ship can have
self._life_pt #Ship's remaining life
#Attack
self._attack_speed
#Rate of fire
self._bullet #Type of projectile => class projectile
def damage(damage: int = 0):
"""Inflict damage due to enemies or player
Args:
damage (int, optional): "Amount" of damage. Defaults to 0.
"""
self.life = self.life - damage
@property
def position(self):
"""Attribute getter
Returns:
(Type of attribute): Value of attribute (Position, int...)
"""
if self._position is not None:
return self._position
@position.setter
def position(self, value):
"""Attribute setter
Args:
value (Type of attribute): Value of attribute (Position, int...)
"""
self._position = value
@position.deleter
def position(self):
"""Attribute "destructor"
"""
self._position = None
@property
def width(self):
"""Attribute getter
Returns:
(Type of attribute): Value of attribute (Position, int...)
"""
if self._width is not None:
return self._width
@width.setter
def width(self, value):
"""Attribute setter
Args:
value (Type of attribute): Value of attribute (Position, int...)
"""
self._width = value
@width.deleter
def width(self):
"""Attribute "destructor"
"""
self._width = None
@property
def height(self):
"""Attribute getter
Returns:
(Type of attribute): Value of attribute (Position, int...)
"""
if self._height is not None:
return self._height
@height.setter
def height(self, value):
"""Attribute setter
Args:
value (Type of attribute): Value of attribute (Position, int...)
"""
self._height = value
@height.deleter
def height(self):
"""Attribute "destructor"
"""
self._height = None
@property
def life_pt(self):
"""Attribute getter
Returns:
(Type of attribute): Value of attribute (Position, int...)
"""
if self._life_pt is not None:
return self._life_pt
@life_pt.setter
def life_pt(self, value):
"""Attribute setter
Args:
value (Type of attribute): Value of attribute (Position, int...)
"""
self._life_pt = value
@life_pt.deleter
def life_pt(self):
"""Attribute "destructor"
"""
self._life_pt = None
@property
def max_life_pt(self):
"""Attribute getter
Returns:
(Type of attribute): Value of attribute (Position, int...)
"""
if self._max_life_pt is not None:
return self._max_life_pt
@max_life_pt.setter
def max_life_pt(self, value):
"""Attribute setter
Args:
value (Type of attribute): Value of attribute (Position, int...)
"""
self._max_life_pt = value
@max_life_pt.deleter
def max_life_pt(self):
"""Attribute "destructor"
"""
self._max_life_pt = None
@property
def attack_speed(self):
"""Attribute getter
Returns:
(Type of attribute): Value of attribute (Position, int...)
"""
if self._attack_speed is not None:
return self._attack_speed
@attack_speed.setter
def attack_speed(self, value):
"""Attribute setter
Args:
value (Type of attribute): Value of attribute (Position, int...)
"""
self._attack_speed = value
@attack_speed.deleter
def attack_speed(self):
"""Attribute "destructor"
"""
self._attack_speed = None
@property
def bullet(self):
"""Attribute getter
Returns:
(Type of attribute): Value of attribute (Position, int...)
"""
if self._bullet is not None:
return self._bullet
@bullet.setter
def bullet(self, value):
"""Attribute setter
Args:
value (Type of attribute): Value of attribute (Position, int...)
"""
self._bullet = value
@bullet.deleter
def bullet(self):
"""Attribute "destructor"
"""
self._bullet = None
class Projectile:
"""Class representing a projectile / bullet in game
"""
_logger = app_logger.get_logger(__name__)
#friend class Jeu;
def __init__(self,
x: int = 0,
y: int = 0,
direction: Direction = RIGHT,
bullet_type: Type_bullet = STANDARD,
attack_pt: int = 1,
p_speed: int = 0):
"""Projectile class constructor
Args:
x (int, optional): Coordinate along the x axis. Defaults to 0.
y (int, optional): Coordinate along the y axis. Defaults to 0.
direction (Direction, optional): Direction of projectile movement. Defaults to RIGHT.
bullet_type (Type_bullet, optional): Type of projectile (standard, strong, ...). Defaults to STANDARD.
attack_pt (int, optional): Damage the projectile can inflict. Defaults to 1.
p_speed (int, optional): Projectile speed. Defaults to 0.
"""
#Location
self._position = Positiion(x, y)
self._direction = direction
self._bullet_type = bullet_type
self._attack_pt = attack_pt
self._projectile_speed = p_speed
@property
def position(self):
"""Attribute getter
Returns:
(Type of attribute): Value of attribute (Position, int...)
"""
if self._position is not None:
return self._position
@position.setter
def position(self, value):
"""Attribute setter
Args:
value (Type of attribute): Value of attribute (Position, int...)
"""
self._position = value
@position.deleter
def position(self):
"""Attribute "destructor"
"""
self._position = None
@property
def direction(self):
"""Attribute getter
Returns:
(Type of attribute): Value of attribute (Position, int...)
"""
if self._direction is not None:
return self._direction
@direction.setter
def direction(self, value):
"""Attribute setter
Args:
value (Type of attribute): Value of attribute (Position, int...)
"""
self._direction = value
@direction.deleter
def direction(self):
"""Attribute "destructor"
"""
self._direction = None
@property
def bullet_type(self):
"""Attribute getter
Returns:
(Type of attribute): Value of attribute (Position, int...)
"""
if self._bullet_type is not None:
return self._bullet_type
@bullet_type.setter
def bullet_type(self, value):
"""Attribute setter
Args:
value (Type of attribute): Value of attribute (Position, int...)
"""
self._bullet_type = value
@bullet_type.deleter
def bullet_type(self):
"""Attribute "destructor"
"""
self._bullet_type = None
@property
def attack_pt(self):
"""Attribute getter
Returns:
(Type of attribute): Value of attribute (Position, int...)
"""
if self._attack_pt is not None:
return self._attack_pt
@attack_pt.setter
def attack_pt(self, value):
"""Attribute setter
Args:
value (Type of attribute): Value of attribute (Position, int...)
"""
self._attack_pt = value
@attack_pt.deleter
def attack_pt(self):
"""Attribute "destructor"
"""
self._attack_pt = None
@property
def projectile_speed(self):
"""Attribute getter
Returns:
(Type of attribute): Value of attribute (Position, int...)
"""
if self._projectile_speed is not None:
return self._projectile_speed
@projectile_speed.setter
def projectile_speed(self, value):
"""Attribute setter
Args:
value (Type of attribute): Value of attribute (Position, int...)
"""
self._projectile_speed = value
@projectile_speed.deleter
def projectile_speed(self):
"""Attribute "destructor"
"""
self._projectile_speed = None
#!/usr/bin/env python3
import requests
import sys
import pprint
import os
from facebook_analytics import FacebookAPI
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import settings
import app_logger
logger = app_logger.get_logger("facebook-log", logfile="facebook.log")
## Get settings:
s = settings.Settings()
stt = s.get_settings()
api = FacebookAPI(stt)
#api.get_user_info()
#api.get_tophashtags()
api.make_output_file()
import json
import requests
from proxy_utils import get_proxy
from rmq_queue import RmqChannel, RmqQueueName
from app_logger import get_logger
from statuses import *
logger = get_logger("hik_scan_worker")
def is_generic_http_server(text: str) -> bool:
text = text.lower()
if "<html" in text:
return True
if "<body" in text:
return True
if "<!doctype" in text:
return True
return False
def is_hikcam(url: str) -> int:
try:
proxy = get_proxy()
r = requests.get(url, allow_redirects=False, timeout=10, proxies=proxy)
try:
if r.ok:
if 'window.location.href = "doc/page/login.asp?_"+nowDate.getTime();' in r.text:
def _organize_and_divide_data():
"""The main rountine to generate the training patches.
"""
logger = app_logger.get_logger('random_divide_data')
# read the configuration file
cfg_file = 'config/SJPath.cfg'
config = ConfigParser.SafeConfigParser()
logger.info('Using the config file: ' + cfg_file)
config.read(cfg_file)
# get the basic dataset and setting info
setting = SJPathSetting()
setting.parse(config)
tumor_anno_files = glob.glob(os.path.join(setting.tumor_anno_folder,
'*.mask'))
tumor_wsi_files = []
normal_wsi_files = []
for t in img_type:
tumor_wsi_files.extend(glob.glob(os.path.join(setting.tumor_wsi_folder, t)))
normal_wsi_files.extend(glob.glob(os.path.join(setting.normal_wsi_folder, t)))
########################################
# normal_anno_files = glob.glob(os.path.join(setting.normal_anno_folder,
# '*.mask'))
# tumor_wsi_files = glob.glob(os.path.join(setting.tumor_wsi_folder,
# '*.svs'))
# normal_wsi_files = glob.glob(os.path.join(setting.normal_wsi_folder,
# '*.svs'))
print len(tumor_wsi_files), len(tumor_anno_files), len(normal_wsi_files)#, len(normal_anno_files)
# normal_svs_to_masks = {}
# for anno_file in normal_anno_files:
# svs_file = file_api.get_mask_info(os.path.basename(anno_file))[0]
# if svs_file not in normal_svs_to_masks:
# normal_svs_to_masks[svs_file] = [anno_file]
# else:
# normal_svs_to_masks[svs_file].append(anno_file)
tumor_svs_to_masks = {}
for anno_file in tumor_anno_files:
svs_file = file_api.get_mask_info(os.path.basename(anno_file))[0]
if svs_file not in tumor_svs_to_masks:
tumor_svs_to_masks[svs_file] = [anno_file]
else:
tumor_svs_to_masks[svs_file].append(anno_file)
print len(tumor_svs_to_masks)#, len(normal_svs_to_masks)
# normal_svs_to_path = {}
tumor_svs_to_path = {}
# for file_path in normal_wsi_files:
# svs_file = os.path.basename(file_path).split('.')[0]
# normal_svs_to_path[svs_file] = file_path
for file_path in tumor_wsi_files:
svs_file = os.path.basename(file_path).split('.')[0]
tumor_svs_to_path[svs_file] = file_path
tumor_files = []
normal_files = []
# some annotation doesn't have corresponding WSIs
for svs_file, mask_files in tumor_svs_to_masks.iteritems():
if svs_file in tumor_svs_to_path:
tumor_files.append((tumor_svs_to_path[svs_file], mask_files))
# for svs_file, mask_files in normal_svs_to_masks.iteritems():
# if svs_file in normal_svs_to_path:
# normal_files.append((normal_svs_to_path[svs_file], mask_files))
for svs_file in normal_wsi_files:
normal_files.append((svs_file, None))
tumor_files = _remove_corrupted_files(tumor_files)
normal_files = _remove_corrupted_files(normal_files)
logger.info('# of annotated normal WSIs: {}'.format(len(normal_files)))
logger.info('# of annotated tumor WSIs: {}'.format(len(tumor_files)))
logger.info('{} of the WSIs will be used for testing'
.format(setting.test_frac))
def change_split(item, name):
item['split'] = name
return item
split_data = []
train_files, test_files, val_files = _split_dataset(tumor_files,
setting.test_frac, setting.val_tumor)
train_tumor_files = map(lambda item: change_split(item, 'Train_tumor'), train_files)
val_tumor_files = map(lambda item: change_split(item, 'Val_tumor'), val_files)
test_tumor_files = map(lambda item: change_split(item, 'Test_tumor'), test_files)
split_data.extend(train_tumor_files)
split_data.extend(val_tumor_files)
split_data.extend(test_tumor_files)
train_files, test_files, val_files = _split_dataset(normal_files,
setting.test_frac, setting.val_normal)
train_normal_files = map(lambda item: change_split(item, 'Train_normal'), train_files)
val_normal_files = map(lambda item: change_split(item, 'Val_normal'), val_files)
test_normal_files = map(lambda item: change_split(item, 'Test_normal'), test_files)
split_data.extend(train_normal_files)
split_data.extend(val_normal_files)
split_data.extend(test_normal_files)
logger.info('# of training WSIs: {}'.format(len(train_tumor_files)
+ len(train_normal_files)))
logger.info('# of validation WSIs: {}'.format(len(val_normal_files)
+ len(val_tumor_files)))
logger.info('# of testing WSIs: {}'.format(len(test_normal_files)
+ len(test_tumor_files)))
with open(setting.split_file, 'w') as f:
json.dump(split_data, f)
logger.info('All the val WSI file paths will be written to {}'
.format(setting.test_file))
with open(setting.test_file, 'w') as f:
for item in test_tumor_files:
f.write(item['WSI'][0] + '\n')
for item in test_normal_files:
f.write(item['WSI'][0] + '\n')
def _generate_mask(self, wsi, mask_files, svs_file, lvl_read):
"""Generates thresholded overview image.
Args:
wsi: An openslide image instance.
Returns:
A 2D numpy array of binary image
"""
logger = app_logger.get_logger('preprocess')
self._wsi = wsi
self._filename = svs_file
# read the lowest WSI resolution
lvl_dim = wsi.level_dimensions[lvl_read]
#print svs_file, wsi.level_dimensions
lvl_img = wsi.read_region((0, 0), lvl_read, lvl_dim)
lvl_threshold = self._threshold_downsample_lvl(lvl_img)
# Initialize all pixels to background
lvl_mask = np.zeros((lvl_dim[1], lvl_dim[0]), np.uint8)
selected_mask = np.zeros((lvl_dim[1], lvl_dim[0]), np.uint8)
# normal_mask = np.zeros((lvl_dim[1], lvl_dim[0]), np.uint8)
tumor_mask = np.zeros((lvl_dim[1], lvl_dim[0]), np.uint8)
assert lvl_mask.shape == (lvl_img.size[1], lvl_img.size[0])
# fill in the corresponding region of the mask file
if mask_files is not None:
for mask_file in mask_files:
# obtain the annotated region from annotation file name
# anno = os.path.basename(mask_file)[: -5]. split('_')
anno = file_api.get_mask_info(
os.path.basename(mask_file.split('.')[0]))
level = int(anno[1])
origin = (int(anno[2]), int(anno[3]))
size = (int(anno[4]), int(anno[5]))
# read annotation file
with open(mask_file, 'rb') as f:
mask_data = f.read()
mask_data = np.frombuffer(mask_data, np.uint8)
mask_data = mask_data.reshape([size[1], size[0]])
new_origin = origin
new_size = size
factor = 1
new_mask = Image.fromarray(mask_data)
anno_lvl_size = wsi.level_dimensions[level]
if (anno_lvl_size[0] == lvl_dim[0]
and anno_lvl_size[1] == lvl_dim[1]):
pass
elif (anno_lvl_size[0] != lvl_dim[0]
and anno_lvl_size[1] != lvl_dim[1]):
factor = lvl_dim[0] / float(anno_lvl_size[0])
assert factor < 1
new_size = [
int(np.ceil(size[0] * factor)),
int(np.ceil(size[1] * factor))
]
new_origin = [
int(np.ceil(origin[0] * factor)),
int(np.ceil(origin[1] * factor))
]
new_mask = new_mask.resize(new_size)
else:
logger.error('Error in WSI: {}'.format(svs_file))
# annotated region
selected_mask[new_origin[1]:new_size[1] + new_origin[1],
new_origin[0]:new_size[0] + new_origin[0]] = 255
new_mask = np.asarray(new_mask)
#print mask_file, factor, wsi.level_dimensions, [new_size[1] + new_origin[1], new_size[0] + new_origin[0]]
# if 'Normal' in svs_file:
# normal_mask[new_origin[1]: new_size[1] + new_origin[1],
# new_origin[0]: new_size[0] + new_origin[0]] = new_mask
# else:
# tumor_mask[new_origin[1]: new_size[1] + new_origin[1],
# new_origin[0]: new_size[0] + new_origin[0]] = new_mask
tumor_mask[new_origin[1]:new_size[1] + new_origin[1],
new_origin[0]:new_size[0] +
new_origin[0]] = new_mask
lvl_mask[selected_mask != 0] = SELECTED
normal_tissue_and = np.logical_and(lvl_threshold, selected_mask)
#print (len(normal_tissue_and != 0))
#print len(tumor_mask != 0)
lvl_mask[normal_tissue_and != 0] = NORMAL
lvl_mask[tumor_mask != 0] = TUMOR
else:
lvl_mask[lvl_threshold != 0] = NORMAL
# lvl_mask[selected_mask != 0] = SELECTED
# if 'Normal' in svs_file:
# lvl_mask[normal_mask != 0] = NORMAL
# else:
# normal_tissue_and = np.logical_and(lvl_threshold, selected_mask)
# #print (len(normal_tissue_and != 0))
# #print len(tumor_mask != 0)
# lvl_mask[normal_tissue_and != 0] = NORMAL
# lvl_mask[tumor_mask != 0] = TUMOR
wsi_dim = wsi.level_dimensions[0]
new_size = (wsi_dim[0] / self._patch_factor,
wsi_dim[1] / self._patch_factor)
if wsi_dim[0] / lvl_dim[0] < self._patch_factor:
lvl_mask = Image.fromarray(lvl_mask)
lvl_mask = lvl_mask.resize(new_size)
lvl_mask = np.asarray(lvl_mask)
#print svs_file
elif wsi_dim[0] / lvl_dim[0] > self._patch_factor:
logger.error('Need a larger patch factor')
if self._save_img:
# save the overview level to check the processing is correct
save_size = (int(np.ceil(self._downsample_rate * lvl_dim[0])),
int(np.ceil(self._downsample_rate * lvl_dim[1])))
save_img = lvl_img.resize(save_size)
save_img.save(
os.path.join(self._img_vis,
os.path.basename(svs_file)[:-4] + self._img_ext))
#lvl_threshold = Image.fromarray(lvl_threshold)
#lvl_threshold.save(os.path.join(self._img_vis,
# os.path.basename(svs_file)[:-4] + '_mask1.png'))
lvl_mask_save = np.zeros((new_size[1], new_size[0], 3), np.uint8)
lvl_mask_save[lvl_mask == TUMOR] = [255, 0, 0]
lvl_mask_save[lvl_mask == NORMAL] = [0, 255, 0]
lvl_mask_save[lvl_mask == SELECTED] = [0, 0, 255]
lvl_mask_save = Image.fromarray(lvl_mask_save)
#save_img = lvl_mask_save.resize(save_size)
lvl_mask_save.save(
os.path.join(
self._img_vis,
os.path.basename(svs_file)[:-4] + '_mask' + self._img_ext))
save_img.close()
lvl_mask_save.close()
# close Image
lvl_img.close()
return lvl_mask
# import requests
import pprint
import telebot
import os
import re
# import ffmpeg
import db
from app_logger import get_logger
from face_detector import detect_faces_quantity
import settings
logger = get_logger(__name__)
tb = telebot.TeleBot(settings.BOT_TOKEN)
def download_file(file_id):
file_path = tb.get_file(file_id).file_path
downloaded_file = tb.download_file(file_path)
return downloaded_file
@tb.message_handler(content_types=['audio', 'voice', 'document'])
def handle_docs_audio(message):
try:
if message.content_type == 'audio':
file_id = message.audio.file_id
elif message.content_type == 'voice':
file_id = message.voice.file_id
elif message.content_type == 'document' and re.match(
r'audio', message.document.mime_type):
"""
runs and supervises vk workers
Example:
spawn_and_supervise("worker.py")
"""
from subprocess import Popen
import time
import app_logger
from typing import List
logger = app_logger.get_logger("supervisor")
def spawn_single_worker(worker_name: str) -> Popen:
""" spawns process for account """
return Popen(['python', worker_name])
def spawn_workers(worker_name: str, n_workers: int) -> List[dict]:
"""
spawns multiple processes
:param worker_name: py filename, e.g. worker.py
:param accounts: accounts to spawn
:return: list of {account:, process:} dictionary
"""
logger.info("running processes...")
processes = []
for i in range(n_workers):
processes.append({'process': spawn_single_worker(worker_name)})
