def post_to_imgur(imgur: ImgurClient, config: configparser, dir: str,
memes: Memes) -> str:
"""
Uploads images and creates a public albumb of the memes.
"""
# Upload images
image_ids: List[str] = []
for meme in memes:
response = imgur.upload_from_path(path=os.path.join(dir, meme),
config=None,
anon=False)
image_ids.append(response["id"])
# Create album
ymd = str(date.today())
title = config.get("daily-memes", "imgur_album_title_prefix")
desc = config.get("daily-memes", "imgur_album_desc")
album = imgur.create_album({
"title": f"{title} {ymd}",
"desc": desc,
"ids": ",".join(image_ids),
"privacy": "public"
})
return album["id"]
def __init__(self, config: configparser, wiki_extpath, model_dir_abspath):
self.wiki_glob_path = os.path.join(wiki_extpath, '**', "wiki_*")
self.prefix = os.path.join(model_dir_abspath,
config.get('train', 'model_name'))
self.vocab_size = config.get('train', 'vocab_size')
if not os.path.exists(model_dir_abspath):
os.mkdir(model_dir_abspath)
# Sentence Piece は、パスに スペースが入ってると動かない...。ので、一旦別の場所に移動する。
self.is_space_in_path = ' ' in model_dir_abspath
if self.is_space_in_path:
print('[確認](SP_Trainer) パスにスペースが入っていたため、回避処理を開始します。')
os.path.splitdrive(os.path.abspath(__file__))
self.root_dir = os.path.join(self.get_root_path(), 'tmp',
'SentencePiece', 'out')
# Wikipedia extractデータ をコピー
if os.path.exists(self.root_dir) and len(os.listdir(
self.root_dir)) == 0:
os.rmdir(self.root_dir)
if not os.path.exists(self.root_dir):
shutil.copytree(wiki_extpath, self.root_dir)
self.wiki_glob_path = os.path.join(self.root_dir, '**', "wiki_*")
# model ファイルは、通常の場所にもコピーする。という方向で。
self.prefix = os.path.join(self.root_dir,
config.get('train', 'model_name'))
self.prefix_tmp = os.path.join(model_dir_abspath,
config.get('train', 'model_name'))
print('[確認](SP_Trainer) パス スペース問題 回避のためのコピー完了。')
def read_config(self):
""" Read config from config file
"""
conf = ConfigParser()
conf.read(self.configfile)
self.host = conf.get("jobs", "host")
self.queue = conf.get("jobs", "queue")
self.username = conf.get("credentials", "username")
self.password = conf.get("credentials", "password")
def read_config(self):
""" Read config from config file
"""
conf = ConfigParser()
conf.read(self.configfile)
self.host = conf.get("jobs", "host")
self.queue = conf.get("jobs", "queue")
self.username = conf.get("credentials", "username")
self.password = conf.get("credentials", "password")
def setup_imgur(config: configparser) -> ImgurClient:
"""
Setups Imgur and logs in the user.
"""
imgur = client = ImgurClient(client_id=config.get("daily-memes",
"imgur_client_id"),
client_secret=config.get(
"daily-memes", "imgur_client_secret"))
imgur.set_user_auth(access_token=config.get("daily-memes",
"imgur_access_token"),
refresh_token=config.get("daily-memes",
"imgur_refresh_token"))
return imgur
def initialC(self):
data = ConfigParser()
data.read('validationData/initial_constants.ini')
name = self.name + '-'\
+ repr(self.index[0]) + '-' + repr(self.index[1])
C = []
for i in range(4):
C.append((float)(data.get(name, 'C' + repr(i + 1))))
return C
def initialC(self):
data = ConfigParser()
data.read('validationData/initial_constants.ini')
name = self.name + '-'\
+ repr(self.index[0]) + '-' + repr(self.index[1])
C = []
for i in range(4):
C.append((float)(data.get(name, 'C' + repr(i + 1))))
return C
def reaload_dbconfig(self, parsed_config: configparser):
"""
reload Database config
:param parsed_config: config Object
:return:
"""
# mysql config
mysql_host = parsed_config.get('mysql', 'Host', fallback='localhost')
mysql_id = parsed_config.getint('mysql', 'ID')
mysql_port = parsed_config.getint('mysql', 'Port', fallback=3306)
mysql_user = parsed_config.get('mysql', 'User')
mysql_passwd = parsed_config.get('mysql', 'Passwd')
mysql_database = parsed_config.get('mysql', 'Database')
mysql_charset = parsed_config.get('mysql', 'Charset')
mysql_pool_size = parsed_config.getint('mysql', 'MaxPoolSize')
# redis config
redis_host = parsed_config.get('redis', 'Host', fallback='localhost')
redis_id = parsed_config.getint('redis', 'ID')
redis_port = parsed_config.getint('redis', 'Port')
redis_passwd = parsed_config.get('redis', 'Passwd')
redis_timeout = parsed_config.getint('redis', 'Timeout')
self.dbconfig.__setitem__(
'mysql',
config.to_formated_dbconfig(mysql_id,
mysql_host,
mysql_port,
mysql_passwd,
database=mysql_database,
user=mysql_user,
charset=mysql_charset,
pool_size=mysql_pool_size))
self.dbconfig.__setitem__(
'redis',
config.to_formated_dbconfig(redis_id,
redis_host,
redis_port,
redis_passwd,
timeout=redis_timeout))
from extract_patches import recompone_overlap
from extract_patches import paint_border
from extract_patches import kill_border
from extract_patches import pred_only_FOV
from extract_patches import get_data_testing
from extract_patches import get_data_testing_overlap
# pre_processing.py
from pre_processing import my_PreProc
#config file to read from
config = ConfigParser()
config.read('configuration.txt')
#===========================================
#run the training on invariant or local
path_data = config.get('data paths', 'path_local')
#original test images (for FOV selection)
DRIVE_test_imgs_original = path_data + config.get('data paths', 'test_imgs_original')
test_imgs_orig = load_hdf5(DRIVE_test_imgs_original)
full_img_height = test_imgs_orig.shape[2]
full_img_width = test_imgs_orig.shape[3]
#the border masks provided by the DRIVE
DRIVE_test_border_masks = path_data + config.get('data paths', 'test_border_masks')
test_border_masks = load_hdf5(DRIVE_test_border_masks)
# dimension of the patches
patch_height = config.getint('data attributes', 'patch_height')
patch_width = config.getint('data attributes', 'patch_width')
#the stride in case output with average
stride_height = config.getint('testing settings', 'stride_height')
stride_width = config.getint('testing settings', 'stride_width')
def _getReflector(self, config: configparser) -> str:
try:
return config.get('reflector', 'reflector')
except ValueError as error:
raise Exception('Unable to parse reflector configuration')
