def __init__(self, settings):
super().__init__(settings)
self.img_field = Image({
'position': (0, 0),
'path': 'static/field',
})
self.img_next = Image({
'position': (300, 77),
'path': 'static/next',
})
self.img_hold = Image({
'position': (450, 77),
'path': 'static/hold',
})
self.txt_score = Text({
'position': (300, 300),
'text': 'Score: 0',
})
self.current = pygame.Surface(
(BOARD_SIZE[0] * TILE, BOARD_SIZE[1] * TILE))
self.base = Base()
self.figure = Figure()
self.next = Figure()
self.hold = None
self.hold_first = True
self.score = 0
self.state = '__WAIT__'
self.field = [[0 for _ in range(BOARD_SIZE[0])]
for _ in range(BOARD_SIZE[1])]
def run_on_days(
app: Base,
callback: Callable[..., None],
day_list: list,
start: datetime.time,
**kwargs
) -> Callable:
"""Run a callback on certain days (at the specified time)."""
handles = []
upcoming_days = []
today = app.date()
todays_event = datetime.datetime.combine(today, start)
if todays_event > app.datetime():
if today.strftime("%A") in day_list:
upcoming_days.append(today)
for day_number in range(1, 8):
day = today + datetime.timedelta(days=day_number)
if day.strftime("%A") in day_list:
if len(upcoming_days) < len(day_list):
upcoming_days.append(day)
for day in upcoming_days:
event = datetime.datetime.combine(day, start)
handles.append(app.run_every(callback, event, 604800, **kwargs))
def cancel():
"""Define a method to cancel all of the handles."""
for handle in handles:
app.cancel_timer(handle)
return cancel
def main(config):
loader = Loader(config)
base = Base(config, loader)
make_dirs(base.output_path)
make_dirs(base.save_logs_path)
make_dirs(base.save_model_path)
logger = Logger(os.path.join(base.save_logs_path, 'log.txt'))
logger(config)
if config.mode == 'train':
if config.resume_train_epoch >= 0:
base.resume_model(config.resume_train_epoch)
start_train_epoch = config.resume_train_epoch
else:
start_train_epoch = 0
if config.auto_resume_training_from_lastest_step:
root, _, files = os_walk(base.save_model_path)
if len(files) > 0:
indexes = []
for file in files:
indexes.append(int(
file.replace('.pkl', '').split('_')[-1]))
indexes = sorted(list(set(indexes)), reverse=False)
base.resume_model(indexes[-1])
start_train_epoch = indexes[-1]
logger(
'Time: {}, automatically resume training from the latest step (model {})'
.format(time_now(), indexes[-1]))
for current_epoch in range(start_train_epoch,
config.total_train_epoch):
base.save_model(current_epoch)
if current_epoch < config.use_graph:
_, result = train_meta_learning(base, loader)
logger('Time: {}; Epoch: {}; {}'.format(
time_now(), current_epoch, result))
if current_epoch + 1 >= 1 and (current_epoch + 1) % 40 == 0:
mAP, CMC = test(config, base, loader)
logger(
'Time: {}; Test on Target Dataset: {}, \nmAP: {} \n Rank: {}'
.format(time_now(), config.target_dataset, mAP, CMC))
else:
_, result = train_with_graph(config, base, loader)
logger('Time: {}; Epoch: {}; {}'.format(
time_now(), current_epoch, result))
if current_epoch + 1 >= 1 and (current_epoch + 1) % 5 == 0:
mAP, CMC = test_with_graph(config, base, loader)
logger(
'Time: {}; Test on Target Dataset: {}, \nmAP: {} \n Rank: {}'
.format(time_now(), config.target_dataset, mAP, CMC))
elif config.mode == 'test':
base.resume_model(config.resume_test_model)
mAP, CMC = test_with_graph(config, base, loader)
logger('Time: {}; Test on Target Dataset: {}, \nmAP: {} \n Rank: {}'.
format(time_now(), config.target_dataset, mAP, CMC))
def start(self):
self.base = Base()
self.figure = Figure()
self.next = Figure()
self.hold = None
self.score = 0
self.txt_score.set_text('Score: 0')
self.state = '__GAME__'
self.field = [[0 for _ in range(BOARD_SIZE[0])]
for _ in range(BOARD_SIZE[1])]
def test_save(self):
from core.TestType import TestType
from core.Base import Base
my_type = str(uuid.uuid4())
doc = [str(uuid.uuid4()), str(uuid.uuid4()), str(uuid.uuid4())]
test_type = TestType(my_type, doc)
test_type.save()
ast = Base().get_all(TestType.collection, {})
assert ast.count() == 1
assert ast[0]['type'] == my_type
assert ast[0]['doc_fields'] == doc
def get_manga_chapt(self, mhash, chash):
with self.db.connect() as connection:
return connection.cursor().execute(
Base.source("sql/get_chapter.sql"), {
"mhash": mhash,
"chash": chash
}).fetchone()
def manga(self) -> dict:
if not self._MANGA:
with self.connection as c:
return c.execute(Base.source("sql/get_manga_by_hash.sql"), {
"hash": self.hash
}).fetchone()
return self._MANGA
def get_queued_by_hash(self, uhash):
with self.db.connect() as connection:
c = connection.cursor()
return c.execute(Base.source("sql/url_exists.sql"), {
"hash": uhash
}).fetchone()
def get_by_cookie(self, cookie):
with self.db.connect() as connection:
c = connection.cursor()
return c.execute(Base.source("sql/user_by_cookie.sql"), {
'cookie': cookie
}).fetchone()
def test_save(self):
from core.Test import Test
from core.Base import Base
test_id = str(uuid.uuid4())
owner = str(uuid.uuid4())
test_type = str(uuid.uuid4())
test = Test(test_id, owner, test_type)
now = datetime.datetime.now()
res = test.save()
assert res
at = Base().get_all(Test.collection, {})
assert at.count() == 1
assert at[0]['_id'] == test_id
assert at[0]['owner'] == owner
assert at[0]['test_id'] == test_id
assert at[0]['type'] == test_type
assert at[0]['last_seen'] < now + datetime.timedelta(seconds=1)
def test_remove(self):
from core.Status import Status
from core.Base import Base
test_id1 = str(uuid.uuid4())
test_status1 = random.choice(['SUCCESS', 'FAILURE'])
test_type = str(uuid.uuid4())
details = {'browser': random.choice(['Firefox', 'Chrome'])}
status1 = Status(test_id1, test_type, test_status1, details=details, last=True)
status1.save()
test_id2 = str(uuid.uuid4())
test_status2 = random.choice(['SUCCESS', 'FAILURE'])
status2 = Status(test_id2, test_type, test_status2, details=details, last=True)
status2.save()
status2.remove()
ast = Base().get_all(Status.collection, {})
assert ast.count() == 1
assert ast[0]['test_id'] == test_id1
def test_save(self):
from core.Index import Index
from core.Base import Base
my_type = str(uuid.uuid4())
field = str(uuid.uuid4())
values = [str(uuid.uuid4()), str(uuid.uuid4()), str(uuid.uuid4())]
index = Index(my_type, field, values)
index.save()
ast = Base().get_all(Index.collection, {})
assert ast.count() == 1
assert ast[0]['type'] == my_type
assert ast[0]['field'] == field
assert ast[0]['values'] == values
index2 = Index(my_type, field)
assert index2._values == None
index2 = index2.get()
assert index2._values == values
def create_tables(self):
connection = self.db.connect()
c = connection.cursor()
c.executescript(Base.source("sql/create_tables.sql"))
connection.commit()
connection.close()
def set_readed(self, uiid, ciid, piid):
with self.connection as c:
c.execute(Base.source("sql/set_page_readed.sql"), {
"miid": self.miid,
"pgid": piid,
"uiid": uiid,
"ciid": ciid
})
def authors(self) -> list:
if not self._AUTHORS:
with self.connection as c:
t = c.execute(Base.source("sql/select_authors_by_miid.sql"), {
"miid": self.miid
}).fetchall()
self._AUTHORS = list(map(lambda x: x["value"], t))
return self._AUTHORS
def genres(self) -> list:
if not self._GENRES:
with self.connection as c:
t = c.execute(Base.source("sql/select_genres_by_miid.sql"), {
"miid": self.miid
}).fetchall()
self._GENRES = list(map(lambda x: x["value"], t))
return self._GENRES
def run_on_weekdays(
app: Base, callback: Callable[..., None], start: datetime.time, **kwargs
) -> list:
"""Run a callback on weekdays (at the specified time)."""
return app.run_on_days(
callback,
["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday"],
start,
**kwargs
)
def get_readed_count(self, ciid, uiid):
if not uiid:
return 0
with self.connection as c:
return c.execute(Base.source("sql/select_readed_count.sql"), {
"miid": self.miid,
"ciid": ciid,
"uiid": uiid
}).fetchone().get("readed", 0)
def test_save(self):
from core.Status import Status
from core.Test import Test
from core.Index import Index
from core.TestType import TestType
from core.Base import Base
test_id = str(uuid.uuid4())
test_status = random.choice(['SUCCESS', 'FAILURE'])
test_type = str(uuid.uuid4())
details = {'browser': random.choice(['Firefox', 'Chrome'])}
status = Status(test_id, test_type, test_status, details=details)
TestType(test_type, doc_fields_to_index=['browser']).save()
now = datetime.datetime.now()
status.save()
ast = Base().get_all(Status.collection, {})
assert ast.count() == 1
assert ast[0]['test_id'] == test_id
assert ast[0]['status'] == test_status
assert ast[0]['details'] == details
assert ast[0]['type'] == test_type
assert ast[0]['on'] < now + datetime.timedelta(seconds=1)
at = Base().get_all(Test.collection, {})
assert at.count() == 1
assert at[0]['test_id'] == test_id
assert at[0]['type'] == test_type
assert at[0]['last_seen'] < ast[0]['on'] + datetime.timedelta(seconds=1)
st = Base().get_one(Index.collection, {})
assert st['type'] == test_type
assert st['field'] == 'browser'
assert st['values'] == [details['browser']]
st = Base().get_one(TestType.collection, {})
assert st['type'] == test_type
assert st['doc_fields'] == ['browser']
test_id = str(uuid.uuid4())
test_status = 'TO_RERUN'
test_type = str(uuid.uuid4())
status = Status(test_id, test_type, test_status)
now = datetime.datetime.now()
status.save()
st = Base().get_one(Status.collection, {'test_id': test_id})
assert st['status'] == 'CUSTOM'
assert st['details']['original_status'] == test_status
def test_update_last(self):
from core.Status import Status
from core.Base import Base
test_id1 = str(uuid.uuid4())
test_status1 = random.choice(['SUCCESS', 'FAILURE'])
test_type = str(uuid.uuid4())
details = {'browser': random.choice(['Firefox', 'Chrome'])}
status1 = Status(test_id1, test_type, test_status1, details=details, last=True)
status1.save()
test_id2 = str(uuid.uuid4())
test_status2 = random.choice(['SUCCESS', 'FAILURE'])
status2 = Status(test_id2, test_type, test_status2, details=details, last=True)
status2.save()
st = Base().get_one(Status.collection, {})
assert st['last'] == True
status2.update_last()
ast = Base().get_all(Status.collection, {'last': True})
assert ast.count() == 1
st = Base().get_one(Status.collection, {'test_id': test_id2})
assert st['last'] == False
def get_readed(self, ciid, uiid):
if not uiid:
return []
with self.connection as c:
readed = c.execute(Base.source("sql/select_readed.sql"), {
"miid": self.miid,
"ciid": ciid,
"uiid": uiid
}).fetchall()
return list(map(lambda x: x["pageid"], readed))
def test_get_last(self):
from core.Status import Status
from core.Test import Test
from core.Base import Base
test_id = str(uuid.uuid4())
test_status1 = 'FAILURE'
details = {'browser': random.choice(['Firefox', 'Chrome'])}
test_type = str(uuid.uuid4())
status1 = Status(test_id, test_type, test_status1, details=details)
status1.save_and_update()
at = Base().get_all(Test.collection, {})
assert at.count() == 1
assert at[0]['test_id'] == test_id
test_status2 = 'SUCCESS'
status2 = Status(test_id, test_type, test_status2, details=details)
status2.save_and_update()
at = Base().get_all(Test.collection, {})
assert at.count() == 1
Base().upsert_by_id(Status.collection, bson.ObjectId(status1._id), {Status._on: datetime.datetime.now() - datetime.timedelta(seconds=3)})
Base().upsert_by_id(Status.collection, bson.ObjectId(status2._id), {Status._on: datetime.datetime.now() - datetime.timedelta(seconds=3)})
sl = Status(test_id).get_last()
assert sl._status == 'SUCCESS'
assert sl._test_id == test_id
at = Base().get_all(Test.collection, {})
assert at.count() == 1
assert at[0]['last_seen'] > Status(base_id=status2._id).get()._on + datetime.timedelta(seconds=1)
def test_index(self):
from core.Index import Index
from core.Status import Status
from core.TestType import TestType
from core.Base import Base
test_id = str(uuid.uuid4())
test_status = random.choice(['SUCCESS', 'FAILURE'])
test_type = str(uuid.uuid4())
field1 = 'browser'
field2 = 'environment'
details1 = {field1: 'Firefox'}
TestType(test_type, doc_fields_to_index=[field1, field2]).save()
status1 = Status(test_id, test_type, test_status, details=details1)
details2 = {field1: 'Chrome'}
status2 = Status(test_id, test_type, test_status, details=details2)
details3 = {field2: 'master'}
status3 = Status(test_id, test_type, test_status, details=details3)
#Index.index(status1)
status1.save()
ast = Base().get_all(Index.collection, {})
assert ast.count() == 1
assert ast[0]['type'] == test_type
assert ast[0]['field'] == field1
assert ast[0]['values'] == ['Firefox']
#Index.index(status2)
status2.save()
ast = Base().get_all(Index.collection, {})
assert ast.count() == 1
assert sorted(ast[0]['values']) == sorted(['Chrome', 'Firefox'])
#Index.index(status3)
status3.save()
ast = Base().get_all(Index.collection, {})
assert ast.count() == 2
ast = Base().get_all(Index.collection, {'field': 'browser'})
assert ast.count() == 1
assert sorted(ast[0]['values']) == sorted(['Chrome', 'Firefox'])
ast = Base().get_all(Index.collection, {'field': 'environment'})
assert ast.count() == 1
assert ast[0]['values'] == ['master']
def test_purge(self):
from core.Status import Status
from core.Base import Base
test_id = str(uuid.uuid4())
test_status1 = 'FAILURE'
details = {'browser': random.choice(['Firefox', 'Chrome'])}
test_type = str(uuid.uuid4())
status1 = Status(test_id, test_type, test_status1, details=details)
status1.save_and_update()
ast = Base().get_all(Status.collection, {})
Base().upsert_by_id(Status.collection, bson.ObjectId(status1._id), {Status._on: datetime.datetime.now() - datetime.timedelta(days=8)})
ast = Base().get_all(Status.collection, {})
test_id2 = str(uuid.uuid4())
test_status2 = 'SUCCESS'
status2 = Status(test_id2, test_type, test_status2, details=details)
status2.save_and_update()
test_status3 = 'SUCCESS'
status3 = Status(test_id, test_type, test_status3, details=details)
status3.save_and_update()
res = status3.purge()
assert res['nb_removed'] == 1
ast = Base().get_all(Status.collection, {})
assert ast.count() == 2
assert sorted([str(st['_id']) for st in ast]) == sorted([status2._id, status3._id])
def test_save_and_update(self):
from core.Status import Status
from core.Base import Base
test_id = str(uuid.uuid4())
test_status1 = 'FAILURE'
test_type = str(uuid.uuid4())
details = {'browser': random.choice(['Firefox', 'Chrome'])}
status1 = Status(test_id, test_type, test_status1, details=details)
status1.save_and_update()
st = Base().get_one(Status.collection, {})
assert st['last'] == True
test_status2 = 'SUCCESS'
status2 = Status(test_id, test_type, test_status2, details=details)
status2.save_and_update()
ast = Base().get_all(Status.collection, {})
assert ast.count() == 2
ast = Base().get_all(Status.collection, {'last': False})
assert ast.count() == 1
assert ast[0]['status'] == 'FAILURE'
ast = Base().get_all(Status.collection, {'last': True})
assert ast.count() == 1
assert ast[0]['status'] == 'SUCCESS'
def main(config):
# init loaders and base
loaders = Loaders(config)
base = Base(config, loaders)
# make directions
make_dirs(base.output_path)
make_dirs(base.save_model_path)
make_dirs(base.save_logs_path)
make_dirs(base.save_visualize_market_path)
make_dirs(base.save_visualize_duke_path)
# init logger
logger = Logger(
os.path.join(os.path.join(config.output_path, 'logs/'), 'log.txt'))
logger('\n' * 3)
logger(config)
if config.mode == 'train': # train mode
# resume model from the resume_train_epoch
start_train_epoch = 0
# automatically resume model from the latest one
if config.auto_resume_training_from_lastest_steps:
root, _, files = os_walk(base.save_model_path)
if len(files) > 0:
# get indexes of saved models
indexes = []
for file in files:
indexes.append(int(
file.replace('.pkl', '').split('_')[-1]))
indexes = sorted(list(set(indexes)), reverse=False)
# resume model from the latest model
base.resume_model(indexes[-1])
#
start_train_epoch = indexes[-1]
logger(
'Time: {}, automatically resume training from the latest step (model {})'
.format(time_now(), indexes[-1]))
# main loop
for current_epoch in range(start_train_epoch,
config.total_train_epochs):
# save model
base.save_model(current_epoch)
# train
base.lr_scheduler.step(current_epoch)
_, results = train_an_epoch(config, base, loaders, current_epoch)
logger('Time: {}; Epoch: {}; {}'.format(time_now(),
current_epoch, results))
# test
testwithVer2(config,
logger,
base,
loaders,
'duke',
use_gcn=True,
use_gm=True)
elif config.mode == 'test': # test mode
# resume from the resume_test_epoch
if config.resume_test_path != '' and config.resume_test_epoch != 0:
base.resume_model_from_path(config.resume_test_path,
config.resume_test_epoch)
else:
assert 0, 'please set resume_test_path and resume_test_epoch '
# test
duke_map, duke_rank = testwithVer2(config,
logger,
base,
loaders,
'duke',
use_gcn=False,
use_gm=False)
logger('Time: {}, base, Dataset: Duke \nmAP: {} \nRank: {}'.format(
time_now(), duke_map, duke_rank))
duke_map, duke_rank = testwithVer2(config,
logger,
base,
loaders,
'duke',
use_gcn=True,
use_gm=False)
logger(
'Time: {}, base+gcn, Dataset: Duke \nmAP: {} \nRank: {}'.format(
time_now(), duke_map, duke_rank))
duke_map, duke_rank = testwithVer2(config,
logger,
base,
loaders,
'duke',
use_gcn=True,
use_gm=True)
logger('Time: {}, base+gcn+gm, Dataset: Duke \nmAP: {} \nRank: {}'.
format(time_now(), duke_map, duke_rank))
logger('')
elif config.mode == 'visualize': # visualization mode
# resume from the resume_visualize_epoch
if config.resume_visualize_path != '' and config.resume_visualize_epoch != 0:
base.resume_model_from_path(config.resume_visualize_path,
config.resume_visualize_epoch)
print('Time: {}, resume model from {} {}'.format(
time_now(), config.resume_visualize_path,
config.resume_visualize_epoch))
# visualization
if 'market' in config.train_dataset:
visualize_ranked_images(config, base, loaders, 'market')
elif 'duke' in config.train_dataset:
visualize_ranked_images(config, base, loaders, 'duke')
else:
assert 0
class Field(Element):
def __init__(self, settings):
super().__init__(settings)
self.img_field = Image({
'position': (0, 0),
'path': 'static/field',
})
self.img_next = Image({
'position': (300, 77),
'path': 'static/next',
})
self.img_hold = Image({
'position': (450, 77),
'path': 'static/hold',
})
self.txt_score = Text({
'position': (300, 300),
'text': 'Score: 0',
})
self.current = pygame.Surface(
(BOARD_SIZE[0] * TILE, BOARD_SIZE[1] * TILE))
self.base = Base()
self.figure = Figure()
self.next = Figure()
self.hold = None
self.hold_first = True
self.score = 0
self.state = '__WAIT__'
self.field = [[0 for _ in range(BOARD_SIZE[0])]
for _ in range(BOARD_SIZE[1])]
def get(self, position, shift=(0, 0)):
if all([
0 <= position[1] + shift[1] < BOARD_SIZE[1],
0 <= position[0] + shift[0] < BOARD_SIZE[0],
]):
return self.field[position[1] + shift[1]][position[0] + shift[0]]
return 1
def set(self, position, value):
self.field[position[1]][position[0]] = value
def start(self):
self.base = Base()
self.figure = Figure()
self.next = Figure()
self.hold = None
self.score = 0
self.txt_score.set_text('Score: 0')
self.state = '__GAME__'
self.field = [[0 for _ in range(BOARD_SIZE[0])]
for _ in range(BOARD_SIZE[1])]
def key_down(self, key):
if key == pygame.K_a:
if self.figure.can_move(self, (-1, 0)):
self.figure.move((-1, 0))
elif key == pygame.K_d:
if self.figure.can_move(self, (1, 0)):
self.figure.move((1, 0))
elif key == pygame.K_q:
if self.figure.can_rotate(self, 3):
self.figure.rotate(3)
elif key == pygame.K_e:
if self.figure.can_rotate(self, 1):
self.figure.rotate(1)
elif key == pygame.K_w:
if not self.hold:
if self.hold_first:
self.hold = self.figure.copy()
self.figure = Figure()
self.hold_first = False
else:
self.figure = self.hold.copy()
self.hold = None
elif key == pygame.K_n:
self.start()
def show(self, surf):
if self.state == '__GAME__':
if self.figure.state == '__STOP__':
if self.figure.can_move(self, (0, 1)):
self.figure.state = '__FALLING__'
else:
self.hold_first = True
for block in self.figure.blocks:
self.set(block.position, 1)
deleted = []
self.base.add_blocks(self.figure)
for i in range(len(self.field)):
if all([
self.get((k, i))
for k in range(len(self.field[i]))
]):
for row in range(i, 0, -1):
self.field[row] = self.field[row - 1]
deleted.append(i)
self.field[0] = [0 for _ in range(BOARD_SIZE[0])]
self.base.set_deleted(deleted)
if any([i == 1 for i in self.field[0]]):
self.state = '__LOSE__'
config.UPDATER.send_result(self.score)
if config.NICKNAME:
rating = config.WINDOW.get('txtRating')
rating.set_text(int(rating.text) + self.score)
best = config.WINDOW.get('txtBest')
best.set_text(max(int(best.text), self.score))
self.score += 2**len(deleted) if deleted else 0
self.txt_score.set_text('Score: ' + str(self.score))
self.figure = self.next
self.next = Figure()
self.current.fill(BACKGROUND)
self.img_field.show(self.current)
if self.state == '__GAME__':
self.figure.show(self.current)
self.base.show(self.current)
self.img_next.show(surf)
self.img_hold.show(surf)
if self.hold and self.state == '__GAME__':
self.hold.static_show(surf, self.img_hold)
if self.state == '__GAME__':
self.next.static_show(surf, self.img_next)
self.txt_score.show(surf)
surf.blit(self.current, self.position)
def run_on_weekend_days(
app: Base, callback: Callable[..., None], start: datetime.time, **kwargs
) -> list:
"""Run a callback on weekend days (at the specified time)."""
return app.run_on_days(callback, ["Friday", "Saturday"], start, **kwargs)
def get_chapter(self, chash):
with self.connection as c:
return c.execute(Base.source("sql/get_chapter.sql"), {
"mhash": self.mhash,
"chash": chash
}).fetchone()
p.add_argument('--bind', '-b', nargs='?', default='0.0.0.0')
p.add_argument('--manga', '-m', nargs='?', default='$HOME/.manga/')
p.add_argument('--install',
action='store_true',
help="Creating database with initial groups and users.")
return p.parse_args()
app_argv = ZeroMangaApplication.args()
BIND_ADDRES = app_argv.bind
PORT_NUMBER = app_argv.listen
data = {"download-threads": 8, "storage": os.path.expandvars(app_argv.manga)}
db = Base(os.path.join(os.path.expandvars(app_argv.manga), 'zeromanga.sqlite'))
data["db"] = db
users = Users(data)
data["users"] = users
if app_argv.install:
ins = Install(data)
ins.run_install()
sys.exit()
data["manga"] = Manga(data)
data["taskman"] = TaskManager(data)
zmApp = ZeroMangaApplication()
def main(config):
# loaders and base
loaders = Loaders(config)
base = Base(config, loaders)
# make dirs
make_dirs(config.save_images_path)
make_dirs(config.save_models_path)
make_dirs(config.save_features_path)
# logger
logger = Logger(os.path.join(config.output_path, 'log.txt'))
logger(config)
if config.mode == 'train':
# automatically resume model from the latest one
start_train_epoch = 0
root, _, files = os_walk(config.save_models_path)
if len(files) > 0:
# get indexes of saved models
indexes = []
for file in files:
indexes.append(int(file.replace('.pkl', '').split('_')[-1]))
# remove the bad-case and get available indexes
model_num = len(base.model_list)
available_indexes = copy.deepcopy(indexes)
for element in indexes:
if indexes.count(element) < model_num:
available_indexes.remove(element)
available_indexes = sorted(list(set(available_indexes)),
reverse=True)
unavailable_indexes = list(
set(indexes).difference(set(available_indexes)))
if len(available_indexes
) > 0: # resume model from the latest model
base.resume_model(available_indexes[0])
start_train_epoch = available_indexes[0] + 1
logger(
'Time: {}, automatically resume training from the latest step (model {})'
.format(time_now(), available_indexes[0]))
else: #
logger('Time: {}, there are no available models')
# main loop
for current_epoch in range(
start_train_epoch, config.warmup_reid_epoches +
config.warmup_gan_epoches + config.train_epoches):
# test
if current_epoch % 10 == 0 and current_epoch > config.warmup_reid_epoches + config.warmup_gan_epoches:
results = test(config, base, loaders, brief=True)
for key in results.keys():
logger('Time: {}\n Setting: {}\n {}'.format(
time_now(), key, results[key]))
# visualize generated images
if current_epoch % 10 == 0 or current_epoch <= 10:
visualize(config, loaders, base, current_epoch)
# train
if current_epoch < config.warmup_reid_epoches: # warmup reid model
results = train_an_epoch(config,
loaders,
base,
current_epoch,
train_gan=True,
train_reid=True,
train_pixel=False,
optimize_sl_enc=True)
elif current_epoch < config.warmup_reid_epoches + config.warmup_gan_epoches: # warmup GAN model
results = train_an_epoch(config,
loaders,
base,
current_epoch,
train_gan=True,
train_reid=False,
train_pixel=False,
optimize_sl_enc=False)
else: # joint train
results = train_an_epoch(config,
loaders,
base,
current_epoch,
train_gan=True,
train_reid=True,
train_pixel=True,
optimize_sl_enc=True)
logger('Time: {}; Epoch: {}; {}'.format(time_now(),
current_epoch, results))
# save model
base.save_model(current_epoch)
# test
results = test(config, base, loaders, brief=False)
for key in results.keys():
logger('Time: {}\n Setting: {}\n {}'.format(
time_now(), key, results[key]))
elif config.mode == 'test':
# resume from pre-trained model and test
base.resume_model_from_path(config.pretrained_model_path,
config.pretrained_model_epoch)
results = test(config, base, loaders, brief=False)
for key in results.keys():
logger('Time: {}\n Setting: {}\n {}'.format(
time_now(), key, results[key]))
def main(config):
# loaders and base
loaders = Loaders(config)
base = Base(config, loaders)
# make dirs
make_dirs(config.save_images_path)
make_dirs(config.save_wp_models_path)
make_dirs(config.save_st_models_path)
make_dirs(config.save_features_path)
logger = setup_logger('adaptation_reid', config.output_path, if_train=True)
if config.mode == 'train':
if config.resume:
# automatically resume model from the latest one
if config.resume_epoch_num == 0:
start_train_epoch = 0
root, _, files = os_walk(config.save_models_path)
if len(files) > 0:
# get indexes of saved models
indexes = []
for file in files:
indexes.append(
int(file.replace('.pkl', '').split('_')[-1]))
# remove the bad-case and get available indexes
model_num = len(base.model_list)
available_indexes = copy.deepcopy(indexes)
for element in indexes:
if indexes.count(element) < model_num:
available_indexes.remove(element)
available_indexes = sorted(list(set(available_indexes)),
reverse=True)
unavailable_indexes = list(
set(indexes).difference(set(available_indexes)))
if len(available_indexes
) > 0: # resume model from the latest model
base.resume_model(available_indexes[0])
start_train_epoch = available_indexes[0] + 1
logger.info(
'Time: {}, automatically resume training from the latest step (model {})'
.format(time_now(), available_indexes[0]))
else: #
logger.info('Time: {}, there are no available models')
else:
start_train_epoch = config.resume_epoch_num
else:
start_train_epoch = 0
# main loop
for current_epoch in range(
start_train_epoch, config.warmup_reid_epoches +
config.warmup_gan_epoches + config.warmup_adaptation_epoches):
# train
if current_epoch < config.warmup_reid_epoches: # warmup reid model
results = train_an_epoch(config,
0,
loaders,
base,
current_epoch,
train_gan=True,
train_reid=True,
self_training=False,
optimize_sl_enc=True,
train_adaptation=False)
elif current_epoch < config.warmup_reid_epoches + config.warmup_gan_epoches: # warmup GAN model
results = train_an_epoch(config,
0,
loaders,
base,
current_epoch,
train_gan=True,
train_reid=False,
self_training=False,
optimize_sl_enc=False,
train_adaptation=False) # joint train
elif current_epoch < config.warmup_reid_epoches + config.warmup_gan_epoches + config.warmup_adaptation_epoches: #warmup adaptation
results = train_an_epoch(config,
0,
loaders,
base,
current_epoch,
train_gan=True,
train_reid=False,
self_training=False,
optimize_sl_enc=False,
train_adaptation=True)
print("another epoch")
logger.info('Time: {}; Epoch: {}; {}'.format(
time_now(), current_epoch, results))
# save model
if current_epoch % config.save_model_interval == 0:
base.save_model(current_epoch, True)
if current_epoch % config.test_model_interval == 0:
visualize(config, loaders, base, current_epoch)
test(config, base, loaders, epoch=0, brief=False)
total_wp_epoches = config.warmup_reid_epoches + config.warmup_gan_epoches
for iter_n in range(config.iteration_number):
src_dataset, src_dataloader, trg_dataset, trg_dataloader = loaders.get_self_train_loaders(
)
trg_labeled_dataloader = generate_labeled_dataset(
base, iter_n, src_dataset, src_dataloader, trg_dataset,
trg_dataloader)
for epoch in range(total_wp_epoches + 1, config.self_train_epoch):
results = train_an_epoch(
config,
iter_n,
loaders,
base,
epoch,
train_gan=True,
train_reid=False,
self_training=True,
optimize_sl_enc=True,
trg_labeled_loader=trg_labeled_dataloader)
logger.info('Time: {}; Epoch: {}; {}'.format(
time_now(), current_epoch, results))
if epoch % config.save_model_interval == 0:
base.save_model(iter_n * config.self_train_epoch + epoch,
False)
elif config.mode == 'test':
# resume from pre-trained model and test
base.resume_model_from_path(config.pretrained_model_path,
config.pretrained_model_epoch)
cmc, map = test(config, base, loaders, epoch=100, brief=False)
def get_queued(self):
with self.db.connect() as connection:
c = connection.cursor()
return c.execute(Base.source("sql/qupload.sql")).fetchall() or []
def get_manga_by_hash(self, mhash):
with self.db.connect() as connection:
c = connection.cursor()
return c.execute(Base.source("sql/get_manga_by_hash.sql"), {
"hash": mhash
}).fetchone()
def add_url(self, uiid, url):
with self.db.connect() as connection:
c = connection.cursor()
c.execute(Base.source("sql/add_url.sql"),
(uiid, url, aux.sha1(url)))
def authorize(self, uiid, cookie, ip):
with self.db.connect() as connection:
c = connection.cursor()
c.execute(Base.source("sql/authorize.sql"), (uiid, ip, cookie))
connection.commit()
def main(config):
# init loaders and base
loaders = ReIDLoaders(config)
base = Base(config)
# make directions
make_dirs(base.output_path)
# init logger
logger = Logger(os.path.join(config.output_path, 'log.txt'))
logger(config)
assert config.mode in ['train', 'test', 'visualize']
if config.mode == 'train': # train mode
# automatically resume model from the latest one
if config.auto_resume_training_from_lastest_steps:
start_train_epoch = base.resume_last_model()
# main loop
for current_epoch in range(start_train_epoch,
config.total_train_epochs):
# save model
base.save_model(current_epoch)
# train
base.lr_scheduler.step(current_epoch)
_, results = train_an_epoch(config, base, loaders)
logger('Time: {}; Epoch: {}; {}'.format(time_now(),
current_epoch, results))
# test
base.save_model(config.total_train_epochs)
mAP, CMC = test(config, base, loaders)
logger('Time: {}; Test Dataset: {}, \nmAP: {} \nRank: {}'.format(
time_now(), config.test_dataset, mAP, CMC))
elif config.mode == 'test': # test mode
base.resume_from_model(config.resume_test_model)
mAP, CMC = test(config, base, loaders)
logger('Time: {}; Test Dataset: {}, \nmAP: {} \nRank: {}'.format(
time_now(), config.test_dataset, mAP, CMC))
elif config.mode == 'visualize': # visualization mode
base.resume_from_model(config.resume_visualize_model)
visualize(config, base, loaders)
def main(config):
# init loaders and base
loaders = Loaders(config)
base = Base(config, loaders)
# make directions
make_dirs(base.output_path)
make_dirs(base.save_model_path)
make_dirs(base.save_logs_path)
make_dirs(base.save_visualize_market_path)
make_dirs(base.save_visualize_duke_path)
# init logger
logger = Logger(
os.path.join(os.path.join(config.output_path, 'logs/'), 'log.txt'))
logger('\n' * 3)
logger(config)
if config.mode == 'train': # train mode
# resume model from the resume_train_epoch
if config.resume_train_epoch >= 0:
base.resume_model(config.resume_train_epoch)
start_train_epoch = config.resume_train_epoch
else:
start_train_epoch = 0
# automatically resume model from the latest one
if config.auto_resume_training_from_lastest_steps:
root, _, files = os_walk(base.save_model_path)
if len(files) > 0:
# get indexes of saved models
indexes = []
for file in files:
indexes.append(int(
file.replace('.pkl', '').split('_')[-1]))
indexes = sorted(list(set(indexes)), reverse=False)
# resume model from the latest model
base.resume_model(indexes[-1])
#
start_train_epoch = indexes[-1]
logger(
'Time: {}, automatically resume training from the latest step (model {})'
.format(time_now(), indexes[-1]))
# main loop
for current_epoch in range(start_train_epoch,
config.total_train_epochs):
# save model
base.save_model(current_epoch)
# train
base.lr_scheduler.step(current_epoch)
_, results = train_an_epoch(config, base, loaders)
logger('Time: {}; Epoch: {}; {}'.format(time_now(),
current_epoch, results))
# test
if (current_epoch + 1) % 40 == 0 and current_epoch + 1 >= 0:
market_map, market_rank = test(config, base, loaders, 'market')
duke_map, duke_rank = test(config, base, loaders, 'duke')
logger(
'Time: {}, Dataset: Market \nmAP: {} \nRank: {}'.format(
time_now(), market_map, market_rank))
logger('Time: {}, Dataset: Duke \nmAP: {} \nRank: {}'.format(
time_now(), duke_map, duke_rank))
logger('')
elif config.mode == 'test': # test mode
# resume from the resume_test_epoch
if config.resume_test_epoch >= 0:
base.resume_model(config.resume_test_epoch)
# test
market_map, market_rank = test(config, base, loaders, 'market')
duke_map, duke_rank = test(config, base, loaders, 'duke')
logger('Time: {}, Dataset: Market \nmAP: {} \nRank: {}'.format(
time_now(), market_map, market_rank))
logger('Time: {}, Dataset: Duke \nmAP: {} \nRank: {}'.format(
time_now(), duke_map, duke_rank))
logger('')
elif config.mode == 'visualize': # visualization mode
# resume from the resume_visualize_epoch
if config.resume_visualize_epoch >= 0:
base.resume_model(config.resume_visualize_epoch)
# visualization
visualize_ranking_list(config, base, loaders, 'market')
visualize_ranking_list(config, base, loaders, 'duke')
