def train_body(self, model, optimizer, lr_scheduler, train_loader, test_loader, unique_image, start_epoch=0):
ocfg = config.get('optm_config')
start = time.time()
ac_t2i_top1_best = 0.0
best_epoch = 0
for epoch in range(start_epoch, ocfg['num_epoches']):
self.train_loop(train_loader, model, optimizer, lr_scheduler, epoch)
if epoch >= 0:
ac_top1_i2t, ac_top5_i2t, ac_top10_i2t, ac_top1_t2i, ac_top5_t2i , ac_top10_t2i, test_time = self.evalution(model, test_loader, unique_image)
state = {'model': model.state_dict(), 'optimizer': optimizer.state_dict(), 'epoch': start_epoch + epoch}
if ac_top1_t2i > ac_t2i_top1_best:
best_epoch = epoch
ac_t2i_top1_best = ac_top1_t2i
self.save_checkpoint(state, config.get('task_id'))
self.logger.write('epoch:{}'.format(epoch))
self.logger.write('top1_t2i: {:.3f}, top5_t2i: {:.3f}, top10_t2i: {:.3f}, top1_i2t: {:.3f}, top5_i2t: {:.3f}, top10_i2t: {:.3f}'.format(
ac_top1_t2i, ac_top5_t2i, ac_top10_t2i, ac_top1_i2t, ac_top5_i2t, ac_top10_i2t))
self.logger.write('Best epoch:{}'.format(best_epoch))
self.logger.write('Train done')
# self.logger.write(config.get('model_config')['checkpoint_dir'])
self.logger.write(config.get('task_id'))
end = time.time()
cost = end - start
cost_h = cost // 3600
cost_m = (cost - cost_h * 3600) // 60
cost_s = cost - cost_h * 3600 - cost_m * 60
self.logger.write('cost time: %d H %d M %d s' % (cost_h, cost_m, cost_s))
def bulid_model(self, class_num):
mconfig = config.get('model_config')
model_name = mconfig['name']
del mconfig['name']
if 'split_bn' in mconfig:
split_bn = mconfig['split_bn']
del mconfig['split_bn']
else:
split_bn = None
net = models.__dict__[model_name]
model = net(num_classes=class_num, **mconfig)
if split_bn:
# convert_dsbn(model)
convert_dsbnConstBatch(
model,
batch_size=config.get('dataset_config')['batch_size'],
constant_batch=32)
# convert_dsbnShare(model, constant_batch=32)
if config.get('debug'):
model = torch.nn.DataParallel(model).cuda()
else:
model = model.cuda()
model = nn.parallel.DistributedDataParallel(
model,
device_ids=[int(os.environ['LOCAL_RANK'])],
find_unused_parameters=True)
cudnn.benchmark = True
self.logger.write(model)
return model
async def _email_to_role(self, ctx, email: str) -> discord.Role:
"""Get role from email address"""
registered = config.get("gatekeeper", "suffixes")
constraints = config.get("gatekeeper", "constraints")
username = email.split("@")[0]
for domain, role_id in list(registered.items())[:-1]:
if not email.endswith(domain):
continue
# found corresponding domain, check constraint
if domain in constraints:
constraint = constraints[domain]
else:
constraint = list(constraints.values())[-1]
match = re.fullmatch(constraint, username)
# return
if match is not None:
return self.getGuild().get_role(role_id)
else:
await self.event.user(
ctx.author, ctx.channel,
f"Rejecting e-mail: {self.sanitise(email)}")
raise BadEmail(constraint=constraint)
# domain not found, fallback to basic guest role
role_id = registered.get(".")
constraint = list(constraints.values())[-1]
match = re.fullmatch(constraint, username)
# return
if match is not None:
return self.getGuild().get_role(role_id)
else:
await self.event.user(ctx.author, ctx.channel,
f"Rejecting e-mail: {self.sanitise(email)}")
raise BadEmail(constraint=constraint)
async def config(self, ctx):
"""See configuration from 'bot' section"""
embed = self.embed(ctx=ctx)
# fmt: off
# hosting
embed.add_field(name="Host machine", value=config.get("bot", "host"))
embed.add_field(name="Loader", value=config.get("bot", "loader"))
# logging
embed.add_field(name="Logging", value=config.get("bot", "logging"))
# extensions
embed.add_field(
name="Default extensions",
value=", ".join(
[x.lower() for x in sorted(config.get("bot", "extensions"))]),
inline=False,
)
embed.add_field(
name="Loaded extensions",
value=", ".join([
x.lower() for x in sorted(self.bot.cogs.keys())
if x != "Errors"
]),
inline=False,
)
# fmt: on
await ctx.send(embed=embed, delete_after=config.delay_embed)
await utils.delete(ctx)
async def _reaction_payload_to_tuple(
self, payload: discord.RawMessageUpdateEvent) -> tuple:
"""Return (channel, message, member, emoji) or None"""
# channel
channel = self.bot.get_channel(payload.channel_id)
if not isinstance(channel, discord.TextChannel):
return
# message
try:
message = await channel.fetch_message(payload.message_id)
except discord.NotFound:
return
# halt if not react-to-role message
# fmt: off
if (channel.id not in config.get("faceshifter",
"react-to-role channels")
and not message.content.startswith(
config.get("faceshifter", "react-to-role prefix"))):
return
# fmt: on
# member
member = message.guild.get_member(payload.user_id)
if member.bot:
return
# emoji
if payload.emoji.is_custom_emoji():
emoji = self.bot.get_emoji(payload.emoji.id) or payload.emoji
else:
emoji = payload.emoji.name
return channel, message, member, emoji
def put_json2(queue, messages):
try:
client = MQ_Client(config.get('rabbitmq', 'host'), queue, messages)
client.do_send()
except:
client = MQ_Client(config.get('rabbitmq', 'bak_host'), queue, messages)
client.do_send()
def main_fun():
"""
main of the function
:return:
"""
credentials = pika.PlainCredentials(
config.get('rcms_activemq', 'username'),
config.get('rcms_activemq', 'password'))
# parameters = pika.ConnectionParameters('223.202.203.52', credentials=credentials, virtual_host='cms3')
parameters = pika.ConnectionParameters(config.get('rcms_activemq', 'host'),
port=5672,
credentials=credentials,
virtual_host='cms3')
connection = pika.BlockingConnection(parameters)
channel = connection.channel()
# channel.exchange_declare(exchange="test_exchange", exchange_type="direct", passive=False, durable=True, auto_delete=False)
channel.queue_declare(queue="CMS3.channel.refresh",
durable=True,
auto_delete=False)
# channel.queue_bind(queue="rabbit.test", exchange="test_exchange", routing_key="standard_key")
channel.basic_qos(prefetch_count=1)
channel.basic_consume(on_message, 'CMS3.channel.refresh')
try:
channel.start_consuming()
except KeyboardInterrupt:
channel.stop_consuming()
def ConverstantEncrypt(uri, startTime, endTime, domains):
access_key_id = config.get("ConverstantApi", "id")
access_key_secret = config.get("ConverstantApi", "secret")
request_method = "POST"
request_timestamp = time.strftime(
"%Y%m%dT%H%M%SZ", time.localtime(time.time() + time.timezone))
nonce = randint(10000, 99999)
request_body = '{"domains":' + str(
json.dumps(domains)
) + ',"startTime":"' + startTime + '", "endTime":"' + endTime + '"}'
print request_body
signing_string = "%s\n%s\n%s\n%d\n%s\n%s" % (request_method.upper(), uri,
request_timestamp, nonce,
access_key_id, request_body)
signature = hex_hmac_sha256(access_key_secret, signing_string)
authorization_val = "HMAC-SHA256 " + access_key_id + ":" + signature
handler = urllib2.HTTPHandler()
opener = urllib2.build_opener(handler)
url = "%s%s" % ("https://cdn-api.swiftfederation.com", uri)
request = urllib2.Request(url, data=request_body)
request.add_header("Authorization", authorization_val)
request.add_header("Content-Type", "application/json; charset=utf-8")
request.add_header("x-sfd-date", request_timestamp)
request.add_header("x-sfd-nonce", nonce)
request.get_method = lambda: request_method
return opener.open(request).read()
def eval_status(self, epoch):
cfg = config.get('dataset_config')
if cfg['train_name'] == 'market1501':
return True
else:
ocfg = config.get('optm_config')
return ocfg.get('epochs') - 10 <= epoch <= ocfg.get('epochs')
async def config(self, ctx):
"""See configuration from 'bot' section"""
embed = self.embed(ctx=ctx)
# hosting
embed.add_field(
name=self.text.get("config_embed", "host"),
value=config.get("bot", "host"),
)
embed.add_field(
name=self.text.get("config_embed", "loader"),
value=config.get("bot", "loader"),
)
# logging
embed.add_field(
name=self.text.get("config_embed", "log_level"),
value=config.get("bot", "logging"),
)
# extensions
embed.add_field(
name=self.text.get("config_embed", "loaded_cogs"),
value=", ".join([x.lower() for x in sorted(self.bot.cogs.keys())]),
inline=False,
)
await ctx.send(embed=embed)
await utils.delete(ctx)
async def _role_remove(self, location: discord.abc.Messageable,
member: discord.Member, role: discord.Role):
if role < self.getLimitProgrammes(
location) and role > self.getLimitInterests(location):
# role is programme, check if user has permission
for programme_role in config.get("faceshifter", "programme roles"):
if programme_role in [r.id for r in member.roles]:
break
else:
await location.send(
text.fill("faceshifter",
"deny role",
mention=member.mention),
delete_after=config.get("delay", "user error"),
)
return
elif role < self.getLimitInterests(location):
# role is below interests limit, continue
pass
else:
# role is limit itself or something above programmes
await location.send(
text.fill("faceshifter",
"deny high role",
mention=member.mention),
delete_after=config.get("delay", "user error"),
)
return
await member.remove_roles(role)
def build_opt_and_lr(self, model):
parameters = model.parameters()
if config.get('debug'):
lr_mul = 1
else:
lr_mul = len(config.get('gpus'))
ocfg = config.get('optm_config')
if ocfg['name'] == 'SGD':
optimizer = torch.optim.SGD(parameters,
float(ocfg['lr']) * lr_mul,
momentum=ocfg['momentum'],
weight_decay=ocfg['weight_decay'])
else:
optimizer = torch.optim.Adam(parameters,
float(ocfg['lr']) * lr_mul,
weight_decay=ocfg['weight_decay'])
if 'multistep' in ocfg and ocfg['multistep']:
lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(
optimizer, ocfg['step'], gamma=ocfg['gamma'], last_epoch=-1)
else:
lr_scheduler = CosineAnnealingWarmUp(
optimizer,
T_0=5,
T_end=ocfg.get('epochs'),
warmup_factor=ocfg.get('warmup_factor'),
last_epoch=-1)
self.logger.write(optimizer)
return optimizer, lr_scheduler
def __init__(self):
super(Bert, self).__init__()
self.tokenizer = BertTokenizer.from_pretrained(os.path.join(config.get('model_config')['language_model_path'], 'bert-base-uncased-vocab.txt'))
modelConfig = BertConfig.from_pretrained(os.path.join(config.get('model_config')['language_model_path'], 'bert_config.json'))
self.textExtractor = BertModel.from_pretrained(
os.path.join(config.get('model_config')['language_model_path'], 'pytorch_model.bin'), config=modelConfig)
async def power_on(self, ctx):
"""Restore"""
jail = self.getGuild().get_channel(config.get("channels", "jail"))
everyone = self.getGuild().default_role
botspam = self.getGuild().get_channel(config.get(
"channels", "botspam"))
visited = []
if jail is not None:
# remove the message
messages = await jail.history(limit=10).flatten()
for message in messages:
if message.content.startswith(
text.get("admin", "poweroff jail")):
await message.delete()
break
# switch to read-write
await jail.set_permissions(everyone,
send_messages=True,
reason="?power on")
visited.append(jail.mention)
if botspam is not None:
# send message
await botspam.send(text.get("admin", "poweron botspam"))
visited.append(botspam.mention)
# send confirmation message
if len(visited) > 0:
await ctx.send(
text.fill("admin", "poweron ok", channels=", ".join(visited)))
else:
await ctx.send(text.fill("admin", "power fail"))
await self.event.sudo(ctx.author, ctx.channel, "Power on")
def extract(self, test_data, model):
model.eval()
res = {}
for p, val_loader in test_data.items():
# if os.path.exists(os.path.join(config.get('task_id'),
# config.get('dataset_config')['name'] + '_' + p + '.mat')):
# return
with torch.no_grad():
paths = []
for i, (input, target, path) in enumerate(val_loader):
# print(input[0])
input = input.cuda(non_blocking=True)
target = target.cuda(non_blocking=True)
# compute output
outputs = model(input)
if isinstance(outputs, (list, tuple)):
if isinstance(outputs[1], (list, tuple)):
feat = normalize(
torch.cat([normalize(x) for x in outputs[1]],
dim=1))
else:
feat = normalize(outputs[1])
else:
feat = normalize(outputs)
#
if config.get('with_flip'):
input_ = input.flip(3)
outputs = model(input_)
if isinstance(outputs, (list, tuple)):
feat_ = normalize(torch.cat(outputs[1], dim=1),
axis=1)
else:
feat_ = normalize(outputs, axis=1)
feat = (feat + feat_) / 2
feat = normalize(feat)
feature = feat.cpu()
target = target.cpu()
nd_label = target.numpy()
nd_feature = feature.numpy()
if i == 0:
all_feature = nd_feature
all_label = nd_label
else:
all_feature = np.vstack((all_feature, nd_feature))
all_label = np.concatenate((all_label, nd_label))
paths.extend(path)
all_label.shape = (all_label.size, 1)
print(all_feature.shape, all_label.shape)
if 'test_name' in config.get('dataset_config'):
test_name = config.get('dataset_config')['test_name']
else:
test_name = config.get('dataset_config')['target_name']
self.save_feature(p, test_name, all_feature, all_label, paths)
res[p] = (all_feature, all_label)
return res
def func_attention_MxN(local_img_query, txt_i_key_expand, txt_i_value_expand, eps=1e-8):
"""
query: (batch, queryL, d)
context: (batch, sourceL, d)
"""
batch_size, queryL, sourceL = txt_i_key_expand.size(
0), local_img_query.size(1), txt_i_key_expand.size(1)
local_img_query_norm = l2norm(local_img_query, dim=-1)
txt_i_key_expand_norm = l2norm(txt_i_key_expand, dim=-1)
# Step 1: preassign attention
# --> (batch, d, queryL)
local_img_queryT = torch.transpose(local_img_query_norm, 1, 2)
# (batch, sourceL, d)(batch, d, queryL)
attn = torch.bmm(txt_i_key_expand_norm, local_img_queryT)
attn = nn.LeakyReLU(0.1)(attn)
attn = l2norm(attn, 2)
# --> (batch, queryL, sourceL)
attn = torch.transpose(attn, 1, 2).contiguous()
# --> (batch*queryL, sourceL)
attn = attn.view(batch_size * queryL, sourceL)
attn = nn.Softmax(dim=1)(attn * config.get('model_config')['lambda_softmax'])
# --> (batch, queryL, sourceL)
attn = attn.view(batch_size, queryL, sourceL)
# print('attn: ', attn)
# Step 2: identify irrelevant fragments
# Learning an indicator function H, one for relevant, zero for irrelevant
if config.get('model_config')['focal_type'] == 'equal':
funcH = focal_equal(attn, batch_size, queryL, sourceL)
elif config.get('model_config')['focal_type'] == 'prob':
funcH = focal_prob(attn, batch_size, queryL, sourceL)
else:
funcH = None
# Step 3: reassign attention
if funcH is not None:
tmp_attn = funcH * attn
attn_sum = torch.sum(tmp_attn, dim=-1, keepdim=True)
attn = tmp_attn / attn_sum
# --> (batch, d, sourceL)
txt_i_valueT = torch.transpose(txt_i_value_expand, 1, 2)
# --> (batch, sourceL, queryL)
attnT = torch.transpose(attn, 1, 2).contiguous()
# (batch x d x sourceL)(batch x sourceL x queryL)
# --> (batch, d, queryL)
weightedContext = torch.bmm(txt_i_valueT, attnT)
# --> (batch, queryL, d)
weightedContext = torch.transpose(weightedContext, 1, 2)
#return weightedContext, attn
return weightedContext
def assemble_node_info_new():
"""
{"status":0,"msg":"Request success","data":{"total":1357,"offset":0,"count":1,
"list":[{"id":10248614,"id_class":"PreservedNode","description":"CHN-GL-a","region":"大陆",
"description_cn":"桂林CHNa","node_code":"010773a","node_status":"Enable 正常运行","business_type":null,
"address":"桂林七星区环城北二路东城小区电信大楼5楼机房","bw_limit":80.0,"department":null,"city":"桂林",
"city_id":903410,"bw_agreement":80.0,"toptier_nodes":[],
"location":{
"city":{"id":903410,"description":"桂林","code":"GL","code_number":"0773",
"notes":"guilin","parent_id":903202},
"province":{"id":903202,"description":"广西",
"code":"GX","code_number":"530000","notes":"GuangXi Zone","parent_id":903145},
"region":{"id":903145,"description":"广州大区","code":"GZ","code_number":"7",
"notes":"GuangZhou Region","parent_id":903128},
"country":{"id":903128,"description":"中国","code":"CN",
"code_number":"86","notes":"China","parent_id":null}},"servers":null,
"switchs":null,
"ipsubnets":null,
"isp":{"description":"CHN","description_cn":"中国电信","isp_code":"01"}}]}}
according rcms, get device node info, re assemble info
:return: {'CHN-BJ-3':{ "cityCode": "0010", "cityName": "BJ", "countryCode": "0086", "countryName": "CN",
"ispCode": "01", "ispName": "CHN", "nodeCode": "0100103", "nodeName": "CHN-BJ-3", "nodeNameZh": "北京CHN3",
"nodeStatus": "OPEN", "provinceCode": "100000", "provinceName": "BJ", "regionCode": "3", "regionName": "BJ" },
'CHN-HZ-1': { "cityCode": "0571", "cityName": "HZ", "countryCode": "0086", "countryName": "CN", "ispCode": "01",
"ispName": "CHN", "nodeCode": "0105711", "nodeName": "CHN-HZ-1", "nodeNameZh": "杭州CHN", "nodeStatus": "SUSPEND",
"provinceCode": "310000", "provinceName": "ZJ", "regionCode": "8", "regionName": "SH" }}
http://wiki.dev.chinacache.com/pages/viewpage.action?pageId=29555120#id-节点信息接口-获取二级节点信息接口
"""
try:
# RMS_NODE_INFO = "https://ris.chinacache.com/v3/resource/node/businessnodes"
if sys.version_info >= (2, 7, 9):
ssl._create_default_https_context = ssl._create_unverified_context
access_id = config.get('rms', 'access_id')
timestamp = int(time.time())
# print "timestamp:%s" % timestamp
# private_key = "4845bdc3e96eade6319fde7582ebe742"
private_key = config.get('rms', 'private_key')
token_str = access_id + private_key + str(timestamp)
logger.debug("token_str:%s" % token_str)
token = get_md5(token_str)
# limit = 0 get all info
rms_node_info_send = RMS_NODE_INFO + '?access_id=' + access_id + '×tamp=' + str(
timestamp) + '&limit=0' '&token=' + token + '&field=location,isp'
logger.debug("rms_node_info:%s" % rms_node_info_send)
# context = ssl._create_unverified_context()
# req = urllib2.Request(rms_node_info)
# print "req:%s" % req
# ssl._create_default_https_context = ssl._create_unverified_context
# res_data = urllib2.urlopen(rms_node_info_send, context=context)
res_data = urllib2.urlopen(rms_node_info_send)
res = res_data.read()
res = json.loads(res)
return assemble_original_info(res)
except Exception, e:
logger.debug("assemble_node_info_new error:%s" %
traceback.format_exc(e))
return {}
def __init__(self):
super(NAFS, self).__init__()
mconfig = config.get('model_config')
oconfig = config.get('optm_config')
self.CMPM = oconfig['CMPM']
self.CMPC = oconfig['CMPC']
self.CONT = oconfig['CONT']
self.epsilon = oconfig['epsilon']
self.num_classes = mconfig['num_classes']
self.W = Parameter(torch.randn(mconfig['feature_size'], mconfig['num_classes']))
nn.init.xavier_uniform_(self.W.data, gain=1)
self.part2 = mconfig['part2']
self.part3 = mconfig['part3']
self.image_model = SfeNet()
self.language_model = Bert()
inp_size = 2048
# shorten the tensor using 1*1 conv
self.conv_images = nn.Conv2d(inp_size, mconfig['feature_size'], 1)
self.conv_text = nn.Conv2d(768, mconfig['feature_size'], 1)
# BN layer before embedding projection
self.bottleneck_image = nn.BatchNorm1d(mconfig['feature_size'])
self.bottleneck_image.bias.requires_grad_(False)
self.bottleneck_image.apply(weights_init_kaiming)
self.bottleneck_text = nn.BatchNorm1d(mconfig['feature_size'])
self.bottleneck_text.bias.requires_grad_(False)
self.bottleneck_text.apply(weights_init_kaiming)
self.local_fc_text_key = nn.Linear(768, mconfig['feature_size'])
self.local_bottleneck_text_key = nn.LayerNorm([98 + 2 + 1, mconfig['feature_size']])
self.local_fc_text_value = nn.Linear(768, mconfig['feature_size'])
self.local_bottleneck_text_value = nn.LayerNorm([98 + 2 + 1, mconfig['feature_size']])
self.global_image_query = nn.Linear(mconfig['feature_size'], mconfig['feature_size'])
self.global_image_value = nn.Linear(mconfig['feature_size'], mconfig['feature_size'])
self.fc_p2_list = nn.ModuleList([nn.Linear(inp_size, mconfig['feature_size']) for i in range(self.part2)])
self.fc_p3_list = nn.ModuleList([nn.Linear(inp_size, mconfig['feature_size']) for i in range(self.part3)])
self.fc_p2_list_query = nn.ModuleList([nn.Linear(mconfig['feature_size'], mconfig['feature_size']) for i in range(self.part2)])
self.fc_p2_list_value = nn.ModuleList([nn.Linear(mconfig['feature_size'], mconfig['feature_size']) for i in range(self.part2)])
self.fc_p3_list_query = nn.ModuleList([nn.Linear(mconfig['feature_size'], mconfig['feature_size']) for i in range(self.part3)])
self.fc_p3_list_value = nn.ModuleList([nn.Linear(mconfig['feature_size'], mconfig['feature_size']) for i in range(self.part3)])
self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
async def submit(self, ctx, code: str):
"""Submit verification code"""
await utils.delete(ctx)
db_user = repo_u.get(ctx.author.id)
if db_user is None or db_user.status in (
"unknown", "unverified") or db_user.code is None:
return await ctx.send(self.text.get("no_code"), delete_after=120)
if db_user.status != "pending":
raise ProblematicVerification(status=db_user.status,
login=db_user.login)
# repair the code
code = code.replace("I", "1").replace("O", "0").upper()
if code != db_user.code:
await ctx.send(self.text.get("WrongVerificationCode",
mention=ctx.author.mention),
delete_after=120)
await self.event.user(
ctx, f"Rejecting code `{code}` (has `{db_user.code}`).")
return
# user is verified now
repo_u.save_verified(ctx.author.id)
# add role
await self._add_verify_roles(ctx.author, db_user)
# send messages
try:
for role_id in config.get("roles", "native"):
if role_id in [x.id for x in ctx.author.roles]:
await ctx.author.send(
self.text.get("verification DM native"))
break
else:
await ctx.author.send(self.text.get("verification DM guest"))
except discord.Forbidden:
await self.event.user(
ctx, "DMs are forbidden, no welcome message sent.")
# announce the verification
await ctx.channel.send(
self.text.get(
"verification public",
mention=ctx.author.mention,
role=db_user.group,
),
delete_after=config.get("delay", "verify"),
)
await self.event.user(
ctx,
f"User {ctx.author.id} verified with group **{db_user.group}**.",
)
def proc(self, image_buffer, is_aug=True):
im = tf.image.decode_jpeg(image_buffer, channels=3, dct_method='INTEGER_ACCURATE')
if config.get('preprocess_config'):
pre_process_im = PreProcessIm(**config.get('preprocess_config'))
else:
pre_process_im = PreProcessIm()
image=pre_process_im(im)
return image
def getMDB(db):
'''
monitor redis
'''
pool = redis.ConnectionPool(host=config.get('monitor', 'cache_host'),
port=6379,
db=db,
password=config.get('monitor',
'cache_password'))
return redis.Redis(connection_pool=pool)
def getDB(db):
pool = redis.ConnectionPool(host=config.get('redis', 'host'),
port=6379,
db=db,
password=config.get('redis', 'password'))
r = RedisWrapper(connection_pool=pool)
r.set_bredis(config.get('redis', 'host_bak'),
db,
password=config.get('redis', 'password'))
return r
async def setVoiceName(self, channel: discord.VoiceChannel):
"""Set voice channel name"""
if len(channel.members) == 0:
await channel.edit(name="Empty")
return
adjs = config.get("voice cog", "adjectives")
nouns = config.get("voice cog", "nouns")
name = "{} {}".format(random.choice(adjs), random.choice(nouns))
await channel.edit(name=name)
def get_learning_rate(self, global_step):
""""""
step_per_epoch = max(self.batches_per_epoch.values())
lr_config = config.get('learning_rate')
lr_base = lr_config['lr_base']
lr_policy = lr_config.get('policy', 'multi_step')
print('task.batches_per_epoch: ', step_per_epoch, ' lr policy: ',
lr_policy)
warmup = lr_config.get('warmup', False)
warmup_step = lr_config.get('warmup_step', None)
warmup_epoch = lr_config.get('warmup_epoch', None)
if warmup:
assert (warmup_step or warmup_epoch)
if warmup_step and warmup_epoch:
raise ValueError(
'warmup_step and warmup_epoch only must be one')
if not warmup_step and warmup_epoch:
warmup_step = warmup_epoch * step_per_epoch
print('warmup_step: ', warmup_step)
if lr_policy == 'multi_step':
boundaries_epoch = lr_config['boundaries']
boundaries_step = [
int(x * step_per_epoch) for x in boundaries_epoch
]
lr_decay = lr_config.get('lr_decay', 0.1)
if warmup:
boundaries_step = [warmup_step] + boundaries_step
rates = [
lr_base * (lr_decay**i)
for i in range(len(boundaries_step))
]
rates = [0.] + rates
else:
rates = [
lr_base * (lr_decay**i)
for i in range(len(boundaries_step) + 1)
]
self.lr = multi_stepping(global_step,
boundaries_step,
rates,
warmup=warmup)
elif lr_policy == 'cosine':
if not warmup:
warmup_step = 0
total_steps = config.get('total_epcohs') * step_per_epoch
self.lr = cosine_decay_with_warmup(
global_step,
learning_rate_base=lr_base,
total_steps=total_steps,
warmup_learning_rate=0,
warmup_steps=warmup_step,
)
return self.lr
def init_seed(self):
random.seed(config.get('seed'))
np.random.seed(config.get('seed'))
torch.manual_seed(config.get('seed'))
torch.cuda.manual_seed_all(config.get('seed'))
cudnn.deterministic = True
warnings.warn('You have chosen to seed training. '
'This will turn on the CUDNN deterministic setting, '
'which can slow down your training considerably! '
'You may see unexpected behavior when restarting '
'from checkpoints.')
async def on_message(self, message):
# fmt: off
if message.channel.id in config.get("faceshifter", "react-to-role channels") \
or message.content.startswith(config.get("faceshifter", "react-to-role prefix")):
emote_channel_list = await self._message_to_tuple_list(message)
for emote_channel in emote_channel_list:
try:
await message.add_reaction(emote_channel[0])
except (discord.errors.Forbidden,
discord.errors.HTTPException):
continue
def evalution(self, model, test_loader):
ckpt = os.path.join(config.get('task_id'), 'best_model.pth')
checkpoint = torch.load(ckpt)
model.load_state_dict(checkpoint['state_dict'], strict=True)
self.logger.write(model)
self.logger.write("=> loading checkpoint '{}'".format(ckpt))
if config.get('convert_to_onnx'):
self.convert_to_onnx(model, test_loader)
# torch.onnx.export(model, )
else:
self.extract_and_eval(test_loader, model)
def save_feature(self, part, data, features, labels, paths):
if not os.path.exists(config.get('task_id')):
os.makedirs(config.get('task_id'))
self.logger.write(
'save at %s' %
os.path.join(config.get('task_id'), data + '_' + part + '.mat'))
sio.savemat(
os.path.join(config.get('task_id'), data + '_' + part + '.mat'), {
'feature': features,
'label': labels,
'path': paths
})
def getBR(db):
pool = redis.ConnectionPool(host=config.get('redis', 'host_bermuda'),
port=6379,
db=db,
password=config.get('redis',
'password_bermuda'),
decode_responses=True) # 10.20.56.91
br = RedisWrapper(connection_pool=pool)
br.set_bredis(config.get('redis', 'host_bermuda_bak'),
db,
password=config.get('redis',
'password_bermuda')) # 10.20.56.87
return br
async def antispam(ctx: commands.Context):
"""Send botroom redirect if needed"""
if not isinstance(ctx.channel, discord.TextChannel):
return
if ctx.channel.id in config.get("channels", "bot allowed"):
return
await ctx.send(
text.fill("server", "botroom redirect"),
user=ctx.author,
channel=config.get("channels", "botspam"),
)
def main():
setup_path()
# Parse command line arguments
args_parser = argparse.ArgumentParser()
args_parser.add_argument(
"--debug",
help="Increase output verbosity",
action="store_true")
args_parser.add_argument("--tweak",
help="Set time and date to tweaked value")
args_parser.add_argument("--chords",
help="Directory to look for scripts in")
args_parser.add_argument( "--history",
help="List previous chord executions" )
args = args_parser.parse_args()
# Set up logging
import core.log as log
logging = log.getLogger(args.debug)
logging.debug('Using Python %s', platform.python_version())
# Load configuration
from core.config import config as conf
# Set up database
db = setup_db()
# Where are the chords kept?
if args.chords:
if os.path.isdir(args.chords):
chordsDirectory = args.chords
else:
try:
chordsDirectoryFromConf = conf.get('ocarina', 'chords')
if os.path.isdir(chordsDirectoryFromConf):
chordsDirectory = chordsDirectoryFromConf
except:
chordsDirectory = os.path.abspath(
os.path.join(currentDirectory, '../chords'))
sys.path.insert(0, chordsDirectory)
# Where are the virtual environments kept?
try:
virtualEnvDirectoryFromConf = conf.get('ocarina', 'virtualenv')
if os.path.isdir(virtualEnvDirectoryFromConf):
virtualEnvDirectory = virtualEnvDirectoryFromConf
except:
virtualEnvDirectory = os.path.abspath(
os.path.join(currentDirectory, '../.virtualenv'))
shouldRunChords = not args.history
if shouldRunChords:
import core.chords as chords
from core.now import Now
now = Now(args.tweak)
chords.run(chordsDirectory, virtualEnvDirectory, now, logging)
if args.history:
from core.history import History
history = History( db )
history.listPrevious( args.history )
