def listen(self):
while True:
try:
read_sockets, write_sockets, error_sockets = select.select(
self.LIST, [], [])
for sock in read_sockets:
if sock == self.gameSocket:
sockfd, address = self.gameSocket.accept()
self.LIST.append(sockfd)
Logger.log(
f'Client [{address[0]}:{address[1]}] connected to the server.',
'CONNECT')
else:
try:
receivedData = sock.recv(40, ).decode()
except:
try:
try:
disconnected_user = self.connectedUsers[
address]
del self.connectedUsers[address]
except:
disconnected_user = '******'
Logger.log(
f'Client [{address[0]}:{address[1]}] ({disconnected_user}) disconnected from the server.',
'DISCONNECT')
except Exception as error:
Logger.error(error)
sock.close()
self.LIST.remove(sock)
continue
if receivedData:
arguments = receivedData.split(';')
if arguments[0] != str(self.randomID):
if arguments[1] == str(
self.gameInstance.getPage()):
self.otherPlayer.setLocation(
round(float(arguments[2]), 2),
round(float(arguments[3]), 2))
else:
self.otherPlayer.hide()
except Exception as error:
Logger.error(error)
def __init__(self, session, js_url, login2ctrip):
super(CtripClickBooking, self).__init__()
self.logger = Logger.getlogger()
self.tasklist = 'PromoteVerify'
self.proxy = {
'url': 'http://proxy.my.com/proxy',
'params': {
"method": "c_hotel",
"group": "cadsl",
"mode": 1,
"url": 1,
"level": "HTTPS",
"expire": -1
}
}
self.js_url = js_url
self.session = session
self.login2ctrip = login2ctrip
self.url_inputneworder = 'http://hotels.ctrip.com/internationalbook/inputneworder.aspx?ctm_ref=hi_0_0_0_0_lst_sr_1_df_ls_1_n_hi_bk_1_0'
class TestGovernLogin(unittest.TestCase):
warnings.simplefilter("ignore", ResourceWarning)
log = Logger()
read_yaml = ReadYaml()
govern_host = read_yaml.get_default_value("govern", "host")
user_name = read_yaml.get_default_value("govern", "account")
pass_word = read_yaml.get_default_value("govern", "password")
driver = webdriver.Chrome()
driver.maximize_window()
govern_login = GovernLogin(driver=driver)
govern_yang_lao = GovernYangLao(driver=driver)
@classmethod
def setUpClass(cls):
cls.log.info(
"---------------------------setUpClass--STA-----------------------------"
)
cls.govern_login = GovernLogin(driver=cls.driver)
cls.driver.get(cls.govern_host + "/Gover")
cls.govern_login.login_info(username=cls.user_name,
password=cls.pass_word)
def test_01_enter_person(self):
self.log.info("--------enter_person-STA-------")
self.govern_yang_lao.person_click()
self.assertEqual("人员信息", self.driver.title)
self.log.info("--------enter_person-END-------")
def test_02_add_person(self):
self.log.info("--------add_person-STA-------")
self.govern_yang_lao.add_person_click()
self.log.info("--------add_person-END-------")
@classmethod
def tearDownClass(cls):
# self.driver.quit()
cls.log.info(
"---------------------------tearDownClass--END-----------------------------"
)
def process(cf):
# Enable log file
sys.stdout = Logger(cf.log_file)
print(' ---> Init experiment: ' + cf.exp_name + ' <---')
# Create the data generators
train_gen, valid_gen, test_gen = Dataset_Generators().make(cf)
# Create the optimizer
print('\n > Creating optimizer...')
optimizer = Optimizer_Factory().make(cf)
# Build model
print('\n > Building model...')
model = Model_Factory().make(cf, optimizer)
# Create the callbacks
print('\n > Creating callbacks...')
cb = Callbacks_Factory().make(cf, valid_gen)
if cf.train_model:
# Train the model
startt = time.time()
model.train(train_gen, valid_gen, cb)
print(' Training time: {}. seconds'.format(time.time() - startt))
if cf.test_model:
# Compute validation metrics
if (valid_gen != None): model.test(valid_gen)
# Compute test metrics
model.test(test_gen)
if cf.pred_model:
# Compute validation metrics
if (valid_gen != None): model.predict(valid_gen, tag='pred')
# Compute test metrics
model.predict(test_gen, tag='pred')
# Finish
print(' ---> Finish experiment: ' + cf.exp_name + ' <---')
def main():
logger = Logger.get_instance(ConfManage.getString("LOG_CRON_NAME"))
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--date', help='日期', type=str)
parser.add_argument('-p', '--pickle', type=str, help='数据集', default='data')
parser.add_argument('estimator', help='算法选择', nargs='?', type=str, default='xgb')
parser.add_argument('predict_target', help='目标值', nargs='?', type=str, default='accept')
parser.add_argument('-f', '--feature-selected', help='特征值选择', action='store_true')
parser.add_argument('-w', '--withhold', help='是否保存数据到bi数据库', action='store_true')
parser.add_argument("-s", "--shift_days", help="The last few days", type=int, default=-1)
args = parser.parse_args()
logger.info('Arguments: estimator=%s, predict-target=%s, feature-selected=%r, withhold-bi-insertion=%r' % \
(args.estimator, args.predict_target, args.feature_selected, args.withhold))
try:
process(logger, args.pickle, args.estimator, args.predict_target, args.withhold, args.date, args.shift_days)
except TestDataEmpty:
logger.error('Test Data Empty!')
except (AttributeError, ValueError) as err:
logger.error(err)
logger.error('Trace: {}'.format(traceback.format_exc()))
except KeyboardInterrupt:
logger.info('Process manually interupted at %s' % arrow.now(tz=ConfManage.getString("ARROW_TIMEZONE")).format(loggable))
logger.info('Releasing Logger...')
def process(cf):
# Enable log file
sys.stdout = Logger(cf.log_file)
print(' ---> Init experiment: ' + cf.exp_name + ' <---')
# Create the data generators
# Train set generator is created but not used
train_gen, valid_gen, test_gen = Dataset_Generators().make(cf)
# Create the optimizer
optimizer = Optimizer_Factory().make(cf)
# Build model
#Need to load different weights for each model
cf.load_pretrained = True
print('\n > Building models...')
models = Model_Factory().make(cf, optimizer)
if cf.test_model:
test_ensemble(models, valid_gen, test_gen, cf)
# Finish
print(' ---> Finish experiment: ' + cf.exp_name + ' <---')
def lobby(self, host=True):
def send(event):
if self.lobbyEntry.get() != '':
self.send('CHAT^{0} ({1}): '.format((testData['information']['username']).upper(), self.gameState) + self.lobbyEntry.get() + '\n')
self.lobbyEntry.delete(0, END)
if host:
Logger.log('Created new lobby successfully.')
self.chat('HOST: Created new lobby\n')
self.gameState = 'HOST-LOBBY'
self.gamePlayers.append(testData['information']['username'])
self.refresh()
else:
Logger.log('Joined a lobby successfully.')
self.gameState = 'JOIN-LOBBY'
Logger.log(f'Set game state to \'{self.gameState}\'.')
self.clear()
self.lobbyChat.place(relx=.052, rely=.26)
self.lobbyEntry.bind('<Return>', send)
self.lobbyEntry.place(relx=.052, rely=.73)
self.lobbyEntry.focus_force()
self.lobbyPlayers.place(relx=.61, rely=.25)
self.playerOne.place(relx=.61, rely=.35)
self.playerTwo.place(relx=.61, rely=.45)
self.playerThree.place(relx=.61, rely=.55)
self.playerFour.place(relx=.61, rely=.65)
self.readyOne.place(relx=.92, rely=.35)
self.readyTwo.place(relx=.92, rely=.45)
self.readyThree.place(relx=.92, rely=.55)
self.readyFour.place(relx=.92, rely=.65)
self.lobbyReady.place(relx=.6075, rely=.75)
self.lobbyLeave.place(relx=.8, rely=.75)
def install(self, command: str):
if self.os_name == 'Windows':
Logger.warn('暂不支持此系统')
elif self.os_name == "Linux":
out = subprocess.check_output("cat /etc/os-release", shell=True)
out = out.decode("utf-8").split('\n')
# 判断系统为ubuntu1804
if "VERSION_ID=\"18.04\"" in out and "NAME=\"Ubuntu\"" in out:
if command == "更新Python3":
UbuntuService().upgrade_python3()
elif command == "更新软件包列表":
UbuntuService().update_source_list()
elif command == "更新系统软件包":
UbuntuService().upgrade_software()
elif command == "更换系统apt源":
UbuntuService().change_apt_source()
elif command == "设置时区":
UbuntuService().set_timezone()
elif command == "安装JDK8":
UbuntuService().install_java()
elif command == "安装emqx":
UbuntuService().install_emqx()
elif command == "安装mosquitto":
UbuntuService().install_mosquitto()
elif command == "安装ssh":
UbuntuService().install_ssh()
elif command == "安装docker":
UbuntuService().install_docker()
elif command == "安装redis":
UbuntuService().install_redis()
elif command == "安装nginx":
UbuntuService().install_nginx()
# TODO: 其他系统
else:
Logger.warn('暂不支持当前系统')
else:
Logger.warn('暂不支持当前系统')
challengedSocket.send(str.encode('CONNECT4<>CHALLENGED<>' + name))
else:
sender.send(b'GAME_ERROR<>MEMBER_NOT_FOUND')
elif args[1] == 'ACCEPTED':
time.sleep(0.1)
challengedSocket = self.extendedUsers[args[2]]
challengedSocket.send(str.encode('<>'.join(args)))
def broadcast(self, message):
global tempMsg
try:
for connectedSocket in self.LIST:
if connectedSocket != self.serverSocket:
connectedSocket.send(str.encode(message))
if message != tempMsg:
Logger.log(message.rstrip().lstrip(), 'BROADCAST')
tempMsg = message
except Exception as error:
Logger.error(error)
tempMsg = ''
if __name__ == '__main__':
try:
Server = GameServer()
Server.run()
except Exception as e:
Logger.error(e)
app_startup_path = os.path.dirname(os.path.realpath(__file__))
# params
MAIN_PROCESS = sys.argv[0][sys.argv[0].rfind(r'/') + 1:]
settings = {'parent_process': MAIN_PROCESS}
params = sys.argv[0:]
for p in params[1:]:
name = p.split('=')[0].strip(' ')
if name == 'num_jspool':
set_jspool(int(p.split('=')[1].strip(' ')))
elif name == 'num_proxy_ip':
set_proxy_ip(int(p.split('=')[1].strip(' ')))
elif name == 'log_path':
LOG_PATH = p.split('=')[1].strip(' ')
logger = Logger.getlogger(level=logging.DEBUG, name=LOG_NAME, path=LOG_PATH)
logger.info('<<<========================= PhantomJS Service This log starting at %s =========================>>>', time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
logger.info('Thread: %s, num_threadpool=%s', thread_name, NUM_THREADPOOL)
logger.info('Thread: %s, num_phantomjspool=%s', thread_name, get_jspool())
# logger.info('Thread: %s, num_proxy_ip=%s', thread_name, get_proxy_ip())
# start threading to update network PROXY
# thread_proxy_daemon = FixProxy()
# thread_proxy_daemon.name = 'thread_proxy_daemon'
# thread_proxy_daemon.setDaemon(True)
# thread_proxy_daemon.start()
# logger.info('Thread: %s, thread_proxy_daemon start... please waiting for few minutes...', thread_name)
# create phantomjs pool
JS_PID = PhantomJSPool.init(num=get_jspool())
def __init__(self):
self.logger = Logger("setting_manager")
self.jobs = []
self.job_id_index = 0
DATA_BEGIN_DATE = setting['DATA_BEGIN_DATE']
CL_INDEXS = setting['CL_INDEXS']
CL_TOP10_HOLDER = setting['CL_TOP10_HOLDER']
CL_TOP10_FLOADHOLDER = setting['CL_TOP10_FLOADHOLDER']
CL_PLEDGE_STAT = setting['CL_PLEDGE_STAT']
CL_REPURCHASE = setting['CL_REPURCHASE']
CL_STK_HOLDERNUMBER = setting['CL_STK_HOLDERNUMBER']
CL_STK_HOLDERTRADE = setting['CL_STK_HOLDERTRADE']
CL_STK_POOL_DAILY = setting['CL_STK_POOL_DAILY']
CL_STK_POOL_CUR = setting['CL_STK_POOL_CUR']
CL_STK_TOP_LIST = setting['CL_STK_TOP_LIST']
CL_STK_POOL_ZZ500 = setting['CL_STK_POOL_ZZ500']
CL_INDEX_ZZ500 = setting['CL_INDEX_ZZ500']
CL_SHORT_8_PCT = setting['CL_SHORT_8_PCT']
LOG = Logger().getlog()
class IndexCode:
INDEX_SH = '000001_SH'
INDEX_SZ = '399001_SZ'
INDEX_ZX = '399005_SZ'
INDEX_CY = '399006_SZ'
INDEX_ZZ = '000905_SH'
_VALUES_TO_NAMES = {
'000001_SH': "INDEX_SH",
'399001_SZ': "INDEX_SZ",
'399005_SZ': "INDEX_ZX",
'399006_SZ': "INDEX_CY",
'000905_SH': "INDEX_ZZ",
}
def __init__(self, error_messages, packager):
self.__error_messages = error_messages
self.__packager = packager
self.__logger = Logger()
def get_emq_status(self):
try:
subprocess.run("sudo emqx_ctl status", shell=True, check=True)
except subprocess.CalledProcessError:
Logger.error('查询状态失败')
class Request:
def __init__(self):
self.log = Logger()
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
warnings.simplefilter("ignore", ResourceWarning)
# 禁用安全请求警告
requests.packages.urllib3.disable_warnings(
urllib3.exceptions.InsecureRequestWarning)
def post_request_data(self, _url, _data, _headers, case_name=None):
response = requests.post(url=_url,
data=_data,
headers=_headers,
verify=False)
self.log.info("【%s - 请求地址】：%s" % (case_name, _url))
self.log.info("【%s - 请求参数】: %s" % (case_name, _data))
self.log.info("【%s - 响应码】: %d" % (case_name, response.status_code))
return response
def post_request_json(self, _url, _json, _headers, case_name=None):
response = requests.post(url=_url,
json=_json,
headers=_headers,
verify=False)
self.log.info("【%s - 请求地址】：%s" % (case_name, _url))
self.log.info("【%s - 请求参数】: %s" % (case_name, _json))
self.log.info("【%s - 响应码】: %d" % (case_name, response.status_code))
return response
def post_request_files(self, _url, _files, _headers, case_name=None):
response = requests.post(url=_url,
files=_files,
headers=_headers,
verify=False)
self.log.info("【%s - 请求地址】：%s" % (case_name, _url))
self.log.info("【%s - 请求参数】: %s" % (case_name, _files))
self.log.info("【%s - 响应码】: %d" % (case_name, response.status_code))
return response
def get_request(self, _url, _headers, _data=None, case_name=None):
response = requests.get(url=_url,
params=_data,
headers=_headers,
verify=False)
self.log.info("【%s - 请求地址】：%s" % (case_name, _url))
self.log.info("【%s - 请求参数】: %s" % (case_name, _data))
self.log.info("【%s - 响应码】: %d" % (case_name, response.status_code))
return response
os.chdir(BASE_DIR)
import signal
from multiprocessing import Process
from dataB import auto_begin as abg
from proxy.clear_proxies import clear_proxies_db
from proxy.get_proxy import GetProxies
from tools.send_mail import SendMail
from tools.auto_analyse_data import AnalyseData
from tools.common import *
from tools.auto_check_articles_list import CheckArticlesList as CAL
from tools.logger import Logger
from dataP.auto_begin import ExpertDataBegin as EB
logger = Logger(logger=__name__).getlog()
process_flag = 0
normal_main_process_time_delta = 1800
abnormal_main_process_time_delta = 300
hostname = os.popen('hostname').read()[:-1]
def get_work_times():
content = get_json_content(SETUP_FILE)
common_times = content['common_actions_time']
real_times = content['real_actions_time']
test_times = content['test_actions_time']
se_times = content['start_end_times']
actions_times = real_times if REAL == 1 else test_times
actions_times.update(se_times)
class CommandManager:
PRIVATE_CHANNEL = "priv"
ORG_CHANNEL = "org"
PRIVATE_MESSAGE = "msg"
def __init__(self):
self.handlers = collections.defaultdict(list)
self.logger = Logger("command_manager")
self.channels = {}
self.ignore_regexes = [
re.compile(" is AFK \(Away from keyboard\) since ", re.IGNORECASE),
re.compile("I am away from my keyboard right now", re.IGNORECASE),
re.compile("Unknown command or access denied!", re.IGNORECASE),
re.compile("I am responding", re.IGNORECASE),
re.compile("I only listen", re.IGNORECASE),
re.compile("Error!", re.IGNORECASE),
re.compile("Unknown command input", re.IGNORECASE),
re.compile("You have been auto invited", re.IGNORECASE),
]
def inject(self, registry):
self.db = registry.get_instance("db")
self.util: Util = registry.get_instance("util")
self.access_manager: AccessManager = registry.get_instance("access_manager")
self.bot: Mangopie = registry.get_instance("mangopie")
self.character_manager: CharacterManager = registry.get_instance("character_manager")
self.setting_manager: SettingManager = registry.get_instance("setting_manager")
self.command_alias_manager = registry.get_instance("command_alias_manager")
def pre_start(self):
self.bot.add_packet_handler(server_packets.PrivateMessage.id, self.handle_private_message)
self.bot.add_packet_handler(server_packets.PrivateChannelMessage.id, self.handle_private_channel_message)
self.register_command_channel("Private Message", self.PRIVATE_MESSAGE)
self.register_command_channel("Org Channel", self.ORG_CHANNEL)
self.register_command_channel("Private Channel", self.PRIVATE_CHANNEL)
def start(self):
# process decorators
for _, inst in Registry.get_all_instances().items():
for name, method in get_attrs(inst).items():
if hasattr(method, "command"):
cmd_name, params, access_level, description, help_file, sub_command = getattr(method, "command")
handler = getattr(inst, name)
module = self.util.get_module_name(handler)
help_text = self.get_help_file(module, help_file)
self.register(handler, cmd_name, params, access_level, description, module, help_text, sub_command)
def register(self, handler, command, params, access_level, description, module, help_text=None, sub_command=None):
command = command.lower()
if sub_command:
sub_command = sub_command.lower()
else:
sub_command = ""
access_level = access_level.lower()
module = module.lower()
command_key = self.get_command_key(command, sub_command)
if help_text is None:
help_text = self.generate_help(command, description, params)
if not self.access_manager.get_access_level_by_label(access_level):
self.logger.error("Could not add command '%s': could not find access level '%s'" % (command, access_level))
return
for channel, label in self.channels.items():
row = self.db.find('command_config', {"command": command, "sub_command": sub_command, 'channel': channel})
if row is None:
# add new command commands
self.db.insert('command_config',
{'command': command, 'sub_command': sub_command, 'access_level': access_level,
'channel': channel, 'module': module, 'verified': 1, 'enabled': 1})
elif 'verified' in row and row['verified']:
if row['module'] != module:
self.logger.warning("module different for different forms of command '%s' and sub_command '%s'" % (
command, sub_command))
else:
# mark command as verified
self.db.update('command_config',
{'module': module, 'command': command, 'channel': channel, 'access_level': access_level,
'sub_command': sub_command},
{'verified': 1})
# save reference to command handler
r = re.compile(self.get_regex_from_params(params), re.IGNORECASE)
self.handlers[command_key].append(
{"regex": r, "callback": handler, "help": help_text, "description": description, "params": params})
def handle_private_message(self, packet: server_packets.PrivateMessage):
# since the command symbol is not required for private messages,
# the command_str must have length of at least 1 in order to be valid,
# otherwise it is ignored
if len(packet.message) < 1 or not self.bot.is_ready():
return
for regex in self.ignore_regexes:
if regex.search(packet.message):
return
if packet.message[:1] == '!':
command_str = packet.message[1:]
else:
command_str = packet.message
self.process_command(
command_str,
"msg",
packet.char_id,
lambda msg: self.bot.send_private_message(packet.char_id, msg))
def handle_private_channel_message(self, packet: server_packets.PrivateChannelMessage):
# since the command symbol is required in the private channel,
# the command_str must have length of at least 2 in order to be valid,
# otherwise it is ignored
if len(packet.message) < 2:
return
symbol = packet.message[:1]
command_str = packet.message[1:]
if symbol == self.setting_manager.get(
"symbol").get_value() and packet.private_channel_id == self.bot.char_id:
self.process_command(
command_str,
"priv",
packet.char_id,
lambda msg: self.bot.send_private_channel_message(msg))
def process_command(self, message: str, channel: str, char_id, reply):
try:
command_str, command_args = self.get_command_parts(message)
# check for command alias
command_alias = self.command_alias_manager.check_for_alias(command_str)
if command_alias:
command_str, command_args = self.get_command_parts(
command_alias + " " + command_args if command_args else command_alias)
cmd_configs = self.get_command_configs(command_str, channel, 1)
cmd_configs = list(cmd_configs)
if cmd_configs:
# given a list of cmd_configs that are enabled, see if one has regex that matches incoming command_str
cmd_config, matches, handler = self.get_matches(cmd_configs, command_args)
if matches:
if self.access_manager.check_access(char_id, cmd_config['access_level']):
sender = MapObject(
{"name": self.character_manager.resolve_char_to_name(char_id), "char_id": char_id})
handler["callback"](channel, sender, reply, self.process_matches(matches, handler["params"]))
else:
self.access_denied_response(char_id, cmd_config, reply)
else:
# handlers were found, but no handler regex matched
help_text = self.get_help_text(char_id, command_str, channel)
if help_text:
reply(self.format_help_text(command_str, help_text))
else:
reply("Error! Invalid syntax.")
else:
reply("Error! Unknown command.")
except Exception as e:
self.logger.error("error processing command: %s" % message, e)
reply("There was an error processing your request.")
def get_help_text(self, char, command_str, channel):
data = self.db.find_all('command_config', {'command': command_str, 'channel': channel, 'enabled': 1})
# filter out commands that character does not have access level for
data = filter(lambda row: self.access_manager.check_access(char, row['access_level']), data)
def read_help_text(row):
command_key = self.get_command_key(row['command'], row['sub_command'])
return filter(lambda x: x is not None, map(lambda handler: handler["help"], self.handlers[command_key]))
content = "\n\n".join(flatmap(read_help_text, data))
return content if content else None
def get_help_file(self, module, help_file):
if help_file:
try:
help_file = "./" + module.replace(".", "/") + "/" + help_file
with open(help_file) as f:
return f.read().strip()
except FileNotFoundError as e:
self.logger.error("Error reading help file", e)
return None
def format_help_text(self, topic, help_text):
return ChatBlob("Help (" + topic + ")", help_text)
def generate_help(self, command, description, params):
return description + ":\n" + "<tab><symbol>" + command + " " + " ".join(map(lambda x: x.get_name(), params))
def access_denied_response(self, char_id, cmd_config, reply):
reply("Error! Access denied.")
def get_command_key(self, command, sub_command):
if sub_command:
return command + ":" + sub_command
else:
return command
def get_command_key_parts(self, command_str):
parts = command_str.split(":", 1)
if len(parts) == 2:
return parts[0], parts[1]
else:
return parts[0], ""
def get_matches(self, cmd_configs, command_args):
if command_args:
command_args = " " + command_args
for row in cmd_configs:
command_key = self.get_command_key(row['command'], row['sub_command'])
handlers = self.handlers[command_key]
for handler in handlers:
matches = handler["regex"].match(command_args)
if matches:
return row, matches, handler
return None, None, None
def process_matches(self, matches, params):
groups = list(matches.groups())
processed = []
for param in params:
processed.append(param.process_matches(groups))
return processed
def format_matches(self, command_args, matches):
# convert matches to list
m = list(matches.groups())
m.insert(0, command_args)
# strip leading spaces for each group, if they group exists
return list(map(lambda x: x[1:] if x else x, m))
def get_command_parts(self, message):
parts = message.split(" ", 1)
if len(parts) == 2:
return parts[0].lower(), parts[1]
else:
return parts[0].lower(), ""
def get_command_configs(self, command, channel=None, enabled=1, sub_command=None):
query = {"command": command}
if channel:
query['channel'] = channel
if enabled:
query['enabled'] = enabled
if sub_command:
query['sub_command'] = sub_command
return self.db.find_all('command_config', query)
def get_handlers(self, command_key):
return self.handlers.get(command_key, None)
def register_command_channel(self, label, value):
if value in self.channels:
self.logger.error("Could not register command channel '%s': command channel already registered" % value)
return
self.logger.debug("Registering command channel '%s'" % value)
self.channels[value] = label
def is_command_channel(self, channel):
return channel in self.channels
def get_regex_from_params(self, params):
# params must be wrapped with line-beginning and line-ending anchors in order to match
# when no params are specified (eg. "^$")
return "^" + "".join(map(lambda x: x.get_regex(), params)) + "$"
def main():
# Get parameters from file parser
parser = argparse.ArgumentParser(description='DeepPolyp model training')
parser.add_argument('-dataset', default='polyps', help='Dataset')
parser.add_argument('-model_name', default='fcn8', help='Model')
parser.add_argument('-model_file', default='weights.hdf5',
help='Model file')
parser.add_argument('-load_pretrained', default=False,
help='Load pretrained model from model file')
parser.add_argument('-learning_rate', default=0.0001, help='Learning Rate')
parser.add_argument('-weight_decay', default=0.,
help='regularization constant')
parser.add_argument('--num_epochs', '-ne', type=int, default=1000,
help='Optional. Int to indicate the max'
'number of epochs.')
parser.add_argument('-max_patience', type=int, default=100,
help='Max patience (early stopping)')
parser.add_argument('-batch_size', type=int, default=10, help='Batch size')
parser.add_argument('--optimizer', '-opt', default='rmsprop',
help='Optimizer')
args = parser.parse_args()
# Parameters
nClasses = 3
w_balance = None # 'median_freq_cost'
crop_size = (224, 224) # (288, 384)
# Experiment name
experiment_name = "tmp"
# Define paths according to user
usr = getuser()
if usr == "michal":
# Michal paths
savepath = '/home/michal/' + experiment_name + '/'
dataset_path = '/home/michal/polyps/polyps_split2/CVC-912/'
train_path = dataset_path + 'train/'
valid_path = dataset_path + 'valid/'
test_path = dataset_path + 'test/'
elif usr == 'vazquezd' or usr == 'romerosa':
shared_dataset_path = '/data/lisa/exp/vazquezd/datasets/polyps_split7/'
dataset_path = '/Tmp/'+usr+'/datasets/polyps_split7/'
# Copy the data to the local path if not existing
if not os.path.exists(dataset_path):
print('The local path {} does not exist. Copying '
'dataset...'.format(dataset_path))
shutil.copytree(shared_dataset_path, dataset_path)
print('Done.')
savepath = '/Tmp/'+usr+'/results/deepPolyp/fcn8/paper/'+experiment_name+'/'
final_savepath = '/data/lisatmp4/' + usr + '/results/deepPolyp/fcn8/' + experiment_name + '/'
train_path = dataset_path + 'train/'
valid_path = dataset_path + 'valid/'
test_path = dataset_path + 'test/'
elif usr == 'dvazquez':
shared_dataset_path = '/home/'+usr+'/Datasets/Polyps/'
dataset_path = '/home/'+usr+'/Datasets/Polyps/'
# Copy the data to the local path if not existing
if not os.path.exists(dataset_path):
print('The local path {} does not exist. Copying '
'dataset...'.format(dataset_path))
shutil.copytree(shared_dataset_path, dataset_path)
print('Done.')
savepath = '/home/'+usr+'/Experiments/deepPolyp/'+experiment_name+'/'
final_savepath = '/home/'+usr+'/Experiments/deepPolyp/'+experiment_name+'/'
train_path = dataset_path + 'train/'
valid_path = dataset_path + 'valid/'
test_path = dataset_path + 'test/'
else:
raise ValueError('User unknown, please add your own paths!')
# Create output folders
if not os.path.exists(savepath):
os.makedirs(savepath)
# Enable log file
sys.stdout = Logger(savepath + "logfile.log")
print (' ---> Experiment: ' + experiment_name + ' <---')
# Train the network
train(dataset=args.dataset,
model_name=args.model_name,
learning_rate=float(args.learning_rate),
weight_decay=float(args.weight_decay),
num_epochs=int(args.num_epochs),
max_patience=int(args.max_patience),
batch_size=int(args.batch_size),
optimizer=args.optimizer,
savepath=savepath,
show_model=False,
train_path=train_path, valid_path=valid_path, test_path=test_path,
crop_size=crop_size,
in_shape=(3, None, None),
n_classes=nClasses+1,
gtSet=nClasses,
void_class=[nClasses],
w_balance=w_balance,
weights_file=savepath+args.model_file if bool(args.load_pretrained) else False,
train_model=True,
plot_hist=False
)
print (' ---> Experiment: ' + experiment_name + ' <---')
print('Copying model and other training files to {}'.format(final_savepath))
start = time.time()
copy_tree(savepath, final_savepath)
open(os.path.join(final_savepath, 'lock'), 'w').close()
print ('Copy time: ' + str(time.time()-start))
class Encoder:
__DEFAULT = {
'FPS': '60/1',
'WIDTH': '1920',
'HEIGHT': '1080',
'TBN': '1/15360',
'HZ': '1/44100'
}
def __init__(self, error_messages, packager):
self.__error_messages = error_messages
self.__packager = packager
self.__logger = Logger()
def commander(self, command, args):
'''
encode (package name[0]):
encodes clips to make all clips has the same parameters options
'''
if command == 'encode':
filter = self.__filter(args, 1)
if filter:
name = args[0]
package = self.__packager.get(name)
if not package:
return True
folder = package.get_data()['clips_folder']
for subdir, dirs, files in os.walk(folder):
files.sort()
for file in files:
mp4_file = Path(folder) / file
opt = self.__check_video(mp4_file)
if opt == '':
continue
else:
self.__logger.log('Re-encoding video {}'.format(mp4_file))
cmd = 'ffmpeg -i ' + str(mp4_file) + opt + str(mp4_file).replace('.mp4', '') + 'converted.mp4'
r = subprocess.run(cmd, capture_output=True, shell=True)
os.remove(mp4_file)
self.__logger.separator()
return True
else:
return False
def __filter(self, args, ammount):
if len(args) >= ammount:
return True
else:
print(ConsoleColors.RED + 'Error: {0}'.format(self.__error_messages[0]) + ConsoleColors.RESET)
def __check_video(self, video):
cmd = 'ffprobe -v error -of json -show_entries stream=time_base,r_frame_rate,width,height ' + str(video)
r = os.popen(cmd).read()
j_streams = json.loads(r)['streams']
streams = list([{}, {}])
for j_stream in j_streams:
if 'width' in j_stream.keys():
streams[0] = j_stream
else:
streams[1] = j_stream
opt = ''
if str(streams[0]['width']) != self.__DEFAULT['WIDTH']:
opt = opt + ' -vf scale=' + self.__DEFAULT['WIDTH'] + ':' + self.__DEFAULT['HEIGHT']
opt = opt + ' -video_track_timescale ' + self.__DEFAULT['TBN'][2:]
opt = opt + ' -r ' + self.__DEFAULT['FPS'][:-2]
else:
if streams[0]['time_base'] != self.__DEFAULT['TBN']:
opt = opt + ' -video_track_timescale ' + self.__DEFAULT['TBN'][2:]
if streams[0]['r_frame_rate'] != self.__DEFAULT['FPS']:
opt = opt + ' -r ' + self.__DEFAULT['FPS'][:-2]
if streams[1]['time_base'] != self.__DEFAULT['HZ']:
opt = opt + ' -ar ' + self.__DEFAULT['HZ'][2:]
if opt != '':
opt = opt + ' '
return opt
def receive(self):
Logger.log(f'Started listening for connections at [{self.EXTERNAL_IP}:{self.PORT}]')
Logger.log(f'Accepting local connections at [127.0.0.1:{self.PORT}]')
while self.serverStatus:
try:
read_sockets, write_sockets, error_sockets = select.select(self.LIST, [], [])
for sock in read_sockets:
if sock == self.serverSocket:
sockfd, address = self.serverSocket.accept()
self.LIST.append(sockfd)
Logger.log(f'Client [{address[0]}:{address[1]}] connected to the server.', 'CONNECT')
else:
try:
receivedData = sock.recv(self.BUFFER_SIZE, )
except:
try:
try:
disconnected_user = self.connectedUsers[address]
del self.connectedUsers[address]
del self.extendedUsers[disconnected_user]
except:
disconnected_user = '******'
Logger.log(f'Client [{address[0]}:{address[1]}] ({disconnected_user}) disconnected from the server.', 'DISCONNECT')
self.broadcast(f'({disconnected_user}) left the server')
self.broadcast(f'LEFT<>{disconnected_user}')
except Exception as error:
Logger.error(error)
sock.close()
self.LIST.remove(sock)
continue
if receivedData:
arguments = receivedData.decode().split('<>')
if 'DISCONNECT' in receivedData.decode():
try:
disconnected_user = self.connectedUsers[address]
del self.connectedUsers[address]
del self.extendedUsers[disconnected_user]
except:
disconnected_user = '******'
Logger.log(f'Client [{address[0]}:{address[1]}] ({disconnected_user}) disconnected from the server.', 'DISCONNECT')
Logger.log(f'Received quit command from client [{address[0]}:{address[1]}]')
self.broadcast(f'({disconnected_user}) left the server')
self.broadcast(f'LEFT<>{disconnected_user}')
sock.close()
self.LIST.remove(sock)
continue
elif arguments[0] == 'USERNAME':
self.connectedUsers[address] = arguments[1]
self.extendedUsers[arguments[1]] = sock
Logger.log(f'Allowed connection from [{address[0]}:{address[1]}] ({arguments[1]})', 'CONNECT')
elif arguments[0] == 'CONNECT4':
self.handle_game_commands(arguments, sock, self.connectedUsers[address])
elif arguments[0] == 'ONLINE':
userList = ''
for user in self.connectedUsers:
userList = userList + self.connectedUsers[user] + ';'
sock.send(str.encode('USER_LIST<>' + userList))
Logger.log(f'[{address[0]}:{address[1]}] ({self.connectedUsers[address]}) requested user list.')
Logger.log(f'Current connected users: {userList.replace(";", " ")}')
elif arguments[0] == 'CLIENT_INFORMATION':
Logger.log(f'Received client information from [{address[0]}:{address[1]}]')
clientInformation = ast.literal_eval(arguments[1])
clientData = []
for field in ['Username', 'Version', 'Rank']:
clientData.append(str([clientInformation['Client Information'][field]][0]))
if clientData[0] == '' or clientData[0] == ' ':
sock.send(b'CONN_ERROR<>Invalid username (username not allowed)')
Logger.log(f'Rejected connection from [{address[0]}:{address[1]}] due to invalid username.')
sock.close()
self.LIST.remove(sock)
else:
userListByName = []
for user in self.connectedUsers:
userListByName.append(self.connectedUsers[user])
if clientData[0] in userListByName:
sock.send(b'CONN_ERROR<>A user with that name is already connected (use a different username)')
Logger.log(f'Rejected connection from [{address[0]}:{address[1]}] ({clientData[0]}) due to duplicate username.', 'DISCONNECT')
sock.close()
self.LIST.remove(sock)
else:
if float(clientData[1]) < self.MIN_VERSION:
sock.send(str.encode(f'CONN_ERROR<>Client is out of date (latest version is {str(self.MIN_VERSION)})'))
Logger.log(f'Rejected connection from [{address[0]}:{address[1]}] ({clientData[0]}) due to outdated client [{clientData[1]}]', 'DISCONNECT')
sock.close()
self.LIST.remove(sock)
else:
self.connectedUsers[address] = clientData[0]
self.extendedUsers[clientData[0]] = sock
sock.send(b'CONN_SUCCESS<>Successfully connected to the server.')
Logger.log(f'Allowed connection from [{address[0]}:{address[1]}] ({clientData[0]}) [{clientData[1]}]', 'CONNECT')
time.sleep(0.10)
start_dt = datetime.strptime(self.serverInformation['Server Information']['Uptime'], '%H:%M:%S')
end_dt = datetime.strptime(datetime.now().strftime('%H:%M:%S'), '%H:%M:%S')
diff = (end_dt - start_dt)
serverInformationTemp = self.serverInformation
serverInformationTemp['Server Information']['Uptime'] = str(diff)
time.sleep(0.10)
sock.send(str.encode(f'SERVER_INFORMATION<>{str(serverInformationTemp)}'))
Logger.log(f'Sent server information to client [{address[0]}:{address[1]}] ({clientData[0]})')
self.serverInformation['Server Information']['Uptime'] = self.LAUNCH_TIME
else:
self.broadcast(receivedData.decode())
except Exception as error:
Logger.error(error)
from src.dax31.ui.testconfig.ui_config import *
from tools.common import *
from tools.logger import Logger
from tools.util import *
# register the logging configuration
test_type_in_module = "ui"
_ui_current_directory = dirname(realpath(__file__))
logging_file_name = abspath(
join(_ui_current_directory, 'log', test_type_in_module, __name__))
logger = Logger(log_name=logging_file_name + '.log',
log_level='1',
logger_name=test_type_in_module).get_log
# test_method_list = list()
test_method_list = generate_valid_test_method_from_apk(
_ui_current_directory,
ui_apk_folder_name,
ui_test_apk_file_name,
_logger_name=test_type_in_module)
# @pytest.mark.parametrize('test_method', test_method_list)
# def test_ui(test_method):
def test_ui_by_method_filter():
global test_method_list
__ui_setup()
_test_class_list = test_method_list
_xml_file_list = list()
def start_emqx(self):
try:
subprocess.run("sudo emqx start", shell=True, check=True)
except subprocess.CalledProcessError:
Logger.error('启动失败')
import json
import time
import requests
from threading import Thread
from settings import lotteries_predict_data_db as lpdb, STAGE_COUNT, avoid_experts_db, PER_EXPERT_ARTICLES_LIST_MAX_PAGE
from settings import EXPERT_URL
from tools.common import get_the_next_stage
from tools.save_data import SaveLotteriesData as SLD
from tools.ua import ua
from settings import saved_db
from settings import LOTTERY_DICT, miss_urls_db
from tools.set_proxies import SetProxies as SP
from tools.logger import Logger
logger = Logger(__name__).logger
class GetExpertsUrls(Thread):
def __init__(self, lottery_name, expert_id, data_type, has_missed_list_urls=0):
super(GetExpertsUrls, self).__init__()
self.lottery_name = lottery_name
self.expert_id = expert_id
self.data_type = data_type
self.max_page = PER_EXPERT_ARTICLES_LIST_MAX_PAGE # 获取文章列表时，page取范围
self.url = EXPERT_URL
self.page = None
self.params = None
self.lottery_id = None
self.urls_db = lpdb[LOTTERY_DICT[self.lottery_name] + '_articles_list'] #
class Downloader:
def __init__(self, error_messages, packager, p_tokens_file):
self.__error_messages = error_messages
self.__packager = packager
self.__p_tokens_file = p_tokens_file
self.__logger = Logger()
def commander(self, command, args):
'''
download (package name[0]):
downloads clips from given package
'''
if command == 'download':
filter = self.__filter(args, 1)
if filter:
name = args[0]
package = self.__packager.get(name)
for streamer in package.get_data()['streamers']:
self.__download(streamer, package)
return True
return False
def __filter(self, args, ammount):
if len(args) >= ammount:
return True
else:
print(ConsoleColors.RED +
'Error: {0}'.format(self.__error_messages[0]) +
ConsoleColors.RESET)
def __check_paths(self, path):
if not os.path.exists(self.__p_packages_file):
os.mkdir(path)
def __download(self, streamer, package):
with open(self.__p_tokens_file, 'r') as f:
client_id = json.load(f)['twitch']['client_id']
access_token, broad_id = Downloader.check_username(streamer).split("&")
if package.get_data()['period'] == 'week':
start = generate(datetime.utcnow().replace(tzinfo=pytz.utc) -
timedelta(days=7))
end = generate(datetime.utcnow().replace(tzinfo=pytz.utc))
elif package.get_data()['period'] == 'month':
start = generate(datetime.utcnow().replace(tzinfo=pytz.utc) -
timedelta(days=30))
end = generate(datetime.utcnow().replace(tzinfo=pytz.utc))
count = 0
pagination = ''
times = list()
game_ids = dict()
while count < int(package.get_data()['limit']):
r = requests.get(
'https://api.twitch.tv/helix/clips?broadcaster_id={}&first={}&started_at={}&ended_at={}&after={}'
.format(broad_id,
package.get_data()['limit'], start, end, pagination),
headers={
'Authorization': 'Bearer ' + access_token,
'Client-ID': client_id
})
clips = r.json()['data']
if len(clips) > 0:
for clip in clips:
if type(clip) == dict:
if clip['game_id'] not in game_ids:
game_ids[clip['game_id']] = 1
else:
game_ids[clip['game_id']] += 1
thumbnail_url = clip['thumbnail_url']
mp4_url = thumbnail_url.split('-preview',
1)[0] + '.mp4'
number = count + 1
if len(str(number)) == 1:
number = '0' + str(number)
mp4_name = str(
Path(package.get_data()['clips_folder']) /
str(str(number) + '.mp4'))
self.__logger.log("Downloading: " + str(mp4_name))
res = requests.get(mp4_url)
with open(mp4_name, 'wb') as f:
f.write(res.content)
cmd = 'ffprobe -show_entries format=duration -v quiet -of csv="p=0" ' + mp4_name
duration = float(os.popen(cmd).read())
created_at = datetime.strptime(clip['created_at'],
'%Y-%m-%dT%H:%M:%SZ')
ended_at = created_at + timedelta(0, duration)
exist = False
if len(times) > 0:
for i in range(len(times)):
if times[i]['created_at'] < created_at < times[
i]['ended_at'] or times[i][
'created_at'] < ended_at < times[
i]['ended_at']:
exist = True
break
else:
time_meta = {
'created_at': created_at,
'ended_at': ended_at
}
times.append(time_meta)
else:
time_meta = {
'created_at': created_at,
'ended_at': ended_at
}
times.append(time_meta)
if exist:
os.remove(mp4_name)
else:
count += 1
if count >= int(package.get_data()['limit']):
break
pagination = r.json()['pagination']['cursor']
else:
raise Exception('Clips not found')
games_sorted = sorted(game_ids.items(),
key=lambda x: x[1],
reverse=True)
stop = 1
if len(games_sorted) > 1:
stop = 2
games = list()
for i in range(stop):
r = requests.get('https://api.twitch.tv/helix/games?id=' +
games_sorted[i][0],
headers={
'Authorization': 'Bearer ' + access_token,
'Client-ID': client_id
})
games.append(r.json()['data'][0]['name'])
package.get_data()['additional_info']['games'] = games
package.update()
self.__logger.separator()
@staticmethod
def check_username(username):
with open(Path('tokens/tokens.json'), 'r') as f:
credentials = json.load(f)['twitch']
try:
r = requests.post(
'https://id.twitch.tv/oauth2/token?client_id={0}&client_secret={1}&grant_type=client_credentials'
.format(credentials['client_id'],
credentials['client_secret']))
access_token = r.json()['access_token']
r = requests.get('https://api.twitch.tv/helix/users?login='******'Authorization': 'Bearer ' + access_token,
'Client-ID': credentials['client_id']
})
broad_id = r.json()['data'][0]['id']
if not broad_id:
print('Bad request, check username or user banned')
return False
except:
print('Failed to connect to Twitch API')
return False
return access_token + '&' + broad_id
def emqx_config_explain(self):
Logger.info("配置路径:")
Logger.info("/etc/emqx")
Logger.info("取消匿名访问模式:")
Logger.info(
"使用sudo nano /etc/emqx/emqx.conf 编辑配置文件,将allow_anonymous设置为false, ctrl o保存"
)
Logger.info("鉴权设置:")
Logger.info(
"使用此命令编辑sudo nano /etc/emqx/etc/plugins/emqx_auth_username.conf, "
"注释掉现有内容, 打开auth.user.1的用户名和密码")
Logger.warn("在启动鉴权时,请先在dashboard中启动鉴权插件")
def __init__(self, error_messages, packager, p_tokens_file):
self.__error_messages = error_messages
self.__packager = packager
self.__p_tokens_file = p_tokens_file
self.__logger = Logger()
from core.predictor import *
from tools.cache import Cache
from tools.pickler import reload_pickle_cache, init_pickle_cache
from tools.po_cache_ret import POCacheRet
from tools.logger import Logger
if sys.version_info[:2] in [(2, 6), (2, 7)]:
reload(sys)
sys.setdefaultencoding('utf-8')
elif sys.version_info[:2] in [(3, 6), (3, 7)]:
# pylint: disable=E0401, E0611, E1101
import importlib
importlib.reload(sys)
logger = Logger.get_instance(ConfManage.getString("LOG_BASE_NAME"))
# Import Thrift generated classes
sys.path.append('gen-py')
sys.path.insert(0, glob.glob('../../lib/py/build/lib*'[0]))
try:
from eta import EstimateTaskDurations
from eta.ttypes import Duration, Range, InvalidInput, ModelMissing, Prediction, Params, \
Prediction_showup, Prediction_quote, Prediction_delivery, PredictionResult, Data
except ImportError:
logger.error('Cannot find thrift classes.')
logger.error('Have (you run `thrift --gen py eta.thrift`?')
raise
def __validate_coordinates__(query):
if query.showup_distance is None:
class Cutout:
def __init__(self, survey, position, radius, **kwargs):
self.survey = survey
self.position = position
self.ra = self.position.ra.to_value(u.deg)
self.dec = self.position.dec.to_value(u.deg)
self.radius = radius
self.basesurvey = kwargs.get('basesurvey', 'racsI')
self.psf = kwargs.get('psf')
self.cmap = kwargs.get('cmap', 'gray_r')
self.color = 'k' if self.cmap == 'hot' else 'black'
self.band = kwargs.get('band', 'g')
level = 'DEBUG' if kwargs.get('verbose') else 'INFO'
self.logger = Logger(__name__, kwargs.get('log'),
streamlevel=level).logger
self.logger.propagate = False
self.kwargs = kwargs
try:
self._get_cutout()
except Exception as e:
msg = f"{survey} failed: {e}"
raise FITSException(msg)
finally:
if 'racs' not in self.survey and 'vast' not in self.survey:
self.plot_sources = False
self.plot_neighbours = False
def __repr__(self):
return f"Cutout({self.survey}, ra={self.ra:.2f}, dec={self.dec:.2f})"
def _get_source(self):
try:
pattern = re.compile(r'\S*(\d{4}[+-]\d{2}[AB])\S*')
selpath = SURVEYS.loc[self.survey]['selavy']
sel = glob.glob(f'{selpath}/*components.txt')
sel = [s for s in sel if pattern.sub(r'\1', self.filepath) in s]
if len(sel) > 1:
df = pd.concat([pd.read_fwf(s, skiprows=[
1,
]) for s in sel])
else:
df = pd.read_fwf(sel[0], skiprows=[
1,
])
coords = SkyCoord(df.ra_deg_cont, df.dec_deg_cont, unit=u.deg)
d2d = self.position.separation(coords)
df['d2d'] = d2d
sources = df.iloc[np.where(d2d.deg < 0.5 * self.radius)[0]]
sources = sources.sort_values('d2d', ascending=True)
if any(sources.d2d < self.pos_err / 3600):
self.source = sources.iloc[0]
self.neighbours = sources.iloc[1:]
self.plot_sources = True
else:
self.source = None
self.neighbours = sources
self.plot_sources = False
self.plot_neighbours = self.kwargs.get('neighbours', True)
self.logger.debug(f'Source: \n {self.source}')
if len(self.neighbours) > 0:
nn = self.neighbours.iloc[0]
self.logger.debug(
f'Nearest neighbour coords: \n {nn.ra_deg_cont, nn.dec_deg_cont}'
)
self.logger.debug(
f'Nearest 5 Neighbours \n {self.neighbours.head()}')
except IndexError:
self.plot_sources = False
self.plot_neighbours = False
self.logger.warning('No nearby sources found.')
def _get_cutout(self):
if not os.path.exists(cutout_cache + self.survey):
msg = f"{cutout_cache}{self.survey} cutout directory does not exist, creating."
self.logger.info(msg)
os.makedirs(cutout_cache + self.survey)
if os.path.isfile(self.survey):
self._get_local_cutout()
elif 'racs' in self.survey or 'vast' in self.survey or 'vlass' in self.survey:
self._get_local_cutout()
elif self.survey == 'skymapper':
self._get_skymapper_cutout()
elif self.survey == 'panstarrs':
self._get_panstarrs_cutout()
elif self.survey == 'decam':
self._get_decam_cutout()
else:
self._get_skyview_cutout()
def _get_local_cutout(self):
"""Fetch cutout data via local FITS images (e.g. RACS / VLASS)."""
fields = self._find_image()
assert len(
fields
) > 0, f"No fields located at {self.position.ra:.2f}, {self.position.dec:.2f}"
closest = fields[fields.dist_field_centre ==
fields.dist_field_centre.min()].iloc[0]
image_path = SURVEYS.loc[self.survey]['images']
if self.survey == 'vlass':
filepath = f'{closest.epoch}/{closest.tile}/{closest.image}/{closest.filename}'
image_path = vlass_path
elif 'racs' in self.survey:
pol = self.survey[-1]
if on_system == 'ada':
filepath = f'RACS_test4_1.05_{closest.field}.fits'
else:
filepath = f'RACS_{closest.field}.EPOCH00.{pol}.fits'
elif 'vast' in self.survey:
pattern = re.compile(r'vastp(\dx*)([IV])')
epoch = pattern.sub(r'\1', self.survey)
pol = pattern.sub(r'\2', self.survey)
filepath = f'VAST_{closest.field}.EPOCH0{epoch}.{pol}.fits'
else:
filepath = f'*{closest.field}*0.restored.fits'
try:
self.filepath = glob.glob(image_path + filepath)[0]
except IndexError:
raise FITSException(
f'Could not match {self.survey} image filepath: \n{image_path + filepath}'
)
with fits.open(self.filepath) as hdul:
self.header, data = hdul[0].header, hdul[0].data
wcs = WCS(self.header, naxis=2)
self.mjd = Time(self.header['DATE']).mjd
try:
cutout = Cutout2D(data[0, 0, :, :],
self.position,
self.radius * u.deg,
wcs=wcs)
except IndexError:
cutout = Cutout2D(data,
self.position,
self.radius * u.deg,
wcs=wcs)
self.data = cutout.data * 1000
self.wcs = cutout.wcs
if 'racs' in self.survey or 'vast' in self.survey:
self.pos_err = SURVEYS.loc[self.basesurvey].pos_err
self._get_source()
else:
# Probably using vlass, yet to include aegean catalogs
self.plot_sources = False
self.plot_neighbours = False
def _get_panstarrs_cutout(self):
"""Fetch cutout data via PanSTARRS DR2 API."""
path = cutout_cache + 'panstarrs/{}_{}arcmin_{}_{}.fits'.format(
self.band,
'{:.3f}',
'{:.3f}',
'{:.3f}',
)
imgpath = path.format(self.radius * 60, self.ra, self.dec)
if not os.path.exists(imgpath):
pixelrad = int(self.radius * 120 * 120)
service = "https://ps1images.stsci.edu/cgi-bin/ps1filenames.py"
url = (
f"{service}?ra={self.ra}&dec={self.dec}&size={pixelrad}&format=fits"
f"&filters=grizy")
table = Table.read(url, format='ascii')
msg = f"No PS1 image at {self.position.ra:.2f}, {self.position.dec:.2f}"
assert len(table) > 0, msg
urlbase = (
f"https://ps1images.stsci.edu/cgi-bin/fitscut.cgi?"
f"ra={self.ra}&dec={self.dec}&size={pixelrad}&format=fits&red="
)
flist = ["yzirg".find(x) for x in table['filter']]
table = table[np.argsort(flist)]
for row in table:
self.mjd = row['mjd']
filt = row['filter']
url = urlbase + row['filename']
path = cutout_cache + 'panstarrs/{}_{}arcmin_{}_{}.fits'.format(
filt,
'{:.3f}',
'{:.3f}',
'{:.3f}',
)
path = path.format(self.radius * 60, self.ra, self.dec)
img = requests.get(url, allow_redirects=True)
if not os.path.exists(path):
with open(path, 'wb') as f:
f.write(img.content)
with fits.open(imgpath) as hdul:
self.header, self.data = hdul[0].header, hdul[0].data
self.wcs = WCS(self.header, naxis=2)
def _get_skymapper_cutout(self):
"""Fetch cutout data via Skymapper API."""
path = cutout_cache + self.survey + '/dr2_jd{:.3f}_{:.3f}arcmin_{:.3f}_{:.3f}'
linka = 'http://api.skymapper.nci.org.au/aus/siap/dr2/'
linkb = 'query?POS={:.5f},{:.5f}&SIZE={:.3f}&BAND=all&RESPONSEFORMAT=CSV'
linkc = '&VERB=3&INTERSECT=covers'
sm_query = linka + linkb + linkc
link = linka + 'get_image?IMAGE={}&SIZE={}&POS={},{}&FORMAT=fits'
table = requests.get(sm_query.format(self.ra, self.dec, self.radius))
df = pd.read_csv(io.StringIO(table.text))
assert len(
df
) > 0, f'No Skymapper image at {self.position.ra:.2f}, {self.position.dec:.2f}'
df = df[df.band == 'z']
self.mjd = df.iloc[0]['mjd_obs']
link = df.iloc[0].get_image
img = requests.get(link)
path = path.format(self.mjd, self.radius * 60, self.ra, self.dec)
if not os.path.exists(path):
with open(path, 'wb') as f:
f.write(img.content)
with fits.open(path) as hdul:
self.header, self.data = hdul[0].header, hdul[0].data
self.wcs = WCS(self.header, naxis=2)
def _get_decam_cutout(self):
"""Fetch cutout data via DECam LS API."""
size = int(self.radius * 3600 / 0.262)
if size > 512:
size = 512
maxradius = size * 0.262 / 3600
self.logger.warning(
f"Using maximum DECam LS cutout radius of {maxradius:.3f} deg")
link = f"http://legacysurvey.org/viewer/fits-cutout?ra={self.ra}&dec={self.dec}"
link += f"&size={size}&layer=dr8&pixscale=0.262&bands={self.band}"
img = requests.get(link)
path = cutout_cache + self.survey + '/dr8_jd{:.3f}_{:.3f}arcmin_{:.3f}_{:.3f}_{}band'
path = path.format(self.mjd, self.radius * 60, self.ra, self.dec,
self.band)
if not os.path.exists(path):
with open(path, 'wb') as f:
f.write(img.content)
with fits.open(path) as hdul:
self.header, self.data = hdul[0].header, hdul[0].data
self.wcs = WCS(self.header, naxis=2)
msg = f"No DECam LS image at {self.position.ra:.2f}, {self.position.dec:.2f}"
assert self.data is not None, msg
def _get_skyview_cutout(self):
"""Fetch cutout data via SkyView API."""
sv = SkyView()
path = cutout_cache + self.survey + '/{:.3f}arcmin_{:.3f}_{:.3f}.fits'
path = path.format(self.radius * 60, self.ra, self.dec)
progress = self.kwargs.get('progress', False)
if not os.path.exists(path):
skyview_key = SURVEYS.loc[self.survey].sv
try:
hdul = sv.get_images(position=self.position,
survey=[skyview_key],
radius=self.radius * u.deg,
show_progress=progress)[0][0]
except IndexError:
raise FITSException('Skyview image list returned empty.')
except ValueError:
raise FITSException(
f'{self.survey} is not a valid SkyView survey.')
except HTTPError:
raise FITSException('No response from Skyview server.')
with open(path, 'wb') as f:
hdul.writeto(f)
with fits.open(path) as hdul:
self.header, self.data = hdul[0].header, hdul[0].data
self.wcs = WCS(self.header, naxis=2)
try:
self.mjd = Time(self.header['DATE']).mjd
except KeyError:
try:
self.epoch = self.kwargs.get('epoch')
msg = "Could not detect epoch, PM correction disabled."
assert self.epoch is not None, msg
self.mjd = self.epoch if self.epoch > 3000 else Time(
self.epoch, format='decimalyear').mjd
except AssertionError as e:
if self.kwargs.get('pm'):
self.logger.warning(e)
self.mjd = None
self.data *= 1000
def _find_image(self):
"""Return DataFrame of survey fields containing coord."""
survey = self.survey.replace('I', '').replace('V', '')
try:
image_df = pd.read_csv(aux_path + f'{survey}_fields.csv')
except FileNotFoundError:
raise FITSException(f"Missing field metadata csv for {survey}.")
beam_centre = SkyCoord(ra=image_df['cr_ra_pix'],
dec=image_df['cr_dec_pix'],
unit=u.deg)
image_df['dist_field_centre'] = beam_centre.separation(
self.position).deg
pbeamsize = 1 * u.degree if self.survey == 'vlass' else 5 * u.degree
return image_df[image_df.dist_field_centre < pbeamsize].reset_index(
drop=True)
def _obfuscate(self):
"""Remove all coordinates and identifying information."""
lon = self.ax.coords[0]
lat = self.ax.coords[1]
lon.set_ticks_visible(False)
lon.set_ticklabel_visible(False)
lat.set_ticks_visible(False)
lat.set_ticklabel_visible(False)
lon.set_axislabel('')
lat.set_axislabel('')
def _plot_setup(self, fig, ax):
"""Create figure and determine normalisation parameters."""
if ax:
self.fig = fig
self.ax = ax
else:
self.fig = plt.figure()
self.ax = self.fig.add_subplot(111, projection=self.wcs)
if self.kwargs.get('grid', True):
self.ax.coords.grid(color='white', alpha=0.5)
self.ax.set_xlabel('RA (J2000)')
self.ax.set_ylabel('Dec (J2000)')
if self.kwargs.get('title', True):
self.ax.set_title(SURVEYS.loc[self.survey]['name'],
fontdict={
'fontsize': 20,
'fontweight': 10
})
if self.kwargs.get('obfuscate', False):
self._obfuscate()
if self.kwargs.get('annotation'):
color = 'white' if self.cmap == 'hot' else 'k'
self.ax.text(0.05,
0.85,
self.kwargs.get('annotation'),
color=color,
weight='bold',
transform=self.ax.transAxes)
def _add_cornermarker(self, ra, dec, span, offset):
color = 'white' if self.cmap != 'gray_r' else 'r'
cosdec = np.cos(np.radians(dec))
raline = Line2D(
xdata=[ra + offset / cosdec, ra + span / cosdec],
ydata=[dec, dec],
color=color,
linewidth=2,
path_effects=[pe.Stroke(linewidth=3, foreground='k'),
pe.Normal()],
transform=self.ax.get_transform('world'))
decline = Line2D(
xdata=[ra, ra],
ydata=[dec + offset, dec + span],
color=color,
linewidth=2,
path_effects=[pe.Stroke(linewidth=3, foreground='k'),
pe.Normal()],
transform=self.ax.get_transform('world'))
self.ax.add_artist(raline)
self.ax.add_artist(decline)
def plot(self, fig=None, ax=None):
"""Plot survey data and position overlay."""
self.sign = self.kwargs.get('sign', 1)
self._plot_setup(fig, ax)
self.data *= self.sign
absmax = max(self.data.max(), self.data.min(), key=abs)
self.logger.debug(f"Max flux in cutout: {absmax:.2f} mJy.")
rms = np.sqrt(np.mean(np.square(self.data)))
self.logger.debug(f"RMS flux in cutout: {rms:.2f} mJy.")
assert (sum((~np.isnan(self.data).flatten())) > 0 and sum(self.data.flatten()) != 0), \
f"No data in {self.survey}"
if self.kwargs.get('maxnorm'):
self.norm = ImageNormalize(self.data,
interval=ZScaleInterval(),
vmax=self.data.max(),
clip=True)
else:
self.norm = ImageNormalize(self.data,
interval=ZScaleInterval(contrast=0.2),
clip=True)
self.im = self.ax.imshow(self.data, cmap=self.cmap, norm=self.norm)
if self.kwargs.get('bar', True):
try:
self.fig.colorbar(self.im,
label=r'Flux Density (mJy beam$^{-1}$)',
ax=self.ax)
except UnboundLocalError:
self.logger.error(
"Colorbar failed. Upgrade to recent version of astropy ")
if self.psf:
try:
self.bmaj = self.header['BMAJ'] * 3600
self.bmin = self.header['BMIN'] * 3600
self.bpa = self.header['BPA']
except KeyError:
self.logger.warning('Header did not contain PSF information.')
try:
self.bmaj = self.psf[0]
self.bmin = self.psf[1]
self.bpa = 0
self.logger.warning(
'Using supplied BMAJ/BMin. Assuming BPA=0')
except ValueError:
self.logger.error('No PSF information supplied.')
rhs = self.wcs.wcs_pix2world(self.data.shape[0], 0, 1)
lhs = self.wcs.wcs_pix2world(0, 0, 1)
# Offset PSF marker by the major axis in pixel coordinates
try:
cdelt = self.header['CDELT1']
except KeyError:
cdelt = self.header['CD1_1']
beamavg = (self.bmaj + self.bmin) / 2
beamsize_pix = beamavg / abs(cdelt) / 3600
ax_len_pix = abs(lhs[0] - rhs[0]) / abs(cdelt) / 3600
beam = self.wcs.wcs_pix2world(beamsize_pix, beamsize_pix, 1)
self.beamx = beam[0]
self.beamy = beam[1]
self.beam = Ellipse((self.beamx, self.beamy),
self.bmin / 3600,
self.bmaj / 3600,
-self.bpa,
facecolor='white',
edgecolor='k',
transform=self.ax.get_transform('world'),
zorder=10)
self.ax.add_patch(self.beam)
# Optionally plot square around the PSF
# Set size to greater of 110% PSF size or 10% ax length
if self.kwargs.get('beamsquare', False):
boxsize = max(beamsize_pix * 1.15, ax_len_pix * .1)
offset = beamsize_pix - boxsize / 2
self.square = Rectangle(
(offset, offset),
boxsize,
boxsize,
facecolor='white',
edgecolor='k',
# transform=self.ax.get_transform('world'),
zorder=5)
self.ax.add_patch(self.square)
if self.plot_sources:
if self.kwargs.get('corner'):
self._add_cornermarker(
self.source.ra_deg_cont, self.source.dec_deg_cont,
self.kwargs.get('corner_span', 20 / 3600),
self.kwargs.get('corner_offset', 10 / 3600))
else:
self.sourcepos = Ellipse(
(self.source.ra_deg_cont, self.source.dec_deg_cont),
self.source.min_axis / 3600,
self.source.maj_axis / 3600,
-self.source.pos_ang,
facecolor='none',
edgecolor='r',
ls=':',
lw=2,
transform=self.ax.get_transform('world'))
self.ax.add_patch(self.sourcepos)
else:
if self.kwargs.get('corner'):
self._add_cornermarker(
self.ra, self.dec, self.kwargs.get('corner_span',
20 / 3600),
self.kwargs.get('corner_offset', 10 / 3600))
else:
self.bmin = 15
self.bmaj = 15
self.bpa = 0
overlay = SphericalCircle(
(self.ra * u.deg, self.dec * u.deg),
self.bmaj * u.arcsec,
edgecolor='r',
linewidth=2,
facecolor='none',
transform=self.ax.get_transform('world'))
self.ax.add_artist(overlay)
if self.plot_neighbours:
for idx, neighbour in self.neighbours.iterrows():
n = Ellipse((neighbour.ra_deg_cont, neighbour.dec_deg_cont),
neighbour.min_axis / 3600,
neighbour.maj_axis / 3600,
-neighbour.pos_ang,
facecolor='none',
edgecolor='c',
ls=':',
lw=2,
transform=self.ax.get_transform('world'))
self.ax.add_patch(n)
def save(self, path, fmt='png'):
"""Save figure with tight bounding box."""
self.fig.savefig(path, format=fmt, bbox_inches='tight')
def savefits(self, path):
"""Export FITS cutout to path"""
header = self.wcs.to_header()
hdu = fits.PrimaryHDU(data=self.data, header=header)
hdu.writeto(path)
para * 4 / 1000 / 1000))
model = model.to(device)
# Set checkpoint
checkpoint = CheckPoint(config.save_path)
# Set optimizer
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=config.lr)
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
milestones=config.step,
gamma=0.1)
# Set trainer
logger = Logger(config.save_path)
trainer = AlexNetTrainer(config.lr, train_loader, valid_loader, model,
optimizer, scheduler, logger, device)
print(model)
time_start = time.time()
for epoch in range(1, config.nEpochs + 1):
cls_loss_, accuracy, accuracy_valid = trainer.train(epoch)
checkpoint.save_model(model, index=epoch)
time_end = time.time()
print(time_end - time_start)
# #计算样本量
import pandas as pd
from keras.losses import *
from loss.pairwise_loss import *
from loss.triplet_loss import *
import loss.pairwise_loss
import loss.triplet_loss
import models
#from distutils.util import strtobool
gpu_count = len(units.get_available_gpus())
dir_path, global_logger = units.getLogger()
from tools.logger import Logger
logger = Logger()
import os
def run(params):
evaluation = []
# params=dataset.classification.process_embedding(reader,params)
qdnn = models.setup(params)
model = qdnn.getModel()
model.summary()
if hasattr(loss.pairwise_loss, params.loss):
loss_func = getattr(loss.pairwise_loss, params.loss)
else:
loss_func = params.loss
optimizer = units.getOptimizer(name=params.optimizer, lr=params.lr)
#
