def __init__(self, client: Client):
Thread.__init__(self)
Logger.__init__(self)
self.client = client
self.stop = False
self.ticker_websocket: WebSocketClientProtocol = None
self.user_webscoket: WebSocketClientProtocol = None
self.ticker_ws_future = None
self.user_ws_future = None
self.mngmt_future = None
self.connection_key = None
self.user_info_cb = None
self.ticker_cb = None
self.symbols = None
if not BinanceWebsocket.__EVENT_LOOP:
self.loop = asyncio.get_event_loop()
BinanceWebsocket.__EVENT_LOOP = self.loop
else:
self.loop = BinanceWebsocket.__EVENT_LOOP
self.time = None
self.name = 'Binance WebSocket Thread'
def close(self):
try:
self.conn.close()
return True
except Exception as e:
Logger.error(name="DB", output=e)
return False
def setUp(self):
self.families = []
self.individuals = []
self.famMap = {}
self.indMap = {}
self.logger = Logger()
self.seed_data()
self.spousecheck = None
def __init__(self, **kwargs):
"""
Basic empty constructor for an EyeTracakerClient class
:param kwargs: list
:return: None
"""
if getattr(self, "defaults", None) is None:
self.defaults = {}
# setting logGaze before constructing Element, so it will
# end up in self.config.logGaze == True
self.defaults["logGaze"] = True
# call Element constructor
super(EyeTrackerClient, self).__init__(**kwargs)
# this is not a visible element!!!
self.hideElement()
""" constructor for the EyeTracker class """
self.status = TRACKER_STATUS.DISCONNECTED
# gazeData is a list of triplets (timeStamp, x, y)
if (getattr(self.config, "smoothWindow", None) == None):
self.config.smoothWindow = 5.0
else:
self.config.smoothWindow = float(self.config.smoothWindow)
self.gazeData = [(0, 0, 0)] * int(self.config.smoothWindow)
self.smoothSampleXY = [0.0, 0.0]
if self.config.logGaze:
# one gaze log per participant
self.gazeLogger = Logger(self.baseTime,
"run/gazeData_" +
self.config.world.participantId + ".log",
mode='w')
else:
self.gazeLogger = Logger(self.baseTime, "noLog")
#self.gazeLogger.startLog()
# create a mutex for accessing the gazeData list
self.gazeMutex = Mutex('gazeMutex')
gazeTex = loader.loadTexture(
'Elements/Game/models/textures/outter_circle.png')
gazeTex.setMinfilter(Texture.FTLinearMipmapLinear)
gazeTex.setAnisotropicDegree(2)
gazeNode = loader.loadModel("Elements/Game/models/plane")
gazeNode.reparentTo(self.hudNP)
gazeNode.setScale(0.1, 1.0, 0.1)
gazeNode.setTransparency(1)
gazeNode.setAlphaScale(0.1)
gazeNode.setTexture(gazeTex)
gazeNode.setPos(-1.7, 0, 0)
self.gazeNode = gazeNode
#w,h = map(float,(cam.screenWidth,cam.screenHeight))
self.normX = base.win.getXSize() / float(
base.win.getYSize()) # self.config.world.getCamera().ratio
self.hudNP.setBin('fixed', 10)
def execute(self, sql: str = None):
if sql is None:
return False
try:
self.c.execute(sql)
return True
except Exception as e:
Logger.error(name="DB", output=e)
return False
class MouseLogger(Element):
"""
Logs the position of the mouse, in a text file using
the participant ID and timestamps that are synchronized with the
rest of the timestamps from other events in the simulation
"""
def __init__(self, **kwargs):
"""
See the Element class to find out what attributes are available
from scratch
"""
super(MouseLogger, self).__init__(**kwargs)
self.mouseLog = Logger(
self.baseTime,
"run/mouseLog_%s.txt" % self.config.world.participantId)
self.left = MouseButton.one()
self.leftDown = False
self.right = MouseButton.two()
self.rightDown = False
self.hideElement()
def enterState(self):
# super class enterState
Element.enterState(self)
#self.mouseLog.startLog()
taskMgr.add(self.logMouseTask,
"mouseLogTask_%s" % self.config.world.participantId)
def exitState(self):
# super class exitState
taskMgr.remove("mouseLogTask_%s" % self.config.world.participantId)
self.mouseLog.stopLog()
Element.exitState(self)
def logMouseTask(self, t):
m = base.mouseWatcherNode
if not m.hasMouse():
return t.cont
try:
self.mouseLog.logEvent("%f %f" % (m.getMouseX(), m.getMouseY()))
if m.is_button_down(self.left):
self.leftDown = True
elif self.leftDown:
self.mouseLog.logEvent("mouse1")
self.leftDown = False
if m.is_button_down(self.right):
self.rightDown = True
elif self.rightDown:
self.mouseLog.logEvent("mouse2")
self.rightDown = False
except:
self.mouseLog.logEvent("mouse outside of the window...")
finally:
return t.cont
def __init__(self, bot):
self.bot = bot
if not hasattr(bot, 'lavalink'): # This ensures the client isn't overwritten during cog reloads.
bot.lavalink = lavalink.Client(bot.user.id)
bot.lavalink.add_node('avexlava.herokuapp.com', 80, 'youshallnotpass', 'us', 'default-node') # Host, Port, Password, Region, Name
bot.add_listener(bot.lavalink.voice_update_handler, 'on_socket_response')
lavalink.add_event_hook(self.track_hook)
self.logger = Logger(False)
self.logger.log(name="Discord", output='Music Cog Loaded!')
class Prime():
nextUpdateTag = 'NextInterestDecisionDate'
logger = None
def __init__(self):
self.logger = Logger()
pass
"""
this method returns an HTML with Table
API return Table with no HTML Tags
"""
def getPrime(self):
self.log('getting prime interest...')
baseUrl = "https://www.boi.org.il/he/BankingSupervision/Data/_layouts/boi/handlers/WebPartHandler.aspx"
params = {
'wp': 'ItemsAggregator',
'PageId': '150',
'CqfDateFrom': '',
'CqfDateTo': '',
'_': '1557755447808'
}
year = str(int(time.strftime("%Y")) - 1)
params['CqfDateFrom'] = time.strftime("%d/%m/") + year
params['CqfDateTo'] = '01/01/' + time.strftime("%Y")
# self.log("Today : " + dateToday + ", First date of the Year " + startOfYear)
# add html and body tags to make this an HTML.
return "<html><body>" + ClientGet(baseUrl, params) + "</body></html>"
"""
this method return a string with a date
date is the next interests table will be updated
"""
def getNextUpdate(self, primeTable):
self.log('getting next Update from xml...')
xmlUrl = "https://www.boi.org.il/HE/BoiLatestPublication.xml"
xmlParser = XmlParser()
xmlResult = ClientGet(xmlUrl, None)
value = xmlParser.findTagInString(xmlResult, self.nextUpdateTag)
return value
def log(self, message):
self.logger.log('Prime', message)
def TEST_Get_Next_Update(self):
prime = Prime()
print(prime.getNextUpdate())
def TEST_Get_Prime_Table(self):
prime = Prime()
print(prime.getPrime())
def __init__(self, **kwargs):
"""
See the Element class to find out what attributes are available
from scratch
"""
super(MouseLogger, self).__init__(**kwargs)
self.mouseLog = Logger(
self.baseTime,
"run/mouseLog_%s.txt" % self.config.world.participantId)
self.left = MouseButton.one()
self.leftDown = False
self.right = MouseButton.two()
self.rightDown = False
self.hideElement()
def main(argv = None):
if(argv is None):
argv = sys.argv
Logger.initLogger()
log = Logger.getLogger()
client = ClientCore()
try:
client.start()
except IOError:
log.error("Client terminated")
except KeyboardInterrupt:
log.info("Client terminated")
def __init__(self, serverName, socket, transactionId):
super(TransactionThread, self).__init__()
self.__name = transactionId
self.__log = Logger.getLogger()
self.__log.debug("Creating transaction thread [%s]", self.__name)
self.__serverName = serverName
self.__socket = socket
self.__transactionId = transactionId
self.__messageParser = MessageParser()
self.__file = FileWorker(self.__serverName)
self.__walRegister = WALRegister(self.__serverName)
self.__resource = ResourceQueue()
self.__resourceOwned = False
# create functions mapping
self.__handlers = {
"begin": self.__begin,
"end": self.__end,
"read": self.__read,
"write": self.__write,
"commit": self.__commit,
"globalCommit": self.__globalCommit,
"abort": self.__abort,
"invalid": self.__reject
}
class Mailer():
server = None
message = ""
emails = ["myMail"]
logger = Logger()
def __init__(self):
pass
def sendMail(self, message, htmlFlag, nextUpdateParam):
if (htmlFlag):
self.message = MIMEMultipart(
"alternative", None,
[MIMEText(message),
MIMEText(message, 'html')])
else:
self.message = MIMEText(message)
self.message[
'Subject'] = 'Prime Interest For this Month, Next Update At ' + nextUpdateParam
self.message['From'] = 'PrimeService - Auto Generated'
self.message['To'] = ','.join(self.emails)
self.server = smtplib.SMTP('smtp.gmail.com', 587)
# self.server.ehlo()
self.server.starttls()
self.server.ehlo()
self.server.login('myMail', 'ikiwrwtovszgxyik')
print('aftrer login and before send - message:')
print(self.message.as_string())
self.server.sendmail("*****@*****.**", self.emails,
self.message.as_string())
self.server.close()
pass
def log(self, message):
self.logger.log('Mailer', message)
def __init__(self): self.__log = Logger.getLogger() self.__servers = ServerList() self.__servers.parseList() self.__commandParser = CommandParser() self.__transacationId = str(self.__generateId()) self.__transactionManager = TransactionManager(self.__transacationId)
def __init__(self, serverName): self.__log = Logger.getLogger() # create thread manager self.__serverName = serverName # create a TCP/IP socket self.__socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
def analyze_once(self):
self.refresh_event.wait()
analysis_logger = self.logger
if not self.settings["log_analysis"]:
analysis_logger = Logger(FileWriter,
PathProvider.pathes["ANALYSIS"].location)
Analyzer(self.cron, self.settings["analyzer_iterations_amount"],
analysis_logger).analyze_all()
def start(self, serverIP, serverPort):
# start logging
log = Logger.getLogger()
log.info("Starting TrackMe-Server...")
# start listening for incomming messages
listener = ConnectionListener()
listener.listen(serverIP, serverPort)
def __init__(self, **kwargs):
super(EventLogger, self).__init__(**kwargs)
self.listener = DirectObject()
# registering some events by hand
self.listener.accept('crossHair',self.logEvent)
uniqueFileName = self.config.logfile +"_"+ self.config.world.participantId + ".log"
self.eventLog = Logger(self.baseTime, uniqueFileName, 'w')
#self.eventLog.startLog()
self.eventLog.logEvent('Event logger started\n')
taskMgr.add( self.updateHooks, 'updateHooks' )
self.registeredEvents = messenger.getEvents()
for e in self.registeredEvents:
self.listener.accept(e, self.logEvent, [e])
self.hideElement()
class Misc(commands.Cog):
def __init__(self, bot):
self.bot = bot
self.logger = Logger(False)
self.logger.log(name="Discord", output="Misc Cog Loaded!")
self.db = DatabaseHandler(db="database.db")
self.db.connect()
@commands.command(name="ping", aliases=["pang", "peng", "pong", "pung"])
async def _ping(self, ctx):
ping = round(self.bot.latency * 1000)
e = discord.Embed()
if ping < 30:
e.colour = 0x39fc03
elif ping < 75:
e.colour = 0xf7ba00
else:
e.colour = 0xf70000
e.title = "Pong! :ping_pong:"
e.description = f"API Latency: {ping}ms"
await ctx.send(embed=e)
@commands.command(name="tier", aliases=["rank"])
async def _tier(self, ctx, member: discord.User = None):
tier = ""
em = discord.Embed()
em.colour = COLOUR
if member is None:
# do invoker
em.title = f"{ctx.author.name}'s Tier"
tier = getUserTier(ctx.author.id)
em.description = f"Your current tier is: {tier}!"
return await ctx.send(embed=em)
em.title = f"{member.name}'s Tier"
tier = getUserTier(member.id)
em.description = f"{member.name} tier is: {tier}!"
await ctx.send(embed=em)
@commands.command(name="test")
async def _test(self, ctx):
await ctx.send(isUserDeveloper(ctx.author.id))
def setUp(self) -> None:
self.testName = unittest.TestCase.id(self)
self.logger = Logger(self.testName)
if not self.logger:
self.fail("No logger!")
self.logger.info("Test started!")
self.driver = webdriver.Chrome()
self.UtilsSelenium = UtilsSelenium(self.driver, self.logger)
self.driver.get("https://www.eobuwie.com.pl/")
WebDriverWait(self.driver, 20).until(
EC.element_to_be_clickable(mpl.main_page_login_button))
self.driver.maximize_window()
# assert self.driver.title == "Modne buty damskie, męskie, dziecięce oraz torebki | eobuwie.pl", f"Loaded page in not main page. Title is {self.driver.title} and should be Modne buty damskie, męskie, dziecięce oraz torebki | eobuwie.pl"
accept_cookies_button = self.driver.find_element(
*mpl.main_page_cookies_accept_button)
accept_cookies_close_button = self.driver.find_element(
*mpl.main_page_cookies_close_button)
if accept_cookies_button.is_displayed():
accept_cookies_close_button.click()
def main(argv = None):
if(argv is None):
argv = sys.argv
Logger.initLogger()
log = Logger.getLogger()
if(len(argv) != 3):
log.error("IP, port or server name not specified!")
log.info("usage: TrackMe-Server <IP> <Port>")
return 1
server = ServerCore()
serverIP = argv[1]
serverPort = int(argv[2])
try:
server.start(serverIP, serverPort)
except IOError:
log.error("TrackMe-Server terminated")
except KeyboardInterrupt:
log.info("TrackMe-Server terminated")
def main(argv = None):
if(argv is None):
argv = sys.argv
Logger.initLogger()
log = Logger.getLogger()
#if(len(argv) != 3):
# log.error("malware directory not specified!")
# log.info("usage: VirusTotalAPI <directory>")
# return 1
#filesDirectory = argv[1]
filesDirectory = "D:\\Virtual Machines\\shared\\Win7\\outputSafe\\EncryptionEngine8x_delay20\\"
try:
virusTotalClient = VirusTotalClient()
virusTotalClient.run(filesDirectory)
except IOError:
log.error("VirusTotal API terminated")
except KeyboardInterrupt:
log.info("VirusTotal API terminated")
def __init__(self):
self.SUCCESS = 1
self.FAILED = 0
self.__log = Logger.getLogger()
self.__responseCode = self.FAILED
self.__writeCounter = 1
self.clear()
# create functions mapping
self.__saveHandlers = {
"txt": self.__saveToTXT
}
class EventLogger(Element):
"""
"""
def __init__(self, **kwargs):
super(EventLogger, self).__init__(**kwargs)
self.listener = DirectObject()
# registering some events by hand
self.listener.accept('crossHair',self.logEvent)
uniqueFileName = self.config.logfile +"_"+ self.config.world.participantId + ".log"
self.eventLog = Logger(self.baseTime, uniqueFileName, 'w')
#self.eventLog.startLog()
self.eventLog.logEvent('Event logger started\n')
taskMgr.add( self.updateHooks, 'updateHooks' )
self.registeredEvents = messenger.getEvents()
for e in self.registeredEvents:
self.listener.accept(e, self.logEvent, [e])
self.hideElement()
def logEvent(self, event=None, args=[]):
if event == 'mouse1':
args = base.mouseWatcherNode.getMouse()
self.eventLog.logEvent("%s;%s\n"%(event,args))
def enterState(self):
# super class enterState
Element.enterState(self)
def exitState(self):
# super class exitState
Element.exitState(self)
taskMgr.remove( 'updateHooks' )
self.eventLog.logEvent('Event logger stopped\n')
self.eventLog.closeLog()
def updateHooks(self, task):
# run every 100 ms
task.delayTime = 0.1
newEvents = [x for x in messenger.getEvents() if x not in self.registeredEvents]
for x in newEvents:
#print "NEW EVENT"
self.listener.accept(x, self.logEvent, [x])
self.registeredEvents.append(x)
return task.cont
def __init__(self):
self.__dbName = RESOURCE_PATH + SERVER_DB_FILE
self.__log = Logger.getLogger()
# check if database file exists
if(os.path.exists(self.__dbName) == False):
self.__log.debug("DB file doesn't exists, creating...")
dbConnection = sqlite3.connect(self.__dbName) # @UndefinedVariable
# check if ResourceQueue table exists within database
c = dbConnection.cursor()
c.execute("CREATE TABLE IF NOT EXISTS ResourceQueue (id VARCHAR(20), timestamp VARCHAR(20))")
dbConnection.close()
def __init__(self):
self.__dbName = RESOURCE_PATH + SERVER_DB_FILE
self.__log = Logger.getLogger()
# check if database file exists
if(os.path.exists(self.__dbName) == False):
self.__log.debug("DB file doesn't exists, creating...")
dbConnection = sqlite3.connect(self.__dbName) # @UndefinedVariable
c = dbConnection.cursor()
# check if tables exist within database
c.execute("CREATE TABLE IF NOT EXISTS UserLocation (userId VARCHAR(30), longtitude VARCHAR(30), latitude VARCHAR(30))")
c.execute("CREATE TABLE IF NOT EXISTS TrackingPermission (userId VARCHAR(30), viewerId VARCHAR(30))")
c.execute("CREATE TABLE IF NOT EXISTS User (userId VARCHAR(30), password VARCHAR(30))")
dbConnection.close()
def __init__(self, serverName):
self.__dbName = RESOURCE_PATH + SERVER_DB_FILE
self.__log = Logger.getLogger()
self.__file = FileWorker(serverName)
self.__resource = ResourceQueue()
# check if database file exists
if(os.path.exists(self.__dbName) == False):
self.__log.debug("DB file doesn't exists, creating...")
dbConnection = sqlite3.connect(self.__dbName) # @UndefinedVariable
c = dbConnection.cursor()
# check if WAL table exists within database
c.execute("CREATE TABLE IF NOT EXISTS WALRegister (id VARCHAR(20), command VARCHAR(5), before VARCHAR(200), after VARCHAR(200), timestamp VARCHAR(20))")
dbConnection.close()
def __init__(self):
self.__log = Logger.getLogger()
self.__servers = ServerList()
self.__servers.parseList()
self.__sockets = {}
for name in self.__servers.getNames():
try:
serverAddress = self.__servers.getAddress(name)
# create a TCP/IP socket and connect to remote server
serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serverSocket.connect(serverAddress)
# save socket
self.__sockets[name] = serverSocket
except socket.error, e:
self.__log.error("Socket error: " + str(e))
raise IOError
def do_init(self):
if self.did_init_osc_server == 0:
self.did_init_osc_server = 1
try:
self.oscServer = RemixNet.OSCServer(self, '127.0.0.1', 9111,
None, 9008)
self.callbackManager = self.oscServer.callbackManager
except:
self.error_log("touchAble: OSCServer init failed")
self.did_init_osc_server = 0
pass
try:
self.tACommon = touchAbleCommon.touchAbleCommon(
self._touchAble__c_instance, self.oscServer, self)
except:
self.did_init = 0
self.error_log("touchAble: tACommon init failed")
pass
try:
self.tAListener = touchAbleListener.touchAbleListener(
self._touchAble__c_instance, self.oscServer, self)
except:
self.did_init = 0
self.error_log("touchAble: tAListener init failed")
pass
try:
self.logger = self._LOG and Logger() or 0
self.tA_log("Logging Enabled")
except:
self.error_log("touchAble: Logger init failed")
pass
try:
with self.component_guard():
pass
#self._session_ring = self._create_session() # SVH Change # Zerodebug Addition
except:
self.error_log("touchAble: _create_session failed")
self.did_init = 1
else:
self.did_init = 0
pass
def __init__(self, socket, clientId):
super(ClientThread, self).__init__()
self.__name = clientId
self.__log = Logger.getLogger()
self.__socket = socket
self.__messageParser = MessageParser()
self.__database = ServerDatabase()
self.__log.debug("Creating client thread [%s]", self.__name)
# create functions mapping
self.__handlers = {
"setLocation": self.__setLocation,
"getLocation": self.__getLocation,
"allowTracking": self.__allowTracking,
"registerUser": self.__registerUser,
"loginUser": self.__loginUser,
"checkPermission": self.__checkPermission,
"invalid": self.__invalidMessage
}
def __init__(self): self.__log = Logger.getLogger() # create a TCP/IP socket self.__socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
def test_func(args,
shared_model,
env_conf,
datasets=None,
tests=None,
shared_dict=None):
ptitle('Valid agent')
if args.valid_gpu < 0:
gpu_id = args.gpu_ids[-1]
else:
gpu_id = args.valid_gpu
env_conf["env_gpu"] = gpu_id
if not args.deploy:
log = {}
logger = Logger(args.log_dir)
create_dir(args.log_dir + "models/")
create_dir(args.log_dir + "tifs/")
create_dir(args.log_dir + "tifs_test/")
os.system("cp *.py " + args.log_dir)
os.system("cp *.sh " + args.log_dir)
os.system("cp models/*.py " + args.log_dir + "models/")
setup_logger('{}_log'.format(args.env),
r'{0}{1}_log'.format(args.log_dir, args.env))
log['{}_log'.format(args.env)] = logging.getLogger('{}_log'.format(
args.env))
d_args = vars(args)
env_conf_log = env_conf
if tests is not None:
if args.testlbl:
test_env = EM_env(tests[0],
env_conf,
type="test",
gt_lbl_list=tests[1])
else:
test_env = EM_env(tests[0], env_conf, type="test")
if not args.deploy:
for k in d_args.keys():
log['{}_log'.format(args.env)].info('{0}: {1}'.format(
k, d_args[k]))
for k in env_conf_log.keys():
log['{}_log'.format(args.env)].info('{0}: {1}'.format(
k, env_conf_log[k]))
torch.manual_seed(args.seed)
if gpu_id >= 0:
torch.cuda.manual_seed(args.seed)
raw_list, gt_lbl_list = datasets
env = EM_env(raw_list, env_conf, type="train", gt_lbl_list=gt_lbl_list)
reward_sum = 0
start_time = time.time()
num_tests = 0
reward_total_sum = 0
player = Agent(None, env, args, None)
player.gpu_id = gpu_id
player.model = get_model(args,
args.model,
env_conf["observation_shape"],
args.features,
atrous_rates=args.atr_rate,
num_actions=2,
split=args.data_channel,
gpu_id=gpu_id,
multi=args.multi)
player.state = player.env.reset()
player.state = torch.from_numpy(player.state).float()
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
player.model = player.model.cuda()
player.state = player.state.cuda()
player.model.eval()
flag = True
if not args.deploy:
create_dir(args.save_model_dir)
recent_episode_scores = ScalaTracker(100)
recent_FgBgDice = ScalaTracker(100)
recent_bestDice = ScalaTracker(100)
recent_diffFG = ScalaTracker(100)
recent_MUCov = ScalaTracker(100)
recent_MWCov = ScalaTracker(100)
recent_AvgFP = ScalaTracker(100)
recent_AvgFN = ScalaTracker(100)
recent_rand_i = ScalaTracker(100)
renderlist = []
renderlist.append(player.env.render())
max_score = 0
# ----------------------------------------- Deploy / Inference -----------------------------------------
if args.deploy:
with torch.cuda.device(gpu_id):
player.model.load_state_dict(shared_model.state_dict())
# inference (args, None, player.model, tests [0], test_env, gpu_id, player.env.rng, len (tests [0]))
if len(tests) == 4:
inference(args, None, player.model, tests[0], test_env, gpu_id,
player.env.rng, len(tests[0]), tests[3])
else:
inference(args, None, player.model, tests[0], test_env, gpu_id,
player.env.rng, len(tests[0]))
return
# ----------------------------------------- End Deploy / Inference -----------------------------------------
merge_ratios = []
split_ratios = []
if args.wctrl == "s2m":
schedule = args.wctrl_schedule
delta = (shared_dict['spl_w'] - shared_dict['mer_w']) / (2 *
len(schedule))
mer_w_delta = delta
mer_w_var = shared_dict['mer_w']
mer_w_scheduler = Scheduler(mer_w_var, schedule, mer_w_delta)
split_delta = -delta / len(args.out_radius)
split_var = shared_dict['spl_w'] / len(args.out_radius)
spl_w_scheduler = Scheduler(split_var, schedule, split_delta)
while True:
if flag:
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
player.model.load_state_dict(shared_model.state_dict())
else:
player.model.load_state_dict(shared_model.state_dict())
player.model.eval()
flag = False
player.action_test()
reward_sum += player.reward.mean()
renderlist.append(player.env.render())
if player.done:
flag = True
num_tests += 1
reward_total_sum += reward_sum
reward_mean = reward_total_sum / num_tests
log['{}_log'.format(args.env)].info(
"VALID: Time {0}, episode reward {1}, num tests {4}, episode length {2}, reward mean {3:.4f}"
.format(
time.strftime("%Hh %Mm %Ss",
time.gmtime(time.time() - start_time)),
reward_sum, player.eps_len, reward_mean, num_tests))
recent_episode_scores.push(reward_sum)
if args.save_max and recent_episode_scores.mean() >= max_score:
max_score = recent_episode_scores.mean()
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
state_to_save = {}
state_to_save = player.model.state_dict()
torch.save(
state_to_save,
'{0}{1}.dat'.format(args.save_model_dir,
'best_model_' + args.env))
if num_tests % args.save_period == 0:
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
state_to_save = player.model.state_dict()
torch.save(
state_to_save,
'{0}{1}.dat'.format(args.save_model_dir,
str(num_tests)))
if num_tests % args.log_period == 0:
if tests is not None and not args.DEBUG:
inference(args, logger, player.model, tests[0], test_env,
gpu_id, player.env.rng, num_tests)
if (np.max(env.lbl) != 0 and np.max(env.gt_lbl) != 0):
bestDice, FgBgDice, diffFG, MWCov, MUCov, AvgFP, AvgFN, rand_i = evaluate(
args, player.env)
recent_FgBgDice.push(FgBgDice)
recent_diffFG.push(abs(diffFG))
recent_bestDice.push(bestDice)
recent_MWCov.push(MWCov)
recent_MUCov.push(MUCov)
recent_AvgFP.push(AvgFP)
recent_AvgFN.push(AvgFN)
recent_rand_i.push(rand_i)
log_info = {
"bestDice": recent_bestDice.mean(),
"FgBgDice": recent_FgBgDice.mean(),
"diffFG": recent_diffFG.mean(),
"MWCov": recent_MWCov.mean(),
"MUCov": recent_MUCov.mean(),
"AvgFP": recent_AvgFP.mean(),
"AvgFN": recent_AvgFN.mean(),
"rand_i": recent_rand_i.mean()
}
for tag, value in log_info.items():
logger.scalar_summary(tag, value, num_tests)
else:
bestDice, FgBgDice, diffFG = 0, 0, 0
MWCov, MUCov, AvgFP, AvgFN = 0, 0, 0, 0
rand_i = 0
print(
"----------------------VALID SET--------------------------"
)
print(args.env)
print("bestDice:", bestDice, "FgBgDice:", FgBgDice, "diffFG:",
diffFG, "MWCov:", MWCov, "MUCov:", MUCov, "AvgFP:",
AvgFP, "AvgFN:", AvgFN, "rand_i:", rand_i)
# print ("mean bestDice")
print("Log test #:", num_tests)
print("rewards: ", player.reward.mean())
print("sum rewards: ", reward_sum)
print("#gt_values:", len(np.unique(player.env.gt_lbl)))
print("values:")
values = player.env.unique()
print(np.concatenate([values[0][None], values[1][None]], 0))
print("------------------------------------------------")
log_img = np.concatenate(renderlist[::-1], 0)
if not "3D" in args.data:
for i in range(3):
player.probs.insert(0, np.zeros_like(player.probs[0]))
while (len(player.probs) - 3 < args.max_episode_length):
player.probs.append(np.zeros_like(player.probs[0]))
probslist = [
np.repeat(np.expand_dims(prob, -1), 3, -1)
for prob in player.probs
]
probslist = np.concatenate(probslist, 1)
probslist = (probslist * 256).astype(np.uint8, copy=False)
# log_img = renderlist [-1]
print(probslist.shape, log_img.shape)
log_img = np.concatenate([probslist, log_img], 0)
log_info = {"valid_sample": log_img}
print(log_img.shape)
io.imsave(
args.log_dir + "tifs/" + str(num_tests) + "_sample.tif",
log_img.astype(np.uint8))
io.imsave(
args.log_dir + "tifs/" + str(num_tests) + "_pred.tif",
player.env.lbl.astype(np.uint8))
io.imsave(args.log_dir + "tifs/" + str(num_tests) + "_gt.tif",
player.env.gt_lbl.astype(np.int32))
if args.seg_scale:
log_info["scaler"] = player.env.scaler
for tag, img in log_info.items():
img = img[None]
logger.image_summary(tag, img, num_tests)
if not args.deploy:
log_info = {
'mean_valid_reward':
reward_mean,
'100_mean_reward':
recent_episode_scores.mean(),
'split_ratio':
player.env.split_ratio_sum.sum() /
np.count_nonzero(player.env.gt_lbl),
'merge_ratio':
player.env.merge_ratio_sum.sum() /
np.count_nonzero(player.env.gt_lbl),
}
if args.wctrl == 's2m':
log_info.update({
'mer_w':
mer_w_scheduler.value(),
'spl_w':
spl_w_scheduler.value() * len(args.out_radius),
})
merge_ratios.append(player.env.merge_ratio_sum.sum() /
np.count_nonzero(player.env.gt_lbl))
split_ratios.append(player.env.split_ratio_sum.sum() /
np.count_nonzero(player.env.gt_lbl))
print("split ratio: ", np.max(player.env.split_ratio_sum),
np.min(player.env.split_ratio_sum))
print("merge ratio: ", np.max(player.env.merge_ratio_sum),
np.min(player.env.merge_ratio_sum))
print("merge ratio: ", merge_ratios)
print("split ratio: ", split_ratios)
for tag, value in log_info.items():
logger.scalar_summary(tag, value, num_tests)
renderlist = []
reward_sum = 0
player.eps_len = 0
if args.wctrl == "s2m":
shared_dict["spl_w"] = spl_w_scheduler.next()
shared_dict["mer_w"] = mer_w_scheduler.next()
player.env.config["spl_w"] = shared_dict["spl_w"]
player.env.config["mer_w"] = shared_dict["mer_w"]
player.clear_actions()
state = player.env.reset(player.model, gpu_id)
renderlist.append(player.env.render())
time.sleep(15)
player.state = torch.from_numpy(state).float()
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
player.state = player.state.cuda()
class US14_32Test(unittest.TestCase):
families = None
individuals = None
famMap = None
indMap = None
logger = None
spousecheck = None
def setUp(self):
self.families = []
self.individuals = []
self.famMap = {}
self.indMap = {}
self.logger = Logger()
self.seed_data()
self.spousecheck = None
def tearDown(self):
self.families = None
self.individuals = None
self.famMap = None
self.indMap = None
self.logger = None
self.spousecheck = None
def seed_data(self):
# seed initial testing data
self.families.append({
"HUSB": ["I1"],
"WIFE": ["I2"],
"CHIL": ["I10"],
"MARR": [1, 1, 2009],
"NOTE": "JAY/GLORIA FAMILY",
"FAM": "F1"
})
self.families.append({
"HUSB": ["I1"],
"WIFE": ["I3"],
"CHIL": ["I4", "I6"],
"MARR": [1, 2, 1968],
"DIV": [1, 2, 2003],
"NOTE": "JAY/DEEDEE",
"FAM": "F2"
})
self.families.append({
"HUSB": ["I8"],
"WIFE": ["I2"],
"CHIL": ["I9"],
"MARR": [1, 1, 1995],
"DIV": [1, 1, 2006],
"NOTE": "JAVIER/GLORIA",
"FAM": "F3"
})
self.families.append({
"HUSB": ["I4", "I5"],
"CHIL": ["I14", "I28", "I29", "I30"],
"MARR": [1, 1, 2014],
"NOTE": "PRITCHETT/TUCKER FAMILY",
"FAM": "F4",
"WIFE": "-"
})
self.families.append({
"HUSB": ["I16"],
"WIFE": ["I17"],
"CHIL": ["I5", "I14"],
"MARR": [1, 1, 1963],
"NOTE": "MERLE/BARB FAMILY",
"FAM": "F5"
})
self.families.append({
"HUSB": ["I7"],
"WIFE": ["I6"],
"CHIL": ["I20", "I24", "I25"],
"MARR": [1, 4, 1993],
"NOTE": "PHIL/CLAIRE FAMILY",
"FAM": "F6"
})
self.families.append({
"HUSB": ["I11"],
"WIFE": ["I13"],
"CHIL": ["I7"],
"MARR": [1, 1, 1965],
"NOTE": "FRANK/GRACE FAMILY",
"FAM": "F7"
})
self.families.append({
"HUSB": ["I11"],
"WIFE": ["I12"],
"MARR": [5, 4, 2017],
"NOTE": "FRANK/LORRAINE FAMILY ",
"FAM": "F8"
})
self.families.append({
"WIFE": ["I14"],
"CHIL": ["I19"],
"NOTE": "PAMERON TUCKER FAMILY",
"FAM": "F9",
"MARR": "-",
"HUSB": "-"
})
self.families.append({
"HUSB": ["I21"],
"WIFE": ["I20"],
"CHIL": ["I22", "I23"],
"MARR": [8, 3, 2019],
"FAM": "F10",
"NOTE": "MARSHALL/DUNPHY FAMILY"
})
self.families.append({
"HUSB": ["I26"],
"WIFE": ["I27"],
"CHIL": ["I26"],
"MARR": [16, 1, 2014],
"NOTE": "MarryToChildFAMILY",
"FAM": "F11"
})
self.individuals.append({
"NAME": "Jay/Pritchett/",
"SEX": "M",
"BIRT": [28, 12, 2021],
"FAMS": ["F1", "F2"],
"AGE": -1,
"INDI": "I1"
})
self.individuals.append({
"NAME": "Gloria/Unknown/",
"SEX": "F",
"BIRT": [10, 5, 1971],
"FAMS": ["F1", "F3"],
"AGE": 49,
"INDI": "I2"
})
self.individuals.append({
"NAME": "DeDe/Pritchett/",
"SEX": "F",
"BIRT": [23, 1, 1947],
"DEAT": [1, 10, 2100],
"FAMS": ["F2"],
"AGE": 143,
"INDI": "I3"
})
self.individuals.append({
"NAME": "Mitchell/Pritchett/",
"SEX": "M",
"BIRT": [1, 6, 1975],
"FAMS": ["F4"],
"FAMC": ["F2"],
"AGE": 45,
"INDI": "I4"
})
self.individuals.append({
"NAME": "Cameron/Tucker/",
"SEX": "M",
"BIRT": [29, 2, 1972],
"FAMS": ["F4"],
"FAMC": ["F5"],
"AGE": 48,
"INDI": "I5"
})
self.individuals.append({
"NAME": "Claire/Pritchett/",
"SEX": "F",
"BIRT": [3, 3, 1970],
"FAMS": ["F6"],
"FAMC": ["F2"],
"AGE": 50,
"INDI": "I6"
})
self.individuals.append({
"NAME": "Phil/Dunphy/",
"SEX": "M",
"BIRT": [3, 4, 1967],
"FAMS": ["F6"],
"FAMC": ["F7"],
"AGE": 53,
"INDI": "I7"
})
self.individuals.append({
"NAME": "Javier/Delgado/",
"SEX": "M",
"BIRT": [1, 1, 1969],
"FAMS": ["F3"],
"AGE": 51,
"INDI": "I8"
})
self.individuals.append({
"NAME": "Manny/Delgado/",
"SEX": "M",
"BIRT": [4, 1, 1999],
"FAMC": ["F3"],
"AGE": 21,
"INDI": "I9"
})
self.individuals.append({
"NAME": "Joe/Pritchett/",
"SEX": "M",
"BIRT": [4, 1, 2013],
"FAMC": ["F1"],
"NOTE": "Duplicatenameandbirthdayindividual",
"AGE": 7,
"INDI": "I10"
})
self.individuals.append({
"NAME": "Jay/Pritchett/",
"SEX": "M",
"BIRT": [28, 12, 2021],
"AGE": -1,
"INDI": "I89"
})
self.individuals.append({
"NAME": "Frank/Dunphy/",
"SEX": "M",
"BIRT": [1, 1, 1945],
"DEAT": [14, 1, 2020],
"FAMS": ["F7", "F8"],
"AGE": 75,
"INDI": "I11"
})
self.individuals.append({
"NAME": "Lorraine/Dunphy/",
"SEX": "F",
"BIRT": [1, 1, 1965],
"FAMS": ["F8"],
"AGE": 55,
"INDI": "I12"
})
self.individuals.append({
"NAME": "Grace/Dunphy/",
"SEX": "F",
"BIRT": [1, 1, 1945],
"DEAT": [1, 1, 2009],
"FAMS": ["F7"],
"AGE": 64,
"INDI": "I13"
})
self.individuals.append({
"NAME": "Lily/Tucker-Pritchett/",
"SEX": "F",
"BIRT": [19, 2, 2008],
"FAMC": ["F4"],
"AGE": 12,
"INDI": "I14"
})
self.individuals.append({
"NAME": "Rexford/Tucker-Pritchett/",
"SEX": "M",
"BIRT": [1, 4, 2020],
"FAMC": ["F4"],
"AGE": 0,
"INDI": "I14"
})
self.individuals.append({
"NAME": "Merle/Tucker/",
"SEX": "M",
"BIRT": [1, 1, 1943],
"FAMS": ["F5"],
"AGE": 77,
"INDI": "I16"
})
self.individuals.append({
"NAME": "Barb/Tucker/",
"SEX": "F",
"BIRT": [1, 1, 1943],
"FAMS": ["F5"],
"AGE": 77,
"INDI": "I17"
})
self.individuals.append({
"NAME": "Pameron/Tucker/",
"SEX": "F",
"BIRT": [1, 1, 1970],
"FAMS": ["F9"],
"FAMC": ["F5"],
"AGE": 50,
"INDI": "I14"
})
self.individuals.append({
"NAME": "Calhoun/Tucker/",
"SEX": "M",
"BIRT": [5, 4, 2017],
"FAMC": ["F9"],
"AGE": 3,
"INDI": "I19"
})
self.individuals.append({
"NAME": "Haley/Dunphy/",
"SEX": "F",
"BIRT": [10, 12, 1993],
"FAMS": ["F10"],
"FAMC": ["F6"],
"AGE": 27,
"INDI": "I20"
})
self.individuals.append({
"NAME": "Dylan/Marshall/",
"SEX": "M",
"BIRT": [3, 4, 1991],
"FAMS": ["F10"],
"AGE": 29,
"INDI": "I21"
})
self.individuals.append({
"NAME": "Poppy/Marshall/",
"SEX": "F",
"BIRT": [8, 5, 2019],
"FAMC": ["F10"],
"AGE": 1,
"INDI": "I22"
})
self.individuals.append({
"NAME": "George/Hastings/",
"SEX": "M",
"BIRT": [8, 5, 2019],
"FAMC": ["F10"],
"AGE": 1,
"INDI": "I23"
})
self.individuals.append({
"NAME": "Alex/Dunphy/",
"SEX": "F",
"BIRT": [14, 1, 1997],
"FAMC": ["F6"],
"AGE": 23,
"INDI": "I24"
})
self.individuals.append({
"NAME": "Luke/Dunphy/",
"SEX": "M",
"BIRT": [28, 11, 1998],
"FAMC": ["F6"],
"NOTE": "JAY/GLORIAFAMILY",
"AGE": 22,
"INDI": "I25"
})
self.individuals.append({
"NAME": "Luke/Hastings/",
"SEX": "M",
"BIRT": [28, 11, 1998],
"FAMS": ["F11"],
"FAMC": ["F11"],
"NOTE": "MarryToChildFAMILY",
"AGE": 22,
"INDI": "I26"
})
self.individuals.append({
"NAME": "Mary/Hastings/",
"SEX": "F",
"BIRT": [28, 11, 1970],
"FAMS": ["F11"],
"NOTE": "MarryToChildFAMILY",
"AGE": 50,
"INDI": "I27"
})
self.individuals.append({
"NAME": "Sammy/Tucker-Pritchett/",
"SEX": "M",
"BIRT": [1, 4, 2020],
"FAMC": ["F4"],
"AGE": 0,
"INDI": "I28"
})
self.individuals.append({
"NAME": "Beatrice/Tucker-Pritchett/",
"SEX": "M",
"BIRT": [1, 4, 2020],
"FAMC": ["F4"],
"AGE": 0,
"INDI": "I29"
})
self.individuals.append({
"NAME": "John/Tucker-Pritchett/",
"SEX": "M",
"BIRT": [1, 4, 2020],
"FAMC": ["F4"],
"AGE": 0,
"INDI": "I30"
})
for ind in self.individuals:
self.indMap[ind["INDI"]] = ind
for fam in self.families:
self.famMap[fam["FAM"]] = normalize_family_entry(fam)
def test_US32_DefaultCount(self):
# should return F4 & F10 family with twins
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck._mult_births(32, 2)
ret = self.logger.get_logs()
self.assertEqual(len(ret), 2, "Did not get the expected results")
expected_ret = [
('Warning', 'Family', 32,
'F4 has 3 children with the same birthday (4/1/2020): I28, I29, I30'
),
('Warning', 'Family', 32,
'F10 has 2 children with the same birthday (5/8/2019): I22, I23')
]
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck.us32_mult_births()
ret = self.logger.get_logs()
self.assertEqual(len(ret), 2, "Did not get the expected results")
expected_ret = [
('Warning', 'Family', 32,
'F4 has 3 children with the same birthday (4/1/2020): I28, I29, I30'
),
('Warning', 'Family', 32,
'F10 has 2 children with the same birthday (5/8/2019): I22, I23')
]
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
def test_ThreeOrMore(self):
# should get the following families with multiple births of 3 or more
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck._mult_births(14, 3)
ret = self.logger.get_logs()
self.assertEqual(len(ret), 1, "Did not get the expected results")
def test_ThreeOrMoreText(self):
# should get the following families with multiple births of 3 or more
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck._mult_births(14, 3)
ret = self.logger.get_logs()
expected_ret = [(
'Warning', 'Family', 14,
'F4 has 3 children with the same birthday (4/1/2020): I28, I29, I30'
)]
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
def test_TwoorMoreMultBirthsinOneFamiy(self):
# add in a set of sixtuplets to family 4 - so they have triplets and twins
for i in range(31, 37):
self.indMap["I" + str(i)] = {
"NAME": "I" + str(i) + "/Tucker-Pritchett/",
"SEX": "F",
"BIRT": [1, 5, 2019],
"FAMC": ["F4"],
"NOTE": "Adding Multiple Birth Child to Family F4",
"INDI": "I" + str(i)
}
self.famMap["F4"]["CHIL"].append("I" + str(i))
# private routine
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck._mult_births(33, 2)
ret = self.logger.get_logs()
self.assertEqual(len(ret), 3, "Did not get the expected results")
expected_ret = [
('Warning', 'Family', 33,
'F4 has 3 children with the same birthday (4/1/2020): I28, I29, I30'
),
('Warning', 'Family', 33,
'F4 has 6 children with the same birthday (5/1/2019): I31, I32, I33, I34, I35, I36'
),
('Warning', 'Family', 33,
'F10 has 2 children with the same birthday (5/8/2019): I22, I23')
]
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
# user story 32
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck.us32_mult_births()
ret = self.logger.get_logs()
self.assertEqual(len(ret), 3, "Did not get the expected results")
expected_ret = [
('Warning', 'Family', 32,
'F4 has 3 children with the same birthday (4/1/2020): I28, I29, I30'
),
('Warning', 'Family', 32,
'F4 has 6 children with the same birthday (5/1/2019): I31, I32, I33, I34, I35, I36'
),
('Warning', 'Family', 32,
'F10 has 2 children with the same birthday (5/8/2019): I22, I23')
]
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
# user story 14
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck.us14_mult_births()
ret = self.logger.get_logs()
self.assertEqual(len(ret), 1, "Did not get the expected results")
expected_ret = [(
'Warning', 'Family', 14,
'F4 has 6 children with the same birthday (5/1/2019): I31, I32, I33, I34, I35, I36'
)]
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
def test_FiveorMoreMultBirthsinOneFamiy(self):
# add in a set of sixtuplets to family 4 - so they have triplets and twins
for i in range(31, 36):
self.indMap["I" + str(i)] = {
"NAME": "I" + str(i) + "/Tucker-Pritchett/",
"SEX": "F",
"BIRT": [1, 5, 2019],
"FAMC": ["F4"],
"NOTE": "Adding Multiple Birth Child to Family F4",
"INDI": "I" + str(i)
}
self.famMap["F4"]["CHIL"].append("I" + str(i))
# user story 14
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck.us14_mult_births()
ret = self.logger.get_logs()
self.assertEqual(len(ret), 0, "Did not get the expected results")
expected_ret = []
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
# user story 32
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck.us32_mult_births()
ret = self.logger.get_logs()
self.assertEqual(len(ret), 3, "Did not get the expected results")
expected_ret = [
('Warning', 'Family', 32,
'F4 has 3 children with the same birthday (4/1/2020): I28, I29, I30'
),
('Warning', 'Family', 32,
'F4 has 5 children with the same birthday (5/1/2019): I31, I32, I33, I34, I35'
),
('Warning', 'Family', 32,
'F10 has 2 children with the same birthday (5/8/2019): I22, I23')
]
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
def test_SixorMoreMultBirthsinOneFamiy(self):
# add in a set of sixtuplets to family 4 - so they have triplets and twins
for i in range(31, 37):
self.indMap["I" + str(i)] = {
"NAME": "I" + str(i) + "/Tucker-Pritchett/",
"SEX": "F",
"BIRT": [1, 5, 2019],
"FAMC": ["F4"],
"NOTE": "Adding Multiple Birth Child to Family F4",
"INDI": "I" + str(i)
}
self.famMap["F4"]["CHIL"].append("I" + str(i))
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck._mult_births(14, 6)
ret = self.logger.get_logs()
self.assertEqual(len(ret), 1, "Did not get the expected results")
expected_ret = [(
'Warning', 'Family', 14,
'F4 has 6 children with the same birthday (5/1/2019): I31, I32, I33, I34, I35, I36'
)]
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck.us14_mult_births()
ret = self.logger.get_logs()
self.assertEqual(len(ret), 1, "Did not get the expected results")
expected_ret = [(
'Warning', 'Family', 14,
'F4 has 6 children with the same birthday (5/1/2019): I31, I32, I33, I34, I35, I36'
)]
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
# user story 32
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck.us32_mult_births()
ret = self.logger.get_logs()
self.assertEqual(len(ret), 3, "Did not get the expected results")
expected_ret = [
('Warning', 'Family', 32,
'F4 has 3 children with the same birthday (4/1/2020): I28, I29, I30'
),
('Warning', 'Family', 32,
'F4 has 6 children with the same birthday (5/1/2019): I31, I32, I33, I34, I35, I36'
),
('Warning', 'Family', 32,
'F10 has 2 children with the same birthday (5/8/2019): I22, I23')
]
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
def test_SevenorMoreMultBirthsinOneFamiy(self):
# add in a set of sixtuplets to family 4 - so they have triplets and twins
# won't find a match on 7
for i in range(31, 37):
self.indMap["I" + str(i)] = {
"NAME": "I" + str(i) + "/Tucker-Pritchett/",
"SEX": "F",
"BIRT": [1, 5, 2019],
"FAMC": ["F4"],
"NOTE": "Adding Multiple Birth Child to Family F4",
"INDI": "I" + str(i)
}
self.famMap["F4"]["CHIL"].append("I" + str(i))
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck._mult_births(14, 7)
ret = self.logger.get_logs()
self.assertEqual(len(ret), 0, "Did not get the expected results")
expected_ret = []
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
# user story 14
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck.us14_mult_births()
ret = self.logger.get_logs()
self.assertEqual(len(ret), 1, "Did not get the expected results")
expected_ret = [
('Warning', 'Family', 14,
'F4 has 6 children with the same birthday (5/1/2019): I31, I32, I33, I34, I35, I36'
),
]
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
# user story 32
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck.us32_mult_births()
ret = self.logger.get_logs()
self.assertEqual(len(ret), 3, "Did not get the expected results")
expected_ret = [
('Warning', 'Family', 32,
'F4 has 3 children with the same birthday (4/1/2020): I28, I29, I30'
),
('Warning', 'Family', 32,
'F4 has 6 children with the same birthday (5/1/2019): I31, I32, I33, I34, I35, I36'
),
('Warning', 'Family', 32,
'F10 has 2 children with the same birthday (5/8/2019): I22, I23')
]
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
def __init__(self, db: str):
self.db = db
self.logger = Logger(write=False)
self.c = ""
class App:
__tabfile_load_text = "loading tabfile"
__settings_load_text = "loading settings"
__wait_tabfile_data_text = "Tabfile is empty. Wait for new data..."
__tabfile_loaded_text = "tabfile data successfully loaded"
__app_start_text = "Starting up application..."
__app_started_text = "Application started successfully"
__app_closing_text = "Clothing application..."
__app_closed_text = "Application closed successfully"
__run_executer_text = "Starting up Executer..."
__stop_executer_text = "Excuter stopped"
__run_analyzer_text = "Starting up Analyzer..."
__stop_analyzer_text = "Analyzer stopped"
__run_refresher_text = "Starting up tabfile refresh..."
__stop_refresher_text = "Tabfile refresh stopped"
__default_exception_text = "An exception of type {0} occurred while {1}. {2}"
def __init__(self):
self.init_components()
self.threads_manager = ThreadsManager(self.logger, self.settings,
self.settings["threads_max"])
self.init_threads()
self.refresh_event = threading.Event()
def init_threads(self):
self.threads_manager.add_thread(
threading.Thread(target=self.run_executer, name="executer"))
self.threads_manager.add_thread(
threading.Thread(target=self.__refresh_crontab, name="refresher"))
self.threads_manager.add_thread(
threading.Thread(target=self.analysis_execution, name="analyzer"))
@staticmethod
def __get_formatted_error(ex, text, additional_info=""):
return App.__default_exception_text.format(
type(ex).__name__, text, additional_info)
def __init_settings(self):
try:
settings = Settings(JSONWriter, JSONReader)
settings.load()
self._settings_obj = settings
self.settings = settings.settings
except Exception as ex:
self.threads_manager.stop_event.set()
self.logger.log_fatal(
App.__get_formatted_error(ex, App.__settings_load_text))
self.close_application(_AppStatus.INIT_SETTINGS_ERROR)
else:
self.logger.log_level = self.settings["log_level"]
self.logger.log_info(App.__app_started_text)
def __refresh_crontab(self):
self.logger.log_info(App.__run_refresher_text)
file_empty = False
empty_delay = 1
while True:
self.refresh_event.clear()
try:
self.cron = CronTab(
tabfile=PathProvider.pathes["TAB"].location,
logger=self.logger,
settings=self.settings)
except ValueError as ex:
if not file_empty:
self.logger.log_error(
App.__get_formatted_error(
ex, App.__tabfile_load_text,
App.__wait_tabfile_data_text))
file_empty = True
time.sleep(empty_delay)
except FileNotFoundError as ex:
self.logger.log_fatal(
App.__get_formatted_error(ex, App.__tabfile_load_text))
self.close_application(_AppStatus.TABFILE_NOT_FOUND_ERROR)
else:
self.logger.log_info(App.__tabfile_loaded_text)
self.refresh_event.set()
if self.settings["tabfile_refresh"]:
time.sleep(self.settings["tabfile_refresh_time"])
else:
break
if self.threads_manager.stop_event.is_set():
return
def init_components(self):
PathProvider.load()
self.logger = Logger(FileWriter, PathProvider.pathes["LOG"].location)
self.logger.log_info(App.__app_start_text)
self.__init_settings()
def analyze_once(self):
self.refresh_event.wait()
analysis_logger = self.logger
if not self.settings["log_analysis"]:
analysis_logger = Logger(FileWriter,
PathProvider.pathes["ANALYSIS"].location)
Analyzer(self.cron, self.settings["analyzer_iterations_amount"],
analysis_logger).analyze_all()
def analysis_execution(self):
self.logger.log_info(App.__run_analyzer_text)
if self.settings["analyze_at_startup_only"]:
self.analyze_once()
else:
while True:
self.analyze_once()
time.sleep(self.settings["analyzer_period"])
def run_executer(self):
self.logger.log_info(App.__run_executer_text)
self.refresh_event.wait()
self.executer = Executer(self.cron, self.logger, self.settings,
self.threads_manager)
self.executer.run(self.refresh_event)
def close_application(self, code):
self.logger.log_info(App.__app_closing_text)
PathProvider.save()
try:
self._settings_obj.save()
except Exception as ex:
self.logger.log_fatal(App.__get_formatted_error(ex, ex.args))
sys.exit(_AppStatus.SAVING_SETTINGS_ERROR)
else:
self.logger.log_info(App.__app_closed_text)
sys.exit(code)
def sigint_handler(self, signum, frame):
self.threads_manager.stop_event.set()
self.threads_manager.stop_all_threads()
self.close_application(_AppStatus.NORMAL_EXIT)
def refresh(self):
self.threads_manager.start_thread("refresher")
def execute(self):
self.threads_manager.start_thread("executer")
def analyze(self):
if self.settings["analyzer_enable"]:
self.threads_manager.start_thread("analyzer")
def train_func(rank,
args,
shared_model,
optimizer,
env_conf,
datasets=None,
shared_dict=None):
if args.deploy:
return
ptitle('Train {0}'.format(rank))
print('Start training agent: ', rank)
if rank == 0:
logger = Logger(args.log_dir[:-1] + '_losses/')
train_step = 0
gpu_id = args.gpu_ids[rank % len(args.gpu_ids)]
env_conf["env_gpu"] = gpu_id
torch.manual_seed(args.seed + rank)
if gpu_id >= 0:
torch.cuda.manual_seed(args.seed + rank)
raw_list, gt_lbl_list = datasets
env = EM_env(raw_list,
env_conf,
type="train",
gt_lbl_list=gt_lbl_list,
seed=args.seed + rank)
if optimizer is None:
if args.optimizer == 'RMSprop':
optimizer = optim.RMSprop(shared_model.parameters(), lr=args.lr)
if args.optimizer == 'Adam':
optimizer = optim.Adam(shared_model.parameters(),
lr=args.lr,
amsgrad=args.amsgrad)
player = Agent(None, env, args, None)
player.gpu_id = gpu_id
player.model = get_model(args,
args.model,
env.observation_space.shape,
args.features,
atrous_rates=args.atr_rate,
num_actions=2,
split=args.data_channel,
gpu_id=gpu_id,
multi=args.multi)
player.state = player.env.reset()
player.state = torch.from_numpy(player.state).float()
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
player.state = player.state.cuda()
player.model = player.model.cuda()
player.model.train()
if rank == 0:
eps_reward = 0
pinned_eps_reward = 0
while True:
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
player.model.load_state_dict(shared_model.state_dict())
else:
player.model.load_state_dict(shared_model.state_dict())
if player.done:
player.eps_len = 0
if rank == 0:
if train_step % args.train_log_period == 0 and train_step > 0:
print("train: step", train_step, "\teps_reward",
eps_reward)
if train_step > 0:
pinned_eps_reward = player.env.sum_reward.mean()
eps_reward = 0
if args.lstm_feats:
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
player.cx, player.hx = player.model.lstm.init_hidden(
batch_size=1, use_cuda=True)
else:
player.cx, player.hx = player.model.lstm.init_hidden(
batch_size=1, use_cuda=False)
elif args.lstm_feats:
player.cx = Variable(player.cx.data)
player.hx = Variable(player.hx.data)
for step in range(args.num_steps):
if rank < args.lbl_agents:
player.action_train(use_lbl=True)
else:
player.action_train()
if rank == 0:
eps_reward = player.env.sum_reward.mean()
if player.done:
break
if player.done:
state = player.env.reset(player.model, gpu_id)
player.state = torch.from_numpy(state).float()
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
player.state = player.state.cuda()
if "3D" in args.data:
R = torch.zeros(1, 1, env_conf["size"][0], env_conf["size"][1],
env_conf["size"][2])
else:
R = torch.zeros(1, 1, env_conf["size"][0], env_conf["size"][1])
if args.lowres:
R = torch.zeros(1, 1, env_conf["size"][0] // 2,
env_conf["size"][1] // 2)
if not player.done:
if args.lstm_feats:
value, _, _ = player.model(
(Variable(player.state.unsqueeze(0)), (player.hx,
player.cx)))
else:
value, _ = player.model(Variable(player.state.unsqueeze(0)))
R = value.data
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
R = R.cuda()
player.values.append(Variable(R))
policy_loss = 0
value_loss = 0
if "3D" in args.data:
gae = torch.zeros(1, 1, env_conf["size"][0], env_conf["size"][1],
env_conf["size"][2])
else:
gae = torch.zeros(1, 1, env_conf["size"][0], env_conf["size"][1])
if args.rew_drop:
keep_map = torch.tensor(player.env.keep_map)
if args.lowres:
gae = torch.zeros(1, 1, env_conf["size"][0] // 2,
env_conf["size"][1] // 2)
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
gae = gae.cuda()
if args.rew_drop:
keep_map = keep_map.cuda()
R = Variable(R)
for i in reversed(range(len(player.rewards))):
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
reward_i = torch.tensor(player.rewards[i]).cuda()
else:
reward_i = torch.tensor(player.rewards[i])
R = args.gamma * R + reward_i
if args.rew_drop:
advantage = R - player.values[i]
value_loss = value_loss + (0.5 * advantage * advantage *
keep_map).mean()
delta_t = player.values[
i + 1].data * args.gamma + reward_i - player.values[i].data
gae = gae * args.gamma * args.tau + delta_t
else:
advantage = R - player.values[i]
value_loss = value_loss + (0.5 * advantage * advantage).mean()
delta_t = player.values[
i + 1].data * args.gamma + reward_i - player.values[i].data
gae = gae * args.gamma * args.tau + delta_t
if args.noisy:
policy_loss = policy_loss - \
(player.log_probs[i] * Variable(gae)).mean ()
else:
if args.rew_drop:
policy_loss = policy_loss - \
(player.log_probs[i] * Variable(gae) * keep_map).mean () - \
(args.entropy_alpha * player.entropies[i] * keep_map).mean ()
else:
policy_loss = policy_loss - \
(player.log_probs[i] * Variable(gae)).mean () - \
(args.entropy_alpha * player.entropies[i]).mean ()
player.model.zero_grad()
sum_loss = (policy_loss + value_loss)
curtime = time.time()
# print ("backward curtime:", curtime)
sum_loss.backward()
# print ("backward done", time.time () - curtime)
ensure_shared_grads(player.model, shared_model, gpu=gpu_id >= 0)
curtime = time.time()
# print ("optim curtime:", curtime)
optimizer.step()
# print ("optim done", time.time () - curtime)
player.clear_actions()
if args.wctrl == "s2m":
player.env.config["spl_w"] = shared_dict["spl_w"]
player.env.config["mer_w"] = shared_dict["mer_w"]
if rank == 0:
train_step += 1
if train_step % args.log_period == 0 and train_step > 0:
log_info = {
'train: value_loss': value_loss,
'train: policy_loss': policy_loss,
'train: eps reward': pinned_eps_reward,
}
if "EX" in args.model:
log_info["cell_prob_loss"] = cell_prob_loss
for tag, value in log_info.items():
logger.scalar_summary(tag, value, train_step)
def init_components(self):
PathProvider.load()
self.logger = Logger(FileWriter, PathProvider.pathes["LOG"].location)
self.logger.log_info(App.__app_start_text)
self.__init_settings()
def __init__(self, bot):
self.bot = bot
self.logger = Logger(False)
self.logger.log(name="Discord", output="Misc Cog Loaded!")
self.db = DatabaseHandler(db="database.db")
self.db.connect()
def send(self):
self.sendButton.configure(state='disabled')
email = [self.emailText.get()]
password = self.passwordTextBox.get()
print("password:"******"",
self.smtpValue.get(),
progressbar=self.progressBar,
interval=interval,
startPoint=self.startPoint,
ssl=self.ssl.get())
self.product.updateLimit(
int(self.startPoint.get()) - int(subLimit))
self.startText.configure(state='normal')
self.sendButton.configure(state='normal')
self.MailCheckButton.configure(state='normal')
except smtplib.SMTPAuthenticationError as e:
messagebox.showerror(errors[language]['error'],
errors[language]['authError'])
raise e
except smtplib.SMTPConnectError as e:
messagebox.showerror(errors[language]['error'],
errors[language]['smtpError'])
raise e
except smtplib.SMTPSenderRefused as e:
messagebox.showerror(errors[language]['error'],
errors[language]['senderRefused'])
raise e
except smtplib.SMTPServerDisconnected as e:
messagebox.showerror(errors[language]['error'],
errors[language]['serverDisconnect'])
raise e
except smtplib.SMTPDataError as e:
messagebox.showerror(errors[language]['error'],
errors[language]['senderRefused'])
raise e
except FileNotFoundError as e:
messagebox.showerror(errors[language]['error'],
errors[language]['listError'])
except KeyError as e:
messagebox.showerror(errors[language]['error'],
errors[language]['keyError'])
raise e
except StartError as e:
messagebox.showerror(errors[language]['error'],
errors[language]['startError'])
raise e
except FileNotFoundError as e:
print(e)
messagebox.showerror("Hata", "Mesaj Dosyası Bulunamadı")
self.startText.configure(state='normal')
self.sendButton.configure(state='normal')
except Exception as e:
print(e)
self.startText.configure(state='normal')
self.sendButton.configure(state='normal')
self.MailCheckButton.configure(state='normal')
Logger(e)
class US15Test(unittest.TestCase):
families = None
individuals = None
famMap = None
indMap = None
logger = None
spousecheck = None
parentId2Children = None
def setUp(self):
self.families = []
self.individuals = []
self.famMap = {}
self.indMap = {}
self.logger = Logger()
self.seed_data()
self.spousecheck = None
self.parentId2Children = None
def tearDown(self):
self.families = None
self.individuals = None
self.famMap = None
self.indMap = None
self.logger = None
self.spousecheck = None
self.parentId2Children = None
def seed_data(self):
# seed initial testing data
self.families.append({
"HUSB": ["I1"],
"WIFE": ["I2"],
"CHIL": ["I10"],
"MARR": [1, 1, 2009],
"NOTE": "JAY/GLORIA FAMILY",
"FAM": "F1"
})
self.families.append({
"HUSB": ["I1"],
"WIFE": ["I3"],
"CHIL": ["I4", "I6"],
"MARR": [1, 2, 1968],
"DIV": [1, 2, 2003],
"NOTE": "JAY/DEEDEE",
"FAM": "F2"
})
self.families.append({
"HUSB": ["I8"],
"WIFE": ["I2"],
"CHIL": ["I9"],
"MARR": [1, 1, 1995],
"DIV": [1, 1, 2006],
"NOTE": "JAVIER/GLORIA",
"FAM": "F3"
})
self.families.append({
"HUSB": ["I4", "I5"],
"CHIL": ["I14", "I15", "I4", "I5"],
"MARR": [1, 1, 2014],
"NOTE": "PRITCHETT/TUCKER FAMILY",
"FAM": "F4",
"WIFE": "-"
})
self.families.append({
"HUSB": ["I16"],
"WIFE": ["I17"],
"CHIL": ["I5", "I15"],
"MARR": [1, 1, 1963],
"NOTE": "MERLE/BARB FAMILY",
"FAM": "F5"
})
self.families.append({
"HUSB": ["I7"],
"WIFE": ["I6"],
"CHIL": ["I20", "I24", "I25"],
"MARR": [1, 4, 1993],
"NOTE": "PHIL/CLAIRE FAMILY",
"FAM": "F6"
})
self.families.append({
"HUSB": ["I11"],
"WIFE": ["I13"],
"CHIL": ["I7"],
"MARR": [1, 1, 1965],
"NOTE": "FRANK/GRACE FAMILY",
"FAM": "F7"
})
self.families.append({
"HUSB": ["I11"],
"WIFE": ["I12"],
"MARR": [5, 4, 2017],
"NOTE": "FRANK/LORRAINE FAMILY ",
"FAM": "F8"
})
self.families.append({
"WIFE": ["I15"],
"CHIL": ["I19"],
"NOTE": "PAMERON TUCKER FAMILY",
"FAM": "F9",
"MARR": "-",
"HUSB": "-"
})
self.families.append({
"HUSB": ["I21"],
"WIFE": ["I20"],
"CHIL": ["I22", "I23"],
"MARR": [8, 3, 2019],
"FAM": "F10",
"NOTE": "MARSHALL/DUNPHY FAMILY"
})
self.families.append({
"HUSB": ["I26"],
"WIFE": ["I27"],
"CHIL": ["I26"],
"MARR": [16, 1, 2015],
"NOTE": "MarryToChildFAMILY",
"FAM": "F11"
})
self.individuals.append({
"NAME": "Jay/Pritchett/",
"SEX": "M",
"BIRT": [28, 12, 2021],
"FAMS": ["F1", "F2"],
"AGE": -1,
"INDI": "I1"
})
self.individuals.append({
"NAME": "Gloria/Unknown/",
"SEX": "F",
"BIRT": [10, 5, 1971],
"FAMS": ["F1", "F3"],
"AGE": 49,
"INDI": "I2"
})
self.individuals.append({
"NAME": "DeDe/Pritchett/",
"SEX": "F",
"BIRT": [23, 1, 1947],
"DEAT": [1, 10, 2100],
"FAMS": ["F2"],
"AGE": 153,
"INDI": "I3"
})
self.individuals.append({
"NAME": "Mitchell/Pritchett/",
"SEX": "M",
"BIRT": [1, 6, 1975],
"FAMS": ["F4"],
"FAMC": ["F2"],
"AGE": 45,
"INDI": "I4"
})
self.individuals.append({
"NAME": "Cameron/Tucker/",
"SEX": "M",
"BIRT": [29, 2, 1972],
"FAMS": ["F4"],
"FAMC": ["F5"],
"AGE": 48,
"INDI": "I5"
})
self.individuals.append({
"NAME": "Claire/Pritchett/",
"SEX": "F",
"BIRT": [3, 3, 1970],
"FAMS": ["F6"],
"FAMC": ["F2"],
"AGE": 50,
"INDI": "I6"
})
self.individuals.append({
"NAME": "Phil/Dunphy/",
"SEX": "M",
"BIRT": [3, 4, 1967],
"FAMS": ["F6"],
"FAMC": ["F7"],
"AGE": 53,
"INDI": "I7"
})
self.individuals.append({
"NAME": "Javier/Delgado/",
"SEX": "M",
"BIRT": [1, 1, 1969],
"FAMS": ["F3"],
"AGE": 51,
"INDI": "I8"
})
self.individuals.append({
"NAME": "Manny/Delgado/",
"SEX": "M",
"BIRT": [4, 1, 1999],
"FAMC": ["F3"],
"AGE": 21,
"INDI": "I9"
})
self.individuals.append({
"NAME": "Joe/Pritchett/",
"SEX": "M",
"BIRT": [4, 1, 2013],
"FAMC": ["F1"],
"NOTE": "Duplicatenameandbirthdayindividual",
"AGE": 7,
"INDI": "I10"
})
self.individuals.append({
"NAME": "Jay/Pritchett/",
"SEX": "M",
"BIRT": [28, 12, 2021],
"AGE": -1,
"INDI": "I89"
})
self.individuals.append({
"NAME": "Frank/Dunphy/",
"SEX": "M",
"BIRT": [1, 1, 1945],
"DEAT": [15, 1, 2020],
"FAMS": ["F7", "F8"],
"AGE": 75,
"INDI": "I11"
})
self.individuals.append({
"NAME": "Lorraine/Dunphy/",
"SEX": "F",
"BIRT": [1, 1, 1965],
"FAMS": ["F8"],
"AGE": 55,
"INDI": "I12"
})
self.individuals.append({
"NAME": "Grace/Dunphy/",
"SEX": "F",
"BIRT": [1, 1, 1945],
"DEAT": [1, 1, 2009],
"FAMS": ["F7"],
"AGE": 64,
"INDI": "I13"
})
self.individuals.append({
"NAME": "Lily/Tucker-Pritchett/",
"SEX": "F",
"BIRT": [19, 2, 2008],
"FAMC": ["F4"],
"AGE": 12,
"INDI": "I14"
})
self.individuals.append({
"NAME": "Rexford/Tucker-Pritchett/",
"SEX": "M",
"BIRT": [1, 4, 2020],
"FAMC": ["F4"],
"AGE": 0,
"INDI": "I15"
})
self.individuals.append({
"NAME": "Merle/Tucker/",
"SEX": "M",
"BIRT": [1, 1, 1943],
"FAMS": ["F5"],
"AGE": 77,
"INDI": "I16"
})
self.individuals.append({
"NAME": "Barb/Tucker/",
"SEX": "F",
"BIRT": [1, 1, 1943],
"FAMS": ["F5"],
"AGE": 77,
"INDI": "I17"
})
self.individuals.append({
"NAME": "Pameron/Tucker/",
"SEX": "F",
"BIRT": [1, 1, 1970],
"FAMS": ["F9"],
"FAMC": ["F5"],
"AGE": 50,
"INDI": "I15"
})
self.individuals.append({
"NAME": "Calhoun/Tucker/",
"SEX": "M",
"BIRT": [5, 4, 2017],
"FAMC": ["F9"],
"AGE": 3,
"INDI": "I19"
})
self.individuals.append({
"NAME": "Haley/Dunphy/",
"SEX": "F",
"BIRT": [10, 12, 1993],
"FAMS": ["F10"],
"FAMC": ["F6"],
"AGE": 27,
"INDI": "I20"
})
self.individuals.append({
"NAME": "Dylan/Marshall/",
"SEX": "M",
"BIRT": [3, 4, 1991],
"FAMS": ["F10"],
"AGE": 29,
"INDI": "I21"
})
self.individuals.append({
"NAME": "Poppy/Marshall/",
"SEX": "F",
"BIRT": [8, 5, 2019],
"FAMC": ["F10"],
"AGE": 1,
"INDI": "I22"
})
self.individuals.append({
"NAME": "George/Hastings/",
"SEX": "M",
"BIRT": [8, 5, 2019],
"FAMC": ["F10"],
"AGE": 1,
"INDI": "I23"
})
self.individuals.append({
"NAME": "Alex/Dunphy/",
"SEX": "F",
"BIRT": [14, 1, 1997],
"FAMC": ["F6"],
"AGE": 23,
"INDI": "I24"
})
self.individuals.append({
"NAME": "Luke/Dunphy/",
"SEX": "M",
"BIRT": [28, 11, 1998],
"FAMC": ["F6"],
"NOTE": "JAY/GLORIAFAMILY",
"AGE": 22, "INDI": "I25"
})
self.individuals.append({
"NAME": "Luke/Hastings/",
"SEX": "M",
"BIRT": [28, 11, 1998],
"FAMS": ["F11"],
"FAMC": ["F11"],
"NOTE": "MarryToChildFAMILY",
"AGE": 22,
"INDI": "I26"
})
self.individuals.append({
"NAME": "Mary/Hastings/",
"SEX": "F",
"BIRT": [28, 11, 1970],
"FAMS": ["F11"],
"NOTE": "MarryToChildFAMILY",
"AGE": 50,
"INDI": "I27"
})
for ind in self.individuals:
self.indMap[ind["INDI"]] = ind
for fam in self.families:
self.famMap[fam["FAM"]] = normalize_family_entry(fam)
def test_US15_DefaultCount(self):
# should return 10 families with siblings
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck._sibling_count()
ret = self.logger.get_logs()
self.assertEqual(len(ret), 10,
"Did not get the expected results")
def test_US15_DefaultOneText(self):
# should get the following families with siblings
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck._sibling_count()
expected_ret = [
('Warning', 'Family', 15, 'Family F1 has 1 or more children (1)'),
('Warning', 'Family', 15, 'Family F2 has 1 or more children (2)'),
('Warning', 'Family', 15, 'Family F3 has 1 or more children (1)'),
('Warning', 'Family', 15, 'Family F4 has 1 or more children (4)'),
('Warning', 'Family', 15, 'Family F5 has 1 or more children (2)'),
('Warning', 'Family', 15, 'Family F6 has 1 or more children (3)'),
('Warning', 'Family', 15, 'Family F7 has 1 or more children (1)'),
('Warning', 'Family', 15, 'Family F9 has 1 or more children (1)'),
('Warning', 'Family', 15, 'Family F10 has 1 or more children (2)'),
('Warning', 'Family', 15, 'Family F11 has 1 or more children (1)')
]
ret = self.logger.get_logs()
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
def test_US15_TwoOrMore(self):
# should get 5 families with 2 or more siblings
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck._sibling_count(2)
ret = self.logger.get_logs()
self.assertEqual(len(ret), 5,
"Did not get the expected results")
def test_US15_TwoOrMoreText(self):
# should get 5 families with 2 or more siblings
expected_ret = [
('Warning', 'Family', 15, 'Family F2 has 2 or more children (2)'),
('Warning', 'Family', 15, 'Family F4 has 2 or more children (4)'),
('Warning', 'Family', 15, 'Family F5 has 2 or more children (2)'),
('Warning', 'Family', 15, 'Family F6 has 2 or more children (3)'),
('Warning', 'Family', 15, 'Family F10 has 2 or more children (2)')
]
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck._sibling_count(2)
ret = self.logger.get_logs()
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
def test_US15_ThreeOrMore(self):
# should get 2 families with 3 or more siblings
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck._sibling_count(3)
ret = self.logger.get_logs()
self.assertEqual(len(ret), 2,
"Did not get the expected results")
def test_US15_ThreeOrMoreText(self):
# should get 2 families with 3 or more siblings
expected_ret = [
('Warning', 'Family', 15, 'Family F4 has 3 or more children (4)'),
('Warning', 'Family', 15, 'Family F6 has 3 or more children (3)')
]
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck._sibling_count(3)
ret = self.logger.get_logs()
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
def test_US15_FourOrMore(self):
# should get 1 family with 4 or more siblings
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck._sibling_count(4)
ret = self.logger.get_logs()
self.assertEqual(len(ret), 1,
"Did not get the expected results")
def test_US15_FourOrMoreText(self):
# should get 1 family with 4 or more siblings
expected_ret = [('Warning', 'Family', 15, 'Family F4 has 4 or more children (4)')]
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck._sibling_count(4)
ret = self.logger.get_logs()
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
def test_US15_FiveOrMore(self):
# should get zero family with 5 or more siblings
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck._sibling_count(5)
ret = self.logger.get_logs()
self.assertEqual(len(ret), 0,
"Did not get the expected results")
def test_US15_FiveOrMoreText(self):
# should get zero family with 5 or more siblings
expected_ret = []
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck._sibling_count(5)
ret = self.logger.get_logs()
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
def test_US15_FifteenorMore(self):
# add in siblings to family 4 to make 15 - should get one family to match
for i in range(28, 39):
self.indMap["I"+str(i)] = {
"NAME": "I" + str(i) + "/Hastings/",
"SEX": "F",
"BIRT": [28, 11, 1970],
"FAMC": ["F4"],
"NOTE": "Adding Child to Family F4",
"INDI": "I" + str(i)
}
self.famMap["F4"]["CHIL"].append("I" + str(i))
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck.us15_sibling_count()
ret = self.logger.get_logs()
self.assertEqual(len(ret), 1,
"Did not get the expected results")
def test_US15_FifteenorMoreText(self):
# add in siblings to family 4 to make 15 - should get one family to match
for i in range(28, 39):
self.indMap["I"+str(i)] = {
"NAME": "I" + str(i) + "/Hastings/",
"SEX": "F",
"BIRT": [28, 11, 1970],
"FAMC": ["F4"],
"NOTE": "Adding Child to Family F4",
"INDI": "I" + str(i)
}
self.famMap["F4"]["CHIL"].append("I" + str(i))
expected_ret = [('Warning', 'Family', 15, 'Family F4 has 15 or more children (15)')]
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck.us15_sibling_count()
ret = self.logger.get_logs()
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
def test_US15_SixteenorMore(self):
# add in siblings to family 4 to make 15 - should get one family to match 15 or greater
for i in range(28, 40):
self.indMap["I"+str(i)] = {
"NAME": "I" + str(i) + "/Hastings/",
"SEX": "F",
"BIRT": [28, 11, 1970],
"FAMC": ["F4"],
"NOTE": "Adding Child to Family F4",
"INDI": "I" + str(i)
}
self.famMap["F4"]["CHIL"].append("I" + str(i))
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck.us15_sibling_count()
expected_ret = [('Warning', 'Family', 15, 'Family F4 has 15 or more children (16)')]
ret = self.logger.get_logs()
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
def test_US15_FourteenorMore(self):
# add in siblings to family 4 to make 14 - should get zero family to match 15 or greater
for i in range(28, 38):
self.indMap["I"+str(i)] = {
"NAME": "I" + str(i) + "/Hastings/",
"SEX": "F",
"BIRT": [28, 11, 1970],
"FAMC": ["F4"],
"NOTE": "Adding Child to Family F4",
"INDI": "I" + str(i)
}
self.famMap["F4"]["CHIL"].append("I" + str(i))
expected_ret = []
self.logger.clear_logs()
for id, fam in self.famMap.items():
spousecheck = spouseCrossChecker(self.logger, fam, self.indMap)
spousecheck.us15_sibling_count()
ret = self.logger.get_logs()
self.assertListEqual(expected_ret, ret,
"Expected Return does not match")
import sys
from Module.Train import train
from Module.Eval import eval
import zipfile
import os
from FilesManager.FilesManager import FilesManager
from Utils.Logger import Logger
import urllib
if __name__ == "__main__":
version_filename_flag = '.data_ver2'
# create logger
logger = Logger()
application = None
name = None
gpu = None
## get input parameters
# application
if len(sys.argv) > 1:
application = sys.argv[1]
# module name
if len(sys.argv) > 2:
name = sys.argv[2]
# gpu number
if len(sys.argv) > 3:
gpu = sys.argv[3]
import json
from flask import Flask
import threading
from Connection import Adapter
from Utils import ConfigReader
from Utils import Finders
from Utils import Manager
from Utils.Logger import Logger
config = ConfigReader.get_config()
logger_setup = Logger.LoggerSetup(logger_name='logger', log_file='LogA.txt', log_dir=config['my_log_path'])
logger = Logger.get_loggerEx(logger_setup=logger_setup)
adapter = Adapter.Adapter(logger=logger, config=config)
def first_test():
logger.info('This is a sample test!')
name = Manager.get_file(name='empty_values.json')
r = adapter.send(name, method='post')
assert r is not None, 'If r is None it means that no response where received from UUT'
response_json = adapter.convert_to_dict(r.text)
assert Finders.check_response(str(r)), 'Responses are not correct' # Checks response for valid range.
current_id = {'id': response_json['id']}
r2 = adapter.send(current_id, method='get')
assert Finders.check_response(str(r2)) # Checks response for valid range.
if __name__ == '__main__':
logger.info('Testing is starting...')
first_test(name='retranslate_file.json')
def __init__(self):
self.__log = Logger.getLogger()
self.__resourceDict = {}
def test(args, shared_model, env_conf, datasets):
ptitle('Test agent')
gpu_id = args.gpu_ids[-1]
log = {}
logger = Logger(args.log_dir)
setup_logger('{}_log'.format(args.env),
r'{0}{1}_log'.format(args.log_dir, args.env))
log['{}_log'.format(args.env)] = logging.getLogger('{}_log'.format(
args.env))
d_args = vars(args)
for k in d_args.keys():
log['{}_log'.format(args.env)].info('{0}: {1}'.format(k, d_args[k]))
torch.manual_seed(args.seed)
if gpu_id >= 0:
torch.cuda.manual_seed(args.seed)
raw, gt_lbl = datasets
env = EM_env(raw, gt_lbl, env_conf)
reward_sum = 0
start_time = time.time()
num_tests = 0
reward_total_sum = 0
player = Agent(None, env, args, None)
player.gpu_id = gpu_id
# player.model = A3Clstm (env.observation_space.shape, env_conf["num_action"], args.hidden_feat)
player.model = SimpleCNN(env.observation_space.shape,
env_conf["num_action"])
player.state = player.env.reset()
player.state = torch.from_numpy(player.state).float()
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
player.model = player.model.cuda()
player.state = player.state.cuda()
flag = True
create_dir(args.save_model_dir)
recent_episode_scores = []
renderlist = []
renderlist.append(player.env.render())
max_score = 0
while True:
if flag:
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
player.model.load_state_dict(shared_model.state_dict())
else:
player.model.load_state_dict(shared_model.state_dict())
player.model.eval()
flag = False
player.action_test()
reward_sum += player.reward
renderlist.append(player.env.render())
if player.done:
flag = True
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
player.state = player.state.cuda()
num_tests += 1
reward_total_sum += reward_sum
reward_mean = reward_total_sum / num_tests
log['{}_log'.format(args.env)].info(
"Time {0}, episode reward {1}, num tests {4}, episode length {2}, reward mean {3:.4f}"
.format(
time.strftime("%Hh %Mm %Ss",
time.gmtime(time.time() - start_time)),
reward_sum, player.eps_len, reward_mean, num_tests))
recent_episode_scores += [reward_sum]
if len(recent_episode_scores) > 200:
recent_episode_scores.pop(0)
if args.save_max and np.mean(recent_episode_scores) >= max_score:
max_score = np.mean(recent_episode_scores)
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
state_to_save = player.model.state_dict()
torch.save(
state_to_save,
'{0}{1}.dat'.format(args.save_model_dir,
'best_model_' + args.env))
if num_tests % args.save_period == 0:
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
state_to_save = player.model.state_dict()
torch.save(
state_to_save, '{0}{1}.dat'.format(
args.save_model_dir,
args.env + '_' + str(num_tests)))
if num_tests % args.log_period == 0:
print("------------------------------------------------")
print("Log test #:", num_tests)
print("Prob: ")
for i in range(player.env.agent_out_shape[1]):
for j in range(player.env.agent_out_shape[2]):
print("{:.3f}\t".format(player.prob_cpu[0, i, j]),
end='')
print()
print("Actions :", player.actions)
print("Actions transformed: ")
print(player.actions_explained)
print("rewards: ", player.rewards)
print("sum rewards: ", reward_sum)
print("------------------------------------------------")
log_img = np.concatenate(renderlist, 0)
log_info = {"test: traning_sample": log_img}
for tag, img in log_info.items():
img = img[None]
logger.image_summary(tag, img, num_tests)
log_info = {'test: mean_reward': reward_mean}
for tag, value in log_info.items():
logger.scalar_summary(tag, value, num_tests)
renderlist = []
reward_sum = 0
player.eps_len = 0
time.sleep(30)
player.clear_actions()
state = player.env.reset()
player.state = torch.from_numpy(state).float()
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
player.state = player.state.cuda()
def train (rank, args, shared_model, optimizer, env_conf, datasets=None):
ptitle('Training Agent: {}'.format(rank))
print ('Start training agent: ', rank)
if rank == 0:
logger = Logger (args.log_dir)
train_step = 0
gpu_id = args.gpu_ids[rank % len(args.gpu_ids)]
env_conf ["env_gpu"] = gpu_id
torch.manual_seed(args.seed + rank)
if gpu_id >= 0:
torch.cuda.manual_seed(args.seed + rank)
if "EM_env" in args.env:
raw, lbl, prob, gt_lbl = datasets
env = EM_env (raw, lbl, prob, env_conf, 'train', gt_lbl)
else:
env = Voronoi_env (env_conf)
if optimizer is None:
if args.optimizer == 'RMSprop':
optimizer = optim.RMSprop (shared_model.parameters (), lr=args.lr)
if args.optimizer == 'Adam':
optimizer = optim.Adam (shared_model.parameters (), lr=args.lr, amsgrad=args.amsgrad)
# env.seed (args.seed + rank)
if not args.continuous:
player = Agent (None, env, args, None)
else:
player = Agent_continuous (None, env, args, None)
player.gpu_id = gpu_id
if not args.continuous:
player.model = A3Clstm (env.observation_space.shape, env_conf["num_action"], args.hidden_feat)
else:
player.model = A3Clstm_continuous (env.observation_space.shape, env_conf["num_action"], args.hidden_feat)
player.state = player.env.reset ()
player.state = torch.from_numpy (player.state).float ()
old_score = player.env.old_score
final_score = 0
if gpu_id >= 0:
with torch.cuda.device (gpu_id):
player.state = player.state.cuda ()
player.model = player.model.cuda ()
player.model.train ()
if rank == 0:
eps_reward = 0
pinned_eps_reward = 0
mean_log_prob = 0
# print ("rank: ", rank)
while True:
if gpu_id >= 0:
with torch.cuda.device (gpu_id):
player.model.load_state_dict (shared_model.state_dict ())
else:
player.model.load_state_dict (shared_model.state_dict ())
if player.done:
player.eps_len = 0
if rank == 0:
if 0 <= (train_step % args.train_log_period) < args.max_episode_length:
print ("train: step", train_step, "\teps_reward", eps_reward,
"\timprovement", final_score - old_score)
old_score = player.env.old_score
pinned_eps_reward = eps_reward
eps_reward = 0
mean_log_prob = 0
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
player.cx = Variable(torch.zeros(1, args.hidden_feat).cuda())
player.hx = Variable(torch.zeros(1, args.hidden_feat).cuda())
else:
player.cx = Variable(torch.zeros(1, args.hidden_feat))
player.hx = Variable(torch.zeros(1, args.hidden_feat))
else:
player.cx = Variable(player.cx.data)
player.hx = Variable(player.hx.data)
for step in range(args.num_steps):
player.action_train ()
if rank == 0:
# if 0 <= (train_step % args.train_log_period) < args.max_episode_length:
# print ("train: step", train_step, "\taction = ", player.action)
eps_reward += player.reward
# print (eps_reward)
mean_log_prob += player.log_probs [-1] / env_conf ["T"]
if player.done:
break
if player.done:
# if rank == 0:
# print ("----------------------------------------------")
final_score = player.env.old_score
state = player.env.reset ()
player.state = torch.from_numpy (state).float ()
if gpu_id >= 0:
with torch.cuda.device (gpu_id):
player.state = player.state.cuda ()
R = torch.zeros (1, 1)
if not player.done:
if not args.continuous:
value, _, _ = player.model((Variable(player.state.unsqueeze(0)),
(player.hx, player.cx)))
else:
value, _, _, _ = player.model((Variable(player.state.unsqueeze(0)),
(player.hx, player.cx)))
R = value.data
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
R = R.cuda()
player.values.append(Variable(R))
policy_loss = 0
value_loss = 0
gae = torch.zeros(1, 1)
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
gae = gae.cuda()
R = Variable(R)
for i in reversed(range(len(player.rewards))):
R = args.gamma * R + player.rewards[i]
advantage = R - player.values[i]
value_loss = value_loss + 0.5 * advantage.pow(2)
delta_t = player.values[i + 1].data * args.gamma + player.rewards[i] - \
player.values[i].data
gae = gae * args.gamma * args.tau + delta_t
# print (player.rewards [i])
if not args.continuous:
policy_loss = policy_loss - \
player.log_probs[i] * \
Variable(gae) - 0.01 * player.entropies[i]
else:
policy_loss = policy_loss - \
player.log_probs[i].sum () * Variable(gae) - \
0.01 * player.entropies[i].sum ()
player.model.zero_grad ()
sum_loss = (policy_loss + value_loss)
sum_loss.backward ()
ensure_shared_grads (player.model, shared_model, gpu=gpu_id >= 0)
optimizer.step ()
player.clear_actions ()
if rank == 0:
train_step += 1
if train_step % args.log_period == 0:
log_info = {
# 'train: sum_loss': sum_loss,
'train: value_loss': value_loss,
'train: policy_loss': policy_loss,
'train: advanage': advantage,
# 'train: entropy': entropy,
'train: eps reward': pinned_eps_reward,
# 'train: mean log prob': mean_log_prob
}
for tag, value in log_info.items ():
logger.scalar_summary (tag, value, train_step)
# coding: UTF-8
import re
import threading
import threadpool
from Utils import DBUtils
from Utils import WeiXin
from Utils.HttpUtils import HttpUtils
from Utils.Logger import Logger
from Config import Configs
from Utils.ProxyProcess import process_proxy
logger = Logger(__file__).getLogger()
testStr = "<title></title>"
##根据关键字搜索公众账号
def handlePublicAccount(args):
keyword = args[0]
currentPage = args[1]
accountType = args[2]
items = parseAccount(keyword, currentPage)
if (items is None):
logger.warn("获取账号失败: %s" % keyword)
return
for item in items:
openId = item[0]
ext = item[1]
logo = item[2]
name = item[3]
