def login_user(self):
does_user_exist = self.check_user_exists()
err_msg = "User '{0}' doesn't exist for '{1}'".format(
config.get(config.ACCOUNT_SECTION,
config.USER_NAME_KEY),
self.idp_server)
assert(does_user_exist), err_msg
# Alway start with this method so as to dodge side effects.
self.reset_browser()
URL = "https://{0}/login".format(self.idp_server)
OpenID = "https://{0}/esgf-idp/openid/".format(self.idp_server)
self.load_page(URL, (By.ID, 'openid_identifier'))
globals.browser.find_element_by_id('openid_identifier').send_keys(OpenID)
globals.browser.find_element_by_xpath("//input[@value='Login']").click()
self.wait_loading('load the login page', (By.ID, 'username'),
(By.CLASS_NAME, 'error-box'))
# After check_user_exists, the page is asking for the user's password.
globals.browser.find_element_by_id('username')\
.send_keys(config.get(config.ACCOUNT_SECTION, config.USER_NAME_KEY))
globals.browser.find_element_by_id('password')\
.send_keys(config.get(config.ACCOUNT_SECTION, config.USER_PASSWORD_KEY))
globals.browser.find_element_by_xpath("//input[@value='SUBMIT']").click()
msg = "login with user '{0}' for '{1}'"\
.format(config.get(config.ACCOUNT_SECTION, config.USER_NAME_KEY), self.idp_server)
#text = 'Invalid OpenID and/or Password combination'
self.wait_loading(msg, not_expected_element=(By.ID, 'null.errors'))
def create_user(self):
# Alway start with this method so as to dodge side effects.
self.reset_browser()
URL = "https://{0}/user/add".format(self.idp_server)
self.load_page(URL, (By.NAME, 'first_name'))
# Filling the form
for element_name in self.elements:
globals.browser.find_element_by_name(element_name).send_keys(self.elements[element_name])
globals.browser.find_element_by_xpath("//input[@value='Submit']").click()
msg = "create user '{0}' in {1}. May be the captcha is on. "\
"Check if USE_CAPTCHA in "\
"/usr/local/cog/cog_config/cog_settings.cfg is set to False and "\
"restart esg-node.".format(config.get(config.ACCOUNT_SECTION,
config.USER_NAME_KEY),
config.get(config.NODES_SECTION,
config.IDP_NODE_KEY))
self.wait_loading(msg, not_expected_element=(By.CLASS_NAME, 'errorlist'))
# message-box and Thank you for creating an account. You can now login.
def __init__(self):
self.idp_server = config.get(config.NODES_SECTION, config.IDP_NODE_KEY)
# Mapping user data to fit to web-fe user creation form
self.elements = {
'first_name':
config.get(config.ACCOUNT_SECTION, config.USER_FIRST_NAME_KEY),
'last_name':
config.get(config.ACCOUNT_SECTION, config.USER_LAST_NAME_KEY),
'email':
config.get(config.ACCOUNT_SECTION, config.USER_EMAIL_KEY),
'username':
config.get(config.ACCOUNT_SECTION, config.USER_NAME_KEY),
'password':
config.get(config.ACCOUNT_SECTION, config.USER_PASSWORD_KEY),
'confirm_password':
config.get(config.ACCOUNT_SECTION, config.USER_PASSWORD_KEY),
'institution':
config.get(config.ACCOUNT_SECTION, config.USER_INSTITUTION_KEY),
'city':
config.get(config.ACCOUNT_SECTION, config.USER_CITY_KEY),
'country':
config.get(config.ACCOUNT_SECTION, config.USER_COUNTRY_KEY)
}
def create_user(self):
# Alway start with this method so as to dodge side effects.
self.reset_browser()
URL = "https://{0}/user/add".format(self.idp_server)
self.load_page(URL, (By.NAME, 'first_name'))
# Filling the form
for element_name in self.elements:
try:
globals.browser.find_element_by_name(element_name).send_keys(
self.elements[element_name])
except NoSuchElementException:
assert (
False
), "{0} is corrupted or not compliant with esgf-test-suite".format(
URL)
globals.browser.find_element_by_xpath(
"//input[@value='Submit']").click()
msg = "create user '{0}' in {1}. May be the captcha is on. "\
"Check if USE_CAPTCHA in "\
"/usr/local/cog/cog_config/cog_settings.cfg is set to False and "\
"restart esg-node.".format(config.get(config.ACCOUNT_SECTION,
config.USER_NAME_KEY),
config.get(config.NODES_SECTION,
config.IDP_NODE_KEY))
self.wait_loading(msg,
not_expected_element=(By.CLASS_NAME, 'errorlist'))
def __init__(self):
AbstractBrowserBasedTest.__init__(self)
self.idp_node = config.get(config.NODES_SECTION, config.IDP_NODE_KEY)
self.username = config.get(config.SLCS_SECTION,
config.ADMIN_USERNAME_KEY)
self.password = config.get(config.SLCS_SECTION,
config.ADMIN_PASSWORD_KEY)
def __init__(self): AbstractBrowserBasedTest.__init__(self) AbstractMyproxyBasedTest.__init__(self) self._tu = cat.ThreddsUtils() self._endpoints = self._tu.get_endpoints() self.data_node = config.get(config.NODES_SECTION, config.DATA_NODE_KEY) self.idp_node = config.get(config.NODES_SECTION, config.IDP_NODE_KEY) self.username = config.get(config.ACCOUNT_SECTION, config.USER_NAME_KEY) self.password = config.get(config.ACCOUNT_SECTION, config.USER_PASSWORD_KEY)
def __init__(self):
AbstractBrowserBasedTest.__init__(self)
AbstractMyproxyBasedTest.__init__(self)
self._tu = cat.ThreddsUtils()
# print("[DEBUG] starting endpoints computation")
self._endpoints = self._tu.get_endpoints()
self.data_node = config.get(config.NODES_SECTION, config.DATA_NODE_KEY)
self.idp_node = config.get(config.NODES_SECTION, config.IDP_NODE_KEY)
self.username = config.get(config.ACCOUNT_SECTION,
config.USER_NAME_KEY)
self.password = config.get(config.ACCOUNT_SECTION,
config.USER_PASSWORD_KEY)
def get_credentials(self):
#try:
# Get credentials (and trustroots)
# username = config.get(config.ACCOUNT_SECTION, config.USER_NAME_KEY)
# password = config.get(config.ACCOUNT_SECTION, config.USER_PASSWORD_KEY)
# self.credentials = self.myproxy.logon(username, password)
#except MyProxyClientGetError as e1:
# err_msg = "wrong username and/or password combination when getting credentials for user '{0}' (reason: {1})"\
# .format(username, e1)
# assert(False), err_msg
#except Exception as e2:
# err = "unable to connect to myproxy server '{0}' (reason: {1})"\
# .format(self.idp_addr, e2)
# assert (False), err
# Write Credentials
#with open(self.credsfile, 'w') as f:
# f.write(self.credentials[0]+self.credentials[1])
# os.chmod(self.credsfile, self.myproxy.PROXY_FILE_PERMISSIONS)
# WORKAROUND SSL-MYPROXYCLIENT PROBLEM
username = config.get(config.ACCOUNT_SECTION, config.USER_NAME_KEY)
password = config.get(config.ACCOUNT_SECTION, config.USER_PASSWORD_KEY)
command = [
'myproxy-logon', '-S', '-T', '-s', self.idp_addr, '-p', self.port,
'-l', username, '-o', self.credsfile, '-b'
]
process = subprocess.run(command,
input=password.encode(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
timeout=60)
assert (
process.returncode == 0
), "fail to get the credentials for {0} (sdtout: {1} ; stderr: {2})".format(
self.idp_addr, process.stdout, process.stderr)
file_content = ''
with open(self.credsfile, 'r') as file:
for line in file.readlines():
file_content += line
certs = re.findall('(-+BEGIN CERTIFICATE-+.+?-+END CERTIFICATE-+\\s+)',
file_content, re.DOTALL)
key = re.findall(
'(-+BEGIN RSA PRIVATE KEY-+.+?-+END RSA PRIVATE KEY-+\\s+)',
file_content, re.DOTALL)
self.credentials = (certs[0], key[0], certs[1])
def test_dl_gridftp(self):
self.gridftp_node = config.get(config.NODES_SECTION,
config.GRIDFTP_NODE_KEY)
gridftp_port = naming.GRIDFTP_PORT_NUMBER
is_enable = networking.ping_tcp_port(self.gridftp_node, gridftp_port)
err_msg = "gridftp server not found at {0} port {1} (reason: {2})"\
.format(self.gridftp_node, gridftp_port, is_enable[2])
assert (is_enable[0]), err_msg
path = self._get_endpoint_path('GridFTP')
url = "gsiftp://{0}:{1}//{2}".format(self.gridftp_node, gridftp_port,
path)
print(("url downloaded: {0}".format(url)))
os.environ['X509_USER_PROXY'] = globals.myproxy_utils.credsfile
os.environ['X509_CERT_DIR'] = globals.myproxy_utils.cacertdir
command = [
'globus-url-copy', '-b', url,
TestDataDownload._DOWNLOADED_FILE_PATH
]
process = subprocess.run(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
timeout=60)
assert (
process.returncode == 0
), "fail to download by GridFTP (sdtout: {0} ; stderr: {1})".format(
process.stdout, process.stderr)
def check_user_exists(self):
# Alway start with this method so as to dodge side effects.
self.reset_browser()
URL = "https://{0}/login".format(self.idp_server)
OpenID = "https://{0}/esgf-idp/openid/{1}".format(
self.idp_server,
config.get(config.ACCOUNT_SECTION, config.USER_NAME_KEY))
self.load_page(URL, (By.ID, 'openid_identifier'))
try:
globals.browser.find_element_by_id('openid_identifier').send_keys(
OpenID)
except NoSuchElementException:
assert (
False
), "{0} is corrupted or not compliant with esgf-test-suite".format(
URL)
globals.browser.find_element_by_xpath(
"//input[@value='Login']").click()
self.wait_loading('load the login confirmation')
#text = 'OpenID Discovery Error: unrecognized by the Identity Provider'
try:
element_present = globals.browser.find_element_by_class_name(
'error-box')
does_user_exist = False
except NoSuchElementException:
does_user_exist = True
return does_user_exist
def __init__(self):
if config.get(config.TEST_SECTION, config.TYPE_KEY).lower() == config.DOCKER_TEST_SET_NAME:
front_ends = TestWebFrontEnds._docker_front_ends
else:
front_ends = TestWebFrontEnds._classic_front_ends
AbstractWebFrontEndTestClass.__init__(self, front_ends, config.IDP_NODE_KEY)
def __init__(self):
if config.get(config.TEST_SECTION, config.TYPE_KEY).lower() == config.DOCKER_TEST_SET_NAME:
front_ends=TestWebFrontEnds._front_ends_docker
else:
front_ends=TestWebFrontEnds._front_ends_installer
AbstractWebFrontEndTestClass.__init__(self, front_ends, config.INDEX_NODE_KEY)
def test_cog_create_user(self):
does_user_exist=self.usr.check_user_exists()
if(does_user_exist):
assert(False), "user '{0}' already exists".format(config.get(config.ACCOUNT_SECTION, config.USER_NAME_KEY))
else:
self.usr.create_user() # Create user
def test_cog_create_user(self):
does_user_exist=self.usr.check_user_exists()
if(does_user_exist):
assert(False), "user '{0}' already exists".format(config.get(config.ACCOUNT_SECTION, config.USER_NAME_KEY))
else:
self.usr.create_user() # Create user
def __get_arg(self):
is_headless = config.get(config.BROWSER_SECTION,
config.BROWSER_IS_HEADLESS_KEY).lower() == naming.TRUE
if(is_headless):
arg_value = '-headless'
else:
arg_value = '-foreground'
return(arg_value)
def login_user(self):
does_user_exist = self.check_user_exists()
err_msg = "User '{0}' doesn't exist for '{1}'".format(
config.get(config.ACCOUNT_SECTION, config.USER_NAME_KEY),
self.idp_server)
assert (does_user_exist), err_msg
# Alway start with this method so as to dodge side effects.
self.reset_browser()
URL = "https://{0}/login".format(self.idp_server)
OpenID = "https://{0}/esgf-idp/openid/".format(self.idp_server)
self.load_page(URL, (By.ID, 'openid_identifier'))
try:
globals.browser.find_element_by_id('openid_identifier').send_keys(
OpenID)
except NoSuchElementException:
assert (
False
), "{0} is corrupted or not compliant with esgf-test-suite".format(
URL)
globals.browser.find_element_by_xpath(
"//input[@value='Login']").click()
self.wait_loading('load the login page', (By.ID, 'username'),
(By.CLASS_NAME, 'error-box'))
# After check_user_exists, the page is asking for the user's password.
globals.browser.find_element_by_id('username')\
.send_keys(config.get(config.ACCOUNT_SECTION, config.USER_NAME_KEY))
globals.browser.find_element_by_id('password')\
.send_keys(config.get(config.ACCOUNT_SECTION, config.USER_PASSWORD_KEY))
globals.browser.find_element_by_xpath(
"//input[@value='SUBMIT']").click()
msg = "login with user '{0}' for '{1}'"\
.format(config.get(config.ACCOUNT_SECTION, config.USER_NAME_KEY), self.idp_server)
#text = 'Invalid OpenID and/or Password combination'
self.wait_loading(msg, not_expected_element=(By.ID, 'null.errors'))
def __get_arg(self):
is_headless = config.get(
config.BROWSER_SECTION,
config.BROWSER_IS_HEADLESS_KEY).lower() == naming.TRUE
if (is_headless):
arg_value = '-headless'
else:
arg_value = '-foreground'
return (arg_value)
def __init__(self):
if config.get(config.TEST_SECTION,
config.TYPE_KEY).lower() == config.DOCKER_TEST_SET_NAME:
front_ends = TestWebFrontEnds._docker_front_ends
else:
front_ends = TestWebFrontEnds._classic_front_ends
AbstractWebFrontEndTestClass.__init__(self, front_ends,
config.DATA_NODE_KEY)
def test_cog_root_login(self):
# Alway start with this method so as to dodge side effects.
self.reset_browser()
index_node=config.get(config.NODES_SECTION, config.INDEX_NODE_KEY)
url = "https://{0}/login2".format(index_node)
self.load_page(url)
globals.browser.find_element_by_id('id_username')\
.send_keys(config.get(config.COG_SECTION, config.ADMIN_USERNAME_KEY))
globals.browser.find_element_by_id('id_password')\
.send_keys(config.get(config.COG_SECTION, config.ADMIN_PASSWORD_KEY))
globals.browser.find_element_by_xpath("//input[@value='Login']").click()
msg = "log onto the Cog admin page of '{0}'"\
.format(index_node)
self.wait_loading(msg, not_expected_element=(By.CLASS_NAME, 'errornote'))
def __init__(self):
self.idp_server = config.get(config.NODES_SECTION, config.IDP_NODE_KEY)
# Mapping user data to fit to web-fe user creation form
self.elements = {'first_name' : config.get(config.ACCOUNT_SECTION,
config.USER_FIRST_NAME_KEY),
'last_name' : config.get(config.ACCOUNT_SECTION,
config.USER_LAST_NAME_KEY),
'email' : config.get(config.ACCOUNT_SECTION,
config.USER_EMAIL_KEY),
'username' : config.get(config.ACCOUNT_SECTION,
config.USER_NAME_KEY),
'password' : config.get(config.ACCOUNT_SECTION,
config.USER_PASSWORD_KEY),
'confirm_password' : config.get(config.ACCOUNT_SECTION,
config.USER_PASSWORD_KEY),
'institution' : config.get(config.ACCOUNT_SECTION,
config.USER_INSTITUTION_KEY),
'city' : config.get(config.ACCOUNT_SECTION,
config.USER_CITY_KEY),
'country' : config.get(config.ACCOUNT_SECTION,
config.USER_COUNTRY_KEY)}
def get_credentials(self):
#try:
# Get credentials (and trustroots)
# username = config.get(config.ACCOUNT_SECTION, config.USER_NAME_KEY)
# password = config.get(config.ACCOUNT_SECTION, config.USER_PASSWORD_KEY)
# self.credentials = self.myproxy.logon(username, password)
#except MyProxyClientGetError as e1:
# err_msg = "wrong username and/or password combination when getting credentials for user '{0}' (reason: {1})"\
# .format(username, e1)
# assert(False), err_msg
#except Exception as e2:
# err = "unable to connect to myproxy server '{0}' (reason: {1})"\
# .format(self.idp_addr, e2)
# assert (False), err
# Write Credentials
#with open(self.credsfile, 'w') as f:
# f.write(self.credentials[0]+self.credentials[1])
# os.chmod(self.credsfile, self.myproxy.PROXY_FILE_PERMISSIONS)
# WORKAROUND SSL-MYPROXYCLIENT PROBLEM
username = config.get(config.ACCOUNT_SECTION, config.USER_NAME_KEY)
password = config.get(config.ACCOUNT_SECTION, config.USER_PASSWORD_KEY)
command = ['myproxy-logon', '-S', '-T', '-s', self.idp_addr, '-p', self.port, '-l', username, '-o', self.credsfile, '-b']
process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False)
stdout, stderr = process.communicate(password)
process.wait()
assert(process.returncode == 0), "fail to get the credentials for {0} (sdtout: {1} ; stderr: {2})".format(self.idp_addr, stdout, stderr)
file_content = ''
with open(self.credsfile, 'r') as file:
for line in file.readlines():
file_content += line
certs = re.findall('(-+BEGIN CERTIFICATE-+.+?-+END CERTIFICATE-+\\s+)', file_content, re.DOTALL)
key = re.findall('(-+BEGIN RSA PRIVATE KEY-+.+?-+END RSA PRIVATE KEY-+\\s+)', file_content, re.DOTALL)
self.credentials = (certs[0], key[0], certs[1])
def test_cog_root_login(self):
# Alway start with this method so as to dodge side effects.
self.reset_browser()
index_node=config.get(config.NODES_SECTION, config.INDEX_NODE_KEY)
url = "https://{0}/login2".format(index_node)
self.load_page(url)
try:
globals.browser.find_element_by_id('id_username')\
.send_keys(config.get(config.COG_SECTION, config.ADMIN_USERNAME_KEY))
except NoSuchElementException:
assert(False), "{0} is corrupted or not compliant with esgf-test-suite".format(url)
globals.browser.find_element_by_id('id_password')\
.send_keys(config.get(config.COG_SECTION, config.ADMIN_PASSWORD_KEY))
globals.browser.find_element_by_xpath("//input[@value='Login']").click()
msg = "log onto the Cog admin page of '{0}'".format(index_node)
self.wait_loading(msg, not_expected_element=(By.CLASS_NAME, 'errornote'))
def __init__(self):
self.cacertdir = os.path.expanduser(naming.CA_CERT_DIR_PATH)
cert_parent_dir_path = os.path.dirname(self.cacertdir)
if False == os.path.exists(cert_parent_dir_path): # WORKAROUND SSL-MYPROXYCLIENT PROBLEM
os.makedirs(cert_parent_dir_path) # WORKAROUND SSL-MYPROXYCLIENT PROBLEM
self.credsfile = os.path.expanduser(naming.CREDENTIALS_FILE_PATH)
self.idp_addr= config.get(config.NODES_SECTION, config.IDP_NODE_KEY).encode('ascii', 'replace')
# self.myproxy = MyProxyClient(hostname=self.idp_addr) # WORKAROUND SSL-MYPROXYCLIENT PROBLEM
# self.myproxy._setCACertDir(self.cacertdir) # WORKAROUND SSL-MYPROXYCLIENT PROBLEM
self.credentials = None
self.trustRoots = None
self.port = '7512'
# Reset the eventually files (for example, after a debugg session).
self.delete_credentials()
self.delete_trustroots()
def test_myproxy_get_trustroots(self):
# Test output from get_trustroots
if(globals.myproxy_utils.trustRoots):
err_msg = "unsupported trusted root certificate format '{0}'".format(globals.myproxy_utils.trustRoots)
assert(isinstance(globals.myproxy_utils.trustRoots, dict)), err_msg
for fileName, fileContents in globals.myproxy_utils.trustRoots.items():
if fileName.endswith('.0'):
# test parsing certificate
cert = crypto.load_certificate(crypto.FILETYPE_PEM, fileContents)
assert(isinstance(cert, crypto.X509)), "unsupported certificate format '{0}'".format(cert)
subj = cert.get_subject()
err_msg = "fail to get the trusted root certificates for '{0}'".format(config.get(config.NODES_SECTION, config.IDP_NODE_KEY))
assert(subj), err_msg
else:
raise SkipTest("unable to check the trustroots")
def test_myproxy_get_trustroots(self):
# Test output from get_trustroots
if(globals.myproxy_utils.trustRoots):
err_msg = "unsupported trusted root certificate format '{0}'".format(globals.myproxy_utils.trustRoots)
assert(isinstance(globals.myproxy_utils.trustRoots, dict)), err_msg
for fileName, fileContents in list(globals.myproxy_utils.trustRoots.items()):
if fileName.endswith('.0'):
# test parsing certificate
cert = crypto.load_certificate(crypto.FILETYPE_PEM, fileContents)
assert(isinstance(cert, crypto.X509)), "unsupported certificate format '{0}'".format(cert)
subj = cert.get_subject()
err_msg = "fail to get the trusted root certificates for '{0}'".format(config.get(config.NODES_SECTION, config.IDP_NODE_KEY))
assert(subj), err_msg
else:
raise SkipTest("unable to check the trustroots")
def __init__(self):
self.cacertdir = os.path.expanduser(naming.CA_CERT_DIR_PATH)
cert_parent_dir_path = os.path.dirname(self.cacertdir)
if False == os.path.exists(
cert_parent_dir_path): # WORKAROUND SSL-MYPROXYCLIENT PROBLEM
os.makedirs(
cert_parent_dir_path) # WORKAROUND SSL-MYPROXYCLIENT PROBLEM
self.credsfile = os.path.expanduser(naming.CREDENTIALS_FILE_PATH)
self.idp_addr = config.get(config.NODES_SECTION,
config.IDP_NODE_KEY).encode(
'ascii', 'replace')
# self.myproxy = MyProxyClient(hostname=self.idp_addr) # WORKAROUND SSL-MYPROXYCLIENT PROBLEM
# self.myproxy._setCACertDir(self.cacertdir) # WORKAROUND SSL-MYPROXYCLIENT PROBLEM
self.credentials = None
self.trustRoots = None
self.port = '7512'
# Reset the eventually files (for example, after a debugg session).
self.delete_credentials()
self.delete_trustroots()
def check_user_exists(self):
# Alway start with this method so as to dodge side effects.
self.reset_browser()
URL = "https://{0}/login".format(self.idp_server)
OpenID = "https://{0}/esgf-idp/openid/{1}".format(self.idp_server,
config.get(config.ACCOUNT_SECTION, config.USER_NAME_KEY))
self.load_page(URL, (By.ID, 'openid_identifier'))
globals.browser.find_element_by_id('openid_identifier').send_keys(OpenID)
globals.browser.find_element_by_xpath("//input[@value='Login']").click()
self.wait_loading('load the login confirmation')
#text = 'OpenID Discovery Error: unrecognized by the Identity Provider'
try:
element_present = globals.browser.find_element_by_class_name('error-box')
does_user_exist = False
except NoSuchElementException:
does_user_exist = True
return does_user_exist
def __init__(self, front_ends, node_key_name): node_name = config.get(config.NODES_SECTION, node_key_name) requests.packages.urllib3.disable_warnings() self._front_ends = front_ends self._node_name = node_name
def train(configuration):
#Setup directories for checkpoints and tensorboard logs
createdirectories(configuration.get('model_path'), exist_ok=True)
createdirectories(pjoin(configuration.get('model_path'), 'logs'),
exist_ok=True)
createdirectories(pjoin(configuration.get('model_path'), 'checkpoints'),
exist_ok=True)
#Initialize tensorboard logger
if configuration.get('enable_log'):
logger = Logger(pjoin(configuration.get('model_path'), 'logs'))
#Choose target architecture
if check_device_availability() and configuration.get('use_gpu'):
target_arch = Arch('cuda:0')
else:
target_arch = Arch('cpu')
network = configuration.get('network').to(target_arch)
phases = configuration.get('phases')
loaders = configuration.get('loaders')
func_loss = configuration.get('loss')
learner = configuration.get('learner')
best_model_wts = Copier(network.state_dict())
best_loss_val = 1e5
train_loss = []
test_loss = []
train_acc = []
test_acc = []
#Main Epoch Loop
for epoch_num in range(configuration.get('max_epoch_val')):
for phase in phases:
if phase == 'train':
network.train()
else:
network.eval()
if not loaders.get(phase):
print(f'Not able to find loaders for phase: {phase}')
continue
loss_val = 0.0
corrects_val = 0
items_val = 0
for idx, (images, targets) in tqdm(enumerate(loaders[phase])):
images = images.to(target_arch)
targets = targets.to(target_arch)
learner.zero_grad()
with disable_backprop(phase == 'train'):
predictions = network(images)
loss_ = func_loss(predictions, targets)
if configuration.get('use_gpu'):
l2_regularization = Tensor(0.).cuda()
else:
l2_regularization = Tensor(0.)
for param in network.parameters():
l2_regularization += Norm(param, 2)**2
loss_ += 1e-3 * l2_regularization
if phase == 'train':
loss_.backward()
learner.step()
loss_val += loss_.item()
items_val += images.size(0)
_, class_preds = Max(predictions, 1)
corrects_val += (targets == class_preds).sum().item()
epoch_loss = loss_val / items_val
epoch_acc = corrects_val / items_val
if phase == 'train':
train_loss.append(epoch_loss)
train_acc.append(epoch_acc)
else:
test_loss.append(epoch_loss)
test_acc.append(epoch_acc)
#Tensorboard logging
logger.add_scalar(phase + '_loss',
epoch_loss,
global_step=epoch_num)
logger.add_scalar(phase + '_acc', epoch_acc, global_step=epoch_num)
print(
f'Epoch {epoch_num} [{phase}]: Loss = {epoch_loss}, Accuracy = {epoch_acc}'
)
#Retain best weights copy for final save based on test set performance
if phase == 'test':
if epoch_loss < best_loss_val:
best_loss_val = epoch_loss
best_model_wts = Copier(network.state_dict())
if epoch_acc > configuration.get('acc_threshold'):
epoch_num = configuration.get('max_epoch_val') + 1
break
# Save checkpoint every 10 epochs
if (epoch_num + 1) % 10 == 0:
Saver(
network.state_dict(),
pjoin(configuration.get('model_path'),
'checkpoints/model_' + str(epoch_num) + '.pth'))
#Stop training if reached limit
if epoch_num > configuration.get('max_epoch_val'):
break
#Save final model
network.load_state_dict(best_model_wts)
Saver(
network.state_dict(),
pjoin(configuration.get('model_path'), 'checkpoints/model_final.pth'))
return train_loss, train_acc, test_loss, test_acc
def problem1c(train_dataset, test_dataset, new_train_dataset,
new_test_dataset):
# lr = 1e-3, decay = 0.001, batch size = 64
configuration['batch_size'] = 64
train_iterator = DataGenerator(
train_dataset,
shuffle=True,
batch_size=configuration.get('batch_size'),
num_workers=configuration.get('num_workers'))
test_iterator = DataGenerator(test_dataset,
shuffle=False,
batch_size=configuration.get('batch_size'),
num_workers=configuration.get('num_workers'))
new_train_iterator = DataGenerator(
new_train_dataset,
shuffle=True,
batch_size=configuration.get('batch_size'),
num_workers=configuration.get('num_workers'))
new_test_iterator = DataGenerator(
new_test_dataset,
shuffle=False,
batch_size=configuration.get('batch_size'),
num_workers=configuration.get('num_workers'))
# Train LeNet on positives and test on negatives
network = ClassifierLN5(
in_channels=1,
number_of_classes=configuration.get('number_of_classes'),
init_method='uniform')
model_configuration = {
'enable_log': configuration.get('enable_log'),
'model_path': pjoin(configuration.get('model_path'), 'mnist_w_neg'),
'network': network,
'use_gpu': configuration.get('use_gpu'),
'phases': ['train', 'test'],
'loaders': {
'train': train_iterator,
'test': test_iterator
},
'loss': CELoss(),
'learner': AdamOptimizer(network.parameters(), lr=1e-3,
weight_decay=0),
'scheduler': None,
'max_epoch_val': 50,
'initialization_method': configuration.get('initialization_method'),
'acc_threshold': 0.99,
}
train_loss, train_acc, test_loss, test_acc = train(model_configuration)
lenet5 = [train_loss, train_acc, test_loss, test_acc]
#Custom classifier training
network = CustomClassifier()
model_configuration = {
'enable_log':
configuration.get('enable_log'),
'model_path':
pjoin(configuration.get('model_path'), 'mnist_new', 'config3'),
'network':
network,
'use_gpu':
configuration.get('use_gpu'),
'phases': ['train', 'test'],
'loaders': {
'train': new_train_iterator,
'test': new_test_iterator
},
'loss':
CELoss(),
'learner':
AdamOptimizer(network.parameters(), lr=1e-3, weight_decay=0.001),
'scheduler':
None,
'max_epoch_val':
configuration.get('max_epoch_val'),
'initialization_method':
configuration.get('initialization_method'),
'acc_threshold':
0.995
}
graphs = train(model_configuration)
results = {'custom_classifier': graphs, 'lenet': lenet5}
with open('problem_1c.json', 'w') as f:
f.write(json.dumps(results))
drawGraphs1c(results)
def __init__(self, front_ends, node_key_name):
node_name = config.get(config.NODES_SECTION, node_key_name)
requests.packages.urllib3.disable_warnings()
self._front_ends = front_ends
self._node_name = node_name
def __init__(self): # init self.in_queue = multiprocessing.JoinableQueue() self.out_queue = multiprocessing.Queue() self.data_node = config.get(config.NODES_SECTION, config.DATA_NODE_KEY)
def problem1b(train_dataset, test_dataset, dataset_name, acc_threshold):
#Final results structure
results = {
'config1': {
'graphs': None,
'test_acc_mean': None,
'test_acc_std': None,
},
'config2': {
'graphs': None,
'test_acc_mean': None,
'test_acc_std': None,
},
'config3': {
'graphs': None,
'test_acc_mean': None,
'test_acc_std': None,
},
'config4': {
'graphs': None,
'test_acc_mean': None,
'test_acc_std': None,
},
'config5': {
'graphs': None,
'test_acc_mean': None,
'test_acc_std': None,
},
}
#Setting 1
configuration['batch_size'] = 64
train_iterator = DataGenerator(
train_dataset,
shuffle=True,
batch_size=configuration.get('batch_size'),
num_workers=configuration.get('num_workers'))
test_iterator = DataGenerator(test_dataset,
shuffle=False,
batch_size=configuration.get('batch_size'),
num_workers=configuration.get('num_workers'))
if dataset_name == 'cifar':
#Different setting 1 for CIFAR
test_acc_config1 = []
graphs = []
for i in range(5):
network = ClassifierLN5(
in_channels=1,
number_of_classes=configuration.get('number_of_classes'),
init_method='uniform')
model_configuration = {
'enable_log':
configuration.get('enable_log'),
'model_path':
pjoin(configuration.get('model_path'), dataset_name,
'config1'),
'network':
network,
'use_gpu':
configuration.get('use_gpu'),
'phases': ['train', 'test'],
'loaders': {
'train': train_iterator,
'test': test_iterator
},
'loss':
CELoss(),
'learner':
RMSOptimizer(network.parameters(), lr=1e-3, weight_decay=0),
'scheduler':
None,
'max_epoch_val':
50,
'initialization_method':
configuration.get('initialization_method'),
'acc_threshold':
acc_threshold,
}
train_loss, train_acc, test_loss, test_acc = train(
model_configuration)
graphs.append({
'train_loss': train_loss,
'train_acc': train_acc,
'test_loss': test_loss,
'test_acc': test_acc
})
max_test_acc = max(test_acc)
print('Max test accuracy [Config 1]:', max_test_acc)
test_acc_config1.append(max_test_acc)
test_acc_config1 = np.array(test_acc_config1)
print('Mean test accuracy [Config 1]:', np.mean(test_acc_config1))
print('Std dev test accuracy [Config 1]:', np.std(test_acc_config1))
results['config1']['graphs'] = graphs
results['config1']['test_acc_mean'] = np.mean(test_acc_config1)
results['config1']['test_acc_std'] = np.std(test_acc_config1)
else:
#Different setting 1 for MNIST/Fashion MNIST
test_acc_config1 = []
graphs = []
for i in range(5):
network = ClassifierLN5(
in_channels=1,
number_of_classes=configuration.get('number_of_classes'),
init_method='uniform')
model_configuration = {
'enable_log':
configuration.get('enable_log'),
'model_path':
pjoin(configuration.get('model_path'), dataset_name,
'config1'),
'network':
network,
'use_gpu':
configuration.get('use_gpu'),
'phases': ['train', 'test'],
'loaders': {
'train': train_iterator,
'test': test_iterator
},
'loss':
CELoss(),
'learner':
AdamOptimizer(network.parameters(), lr=1e-3, weight_decay=0),
'scheduler':
None,
'max_epoch_val':
50,
'initialization_method':
configuration.get('initialization_method'),
'acc_threshold':
acc_threshold,
}
train_loss, train_acc, test_loss, test_acc = train(
model_configuration)
graphs.append({
'train_loss': train_loss,
'train_acc': train_acc,
'test_loss': test_loss,
'test_acc': test_acc
})
max_test_acc = max(test_acc)
print('Max test accuracy [Config 1]:', max_test_acc)
test_acc_config1.append(max_test_acc)
test_acc_config1 = np.array(test_acc_config1)
print('Mean test accuracy [Config 1]:', np.mean(test_acc_config1))
print('Std dev test accuracy [Config 1]:', np.std(test_acc_config1))
results['config1']['graphs'] = graphs
results['config1']['test_acc_mean'] = np.mean(test_acc_config1)
results['config1']['test_acc_std'] = np.std(test_acc_config1)
#Setting 2
configuration['batch_size'] = 256
train_iterator = DataGenerator(
train_dataset,
shuffle=True,
batch_size=configuration.get('batch_size'),
num_workers=configuration.get('num_workers'))
test_iterator = DataGenerator(test_dataset,
shuffle=False,
batch_size=configuration.get('batch_size'),
num_workers=configuration.get('num_workers'))
test_acc_config2 = []
graphs = []
for i in range(5):
network = ClassifierLN5(
in_channels=1,
number_of_classes=configuration.get('number_of_classes'),
init_method='xavier_normal')
model_configuration = {
'enable_log':
configuration.get('enable_log'),
'model_path':
pjoin(configuration.get('model_path'), dataset_name, 'config2'),
'network':
network,
'use_gpu':
configuration.get('use_gpu'),
'phases': ['train', 'test'],
'loaders': {
'train': train_iterator,
'test': test_iterator
},
'loss':
CELoss(),
'learner':
AdamOptimizer(network.parameters(), lr=1e-2, weight_decay=0),
'scheduler':
None,
'max_epoch_val':
configuration.get('max_epoch_val'),
'initialization_method':
configuration.get('initialization_method'),
'acc_threshold':
acc_threshold,
}
train_loss, train_acc, test_loss, test_acc = train(model_configuration)
graphs.append({
'train_loss': train_loss,
'train_acc': train_acc,
'test_loss': test_loss,
'test_acc': test_acc
})
max_test_acc = max(test_acc)
print('Max test accuracy [Config 2]:', max_test_acc)
test_acc_config2.append(max_test_acc)
test_acc_config2 = np.array(test_acc_config2)
print('Mean test accuracy [Config 2]:', np.mean(test_acc_config2))
print('Std dev test accuracy [Config 2]:', np.std(test_acc_config2))
results['config2']['graphs'] = graphs
results['config2']['test_acc_mean'] = np.mean(test_acc_config2)
results['config2']['test_acc_std'] = np.std(test_acc_config2)
# Setting 3
configuration['batch_size'] = 64
train_iterator = DataGenerator(
train_dataset,
shuffle=True,
batch_size=configuration.get('batch_size'),
num_workers=configuration.get('num_workers'))
test_iterator = DataGenerator(test_dataset,
shuffle=False,
batch_size=configuration.get('batch_size'),
num_workers=configuration.get('num_workers'))
test_acc_config3 = []
graphs = []
for i in range(5):
network = ClassifierLN5(
in_channels=1,
number_of_classes=configuration.get('number_of_classes'),
init_method='he_normal')
model_configuration = {
'enable_log':
configuration.get('enable_log'),
'model_path':
pjoin(configuration.get('model_path'), dataset_name, 'config3'),
'network':
network,
'use_gpu':
configuration.get('use_gpu'),
'phases': ['train', 'test'],
'loaders': {
'train': train_iterator,
'test': test_iterator
},
'loss':
CELoss(),
'learner':
AdamOptimizer(network.parameters(), lr=1e-3, weight_decay=0.01),
'scheduler':
None,
'max_epoch_val':
configuration.get('max_epoch_val'),
'initialization_method':
configuration.get('initialization_method'),
'acc_threshold':
acc_threshold,
}
train_loss, train_acc, test_loss, test_acc = train(model_configuration)
graphs.append({
'train_loss': train_loss,
'train_acc': train_acc,
'test_loss': test_loss,
'test_acc': test_acc
})
max_test_acc = max(test_acc)
print('Max test accuracy [Config 3]:', max_test_acc)
test_acc_config3.append(max_test_acc)
test_acc_config3 = np.array(test_acc_config3)
print('Mean test accuracy [Config 3]:', np.mean(test_acc_config3))
print('Std dev test accuracy [Config 3]:', np.std(test_acc_config3))
results['config3']['graphs'] = graphs
results['config3']['test_acc_mean'] = np.mean(test_acc_config3)
results['config3']['test_acc_std'] = np.std(test_acc_config3)
#Setting 4
configuration['batch_size'] = 64
train_iterator = DataGenerator(
train_dataset,
shuffle=True,
batch_size=configuration.get('batch_size'),
num_workers=configuration.get('num_workers'))
test_iterator = DataGenerator(test_dataset,
shuffle=False,
batch_size=configuration.get('batch_size'),
num_workers=configuration.get('num_workers'))
test_acc_config4 = []
graphs = []
for i in range(5):
network = ClassifierLN5(
in_channels=1,
number_of_classes=configuration.get('number_of_classes'),
init_method='he_normal')
model_configuration = {
'enable_log':
configuration.get('enable_log'),
'model_path':
pjoin(configuration.get('model_path'), dataset_name, 'config4'),
'network':
network,
'use_gpu':
configuration.get('use_gpu'),
'phases': ['train', 'test'],
'loaders': {
'train': train_iterator,
'test': test_iterator
},
'loss':
CELoss(),
'learner':
RMSOptimizer(network.parameters(), lr=1e-3, weight_decay=0.01),
'scheduler':
None,
'max_epoch_val':
configuration.get('max_epoch_val'),
'initialization_method':
configuration.get('initialization_method'),
'acc_threshold':
acc_threshold,
}
train_loss, train_acc, test_loss, test_acc = train(model_configuration)
graphs.append({
'train_loss': train_loss,
'train_acc': train_acc,
'test_loss': test_loss,
'test_acc': test_acc
})
max_test_acc = max(test_acc)
print('Max test accuracy [Config 4]:', max_test_acc)
test_acc_config4.append(max_test_acc)
test_acc_config4 = np.array(test_acc_config4)
print('Mean test accuracy [Config 4]:', np.mean(test_acc_config4))
print('Std dev test accuracy [Config 4]:', np.std(test_acc_config4))
results['config4']['graphs'] = graphs
results['config4']['test_acc_mean'] = np.mean(test_acc_config4)
results['config4']['test_acc_std'] = np.std(test_acc_config4)
# Setting 5
configuration['batch_size'] = 128
train_iterator = DataGenerator(
train_dataset,
shuffle=True,
batch_size=configuration.get('batch_size'),
num_workers=configuration.get('num_workers'))
test_iterator = DataGenerator(test_dataset,
shuffle=False,
batch_size=configuration.get('batch_size'),
num_workers=configuration.get('num_workers'))
test_acc_config5 = []
graphs = []
for i in range(5):
network = ClassifierLN5(
in_channels=1,
number_of_classes=configuration.get('number_of_classes'),
init_method='he_uniform')
model_configuration = {
'enable_log':
configuration.get('enable_log'),
'model_path':
pjoin(configuration.get('model_path'), dataset_name, 'config5'),
'network':
network,
'use_gpu':
configuration.get('use_gpu'),
'phases': ['train', 'test'],
'loaders': {
'train': train_iterator,
'test': test_iterator
},
'loss':
CELoss(),
'learner':
SGDOptimizer(network.parameters(), lr=1e-3, weight_decay=0.001),
'scheduler':
None,
'max_epoch_val':
configuration.get('max_epoch_val'),
'initialization_method':
configuration.get('initialization_method'),
'acc_threshold':
acc_threshold,
}
train_loss, train_acc, test_loss, test_acc = train(model_configuration)
graphs.append({
'train_loss': train_loss,
'train_acc': train_acc,
'test_loss': test_loss,
'test_acc': test_acc
})
max_test_acc = max(test_acc)
print('Max test accuracy [Config 5]:', max_test_acc)
test_acc_config5.append(max_test_acc)
test_acc_config5 = np.array(test_acc_config4)
print('Mean test accuracy [Config 5]:', np.mean(test_acc_config5))
print('Std dev test accuracy [Config 5]:', np.std(test_acc_config5))
results['config5']['graphs'] = graphs
results['config5']['test_acc_mean'] = np.mean(test_acc_config5)
results['config5']['test_acc_std'] = np.std(test_acc_config5)
with open(dataset_name + '_results.json', 'w') as f:
json.dump(results, f)
drawGraphs1b(results, dataset_name)
def __init__(self): # init self.in_queue = multiprocessing.JoinableQueue() self.out_queue = multiprocessing.Queue() self.data_node = config.get(config.NODES_SECTION, config.DATA_NODE_KEY)
import utils.naming as naming import utils.configuration as config is_debug = config.get(config.SYSTEM_SECTION, config.IS_DEBUG_KEY).lower() == naming.TRUE # At the begining of a run, nosetest instanciates the class of the tests. But, it instanciates # as many time the same class as this class has tests. Abstract classes, in utils # package, manage this problem. These global variables are centralized so as to # optimize the test suite. # Create only one instance of the browser so as to optimized the wall clock time. # As the tests share the same browser instance, AbstractBrowserBasedTest class # offers a convenient way to create a new session of the browser instance # (see reset_browser). browser = None # Create only one instance of the myproxy utils so as to optimized the wall clock time. myproxy_utils = None
