def open_file(file_path, privs):
"""
Open and return the file handle to the file identified by the provided 'file_path' with the provided 'privs'.
If an error occurs during the opening of the file then it will be logged and 'None' will be returned.
:type file_path str
:type privs str
:param file_path: the path to be opened.
:param privs: the privs with which to open the provided file path.
:return: a file handle ( object ) if the open operation is successful else None.
"""
if file_path and isinstance(file_path, basestring) and privs and isinstance(privs, basestring):
# ADD: Some verbosity
if args.verbose:
logging.info("Opening with privs [{0}]: {1}", privs, file_path)
try:
# ATTEMPT: to open the file identified by the provided file_path
file_handle = open(file_path, privs)
# LET: the verbose users know we succeeded
if args.verbose:
logging.info('Successfully preformed open [{0}] on {1}', privs, file_path)
# RETURN: the file_handle
return file_handle
except IOError, e:
logging.error('Unable to open file: {0} due to {1}: {2}', file_path, e.args[0], e.args[1])
return None
def off_parsed(args):
"""
Handles the appropriate execution of an 'Off' mode request given
the provided command line arguments.
"""
data = {
'application': args.application if args.application else ''
}
try:
result = akrrrestclient.put(
'/resources/{0}/off'.format(args.resource),
data=data)
if result.status_code == 200:
message = 'Successfully disabled {0} -> {1}.\n{2}' if args.application and args.resource \
else 'Successfully disabled all applications on {0}.\n{1}'
parameters = (args.application, args.resource, result.text) if args.application and args.resource \
else (args.resource, result.text)
log.info(message, *parameters)
else:
log.error(
'something went wrong. {0}:{1}',
result.status_code,
result.text)
except StandardError, e:
log.error('''
An error occured while communicating
with the REST API.
{0}: {1}
''',
e.args[0] if len(e.args) > 0 else '',
e.args[1] if len(e.args) > 1 else '')
def is_api_up():
request = requests.get(api_url + "/scheduled_tasks", auth=(token, ""), verify=ssl_verify, cert=ssl_cert)
if request.status_code == 200:
return True
else:
log.error('Unable to successfully contact the REST API: {0}: {1}', request.status_code, request.text)
return False
def write_to_file(file_handle, lines):
"""
Write the provided lines to the provided file_handle. If an error occurs an error message will be logged to stdout.
:type lines: list
:type file_handle: file
:param file_handle: the file that the provided lines should be written
:param lines: the lines that should be written to the provided file.
:return: void
"""
if not file_handle or not isinstance(file_handle, file):
logging.error('Received an invalid file reference.')
return
if lines:
try:
if args.verbose:
logging.info("Writing {0} lines to {1}", len(lines), file_handle.name)
file_handle.writelines(lines)
if args.verbose:
logging.info("Successfully wrote {0} lines to {1}", len(lines), file_handle.name)
except IOError, e:
logging.error('There was an error while writing to {0}. {1}: {2}', file_handle.name, e.args[0], e.args[1])
def read_from_file(file_handle):
"""
Read the contents from the provided file_handle and return them. If there is an error then a message detailing the
problem will be written to stdout and None will be returned.
:type file_handle file
:param file_handle: the file object to read from.
:return: the contents of the file or, if an error is encountered, None.
"""
if file_handle and isinstance(file_handle, file):
# ADDING: verbose logging before the operation.
if args.verbose:
logging.info('Attempting to read in contents of {0}', file_handle.name)
try:
# ATTEMPT: to read the contents of the file.
contents = file_handle.read()
# PROVIDE: the user some verbose success logging.
if args.verbose:
logging.info('Successfully read the contents of {0}', file_handle.name)
# RETURN: the contents of the file.
return contents
except IOError, e:
logging.error('There was a problem reading from {0}. {1}: {2}', file_handle.name, e.args[0], e.args[1])
return None
def generate_self_signed_certificate(self):
log.info("Generating self-signed certificate for REST-API")
try:
output=subprocess.check_output("which openssl", shell=True)
except Exception,e:
log.error("""openssl program is not available. Install it!
For example by running on Ubuntu:
sudo apt-get install openssl""")
exit(1)
def getSytemCharacteristics():
global ppn
while True:
try:
logging.input("Enter processors (cores) per node count:")
ppn=int(raw_input(""))
break
except Exception,e:
logging.error("Incorrect entry, try again.")
def populate_token():
request = requests.get(api_url + "/token", auth=HTTPBasicAuth(akrr.restapi_rw_username, akrr.restapi_rw_password), verify=ssl_verify, cert=ssl_cert)
if request.status_code == 200:
global token
token = request.json()['data']['token']
else:
log.error('Something went wrong when attempting to contact the REST API.')
return token is not None
def get_json_request_response(url: str):
response = requests.get(url)
if not response.ok:
logging.error(
f"Steam API response not OK: {response.status_code} [{url}]")
return None
json = response.json()
return json
def get_inspirational_quote() -> str:
response = requests.get("http://inspirobot.me/api?generate=true")
if not response.ok:
logging.error(
f"Inspirobot API response not OK: {response.status_code} [http://inspirobot.me/api?generate=true]"
)
return None
return response.text
def new_session(self):
"""
Add new clients to the list of connected sessions
"""
try:
session = Session(self)
game.session_manager.connections.append(session)
except Exception as e:
log.error("Error caught (connection.py): " + str(e))
def add_room(self, room):
"""
Add a room to the collection of rooms
:param room:
:return:
"""
if room.id not in self.rooms:
self.rooms[room.id] = room
else:
log.error("Cannot add this room to the rooms collection")
def check_previous_installation(self):
if(os.path.exists(os.path.join(akrr_home,"cfg","akrr.inp.py"))):
msg="This is a fresh installation script. "+akrr_home+\
" contains previous AKRR installation. Either uninstall it or see documentation on updates.\n\n";
msg+="To uninstall AKRR manually:\n\t1)remove $AKRR_HOME/cfg/akrr.inp.py\n\t\trm $AKRR_HOME/cfg/akrr.inp.py\n"
msg+="\t2) (optionally for totally fresh start) drop mod_akrr and mod_appkernel database\n"
msg+="\t\tDROP DATABASE mod_appkernel;\n"
msg+="\t\tDROP DATABASE mod_akrr;\n\n"
log.error(msg)
exit(1)
def load(file: str) -> bool:
global settings
filepath = file
try:
with open(file, "r") as sett_file:
settings = json.load(sett_file)
logging.info(f"Trying {filepath}.....Success!")
return True
except Exception as e:
logging.info(f"Trying {filepath}.....Failed!")
logging.error(str(e))
return False
def resolve_vanity_url(vanity_url: str) -> str:
url = assemble_url("ISteamUser", "ResolveVanityURL",
1) + f"&vanityurl={vanity_url}"
json = get_json_request_response(url)
success = json["response"]["success"]
if success != 1:
logging.error(
f"Something went wrong while resolving Vanity URL: {success}")
return None
return json["response"]["steamid"]
def image_of_the_day() -> str:
url = f"https://api.nasa.gov/planetary/apod?api_key={api_key}"
response = requests.get(url)
if not response.ok:
logging.error(
f"Failed to fetch Image of the day (NASA API): {response.status_code}"
)
return
image_link = response.json()["hdurl"]
return image_link
def wall_time_parsed(args):
if not args.list and not (args.resource and
args.appkernel and
args.nodes and
args.walltime):
parser.error(
'Please provide a resource, app, node count and wall time.')
exit(1)
listing = args.list
resource = args.resource
app = args.appkernel
nodes = args.nodes
walltime = args.walltime
comments = args.comments
node_list = [node.strip() for node in nodes.split(',')] if ',' in nodes else list(nodes)
for nodes in node_list:
data = {
'resource_params': "{'nnodes':%d}" % (int(nodes),) if nodes else "{}",
'app_param':'{}',
'walltime': walltime,
'comments':comments
}
try:
result = akrrrestclient.post(
'/walltime/%s/%s'%(resource,app),
data=data) if not listing else \
akrrrestclient.get(
'/walltime/%s/%s'%(resource,app),
data=data)
if result.status_code == 200:
if not listing:
log.info('Successfully updated wall time (resource %s: application kernel: %s nodes: %d).'%(resource,app,nodes))
else:
log.info(
'Successfully queried walltime records. \n{0}',
result.text)
else:
log.error('something went wrong. {0}:{1}',
result.status_code,
result.text)
except StandardError, e:
import traceback
log.error('''
An error occured while communicating
with the REST API.
{0}: {1}
'''.strip(),
e.args[0] if len(e.args) > 0 else '',
e.args[1] if len(e.args) > 1 else '')
print traceback.print_exc()
def new_task_parsed(args):
"""
Handles the appropriate execution of a 'New Task' mode request
given the provided command line arguments.
"""
if not (args.resource and
args.appkernel and
args.nodes):
parser.error(
'Please provide a resource, application and node count.')
exit(1)
resource = args.resource
app = args.appkernel
time_to_start=args.start_time
time_start = args.time_start# if args.time_start else '01:00'
time_end = args.time_end# if args.time_end else '05:00'
repeat_in = args.periodicity
nodes = args.nodes
node_list = [node.strip() for node in nodes.split(',')] if ',' in nodes else list(nodes)
for node in node_list:
if time_start!=None and time_end!=None:
time_to_start = calculate_random_start_time(
args.start_time,
repeat_in,
time_start,
time_end)
data = {
'resource': resource,
'app': app,
'time_to_start': time_to_start,
'repeat_in': repeat_in,
'resource_param': "{'nnodes':%s}" % (node,)
}
try:
result = akrrrestclient.post(
'/scheduled_tasks',
data=data)
if result.status_code == 200:
log.info('Successfully submitted new task')
else:
log.error(
'something went wrong. {0}:{1}',
result.status_code,
result.text)
except StandardError, e:
log.error('''
An error occured while communicating
with the REST API.
{0}: {1}
''',
e.args[0] if len(e.args) > 0 else '',
e.args[1] if len(e.args) > 1 else '')
def incoming_message(self, client, message_id, message_data):
if message_data is not None:
if message_id in self.events:
log.line("[PACKET] " + str(message_id) + " : waiting...")
try:
self.events[message_id].handle(client, message_data)
log.line("[PACKET] " + str(message_id) + " : OK!")
except Exception as e:
log.line("[PACKET] " + message_id + " : doesn't work!")
log.error(str(e))
else:
log.error("This packet id is not registered!")
def read_sql_root_creds(self):
while True:
log.input("""Please provide an administrative database user under which the installation sql script should
run (This user must have privileges to create users and databases):""")
self.sql_root_name=raw_input()
log.input("Please provide the password for the the user which you previously entered:")
self.sql_root_password=getpass.getpass()
try:
self.get_db(user=self.sql_root_name,password=self.sql_root_password)
break
except Exception,e:
log.error("Entered credential is not valid. Please try again.")
def get_steam_level(vanity_url: str) -> str:
steamid = resolve_vanity_url(vanity_url)
if steamid is None:
logging.error("Could not resolve Vanity URL")
return None
url = assemble_url("IPlayerService", "GetSteamLevel",
1) + f"&steamid={steamid}"
json = get_json_request_response(url)
if len(json["response"]) == 0:
return None
return json["response"]["player_level"]
def validate_resource_name(resource_name):
if resource_name.strip()=="":
logging.error("Bad name for resource, try a different name")
return False
#check config file presence
file_path = os.path.abspath(os.path.join(resources_dir, resource_name))
if os.path.exists(file_path):
logging.error("Resource configuration directory (%s) for resource with name %s already present on file system, try a different name"%(file_path,resource_name,))
return False
#check the entry in mod_appkernel
dbAK,curAK=akrr.getAKDB(True)
curAK.execute('''SELECT * FROM resource WHERE nickname=%s''', (resource_name,))
resource_in_AKDB = curAK.fetchall()
if len(resource_in_AKDB)!=0:
logging.error("Resource with name %s already present in mod_appkernel DB, try a different name"%(resource_name,))
return False
#check the entry in mod_akrr
db,cur=akrr.getDB(True)
cur.execute('''SELECT * FROM resources WHERE name=%s''', (resource_name,))
resource_in_DB = cur.fetchall()
if len(resource_in_DB)!=0:
logging.error("Resource with name %s already present in mod_akrr DB, try a different name"%(resource_name,))
return False
return True
def init_mysql_dbs(self):
try:
log.info("Creating AKRR databases and granting permissions for AKRR user.")
db_root,cur_root=self.get_db(user=self.sql_root_name,password=self.sql_root_password)
cur_root.execute("SHOW DATABASES")
dbsNames=[v['Database'] for v in cur_root.fetchall()]
cur_root.execute("SELECT @@hostname")
results=cur_root.fetchall()
hostname=results[0]['@@hostname']
#create user if needed
#cur_root.execute("SELECT * FROM mysql.user WHERE User=%s",(self.akrr_user_name,))
#results=cur_root.fetchall()
# ENSURE: That the `mod_akrr` database is created.
if 'mod_akrr' not in dbsNames:
cur_root.execute("CREATE DATABASE IF NOT EXISTS mod_akrr")
while cur_root.nextset() is not None: pass
# ENSURE: That the `mod_appkernel` database is created.
if 'mod_appkernel' not in dbsNames:
cur_root.execute("CREATE DATABASE IF NOT EXISTS mod_appkernel")
while cur_root.nextset() is not None: pass
# ENSURE: That the user that will be used by AKRR is created with the correct privileges.
cur_root.execute("GRANT ALL ON mod_akrr.* TO %s@%s IDENTIFIED BY %s",(self.akrr_user_name, '%', self.akrr_user_password))
cur_root.execute("GRANT ALL ON mod_akrr.* TO %s@%s IDENTIFIED BY %s",(self.akrr_user_name, 'localhost', self.akrr_user_password))
cur_root.execute("GRANT ALL ON mod_akrr.* TO %s@%s IDENTIFIED BY %s",(self.akrr_user_name, hostname, self.akrr_user_password))
while cur_root.nextset() is not None: pass
# ENSURE: That the AKRR user has the correct privileges to the `mod_appkernel` database.
cur_root.execute("GRANT ALL ON mod_appkernel.* TO %s@%s IDENTIFIED BY %s",(self.akrr_user_name, '%', self.akrr_user_password))
cur_root.execute("GRANT ALL ON mod_appkernel.* TO %s@%s IDENTIFIED BY %s",(self.akrr_user_name, 'localhost', self.akrr_user_password))
cur_root.execute("GRANT ALL ON mod_appkernel.* TO %s@%s IDENTIFIED BY %s",(self.akrr_user_name, hostname, self.akrr_user_password))
while cur_root.nextset() is not None: pass
# ENSURE: That the AKRR modw user is created w/ the correct privileges
cur_root.execute("GRANT SELECT ON modw.resourcefact TO %s@%s IDENTIFIED BY %s",(self.xd_user_name, '%', self.xd_user_password))
cur_root.execute("GRANT SELECT ON modw.resourcefact TO %s@%s IDENTIFIED BY %s",(self.xd_user_name, 'localhost', self.xd_user_password))
cur_root.execute("GRANT SELECT ON modw.resourcefact TO %s@%s IDENTIFIED BY %s",(self.xd_user_name, hostname, self.xd_user_password))
while cur_root.nextset() is not None: pass
# ENSURE: That the newly granted privileges are flushed into active service.
cur_root.execute("FLUSH PRIVILEGES")
while cur_root.nextset() is not None: pass
db_root.commit()
except Exception,e:
log.error("Can not execute the sql install script: "+str(e))
exit(1)
def read_akrr_creds(self):
log.info("Before Installation continues we need to setup the database.")
log.input("Please specify a database user for AKRR (This user will be created if it does not already exist):")
self.akrr_user_name=raw_input('[{0}] '.format(self.default_akrr_user))
if self.akrr_user_name=='':self.akrr_user_name=self.default_akrr_user
while True:
log.input("Please specify a password for the AKRR database user:"******"Please reenter password:"******"Entered passwords do not match. Please try again.")
def retrieve_resources():
"""
Retrieve the applicable contents of the `modw`.`resourcefact` table.
:return: a tuple of strings containing the name of the resources.
"""
try:
connection, cursor = akrr.getXDDB()
# PROVIDES: automatic resource cleanup
with connection:
cursor = connection.cursor()
cursor.execute("SELECT `name`,`id` FROM `modw`.`resourcefact`")
rows = cursor.fetchall()
except MySQLdb.Error, e:
logging.error("MySQL Error: {0}: {1}", e.args[0]. e.args[1])
sys.exit(1)
def read_modw_creds(self):
log.input("Please specify the user that will be connecting to the XDMoD database (modw):")
self.xd_user_name=raw_input('[{0}] '.format(self.default_akrr_user))
if self.xd_user_name=='':self.xd_user_name=self.default_akrr_user
if self.xd_user_name==self.akrr_user_name:
log.info("Same user as for AKRR database user, will set same password")
self.xd_user_password=self.akrr_user_password
else:
while True:
log.input("Please specify the password:"******"Please reenter password:"******"Entered passwords do not match. Please try again.")
def close_file(file_handle):
"""
Close the provided file_handle. If an error is encountered than a message will be logged to stdout.
:type file_handle file
:param file_handle: the file to be closed.
:return: void
"""
if file_handle and isinstance(file_handle, file):
try:
if args.verbose:
logging.info('Attempting to close the file {0}', file_handle.name)
file_handle.close()
if args.verbose:
logging.info('Successfully closed the file {0}', file_handle.name)
except IOError, e:
logging.error('There was an error encountered while closing {0}. {1}: {2}', file_handle.name, e.args[0], e.args[1])
async def on_ready():
# Try to load cogs
logging.info("--- Loading cogs ---\n")
total = 0
failed = 0
# Load all cogs on startup
# Load "fun" cogs
for filename in os.listdir("./cogs/fun"):
if filename.endswith(".py"):
total += 1
cog = f"cogs.fun.{filename[:-3]}"
try:
client.load_extension(cog)
logging.info(f"Trying {cog}.....Success!")
except Exception as e:
logging.info(f"Trying {cog}.....Failed!")
logging.error(str(e))
failed += 1
# Load "api" cogs
for filename in os.listdir("./cogs/api"):
if filename.endswith(".py"):
total += 1
cog = f"cogs.api.{filename[:-3]}"
try:
client.load_extension(cog)
logging.info(f"Trying {cog}.....Success!")
except Exception as e:
logging.info(f"Trying {cog}.....Failed!")
logging.error(str(e))
failed += 1
logging.info("Finished loading cogs.")
logging.info(f"Total {total}, Failed {failed}\n")
logging.info("Deep Blue finished loading.")
def akrrdb_check():
# CHECK: to make sure that we have MySQL drivers before continuing.
if not mysql_available:
log.error("Unable to find MySQLdb. Please install MySQLdb for python before running this script again.")
exit(1)
# CHECK: the akrr db
akrr_ok = check_rw_db(
akrr.getDB,
"Checking 'mod_akrr' Database / User privileges...",
"'mod_akrr' Database check complete - Status: {0}",
)
# Check: the app_kernel db
app_kernel_ok = check_rw_db(
akrr.getAKDB,
"Checking 'mod_appkernel' Database / User privileges...",
"'mod_appkernel' Database check complete - Status: {0}",
)
# CHECK: the XDMoD db
xdmod_ok = check_r_db(
akrr.getXDDB, "Checking 'modw' Database / User privileges...", "'modw' Database check complete - Status: {0}"
)
# DETERMINE: whether or not everything passed.
overall_success = akrr_ok and app_kernel_ok and xdmod_ok
if overall_success:
log.info("All Databases / User privileges check out!")
else:
log.error(
"One or more of the required databases and their required users ran into a problem. Please take note of the previous messages, correct the issue and re-run this script."
)
exit(1)
def batch_job_parsed(args):
if not (args.resource and
args.appkernel and
args.nodes):
parser.error(
'Please provide a resource, application kernel and node count.')
exit(1)
resource = akrr.FindResourceByName(args.resource)
app = akrr.FindAppByName(args.appkernel)
nodes = args.nodes
node_list = [node.strip() for node in nodes.split(',')] if ',' in nodes else [int(nodes)]
print_only=args.print_only
verbose=args.verbose
str_io=cStringIO.StringIO()
if not verbose:
sys.stdout = sys.stderr = str_io
from akrrtaskappker import akrrTaskHandlerAppKer
# taskHandler=akrrTaskHandlerAppKer(1,resource['name'],app['name'],"{'nnodes':%s}" % (node_list[0],),"{}","{}")
# test arbitrary resourceParam like WLMheader stuff
taskHandler=akrrTaskHandlerAppKer(1,resource['name'],app['name'],"{'SlUrM':'--foo','slurm':'#SBATCH --mail=maya','nnodes':%s}" % (node_list[0],),"{}","{}")
if print_only:
taskHandler.GenerateBatchJobScript()
else:
taskHandler.CreateBatchJobScriptAndSubmitIt(doNotSubmitToQueue=True)
sys.stdout=sys.__stdout__
sys.stderr=sys.__stderr__
if taskHandler.status.count("ERROR")>0:
log.error('Batch job script was not generated see log below!')
print str_io.getvalue()
log.error('Batch job script was not generated see log above!')
jobScriptFullPath=os.path.join(taskHandler.taskDir,"jobfiles",taskHandler.JobScriptName)
if os.path.isfile(jobScriptFullPath):
fin=open(jobScriptFullPath,"r")
jobScriptContent=fin.read()
fin.close()
if print_only:
log.info('Below is content of generated batch job script:')
print jobScriptContent
else:
log.info("Local copy of batch job script is "+jobScriptFullPath)
print
log.info("Application kernel working directory on "+resource['name']+" is "+taskHandler.remoteTaskDir)
log.info("Batch job script location on "+resource['name']+" is "+os.path.join(taskHandler.remoteTaskDir,taskHandler.JobScriptName))
else:
log.error('Batch job script was not generated see messages above!')
if print_only:
log.info('Removing generated files from file-system as only batch job script printing was requested')
taskHandler.DeleteLocalFolder()
def retrieve_tasks(resource, application):
"""
Retrieve the list of currently scheduled tasks ( resource / application pairings ) from
mod_akrr.
:type resource str
:type application str
:param resource: filter the results by the provided resource
:param application: filter the results by the provided application
:return: a dict representation of the mod_akrr.SCHEDULEDTASKS table
"""
data = {
'application': application,
'resource': resource
}
try:
akrrrestclient.get_token()
except StandardError:
log.error('''
An error occured while attempting to retrieve a token
from the REST API.
''')
try:
result = akrrrestclient.get(
'/scheduled_tasks',
data=data)
if result.status_code == 200:
log.info('Successfully Completed Task Retrieval.\n{0}', result.text)
else:
log.error(
'something went wrong. {0}:{1}',
result.status_code,
result.text)
return result
except StandardError, e:
log.error('''
An error occured while communicating
with the REST API.
{0}: {1}
''',
e.args[0] if len(e.args) > 0 else '',
e.args[1] if len(e.args) > 1 else '')
def check_rw_db(connection_func, pre_msg, post_msg):
"""
Check that the user has the correct privileges to the database
at the end of the connection provided by 'connection_func'. Specifically, checking
for read / write permissions ( and create table ).
:type connection_func function
:type pre_msg str
:type post_msg str
:param connection_func: the function that will provide a (connection, cursor) tuple.
:param pre_msg: a message to be provided to the user before the checks begin.
:param post_msg: a message to be provided to the user after the checks are successful
:return: true if the database is available / the provided user has the correct privileges.
"""
success = False
log.info(pre_msg)
try:
connection, cursor = connection_func()
try:
with connection:
result = cursor.execute("CREATE TABLE CREATE_ME(`id` INT NOT NULL PRIMARY KEY, `name` VARCHAR(48));")
success = True if result == 0 else False
if success:
log.info(post_msg, success)
else:
log.error(post_msg, success)
except MySQLdb.MySQLError, e:
log.error("Unable to create a table w/ the provided username. {0}: {1}", e.args[0], e.args[1])
connection, cursor = connection_func()
try:
with connection:
cursor.execute("DROP TABLE CREATE_ME;")
except MySQLdb.MySQLError, e:
log.error("Unable to drop the table created to check permissions. {0}: {1}", e.args[0], e.args[1])
def check_r_db(connection_func, pre_msg, post_msg):
"""
Check that the user has the correct privileges to the database
at the end of the connection provided by 'connection_func'.
Specifically checking for read permissions.
:type connection_func function
:type pre_msg str
:type post_msg str
:param connection_func: the function that will provide a (connection, cursor) tuple.
:param pre_msg: a message to be provided to the user before the checks begin.
:param post_msg: a message to be provided to the user after the checks are successful
:return: true if the database is available / the provided user has the correct privileges.
"""
success = False
log.info(pre_msg)
try:
connection, cursor = connection_func()
try:
with connection:
result = cursor.execute("SELECT COUNT(*) FROM `modw`.`resourcefact`;")
success = True if result >= 0 else False
if success:
log.info(post_msg, success)
else:
log.error(post_msg, success)
except MySQLdb.MySQLError, e:
log.error("Unable to select from `modw`.`resourcefact`. {0}: {1}", e.args[0], e.args[1])
except MySQLdb.MySQLError, e:
log.error("Unable to connect to Database. {0}: {1}", e.args[0], e.args[1])
from database.connection import Database
import manager
log.line("#####################")
log.line("# #")
log.line("# Scuti Server #")
log.line("# #")
log.line("#####################")
scuti.clients = {}
try:
scuti.db = Database("localhost", "root", "", "scuti")
log.info("Connected to database!")
# Room
manager.room.load_rooms()
log.info("Room manager loaded!")
# Users
manager.user.load_users()
log.info("User manager loaded!")
except Exception as e:
log.error(str(e))
exit()
log.line()
log.info("Server online!")
server = SimpleWebSocketServer("", 3000, Server)
server.serveforever()
log.info(post_msg, success)
else:
log.error(post_msg, success)
except MySQLdb.MySQLError, e:
log.error("Unable to create a table w/ the provided username. {0}: {1}", e.args[0], e.args[1])
connection, cursor = connection_func()
try:
with connection:
cursor.execute("DROP TABLE CREATE_ME;")
except MySQLdb.MySQLError, e:
log.error("Unable to drop the table created to check permissions. {0}: {1}", e.args[0], e.args[1])
except MySQLdb.MySQLError, e:
log.error("Unable to connect to Database. {0}: {1}", e.args[0], e.args[1])
return success
def check_r_db(connection_func, pre_msg, post_msg):
"""
Check that the user has the correct privileges to the database
at the end of the connection provided by 'connection_func'.
Specifically checking for read permissions.
:type connection_func function
:type pre_msg str
:type post_msg str
:param connection_func: the function that will provide a (connection, cursor) tuple.
cmd = kwargs.get("args")
if cmd is None:
cmd = popenargs[0]
raise subprocess.CalledProcessError(retcode, cmd)
return output
subprocess.check_output = f
from util import logging as log
try:
import MySQLdb
import MySQLdb.cursors
except Exception,e:
log.error("""python module MySQLdb is not available. Install it!
For example by running on Ubuntu:
sudo apt-get install python2.7-mysqldb""")
exit(1)
class InstallAKRR:
default_akrr_user='******'
def __init__(self):
self.akrr_user_name=None
self.akrr_user_password=None
self.xd_user_name=None
self.xd_user_password=None
self.sql_root_name=None
self.sql_root_password=None
self.cronemail=None
def getFileSytemAccessPoints():
global networkScratch
global localScratch
global akrrData
global appKerDir
homeDir=akrr.sshCommand(rsh,"echo $HOME").strip()
scratchNetworkDir=akrr.sshCommand(rsh,"echo $SCRATCH").strip()
#localScratch
localScratchDefault="/tmp"
while True:
logging.input("Enter location of local scratch (visible only to single node):")
localScratch=raw_input("[%s]"%localScratchDefault)
if localScratch.strip()=="":
localScratch=localScratchDefault
status,msg=resource_validation_and_deployment.CheckDirSimple(rsh, localScratch)
if status:
logging.info(msg)
print
break
else:
logging.warning(msg)
logging.warning('local scratch might be have a different location on head node, so if it is by design it is ok')
print
break
localScratch=akrr.sshCommand(rsh,"echo %s"%(localScratch,)).strip()
#networkScratch
networkScratchDefault=""
if scratchNetworkDir!="":
networkScratchDefault=scratchNetworkDir
networkScratchVisible=False
while True:
logging.input("Enter location of network scratch (visible only to all nodes), used for temporary storage of app kernel input/output:")
if networkScratchDefault!="":
networkScratch=raw_input("[%s]"%networkScratchDefault)
if networkScratch.strip()=="":
networkScratch=networkScratchDefault
else:
networkScratch=raw_input("")
if networkScratch=="":
logging.error("Incorrect value for networkScratch, try again")
continue
status,msg=resource_validation_and_deployment.CheckDir(rsh, networkScratch,exitOnFail=False,tryToCreate=True)
if status:
logging.info(msg)
networkScratchVisible=True
print
break
else:
logging.warning(msg)
#logging.warning('network scratch might be have a different location on head node, so if it is by design it is ok')
#print
break
networkScratch=akrr.sshCommand(rsh,"echo %s"%(networkScratch,)).strip()
#appKerDir
appKerDirDefault=os.path.join(homeDir,"appker",resourceName)
while True:
logging.input("Enter future location of app kernels input and executable files:")
appKerDir=raw_input("[%s]"%appKerDirDefault)
if appKerDir.strip()=="":
appKerDir=appKerDirDefault
status,msg=resource_validation_and_deployment.CheckDir(rsh, appKerDir,exitOnFail=False,tryToCreate=True)
if status:
logging.info(msg)
print
break
else:
logging.error(msg)
appKerDir=akrr.sshCommand(rsh,"echo %s"%(appKerDir,)).strip()
#akrrData
akrrDataDefault=os.path.join(homeDir,"akrrdata",resourceName)
if networkScratchVisible:
akrrDataDefault=os.path.join(networkScratch,"akrrdata",resourceName)
while True:
logging.input("Enter future locations for app kernels working directories (can or even should be on scratch space):")
akrrData=raw_input("[%s]"%akrrDataDefault)
if akrrData.strip()=="":
akrrData=akrrDataDefault
status,msg=resource_validation_and_deployment.CheckDir(rsh, akrrData,exitOnFail=False,tryToCreate=True)
if status:
logging.info(msg)
print
break
else:
logging.error(msg)
akrrData=akrr.sshCommand(rsh,"echo %s"%(akrrData,)).strip()
"""
Data access object class
author: TheAmazingAussie (Alex)
"""
import util.logging as log
from database.database_connect import DatabaseConnection
from database.access_objects.user_dao import UserDao
from database.access_objects.navigator_dao import NavigatorDao
from database.access_objects.room_dao import RoomDao
user = None
navigator = None
room_dao = None
try:
user = UserDao(DatabaseConnection())
navigator = NavigatorDao(DatabaseConnection())
room_dao = RoomDao(DatabaseConnection())
except Exception as e:
log.error("Error caught (" + __file__ + "): " + str(e))
def init_dao():
"""
Query database after DAO class has initialised
:return:
"""
room_dao.get_models()
def getRemoteAccessMethod():
global remoteAccessNode
global remoteAccessMethod
global remoteCopyMethod
global sshUserName
global sshPassword
global sshPassword4thisSession
global sshPrivateKeyFile
global sshPrivateKeyPassword
global rsh
#set remoteAccessNode
while True:
logging.input("Enter Resource head node (access node) full name (e.g. headnode.somewhere.org):")
remoteAccessNode=raw_input("[%s] "%resourceName)
if remoteAccessNode.strip()=="":
remoteAccessNode=resourceName
response = os.system("ping -c 1 -w2 " + remoteAccessNode + " > /dev/null 2>&1")
if response==0:
break
else:
logging.error("Incorrect head node name (can not ping %s), try again"%remoteAccessNode)
#set sshUserName
curentuser=getpass.getuser()
askForUserName=True
successfullyConnected=False
while True:
if askForUserName:
logging.input("Enter username for resource access:")
sshUserName=raw_input("[%s] "%curentuser)
if sshUserName.strip()=="":
sshUserName=curentuser
curentuser=sshUserName
#check passwordless access
if sshPassword==None:
logging.info("Checking for password-less access")
else:
logging.info("Checking for resource access")
successfullyConnected=checkConnectionToResource()
if successfullyConnected:
if sshPassword==None:
logging.info("Can access resource without password")
else:
logging.info("Can access resource")
if successfullyConnected==False:
logging.info("Can not access resource without password")
actionList=[]
actionList.append(["TryAgain","The private and public keys was generated manually, right now. Try again."])
#check private keys
userHomeDir = os.path.expanduser("~")
privateKeys = [ os.path.join(userHomeDir,'.ssh',f[:-4]) for f in os.listdir(os.path.join(userHomeDir,'.ssh')) if os.path.isfile(os.path.join(userHomeDir,'.ssh',f)) \
and f[-4:]=='.pub' and os.path.isfile(os.path.join(userHomeDir,'.ssh',f[:-4]))]
if len(privateKeys)>0:
actionList.append(["UseExistingPrivateKey","Use existing private and public key."])
actionList.append(["GenNewKey","Generate new private and public key."])
actionList.append(["UsePassword","Use password directly."])
print
print "Select authentication method:"
for i in range(len(actionList)):
print "%3d %s"%(i,actionList[i][1])
while True:
logging.input("Select option from list above:")
try:
action=raw_input("[2] ")
if action.strip()=="":action=2
else: action=int(action)
if action<0 or action>=len(actionList):
raise
break
except Exception,e:
logging.error("Incorrect entry, try again.")
#do the action
print
if actionList[action][0]=="TryAgain":
continue
if actionList[action][0]=="UsePassword":
logging.input("Enter password for %s@%s:"%(sshUserName,remoteAccessNode))
sshPassword=getpass.getpass("")
askForUserName=not askForUserName
continue
if actionList[action][0]=="UseExistingPrivateKey":
print "Available private keys:"
for i in range(len(privateKeys)):
print "%3d %s"%(i,privateKeys[i])
while True:
logging.input("Select key number from list above:")
try:
iKey=raw_input("")
iKey=int(iKey)
if iKey<0 or iKey>=len(privateKeys):
raise
break
except Exception,e:
logging.error("Incorrect entry, try again.")
sshPrivateKeyFile=privateKeys[iKey]
askForUserName=not askForUserName
continue
if actionList[action][0]=="GenNewKey":
count=0
while True:
logging.input("Enter password for %s@%s (will be used only during this session):"%(sshUserName,remoteAccessNode))
sshPassword4thisSession=getpass.getpass("")
sshPassword=sshPassword4thisSession
if checkConnectionToResource():
break
count+=1
if count>=3:
break
sshPassword=None
#generate keys
logging.input("Enter private key name:")
sshPrivateKeyFile=raw_input("[id_rsa_%s]"%resourceName)
if sshPrivateKeyFile.strip()=="":
sshPrivateKeyFile="id_rsa_%s"%resourceName
sshPrivateKeyFile=os.path.join(userHomeDir,'.ssh',sshPrivateKeyFile)
logging.input("Enter passphrase for new key (leave empty for passwordless access):")
sshPrivateKeyPassword=getpass.getpass("")
os.system("ssh-keygen -t rsa -N \"%s\" -f %s"%(sshPrivateKeyPassword,sshPrivateKeyFile))
if sshPrivateKeyPassword.strip()=="":
sshPrivateKeyPassword=None
#copy keys
akrr.sshAccess(remoteAccessNode, ssh='ssh-copy-id', username=sshUserName, password=sshPassword4thisSession,
PrivateKeyFile=sshPrivateKeyFile, PrivateKeyPassword=None, logfile=sys.stdout,
command='')
askForUserName=not askForUserName
continue
def checkConnectionToResource():
successfullyConnected=False
global remoteAccessNode
global remoteAccessMethod
global remoteCopyMethod
global sshUserName
global sshPassword
global sshPassword4thisSession
global sshPrivateKeyFile
global sshPrivateKeyPassword
passphraseEntranceCount=0
authorizeKeyCount=0
while True:
str_io=cStringIO.StringIO()
try:
sys.stdout = sys.stderr = str_io
akrr.sshAccess(remoteAccessNode, ssh=remoteAccessMethod, username=sshUserName, password=sshPassword,
PrivateKeyFile=sshPrivateKeyFile, PrivateKeyPassword=sshPrivateKeyPassword, logfile=str_io,
command='ls')
sys.stdout=sys.__stdout__
sys.stderr=sys.__stderr__
successfullyConnected=True
break
except Exception,e:
sys.stdout=sys.__stdout__
sys.stderr=sys.__stderr__
if args.verbose:
logging.info("Had attempted to access resource without password and failed, below is resource response")
print "="*80
print str_io.getvalue()
print traceback.format_exc()
print "="*80
#check if it asking for passphrase
m=re.search(r"Enter passphrase for key '(.*)':",str_io.getvalue())
if m:
if passphraseEntranceCount>=3:
sshPrivateKeyPassword=None
sshPrivateKeyFile=None
break
if passphraseEntranceCount>0:
logging.error("Incorrect passphrase try again")
sshPrivateKeyFile=m.group(1)
logging.input("Enter passphrase for key '%s':"%sshPrivateKeyFile)
sshPrivateKeyPassword=getpass.getpass("")
passphraseEntranceCount+=1
continue
m2=re.search(r"[pP]assword:",str_io.getvalue())
if m==None and sshPrivateKeyFile!=None and m2:
logging.warning("Can not login to head node. Probably the public key of private key was not authorized on head node")
print "Will try to add public key to list of authorized keys on head node"
while True:
try:
authorizeKeyCount+=1
logging.input("Enter password for %s@%s (will be used only during this session):"%(sshUserName,remoteAccessNode))
sshPassword4thisSession=getpass.getpass("")
print
str_io=cStringIO.StringIO()
sys.stdout = sys.stderr = str_io
akrr.sshAccess(remoteAccessNode, ssh='ssh-copy-id', username=sshUserName, password=sshPassword4thisSession,
PrivateKeyFile=sshPrivateKeyFile, PrivateKeyPassword=None, logfile=str_io,
command='')
sys.stdout=sys.__stdout__
sys.stderr=sys.__stderr__
print str_io.getvalue()
#successfullyConnected=True
logging.info("Have added public key to list of authorized keys on head node, will attempt to connect again.")
print
break
except Exception,e:
sys.stdout=sys.__stdout__
sys.stderr=sys.__stderr__
if args.verbose:
logging.info("Had attempted to add public key to list of authorized keys on head node and failed, below is resource response")
print "="*80
print str_io.getvalue()
print traceback.format_exc()
print "="*80
logging.info("Incorrect password try again.")
if authorizeKeyCount>=3:
break
if authorizeKeyCount<3:
continue
break
def populate_token():
request = requests.get(api_url + "/token", auth=HTTPBasicAuth(akrr.restapi_rw_username, akrr.restapi_rw_password), verify=ssl_verify, cert=ssl_cert)
if request.status_code == 200:
global token
token = request.json()['data']['token']
else:
log.error('Something went wrong when attempting to contact the REST API.')
return token is not None
def is_api_up():
request = requests.get(api_url + "/scheduled_tasks", auth=(token, ""), verify=ssl_verify, cert=ssl_cert)
if request.status_code == 200:
return True
else:
log.error('Unable to successfully contact the REST API: {0}: {1}', request.status_code, request.text)
return False
if __name__ == '__main__':
log.info('Beginning check of the AKRR Rest API...')
token_populated = populate_token()
if token_populated:
is_up = is_api_up()
if is_up:
log.info('REST API is up and running!')
else:
exit(1)
else:
log.error('Unable to retrieve authentication token.')
exit(1)
def add_user(self, user):
if user.id not in self.users:
self.users[user.id] = user
else:
log.error("Cannot add this user to the users collection")
logging.input("Enter passphrase for new key (leave empty for passwordless access):")
sshPrivateKeyPassword=getpass.getpass("")
os.system("ssh-keygen -t rsa -N \"%s\" -f %s"%(sshPrivateKeyPassword,sshPrivateKeyFile))
if sshPrivateKeyPassword.strip()=="":
sshPrivateKeyPassword=None
#copy keys
akrr.sshAccess(remoteAccessNode, ssh='ssh-copy-id', username=sshUserName, password=sshPassword4thisSession,
PrivateKeyFile=sshPrivateKeyFile, PrivateKeyPassword=None, logfile=sys.stdout,
command='')
askForUserName=not askForUserName
continue
if successfullyConnected:
break
else:
logging.error("Incorrect resource access credential")
if successfullyConnected:
print
logging.info("Connecting to "+resourceName)
str_io=cStringIO.StringIO()
sys.stdout = sys.stderr = str_io
rsh=akrr.sshAccess(remoteAccessNode, ssh=remoteAccessMethod, username=sshUserName, password=sshPassword,
PrivateKeyFile=sshPrivateKeyFile, PrivateKeyPassword=sshPrivateKeyPassword, logfile=sys.stdout,
command=None)
sys.stdout=sys.__stdout__
sys.stderr=sys.__stderr__
logging.info(" Done")
print
return successfullyConnected