def _create_config_rule(entity_name: str,
rule_json: Dict) -> Optional[ConfigRule]:
sources_json = rule_json.get("sources", [])
if entity_name != "default" and not sources_json:
logging.warning(
f"Encountered invalid rule with missing sources for config entry named {entity_name}",
"metadata-rule-missing-sources")
return None
sources = _create_sources(sources_json)
if entity_name != "default" and not sources:
logging.warning(
f"Encountered invalid rule with invalid sources for config entry named {entity_name}: {sources_json}",
"metadata-rule-invalid-sources")
return None
attributes = _create_attributes(rule_json.get("attributes", []))
aws_loggroup_pattern = rule_json.get("aws", {}).get("logGroup", None)
log_content_parse_type = rule_json.get("aws",
{}).get("logContentParseAs", None)
return ConfigRule(entity_type_name=entity_name,
source_matchers=sources,
attributes=attributes,
aws_loggroup_pattern=aws_loggroup_pattern,
log_content_parse_type=log_content_parse_type)
def list_parsed(args):
"""
Handles the appropriate execution of a 'List' mode request given
the provided command line arguments.
"""
verbose = args.verbose
resource = args.resource
application = args.application
status = args.status
if verbose:
log.info("Attempting to complete the requested Operation: ")
if status:
results = {}
pass
else:
results = retrieve_tasks(resource, application)
if results:
log.info('Retrieved the following: ')
[log.info("[{:<8}] Resource: {:<15} App:{:<24}",r['task_id'], r['resource'], r['app']) for r in results]
else:
log.warning('No records returned.')
if verbose:
log.info("Requested Operation Completed")
def generate_default_templates(resources):
"""
Create the default templates per resource that was retrieved from the `modw`.`resourcefact` table.
:param resources: a tuple of (string,int) [name, id] retrieved from the `modw`.`resourcefact` table.
:return: void
"""
if not resources:
logging.warning("No resources found. No files to create.")
else:
slurm_template_contents = retrieve_queue_template(os.path.join(akrr.curdir, 'templates','template.{0}.inp.py'), 'slurm')
pbs_template_contents = retrieve_queue_template(os.path.join(akrr.curdir, 'templates', 'template.{0}.inp.py'), 'pbs')
queues = {'slurm': slurm_template_contents, 'pbs': pbs_template_contents}
for resource in resources:
if args.verbose:
logging.info("Creating Resource Template: {0} ", resource[0] + "")
if not args.test:
for queue, contents in queues.iteritems():
file_path = os.path.join(resources_dir, resource[0] + 'resource.inp.py')
create_resource_template(file_path, queue, contents)
logging.info("Resource Template Generation Complete!")
def install_cron_scripts(self):
"""Install cron scripts."""
if self.cronemail:mail="MAILTO = "+self.cronemail
else: mail=None
restart="50 23 * * * "+akrr_home+"/bin/restart.sh"
checknrestart="33 * * * * "+akrr_home+"/bin/checknrestart.sh"
try:
crontanContent=subprocess.check_output("crontab -l", shell=True)
crontanContent=crontanContent.splitlines(True)
except:
log.info("Crontab does not have user's crontab yet")
crontanContent=[]
mailUpdated=False
mailThere=False
restartThere=False
checknrestartThere=False
for i in range(len(crontanContent)):
l=crontanContent[i]
if len(l.strip())>1 and l.strip()[0]!="#":
m=re.match(r'^MAILTO\s*=\s*(.*)',l.strip())
if m:
cronemail=m.group(1)
cronemail=self.cronemail.replace('"','')
mailThere=True
if self.cronemail!=cronemail:
if mail:
crontanContent[i]=mail
else:
crontanContent[i]="#"+crontanContent[i]
mailUpdated=True
if l.count("restart.sh")>0:
restartThere=True
if l.count("checknrestart.sh")>0:
checknrestartThere=True
if mailUpdated:
log.info("Cron's MAILTO was updated")
if ((self.cronemail!=None and mailThere) or (self.cronemail==None and mailThere==False)) and restartThere and checknrestartThere and mailUpdated==False:
log.warning("All entries found. No modifications necessary.")
return
if self.cronemail!=None and mailThere==False:
crontanContent.insert(0, mail+"\n")
if restartThere==False:
crontanContent.append(restart+"\n")
if checknrestartThere==False:
crontanContent.append(checknrestart+"\n")
with open(os.path.expanduser('.crontmp'),'w') as f:
for l in crontanContent:
f.write(l)
subprocess.call("crontab .crontmp", shell=True)
os.remove(".crontmp")
log.info("Cron Scripts Processed!")
def parse_aws_loggroup_with_grok_pattern(loggroup, pattern) -> dict:
grok = get_grok(pattern)
extracted_values = grok.match(loggroup)
if not extracted_values:
logging.warning(
f"Failed to match logGroup '{loggroup}' against the pattern '{pattern}'",
"loggroup-pattern-matching-failure")
return {}
return extracted_values
def socketcall_debug_printer(pid, orig_eax, syscall_object):
subcall_debug_printers = {
1: socket_debug_printer,
9: send_debug_printer,
13: shutdown_debug_printer
}
subcall_id = cint.peek_register(pid, cint.EBX)
logging.debug('Got subcall {} {}'.format(subcall_id,
SOCKET_SUBCALLS[subcall_id]))
try:
subcall_debug_printers[subcall_id](pid, syscall_object)
except KeyError as e:
logging.warning(
'This subcall ({}) has no debug printer'.format(subcall_id))
raise e
def _create_attributes(attributes_json: List[Dict]) -> List[Attribute]:
result = []
for source_json in attributes_json:
key = source_json.get("key", None)
priority = source_json.get("priority", None)
pattern = source_json.get("pattern", None)
if key and pattern:
result.append(Attribute(key, priority, pattern))
else:
logging.warning(
f"Encountered invalid rule attribute with missing parameter, parameters were: key = {key}, pattern = {pattern}",
"metadata-attribute-missing-parameter")
# attributes without priority are executed last
result.sort(key=lambda attribute: attribute.priority
if attribute.priority is not None else inf)
return result
def _create_sources(sources_json: List[Dict]) -> List[SourceMatcher]:
result = []
for source_json in sources_json:
source = source_json.get("source", None)
condition = source_json.get("condition", None)
source_matcher = None
if source and condition:
source_matcher = SourceMatcher(source, condition)
if source_matcher and source_matcher.valid:
result.append(source_matcher)
else:
logging.warning(
f"Encountered invalid rule source, parameters were: source= {source}, condition = {condition}",
"metadata-invalid-rule-source")
return []
return result
def __init__(self, source: str, condition: str):
self.source = source
self.condition = condition
for key in _CONDITION_COMPARATOR_MAP.keys():
if condition.startswith(key):
self._evaluator = _CONDITION_COMPARATOR_MAP[key]
break
operands = re.findall(r"'(.*?)'", condition, re.DOTALL)
self._operand = operands[0] if operands else None
self._source_value_extractor = _SOURCE_VALUE_EXTRACTOR_MAP.get(
source.casefold(), None)
if not self._source_value_extractor:
logging.warning(f"Unsupported source type: '{source}'",
"metadata-unsupported-source-type")
self.valid = False
if not self._evaluator or not self._operand:
logging.warning(
f"Failed to parse condition macro for expression: '{condition}'",
"metadata-condition-parsing-failure")
self.valid = False
def getFileSytemAccessPoints():
global networkScratch
global localScratch
global akrrData
global appKerDir
homeDir=akrr.sshCommand(rsh,"echo $HOME").strip()
scratchNetworkDir=akrr.sshCommand(rsh,"echo $SCRATCH").strip()
#localScratch
localScratchDefault="/tmp"
while True:
logging.input("Enter location of local scratch (visible only to single node):")
localScratch=raw_input("[%s]"%localScratchDefault)
if localScratch.strip()=="":
localScratch=localScratchDefault
status,msg=resource_validation_and_deployment.CheckDirSimple(rsh, localScratch)
if status:
logging.info(msg)
print
break
else:
logging.warning(msg)
logging.warning('local scratch might be have a different location on head node, so if it is by design it is ok')
print
break
localScratch=akrr.sshCommand(rsh,"echo %s"%(localScratch,)).strip()
#networkScratch
networkScratchDefault=""
if scratchNetworkDir!="":
networkScratchDefault=scratchNetworkDir
networkScratchVisible=False
while True:
logging.input("Enter location of network scratch (visible only to all nodes), used for temporary storage of app kernel input/output:")
if networkScratchDefault!="":
networkScratch=raw_input("[%s]"%networkScratchDefault)
if networkScratch.strip()=="":
networkScratch=networkScratchDefault
else:
networkScratch=raw_input("")
if networkScratch=="":
logging.error("Incorrect value for networkScratch, try again")
continue
status,msg=resource_validation_and_deployment.CheckDir(rsh, networkScratch,exitOnFail=False,tryToCreate=True)
if status:
logging.info(msg)
networkScratchVisible=True
print
break
else:
logging.warning(msg)
#logging.warning('network scratch might be have a different location on head node, so if it is by design it is ok')
#print
break
networkScratch=akrr.sshCommand(rsh,"echo %s"%(networkScratch,)).strip()
#appKerDir
appKerDirDefault=os.path.join(homeDir,"appker",resourceName)
while True:
logging.input("Enter future location of app kernels input and executable files:")
appKerDir=raw_input("[%s]"%appKerDirDefault)
if appKerDir.strip()=="":
appKerDir=appKerDirDefault
status,msg=resource_validation_and_deployment.CheckDir(rsh, appKerDir,exitOnFail=False,tryToCreate=True)
if status:
logging.info(msg)
print
break
else:
logging.error(msg)
appKerDir=akrr.sshCommand(rsh,"echo %s"%(appKerDir,)).strip()
#akrrData
akrrDataDefault=os.path.join(homeDir,"akrrdata",resourceName)
if networkScratchVisible:
akrrDataDefault=os.path.join(networkScratch,"akrrdata",resourceName)
while True:
logging.input("Enter future locations for app kernels working directories (can or even should be on scratch space):")
akrrData=raw_input("[%s]"%akrrDataDefault)
if akrrData.strip()=="":
akrrData=akrrDataDefault
status,msg=resource_validation_and_deployment.CheckDir(rsh, akrrData,exitOnFail=False,tryToCreate=True)
if status:
logging.info(msg)
print
break
else:
logging.error(msg)
akrrData=akrr.sshCommand(rsh,"echo %s"%(akrrData,)).strip()
def checkConnectionToResource():
successfullyConnected=False
global remoteAccessNode
global remoteAccessMethod
global remoteCopyMethod
global sshUserName
global sshPassword
global sshPassword4thisSession
global sshPrivateKeyFile
global sshPrivateKeyPassword
passphraseEntranceCount=0
authorizeKeyCount=0
while True:
str_io=cStringIO.StringIO()
try:
sys.stdout = sys.stderr = str_io
akrr.sshAccess(remoteAccessNode, ssh=remoteAccessMethod, username=sshUserName, password=sshPassword,
PrivateKeyFile=sshPrivateKeyFile, PrivateKeyPassword=sshPrivateKeyPassword, logfile=str_io,
command='ls')
sys.stdout=sys.__stdout__
sys.stderr=sys.__stderr__
successfullyConnected=True
break
except Exception,e:
sys.stdout=sys.__stdout__
sys.stderr=sys.__stderr__
if args.verbose:
logging.info("Had attempted to access resource without password and failed, below is resource response")
print "="*80
print str_io.getvalue()
print traceback.format_exc()
print "="*80
#check if it asking for passphrase
m=re.search(r"Enter passphrase for key '(.*)':",str_io.getvalue())
if m:
if passphraseEntranceCount>=3:
sshPrivateKeyPassword=None
sshPrivateKeyFile=None
break
if passphraseEntranceCount>0:
logging.error("Incorrect passphrase try again")
sshPrivateKeyFile=m.group(1)
logging.input("Enter passphrase for key '%s':"%sshPrivateKeyFile)
sshPrivateKeyPassword=getpass.getpass("")
passphraseEntranceCount+=1
continue
m2=re.search(r"[pP]assword:",str_io.getvalue())
if m==None and sshPrivateKeyFile!=None and m2:
logging.warning("Can not login to head node. Probably the public key of private key was not authorized on head node")
print "Will try to add public key to list of authorized keys on head node"
while True:
try:
authorizeKeyCount+=1
logging.input("Enter password for %s@%s (will be used only during this session):"%(sshUserName,remoteAccessNode))
sshPassword4thisSession=getpass.getpass("")
print
str_io=cStringIO.StringIO()
sys.stdout = sys.stderr = str_io
akrr.sshAccess(remoteAccessNode, ssh='ssh-copy-id', username=sshUserName, password=sshPassword4thisSession,
PrivateKeyFile=sshPrivateKeyFile, PrivateKeyPassword=None, logfile=str_io,
command='')
sys.stdout=sys.__stdout__
sys.stderr=sys.__stderr__
print str_io.getvalue()
#successfullyConnected=True
logging.info("Have added public key to list of authorized keys on head node, will attempt to connect again.")
print
break
except Exception,e:
sys.stdout=sys.__stdout__
sys.stderr=sys.__stderr__
if args.verbose:
logging.info("Had attempted to add public key to list of authorized keys on head node and failed, below is resource response")
print "="*80
print str_io.getvalue()
print traceback.format_exc()
print "="*80
logging.info("Incorrect password try again.")
if authorizeKeyCount>=3:
break
if authorizeKeyCount<3:
continue
break
# ATTEMPT: to open the file identified by the provided file_path
file_handle = open(file_path, privs)
# LET: the verbose users know we succeeded
if args.verbose:
logging.info('Successfully preformed open [{0}] on {1}', privs, file_path)
# RETURN: the file_handle
return file_handle
except IOError, e:
logging.error('Unable to open file: {0} due to {1}: {2}', file_path, e.args[0], e.args[1])
return None
# IF: we've reached this point than one of the parameters was incorrect.
logging.warning('Error 62: Internal error. Please contact support.')
return None
def read_from_file(file_handle):
"""
Read the contents from the provided file_handle and return them. If there is an error then a message detailing the
problem will be written to stdout and None will be returned.
:type file_handle file
:param file_handle: the file object to read from.
:return: the contents of the file or, if an error is encountered, None.
"""
if file_handle and isinstance(file_handle, file):