def _create_config_rule(entity_name: str,
                        rule_json: Dict) -> Optional[ConfigRule]:
    sources_json = rule_json.get("sources", [])
    if entity_name != "default" and not sources_json:
        logging.warning(
            f"Encountered invalid rule with missing sources for config entry named {entity_name}",
            "metadata-rule-missing-sources")
        return None
    sources = _create_sources(sources_json)
    if entity_name != "default" and not sources:
        logging.warning(
            f"Encountered invalid rule with invalid sources for config entry named {entity_name}: {sources_json}",
            "metadata-rule-invalid-sources")
        return None
    attributes = _create_attributes(rule_json.get("attributes", []))

    aws_loggroup_pattern = rule_json.get("aws", {}).get("logGroup", None)
    log_content_parse_type = rule_json.get("aws",
                                           {}).get("logContentParseAs", None)

    return ConfigRule(entity_type_name=entity_name,
                      source_matchers=sources,
                      attributes=attributes,
                      aws_loggroup_pattern=aws_loggroup_pattern,
                      log_content_parse_type=log_content_parse_type)
Esempio n. 2
0
def list_parsed(args):
    """
    Handles the appropriate execution of a 'List' mode request given
    the provided command line arguments.
    """

    verbose = args.verbose
    resource = args.resource
    application = args.application
    status = args.status

    if verbose:
        log.info("Attempting to complete the requested Operation: ")

    if status:
        results = {}
        pass
    else:
        results = retrieve_tasks(resource, application)
        if results:
            log.info('Retrieved the following: ')
            [log.info("[{:<8}] Resource: {:<15} App:{:<24}",r['task_id'],  r['resource'], r['app']) for r in results]
        else:
            log.warning('No records returned.')

    if verbose:
        log.info("Requested Operation Completed")
Esempio n. 3
0
def generate_default_templates(resources):
    """
    Create the default templates per resource that was retrieved from the `modw`.`resourcefact` table.

    :param resources: a tuple of (string,int) [name, id] retrieved from the `modw`.`resourcefact` table.
    :return: void
    """

    if not resources:
        logging.warning("No resources found. No files to create.")
    else:

        slurm_template_contents = retrieve_queue_template(os.path.join(akrr.curdir, 'templates','template.{0}.inp.py'), 'slurm')
        pbs_template_contents = retrieve_queue_template(os.path.join(akrr.curdir, 'templates', 'template.{0}.inp.py'), 'pbs')

        queues = {'slurm': slurm_template_contents, 'pbs': pbs_template_contents}

        for resource in resources:
            if args.verbose:
                logging.info("Creating Resource Template: {0} ", resource[0] + "")

            if not args.test:
                for queue, contents in queues.iteritems():

                    file_path = os.path.join(resources_dir, resource[0] + 'resource.inp.py')

                    create_resource_template(file_path, queue, contents)

        logging.info("Resource Template Generation Complete!")
Esempio n. 4
0
File: setup.py Progetto: ubccr/akrr
 def install_cron_scripts(self):
     """Install cron scripts."""
     if self.cronemail:mail="MAILTO = "+self.cronemail
     else: mail=None
     restart="50 23 * * * "+akrr_home+"/bin/restart.sh"
     checknrestart="33 * * * * "+akrr_home+"/bin/checknrestart.sh"
     
     try:
         crontanContent=subprocess.check_output("crontab -l", shell=True)
         crontanContent=crontanContent.splitlines(True)
     except:
         log.info("Crontab does not have user's crontab yet")
         crontanContent=[]
     
     mailUpdated=False
     mailThere=False
     restartThere=False
     checknrestartThere=False
     
     for i in range(len(crontanContent)):
         l=crontanContent[i]
         if len(l.strip())>1 and l.strip()[0]!="#":
             m=re.match(r'^MAILTO\s*=\s*(.*)',l.strip())
             if m:
                 cronemail=m.group(1)
                 cronemail=self.cronemail.replace('"','')
                 mailThere=True
                 if self.cronemail!=cronemail:
                     if mail:
                         crontanContent[i]=mail
                     else:
                         crontanContent[i]="#"+crontanContent[i]
                     mailUpdated=True
             if l.count("restart.sh")>0:
                 restartThere=True
             if l.count("checknrestart.sh")>0:
                 checknrestartThere=True
     if mailUpdated:
         log.info("Cron's MAILTO was updated")
     if ((self.cronemail!=None and mailThere) or (self.cronemail==None and mailThere==False)) and restartThere and checknrestartThere and mailUpdated==False:
         log.warning("All entries found. No modifications necessary.")
         return
     if self.cronemail!=None and mailThere==False:
         crontanContent.insert(0, mail+"\n")
     if restartThere==False:
         crontanContent.append(restart+"\n")
     if checknrestartThere==False:
         crontanContent.append(checknrestart+"\n")
     
     with open(os.path.expanduser('.crontmp'),'w') as f:
         for l in crontanContent:
             f.write(l)
     subprocess.call("crontab .crontmp", shell=True)
     os.remove(".crontmp")
     log.info("Cron Scripts Processed!")
def parse_aws_loggroup_with_grok_pattern(loggroup, pattern) -> dict:
    grok = get_grok(pattern)
    extracted_values = grok.match(loggroup)

    if not extracted_values:
        logging.warning(
            f"Failed to match logGroup '{loggroup}' against the pattern '{pattern}'",
            "loggroup-pattern-matching-failure")
        return {}

    return extracted_values
Esempio n. 6
0
def socketcall_debug_printer(pid, orig_eax, syscall_object):
    subcall_debug_printers = {
        1: socket_debug_printer,
        9: send_debug_printer,
        13: shutdown_debug_printer
    }
    subcall_id = cint.peek_register(pid, cint.EBX)
    logging.debug('Got subcall {} {}'.format(subcall_id,
                                             SOCKET_SUBCALLS[subcall_id]))
    try:
        subcall_debug_printers[subcall_id](pid, syscall_object)
    except KeyError as e:
        logging.warning(
            'This subcall ({}) has no debug printer'.format(subcall_id))
        raise e
def _create_attributes(attributes_json: List[Dict]) -> List[Attribute]:
    result = []

    for source_json in attributes_json:
        key = source_json.get("key", None)
        priority = source_json.get("priority", None)
        pattern = source_json.get("pattern", None)

        if key and pattern:
            result.append(Attribute(key, priority, pattern))
        else:
            logging.warning(
                f"Encountered invalid rule attribute with missing parameter, parameters were: key = {key}, pattern = {pattern}",
                "metadata-attribute-missing-parameter")

    # attributes without priority are executed last
    result.sort(key=lambda attribute: attribute.priority
                if attribute.priority is not None else inf)
    return result
def _create_sources(sources_json: List[Dict]) -> List[SourceMatcher]:
    result = []

    for source_json in sources_json:
        source = source_json.get("source", None)
        condition = source_json.get("condition", None)
        source_matcher = None

        if source and condition:
            source_matcher = SourceMatcher(source, condition)

        if source_matcher and source_matcher.valid:
            result.append(source_matcher)
        else:
            logging.warning(
                f"Encountered invalid rule source, parameters were: source= {source}, condition = {condition}",
                "metadata-invalid-rule-source")
            return []

    return result
    def __init__(self, source: str, condition: str):
        self.source = source
        self.condition = condition
        for key in _CONDITION_COMPARATOR_MAP.keys():
            if condition.startswith(key):
                self._evaluator = _CONDITION_COMPARATOR_MAP[key]
                break
        operands = re.findall(r"'(.*?)'", condition, re.DOTALL)
        self._operand = operands[0] if operands else None
        self._source_value_extractor = _SOURCE_VALUE_EXTRACTOR_MAP.get(
            source.casefold(), None)

        if not self._source_value_extractor:
            logging.warning(f"Unsupported source type: '{source}'",
                            "metadata-unsupported-source-type")
            self.valid = False
        if not self._evaluator or not self._operand:
            logging.warning(
                f"Failed to parse condition macro for expression: '{condition}'",
                "metadata-condition-parsing-failure")
            self.valid = False
Esempio n. 10
0
def getFileSytemAccessPoints():
    global networkScratch
    global localScratch
    global akrrData
    global appKerDir
    
    homeDir=akrr.sshCommand(rsh,"echo $HOME").strip()
    scratchNetworkDir=akrr.sshCommand(rsh,"echo $SCRATCH").strip()
    
    #localScratch
    localScratchDefault="/tmp"
    while True:                    
        logging.input("Enter location of local scratch (visible only to single node):")
        localScratch=raw_input("[%s]"%localScratchDefault)
        if localScratch.strip()=="":
            localScratch=localScratchDefault
        status,msg=resource_validation_and_deployment.CheckDirSimple(rsh, localScratch)
        if status:
            logging.info(msg)
            print
            break
        else:
            logging.warning(msg)
            logging.warning('local scratch might be have a different location on head node, so if it is by design it is ok')
            print
            break
    localScratch=akrr.sshCommand(rsh,"echo %s"%(localScratch,)).strip()
    #networkScratch
    networkScratchDefault=""
    if scratchNetworkDir!="":
        networkScratchDefault=scratchNetworkDir
    networkScratchVisible=False
    while True:                    
        logging.input("Enter location of network scratch (visible only to all nodes), used for temporary storage of app kernel input/output:")
        if networkScratchDefault!="":
            networkScratch=raw_input("[%s]"%networkScratchDefault)
            if networkScratch.strip()=="":
                networkScratch=networkScratchDefault
        else:
            networkScratch=raw_input("")
            
        if networkScratch=="":
            logging.error("Incorrect value for networkScratch, try again")
            continue
        
        
        status,msg=resource_validation_and_deployment.CheckDir(rsh, networkScratch,exitOnFail=False,tryToCreate=True)
        if status:
            logging.info(msg)
            networkScratchVisible=True
            print
            break
        else:
            logging.warning(msg)
            #logging.warning('network scratch might be have a different location on head node, so if it is by design it is ok')
            #print
            break
    networkScratch=akrr.sshCommand(rsh,"echo %s"%(networkScratch,)).strip()
    #appKerDir
    appKerDirDefault=os.path.join(homeDir,"appker",resourceName)   
    while True:                    
        logging.input("Enter future location of app kernels input and executable files:")
        appKerDir=raw_input("[%s]"%appKerDirDefault)
        if appKerDir.strip()=="":
            appKerDir=appKerDirDefault
        status,msg=resource_validation_and_deployment.CheckDir(rsh, appKerDir,exitOnFail=False,tryToCreate=True)
        if status:
            logging.info(msg)
            print
            break
        else:
            logging.error(msg)
    appKerDir=akrr.sshCommand(rsh,"echo %s"%(appKerDir,)).strip()
    #akrrData
    akrrDataDefault=os.path.join(homeDir,"akrrdata",resourceName)
    if networkScratchVisible:
        akrrDataDefault=os.path.join(networkScratch,"akrrdata",resourceName)
    while True:                    
        logging.input("Enter future locations for app kernels working directories (can or even should be on scratch space):")
        akrrData=raw_input("[%s]"%akrrDataDefault)
        if akrrData.strip()=="":
            akrrData=akrrDataDefault
        status,msg=resource_validation_and_deployment.CheckDir(rsh, akrrData,exitOnFail=False,tryToCreate=True)
        if status:
            logging.info(msg)
            print
            break
        else:
            logging.error(msg) 
    akrrData=akrr.sshCommand(rsh,"echo %s"%(akrrData,)).strip()
Esempio n. 11
0
def checkConnectionToResource():
    successfullyConnected=False
    global remoteAccessNode
    global remoteAccessMethod
    global remoteCopyMethod
    global sshUserName
    global sshPassword
    global sshPassword4thisSession
    global sshPrivateKeyFile
    global sshPrivateKeyPassword
    passphraseEntranceCount=0
    authorizeKeyCount=0
    while True:
        str_io=cStringIO.StringIO()
        try:
            sys.stdout = sys.stderr = str_io
            akrr.sshAccess(remoteAccessNode, ssh=remoteAccessMethod, username=sshUserName, password=sshPassword,
                    PrivateKeyFile=sshPrivateKeyFile, PrivateKeyPassword=sshPrivateKeyPassword, logfile=str_io,
                    command='ls')
            
            sys.stdout=sys.__stdout__
            sys.stderr=sys.__stderr__
            
            successfullyConnected=True
            break
        except Exception,e:
            sys.stdout=sys.__stdout__
            sys.stderr=sys.__stderr__
            if args.verbose:
                logging.info("Had attempted to access resource without password and failed, below is resource response")
                print "="*80
                print str_io.getvalue()
                print traceback.format_exc()
                print "="*80
            #check if it asking for passphrase
            m=re.search(r"Enter passphrase for key '(.*)':",str_io.getvalue())
            if m:
                if passphraseEntranceCount>=3:
                    sshPrivateKeyPassword=None
                    sshPrivateKeyFile=None
                    break
                if passphraseEntranceCount>0:
                    logging.error("Incorrect passphrase try again")
                sshPrivateKeyFile=m.group(1)
                logging.input("Enter passphrase for key '%s':"%sshPrivateKeyFile)
                sshPrivateKeyPassword=getpass.getpass("")
                passphraseEntranceCount+=1
                continue
            m2=re.search(r"[pP]assword:",str_io.getvalue())
            if m==None and sshPrivateKeyFile!=None and m2:
                logging.warning("Can not login to head node. Probably the public key of private key was not authorized on head node")
                print "Will try to add public key to list of authorized keys on head node"
                while True:
                    try:
                        authorizeKeyCount+=1
                        logging.input("Enter password for %s@%s (will be used only during this session):"%(sshUserName,remoteAccessNode))
                        sshPassword4thisSession=getpass.getpass("")
                        print
                        str_io=cStringIO.StringIO()
                        sys.stdout = sys.stderr = str_io
                        akrr.sshAccess(remoteAccessNode, ssh='ssh-copy-id', username=sshUserName, password=sshPassword4thisSession,
                            PrivateKeyFile=sshPrivateKeyFile, PrivateKeyPassword=None, logfile=str_io,
                            command='')
                    
                        sys.stdout=sys.__stdout__
                        sys.stderr=sys.__stderr__
                        print str_io.getvalue()
                        #successfullyConnected=True
                        logging.info("Have added public key to list of authorized keys on head node, will attempt to connect again.")
                        print
                        break
                    except Exception,e:
                        sys.stdout=sys.__stdout__
                        sys.stderr=sys.__stderr__
                        if args.verbose:
                            logging.info("Had attempted to add public key to list of authorized keys on head node and failed, below is resource response")
                            print "="*80
                            print str_io.getvalue()
                            print traceback.format_exc()
                            print "="*80
                        logging.info("Incorrect password try again.")
                        if authorizeKeyCount>=3:
                            break
                if authorizeKeyCount<3:
                     continue       
            break
Esempio n. 12
0
            # ATTEMPT: to open the file identified by the provided file_path
            file_handle = open(file_path, privs)

            # LET: the verbose users know we succeeded
            if args.verbose:
                logging.info('Successfully preformed open [{0}] on {1}', privs, file_path)

            # RETURN: the file_handle
            return file_handle
        except IOError, e:
            logging.error('Unable to open file: {0} due to {1}: {2}', file_path, e.args[0], e.args[1])
            return None

    # IF: we've reached this point than one of the parameters was incorrect.
    logging.warning('Error 62: Internal error. Please contact support.')
    return None


def read_from_file(file_handle):
    """
    Read the contents from the provided file_handle and return them. If there is an error then a message detailing the
    problem will be written to stdout and None will be returned.

    :type file_handle file

    :param file_handle: the file object to read from.
    :return: the contents of the file or, if an error is encountered, None.
    """
    if file_handle and isinstance(file_handle, file):