Пример #1
0
def trimFile_btwn(sFile, string_srt, string_end):
    sNAMEFUNC = 'trimFile_btwn'
    sTxt = "Called... "
    sndMSG(sTxt, 'INFO', sNAMEFUNC)

    iCnt = 0
    iSrt = 0
    iEnd = 0
    with open(sFile, 'r+') as data:
        lines = data.readlines()

        data.seek(0)
        for line in lines:
            if string_srt in line:
                iSrt = iCnt
            if string_end in line:
                iEnd = iCnt
            iCnt += 1

        data.truncate()
        for i in range(iSrt, iEnd + 1):
            data.write(lines[i])

        lines = None
        data.close()

    return (False)
Пример #2
0
    def decompress(self, srcFile, dstFile, cmpType):

        if not cmpType in self.CompressionTypes:
            sTxt = 'Unknown decompress type: ' + cmpType
            sndMSG(sTxt, 'ERROR', self._strClassName)
            return (None)

        if cmpType == 'gz':
            sTxt = 'Attempting to decompress ' + cmpType + ' file '
            sndMSG(sTxt, 'INFO', self._strClassName)
            return (True)

        if cmpType == 'zip':
            sTxt = 'Attempting to decompress ' + cmpType + ' file '
            sndMSG(sTxt, 'INFO', self._strClassName)
            return (True)

        if cmpType == 'bz':
            sTxt = 'Attempting to decompress ' + cmpType + ' file '
            sndMSG(sTxt, 'INFO', self._strClassName)
            return (True)

        if cmpType == 'tar':
            sTxt = 'Attempting to decompress ' + cmpType + ' file '
            sndMSG(sTxt, 'INFO', self._strClassName)
            return (True)

        return (False)
Пример #3
0
def getFile_CSV2List(sFile, dialect=None, sCommentFlag=None):
    """
    Opens received File and parses it based on the received dialect
    
    Keyword arguments: 
    sFile -- File Path and File name to be parsed
        exmple: ./test/test.csv
    dialect -- Is an instantiates of mngLoc_File.clsCSVDialect 
        containing the CSV parsing parameters
    sCommentChar -- Is char that begins a line to be ingnored
    
    Returns:
        List object containing a List for each line 
        example:
        [['date','ip1','domain1'],
         ['date','ip2','domain2']
        ]
    """

    import csv
    sFuncName = 'mngFiles.getFile_CSV2List'
    sTxt = "Attemping to parse file as CSV: " + sFile
    sndMSG(sTxt, 'INFO', sFuncName)

    if dialect == None:
        sTxt = "\--> NOTE: No csv dialect was passed, default parsing is by comma "
        sndMSG(sTxt, 'INFO', sFuncName)

    if sCommentFlag == None:
        sCommentFlag = '#'
        sTxt = "\--> NOTE: No comment flag was passed, default is '#' "
        sndMSG(sTxt, 'INFO', sFuncName)

    try:
        #objFile = _getFile_OBJ(sFile)
        objFile = open(sFile)
        if objFile == None:
            return (None)
        data = csv.reader(objFile, dialect)
        sList = []
        for line in data:
            if sCommentFlag in str(line):
                # TODO: Needs better comment handling,
                # As is it will remove line in line comments
                # this need to be fixed to just remove commented section
                # and still keep the data before it
                continue
            sList.append(line)
        #sList.sort()
        objFile.close
        return (sList)
    except IOError as e:
        sTxt = str("\--> I/O error({0}): {1}".format(e.errno, e.strerror))
        sndMSG(sTxt, 'ERROR', sFuncName)
        return (None)
    except:
        sTxt = "\--> Unexpected error: " + str(sys.exc_info()[0])
        sndMSG(sTxt, 'ERROR', sFuncName)
        return (None)
Пример #4
0
def sndFile(sData, sFile):
    import sys
    sFuncName = 'mngFiles.sndFile'
    sTxt = "Trying to write to this file : " + sFile
    sndMSG(sTxt, 'INFO', sFuncName)

    with open(sFile, "w") as outfile:
        for line in sData:
            outfile.write(line)
    outfile.close()
    return (0)
Пример #5
0
def decompressor(srcFile, dstFile, cmpFormat):
    _strFuncName = 'mngFiles.decompressor'
    _strFuncVer = '0.0.0'

    if srcFile == None or dstFile == None or cmpFormat == None:
        return (None)

    if srcFile:
        sTxt = 'decompressing ' + srcFile
        sndMSG(sTxt, 'INFO', _strFuncName)
        return (True)

    return (None)
    def unpck_file2file(self):
            
         if self._dicParsingArgs['format'][0] in self._lstCompressionTypes:
            sTxt = 'Attempting to decompression format: ' + self._dicParsingArgs['format'][0]
            sndMSG(sTxt,'INFO',self._strClassName)
         
            ### Run decompressor
            from mngFiles import decompressor
            from mngFiles import Decompressor
            
            sFile = self._strFilePath + self.fileName + "." 
            
            srcFile = sFile + self._dicParsingArgs['format'][0]
            dstFile = sFile + self._dicParsingArgs['format'][1]
            cmpFrmt = self._dicParsingArgs['format'][0]
            oDcmp = Decompressor()
            bStatus = oDcmp.decompress(srcFile,dstFile,cmpFrmt)
            
            if bStatus == True:
                self._dicParsingArgs['format'].pop(0)

            elif bStatus == False:
                sTxt = 'Failed to unpack ' + self._lstFileEncapFormats(0) + ' file '
                sndMSG(sTxt,'ERROR',self._strClassName)
            
            else:
                sTxt = 'Unpacker returned unexpected result'
                sndMSG(sTxt,'ERROR',self._strClassName)
         
            ### This causes a loop to interater through self._lstFileEncapFormats
            self.cnvt_SrcFile2Dict()
         else:
            sTxt = 'Unrecognized file compression format: ' + self._lstFileEncapFormats(0)
            sndMSG(sTxt,'ERROR',self._strClassName)
            return(None)
Пример #7
0
def getUTCTime():
    '''
    - Used to get UTC in a standardized format
    
    Keyword arguments: 
   
    Returns:
    '''
    from datetime import datetime
    sFuncName = 'getUTCTime()'
    try:
        sDateTime = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
        return (sDateTime)
    except:
        sTxt = "Unexpected error: " + str(sys.exc_info()[0])
        sndMSG(sTxt, 'ERROR', sFuncName)
        return (None)
Пример #8
0
def cnvt_XML2Dict(srcXML, dstFile=None):
    '''
    {Description}
    
    Keyword arguments: 
   
    Returns:
    '''

    import urllib2
    import xmltodict

    ### Setup function
    sFuncName = 'cnvt_XML2Dict'
    if not srcXML:
        sTxt = 'XML Source Required'
        return (None)

    if not "http" in srcXML:
        srcXML = "file://" + srcXML

    #rssData = urllib2.urlopen(srcXML).read()

    try:
        rssData = urllib2.urlopen(srcXML).read()
    except IOError as e:
        sTxt = str("\--> I/O error({0}): {1}".format(e.errno, e.strerror))
        sndMSG(sTxt, 'ERROR', sFuncName)
        return (None)
    except:
        sTxt = "\--> Unexpected error: " + str(sys.exc_info()[0])
        sndMSG(sTxt, 'ERROR', sFuncName)
        return (None)

    #dataDict = xmltodict.parse(rssData)
    try:
        dataDict = xmltodict.parse(rssData)
        pass
    except IOError as e:
        sTxt = str("\--> I/O error({0}): {1}".format(e.errno, e.strerror))
        sndMSG(sTxt, 'ERROR', sFuncName)
        return (None)
    except:
        sTxt = "\--> Unexpected error: " + str(sys.exc_info()[0])
        sndMSG(sTxt, 'ERROR', sFuncName)
        return (None)

    if dstFile:
        return (sndFile_Dict2JSON(dataDict, dstFile))
    else:
        return (dataDict)

    return (None)
Пример #9
0
def getFile_JSON2Dict(sFile):
    '''
    {Description}
    Keyword arguments: 
    Returns:
    '''
    import json
    sFuncName = 'mngFiles.getFile_JSON2Dict'
    sTxt = "Attemping to parse file as JSON: " + sFile
    sndMSG(sTxt, 'INFO', sFuncName)

    try:

        #json_data = _getFile_OBJ(sFile)
        json_data = open(sFile)
        if json_data == None:
            return (None)
        data = json.load(json_data)
        json_data.close()
        return (data)
    except IOError as e:
        sTxt = str("\--> I/O error({0}): {1}".format(e.errno, e.strerror))
        sndMSG(sTxt, 'ERROR', sFuncName)
        return (None)
    except:
        sTxt = "\--> Unexpected error: " + str(sys.exc_info()[0])
        sndMSG(sTxt, 'ERROR', sFuncName)
        return (None)
Пример #10
0
def getRmt_File(dicCreds,sFile):
    '''
    - Simple URL Retrieval function based on python's urllib
    
    Keyword arguments: 
   
    Returns:
    '''
    import urllib
    sFuncName = 'getRmt_File'
    rmtSrcFile = dicCreds['URI']
    
    sTxt = "Connecting to " + rmtSrcFile
    sndMSG(sTxt,'INFO',sFuncName)
    
    try:
        urllib.urlretrieve(rmtSrcFile,sFile)
        return(True)
    except:
        sTxt = "Unexpected error: " + str(sys.exc_info()[0])
        sndMSG(sTxt,'ERROR',sFuncName)
        return(False)
    return False
    def cnvt_SrcFile2Dict(self):

        if self._dicParsingArgs['format'][0] in self._lstCompressionTypes:
            self.unpck_file2file()      
            
        if self._dicParsingArgs['format'][0] in self._lstFileFormatTypes:
            sFrmt = self._dicParsingArgs['format'][0]
            sFileSuffix = self._strFilePath + self._strFileNameSuffix 
            
               
            if sFrmt == 'txt' or sFrmt == 'csv':
                from cnvtFiles import cnvt_CSV2Dict
                from mngFiles  import clsCSVDialect_01
                oDialect = clsCSVDialect_01()
                oDialect.from_dict(self._dicParsingArgs)
                sFile = self.fileName + "." + sFrmt
                
                return(cnvt_CSV2Dict(sFile,oDialect,strEncoding=self._strUnicodeEncoding))
            
            elif sFrmt == 'xml' or sFrmt == 'rss':
                from mngFiles import cnvt_XML2Dict
                return(cnvt_XML2Dict(sFileSuffix + sFrmt))
                
            elif sFrmt == 'htm' or sFrmt == 'html':
                from mngFiles import cnvt_HTML2Dict
                sArg = self._dicParseArg["format"]["htmFltr"]
                return(cnvt_HTML2Dict(sFileSuffix + sFrmt,sArg))
                    
            else:
                sTxt = "Format not recognized: " + sFrmt
                sndMSG(sTxt,'ERROR',self._strClassName)                
        
        else:
            sTxt = 'Unrecognized file compression format: ' + self._dicParsingArgs['format'][0]
            sndMSG(sTxt,'ERROR',self._strClassName)
            return(None)
Пример #12
0
def sndFile_Dict2JSON(sData, sFile, isCompact=False):
    '''
    {Description}
    
    Keyword arguments: 
   
    Returns:
    '''
    import json
    sFuncName = 'mngFiles.sndFile_Dict2JSON'
    sTxt = "Trying to write to this file : " + sFile
    sndMSG(sTxt, 'INFO', sFuncName)

    chkFile(sFile, True)

    try:
        with open(sFile, "w") as outfile:

            try:
                if isCompact == 'True':
                    json.dump(sData, outfile)
                else:
                    json.dump(sData, outfile, indent=4)
            except:
                sTxt = "Unexpected error: " + str(sys.exc_info()[0])
                sndMSG(sTxt, 'ERROR', sFuncName)
                return (False)
        outfile.close()
        return (True)
    except IOError as e:
        sTxt = str("I/O error({0}): {1}".format(e.errno, e.strerror))
        sndMSG(sTxt, 'ERROR', sFuncName)
        return (None)
    except:
        sTxt = "Unexpected error: " + str(sys.exc_info()[0])
        sndMSG(sTxt, 'ERROR', sFuncName)
        return (None)
Пример #13
0
def getRmt_File2(dicCreds,sFile):
    '''
    - Simple URL Retreval funtion based on python's urllib2
    - With chunking and download update 
    
    Keyword arguments: 
   
    Returns:
    '''
    import urllib2
    sFuncName = 'getRmtFile2'
    rmtFile = dicCreds['URI']
    
    #"Mozilla/5.0 (X11 U Linux i686) Gecko/20071127 Firefox/2.0.0.11"
    try:
        ### Add Proxy
        # SOURCE: https://docs.python.org/2.4/lib/urllib2-examples.html
        # proxy_handler = urllib2.ProxyHandler({'http': 'http://www.example.com:3128/'})
        # proxy_auth_handler = urllib2.HTTPBasicAuthHandler()
        # proxy_auth_handler.add_password('realm', 'host', 'username', 'password')
        #
        # opener = build_opener(proxy_handler, proxy_auth_handler)
        # This time, rather than install the OpenerDirector, we use it directly:
        # opener.open('http://www.example.com/login.html'
        #
        # ## Mixed HTTPS HTTP enviroment
        # urllib2.ProxyHandler({'https': 'http://*****:*****@proxy:3128' }))
        ###
        
        ### Basic HTTP Authentication
        # SOURCE: https://docs.python.org/2.4/lib/urllib2-examples.html
        # ##Create an OpenerDirector with support for Basic HTTP Authentication...
        #   auth_handler = urllib2.HTTPBasicAuthHandler()
        #   auth_handler.add_password('realm', 'host', 'username', 'password')
        #   opener = urllib2.build_opener(auth_handler)
        # ##...and install it globally so it can be used with urlopen.
        #   urllib2.install_opener(opener)
        #   urllib2.urlopen('http://www.example.com/login.html')
        ###
        
        ### Cookie Handler
        # jar = cookielib.FileCookieJar("cookies")
        # opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(jar))
        ### 

        ### Modifiy User Agent
        # opener = urllib2.build_opener()
        # opener.addheaders = [('User-agent', 'Mozilla/5.0')]
        # opener.open('http://www.example.com/')
        ###
        
        objURL = urllib2.urlopen(rmtFile)
    except:
        sTxt = "Unexpected error: " + str(sys.exc_info()[0])
        sndMSG(sTxt,'ERROR',sFuncName)
        return(False)
    
    objFile = open(sFile, 'wb')
    meta = objURL.info()
    file_size = int(meta.getheaders("Content-Length")[0])
    sTxt = "Downloading: %s Bytes: %s" % (sFile, file_size)
    sndMSG(sTxt,'INFO',sFuncName)

    file_size_dl = 0
    block_sz = 8192
    while True:
        buffer = objURL.read(block_sz)
        if not buffer:
            break

        file_size_dl += len(buffer)
        objFile.write(buffer)
        status = r"%10d  [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size)
        status = status + chr(8)*(len(status)+1)
        sndMSG(status,'INFO',sFuncName)

    objFile.close()
    return(True)
Пример #14
0
def cnvt_CSV2Dict(srcCSV, dialect=None, sCommentFlag=None, strEncoding=None):
    _sFuncName = 'cnvtFiles.cnvt_CSV2Dict'
    sTxt = "Called... "
    sndMSG(sTxt, 'INFO', _sFuncName)

    bDebug = False

    if srcCSV == None or dialect == None:
        return (None)

    if sCommentFlag == None:
        sCommentFlag = '#'

    if strEncoding == None:
        strEncoding = 'utf-8'

    if dialect.header == True:
        #TODO: Change Col header from number to Header Names

        pass

    if bDebug:
        sTxt = "\--[ CSV File Location ]--> " + str(srcCSV)
        sndMSG(sTxt, 'INFO', _sFuncName)

    with open(srcCSV, 'rb') as dataSrc:

        if dataSrc == None:
            return (None)

        if bDebug:
            sTxt = "\--[ input File charCnt ]--> " + str(len(dataSrc.read()))
            sndMSG(sTxt, 'INFO', _sFuncName)

        dataSrc.seek(0)
        data = csv.reader(dataSrc, dialect=dialect)
        if bDebug:
            sTxt = "\--[ output CSV RowCnt ]--> " + str(data)
            sndMSG(sTxt, 'INFO', _sFuncName)

        csvDict = {}
        listHeaders = []
        iRow = 0
        iCol = 0

        for row in data:

            if len(row) < 1:
                continue

            if sCommentFlag in str(row[0]):
                continue

            if dialect.header == True and len(listHeaders) < 1:
                listHeaders = row
                continue

            csvDict.update({iRow: {}})
            for col in row:
                try:
                    col = unicode(col, strEncoding)
                except:
                    col = " (NOTE: This data was modified from origin, due to non " + strEncoding + " compatiable chars) " + unicode(
                        col, "utf-8", errors='ignore')

                if len(listHeaders) > 1:
                    csvDict[iRow].update({listHeaders[iCol]: col})
                else:
                    csvDict[iRow].update({iCol: col})

                iCol += 1

            iCol = 0
            iRow += 1

    if bDebug:
        sTxt = "\--[ output Dict RowCnt ]--> " + str(len(csvDict))
        sndMSG(sTxt, 'INFO', _sFuncName)

    dataSrc.close
    return (csvDict)
Пример #15
0
def _getFile_OBJ(sFile,
                 MAXFILESIZE=None,
                 FILEENCODING=None,
                 sPassFuncName=None):
    """
        Code Broken, do not use
    """

    #if sPAssFuncName == None:
    #    sFuncName = 'mngFiles._getFile_OBJ'
    #else:
    #    sFuncName = sPAssFuncName + "(" + sFuncName + ")"

    sFuncName = 'mngFiles._getFile_OBJ'

    sTxt = "Attemping to open this file: " + sFile
    sndMSG(sTxt, 'INFO', sFuncName)

    if MAXFILESIZE == None:
        MAXFILESIZE = 1000000000  # => 1GB
    else:
        sTxt = "\--> MAXFILESIZE was changed to: " + MAXFILESIZE
        sndMSG(sTxt, 'INFO', sFuncName)

    if FILEENCODING == None:
        FILEENCODING = 'utf-8'
    else:
        sTxt = "\--> FILEENCODING was changed to: " + FILEENCODING
        sndMSG(sTxt, 'INFO', sFuncName)

    #check Existance of file
    bFlag = os.path.exists(sFile)
    if bFlag == False:
        sTxt = "\--> This file or file path does not exist: " + sFile
        sndMSG(sTxt, 'ERROR', sFuncName)
        return (None)

    #check File size
    #iSize = os.path.getsize(sFile)
    #if iSize > MAXFILESIZE:
    #sTxt = "\--> This file is larger than MAXFILESIZE: " + sFile
    #sndMSG(sTxt,'ERROR',sFuncName)
    #return(None)

    try:
        with open(sFile) as data:
            oFile = data.read()

        data.close()
    except IOError as e:
        sTxt = str("\--> I/O error({0}): {1}".format(e.errno, e.strerror))
        sndMSG(sTxt, 'ERROR', sFuncName)
        return (None)
    except:
        sTxt = "\--> Unexpected error: " + str(sys.exc_info()[0])
        sndMSG(sTxt, 'ERROR', sFuncName)
        return (None)

    sTxt = "\--> File was successfully Opened"
    sndMSG(sTxt, 'INFO', sFuncName)
    return (oFile)