Exemple #1
0
# Get collectors where field contains some specified string
#
# python get-collectors.py <accessId> <accessKey> <field> <string>

import sys

from sumologic import SumoLogic

args = sys.argv
sumo = SumoLogic(args[1], args[2])
field, string = args[3], args[4]
cs = sumo.collectors()

for c in cs:
    if field in c and string in c[field]:
        print sumo.sources(c['id'])
# Renames a category across all collectors and sources in a given account.
#
# python mv-cat.py <accessId/email> <accessKey/password> <fromName> <toName>
#
# TODO update query category constraints
# TODO regex

import sys

from sumologic import SumoLogic

args = sys.argv
sumo = SumoLogic(args[1], args[2])
fromCat, toCat = args[3], args[4]
cs = sumo.collectors()

for c in cs:
    if 'category' in c and c['category'] == fromCat:
        cv, etag = sumo.collector(c['id'])
        cv['collector']['category'] = toCat
        print sumo.update_collector(cv, etag).text
    ss = sumo.sources(c['id'])
    for s in ss:
        if s['category'] == fromCat:
            sv, etag = sumo.source(c['id'], s['id'])
            sv['source']['category'] = toCat
            print sumo.update_source(c['id'], sv, etag).text
# Get collectors where field contains some specified string
#
# python get-collectors.py <accessId> <accessKey> <field> <string>

import sys

from sumologic import SumoLogic

args = sys.argv
sumo = SumoLogic(args[1], args[2])
field, string = args[3], args[4]
cs = sumo.collectors()

for c in cs:
	if field in c and string in c[field]:
		print(sumo.sources(c['id']))
# Get collectors where field contains some specified string
#
# python get-collectors.py <accessId> <accessKey> <field> <string>

import sys

from sumologic import SumoLogic

args = sys.argv
sumo = SumoLogic(args[1], args[2])
field, string = args[3], args[4]
cs = sumo.collectors()

for c in cs:
	if field in c and string in c[field]:
		print sumo.sources(c['id'])
Exemple #5
0
def getSources():

    """
    We're going to build a big ol' data structure that, in the end,
    should look something like this:

    sourceinfo = {
        'all_sources': {
            8583428: [{COMPLETE SOURCE OBJECT}],
            8583111: [{COMPLETE SOURCE OBJECT}],
            8583846: [{COMPLETE SOURCE OBJECT}],
            8583298: [{COMPLETE SOURCE OBJECT}]
        },
        'source_names': {
            'Friendly Source Name #1': {
                'memberids': [8583428, 8583111],
                'flattened' : {FLATTENED OBJECT VALUES},
                'ignore': {},
                'collectors': ['prodweb01', 'stagedb02']
            },
            'Friendly Source Name #2': {
                'memberids': [8583846, 8583298],
                'flattened': {
                    'something like this': "",
                    'category': ['System', 'System2'],
                    'encoding': ['UTF-8', 'ISO-8859-1']
                },
            }
        },
        'collector_map': {
            8583422: [8583428, 8583111],
            8583401: [8583846, 8583298],
        }
    }
    """

    sourceinfo = {"all_sources": {}, "source_names": {}, "collector_map": {}}

    data = request.json
    sumo = SumoLogic(data["apiid"], data["apikey"])

    for collector in data["collectors"]:
        # Get each collector's source from Sumo & store locally:

        sources = sumo.sources(collector["id"])
        sleep(0.15)

        # Add a key of the collector's id to the map. This map maps
        # sources to collectors so we don't have to keep asking Sumo.
        if collector["id"] not in sourceinfo["collector_map"]:
            sourceinfo["collector_map"][collector["id"]] = []

        for source in sources:

            # Add to the main sourceinfo list of all sources
            if source["id"] not in sourceinfo["all_sources"]:
                sourceinfo["all_sources"][source["id"]] = source

            # We need to group the sources by their name, but keep their
            # data in the original format, so we'll make a dictionary of
            # names and a list of source ids that have them:
            if source["name"] not in sourceinfo["source_names"]:
                sourceinfo["source_names"][source["name"]] = {
                    "memberids": [],
                    "flattened": {},
                    "ignore": {},
                    "collectors": [],
                }

            sourceinfo["source_names"][source["name"]]["memberids"].append(source["id"])
            sourceinfo["source_names"][source["name"]]["collectors"].append(collector["name"])

            # Update the collector_map with this source's key
            sourceinfo["collector_map"][collector["id"]].append(source["id"])

    # Finally, we're grouping the sources together by name for bulk editing
    # so we'll check to see if there are any sources that differ from
    # each other using sets.  The best way to do this is to go through the
    # sources that are listed with the same source_names:

    for sourcename in sourceinfo["source_names"]:

        # Each unique source name has members. Those member's
        # values will be flattened into the flattener!
        flattener = defaultdict(set)
        for memberid in sourceinfo["source_names"][sourcename]["memberids"]:
            for srckey, srcval in sourceinfo["all_sources"][memberid].iteritems():
                # Sets don't like lists. Unpacking accomplishes almost the same goal
                if isinstance(srcval, list):
                    for item in srcval:
                        flattener[srckey].add(item)
                else:
                    flattener[srckey].add(srcval)

        # What got flattened?
        # Since we have to tie the Angular data model to a flattened version (sigh)
        # we build this flattened dict instead of using the original sources. It
        # makes things hard in the long run, but it kinda has to be that way:
        for k, v in flattener.iteritems():
            if k not in uneditableSourceFields:
                if len(v) == 1:
                    sourceinfo["source_names"][sourcename]["flattened"][k] = list(v)[0]
                elif len(v) > 1:
                    sourceinfo["source_names"][sourcename]["flattened"][k] = list(v)
                elif len(v) == 0:
                    sourceinfo["source_names"][sourcename]["flattened"][k] = []

        # Not all values are returned with every API call.
        # Blacklist is one of them, but it needs to be here
        # in order to be rendered in the template.  This
        # should be handled better eventually. TODO
        if "blacklist" not in flattener:
            sourceinfo["source_names"][sourcename]["flattened"]["blacklist"] = []

    # That's it.  Just return that to Angular
    return jsonify(results=sourceinfo)
# Renames a category across all collectors and sources in a given account.
#
# python mv-cat.py <accessId/email> <accessKey/password> <fromName> <toName>
#
# TODO update query category constraints
# TODO regex

import sys

from sumologic import SumoLogic

args = sys.argv
sumo = SumoLogic(args[1], args[2])
fromCat, toCat = args[3], args[4]
cs = sumo.collectors()

for c in cs:
	if 'category' in c and c['category'] == fromCat:
		cv, etag = sumo.collector(c['id'])
		cv['collector']['category'] = toCat
		print sumo.update_collector(cv, etag).text
	ss = sumo.sources(c['id'])
	for s in ss:
		if s['category'] == fromCat:
			sv, etag = sumo.source(c['id'], s['id'])
			sv['source']['category'] = toCat
			print sumo.update_source(c['id'], sv, etag).text
class sumotoolbox(QtGui.QMainWindow, Ui_MainWindow):

    def __init__(self):
        QtGui.QMainWindow.__init__(self)
        Ui_MainWindow.__init__(self)
        # detect if we are running in a pyinstaller bundle and set the base directory for file loads"
        if getattr(sys,'frozen', False):
            self.basedir = sys._MEIPASS
        else:
            self.basedir = os.path.dirname(__file__)

        self.setupUi(self)
        self.initModels()   #load all the comboboxes and such with values
        self.loadcredentials()  #if a credential file exists populate the creds with values
        # init all of the dialogs we'll be using
        self.collectorcopyUI = uic.loadUi(qtCollectorCopyDialogUI)
        self.restoresourcesUI = uic.loadUi(qtRestoreSourcesDialogUI)
        self.deletesourceUI = uic.loadUi(qtDeleteSourcesDialogUI)
        # connect all of the UI button elements to their respective methods
        self.pushButtonUpdateListSource.clicked.connect(self.updatecollectorlistsource)
        self.pushButtonUpdateListDestination.clicked.connect(self.updatecollectorlistdestination)
        self.pushButtonCopyCollectorsToDest.clicked.connect(self.copysourcesfromsourcetodestinationdialog)
        self.pushButtonStartSearch.clicked.connect(self.runsearch)
        self.pushButtonBackupCollector.clicked.connect(self.backupcollector)
        self.pushButtonRestoreSources.clicked.connect(self.restoresources)
        self.pushButtonDeleteSources.clicked.connect(self.deletesource)

    def loadcredentials(self):

        #look to see if the credential file exists and load credentials if it does
        #fail if anything at all goes wrong
        if os.path.isfile(os.path.join(self.basedir,'data/credentials.json')):
            try:
                with open(os.path.join(self.basedir,'data/credentials.json'), 'r') as filepointer:
                    credentials = json.load(filepointer)
                self.SourceUserName.setText(credentials['source']['user'])
                self.SourcePassword.setText(credentials['source']['password'])
                self.DestinationUserName.setText(credentials['destination']['user'])
                self.DestinationPassword.setText(credentials['destination']['password'])
            except:
                pass

    def updatecollectorlistsource(self):
        self.listWidgetSourceCollectors.clear()     #clear the list first since it might already be populated
        sourceurl = self.loadedapiurls[str(self.comboBoxSourceRegion.currentText())] #get the selected API URL
        sourceusername = str(self.SourceUserName.text()) #get username
        sourcepassword = str(self.SourcePassword.text()) #get password
        self.sourcecollectordict = {} #init this so we can store a dict of collectors (easier to access than list)
        regexprog = re.compile(r'\S+') # make sure username and password have something in them
        if (re.match(regexprog,sourceusername) != None) and (re.match(regexprog,sourcepassword) != None):
            #access the API with provided credentials
            self.sumosource = SumoLogic(sourceusername, sourcepassword, endpoint=sourceurl)
            try:
                self.sourcecollectors = self.sumosource.collectors() #get list of collectors
                for collector in self.sourcecollectors:
                    self.sourcecollectordict[collector['name']]=collector['id'] #make a dict with just names and ids

                for collector in self.sourcecollectordict:
                    self.listWidgetSourceCollectors.addItem(collector) #populate the list widget in the GUI

                #set up a signal to update the source list if anything is changed
                self.listWidgetSourceCollectors.currentItemChanged.connect(self.updatesourcelistsource)
            except:
                self.errorbox('Incorrect Credentials.')
        else:
           self.errorbox('No user and/or password.')

    def updatecollectorlistdestination(self):
        self.listWidgetDestinationCollectors.clear() #clear the list first since it might already be populated
        destinationurl = self.loadedapiurls[str(self.comboBoxDestinationRegion.currentText())] #get the selected API URL
        destinationusername = str(self.DestinationUserName.text()) #get username
        destinationpassword = str(self.DestinationPassword.text()) #get password
        self.destinationcollectordict = {} #init this so we can store a dict of collectors (easier to access than list)
        regexprog = re.compile(r'\S+') # make sure username and password have something in them
        if (re.match(regexprog, destinationusername) is not None) and (re.match(regexprog,destinationpassword) is not None):
            #access the API with provided credentials
            self.sumodestination = SumoLogic(destinationusername, destinationpassword, endpoint=destinationurl)
            try:
                self.destinationcollectors = self.sumodestination.collectors() #get list of collectors
                for collector in self.destinationcollectors:
                    self.destinationcollectordict[collector['name']]=collector['id'] #make a dict with just names and ids

                for collector in self.destinationcollectordict:
                    self.listWidgetDestinationCollectors.addItem(collector) #populate the list widget in the GUI

                #set up a signal to update the source list if anything is changed
                self.listWidgetDestinationCollectors.currentItemChanged.connect(self.updatedestinationlistsource)
            except:
                self.errorbox('Incorrect Credentials.')
        else:
            self.errorbox('No user and/or password.')

    def updatesourcelistsource(self, currentcollector, prevcollector):

        self.listWidgetSourceSources.clear() #clear the list first since it might already be populated
        if currentcollector != None: #check to make sure that there is a collector selected
            self.sourcesourcesdict = {}
            # populate the list of sources
            self.sourcesources = self.sumosource.sources(self.sourcecollectordict[str(currentcollector.text())])
            for source in self.sourcesources:
                self.sourcesourcesdict[source['name']]='' #this is sloppy but I just want a dict of names
            for source in self.sourcesourcesdict:
                self.listWidgetSourceSources.addItem(source) #populate the display with sources

    def updatedestinationlistsource(self, currentcollector, prevcollector):

        self.listWidgetDestinationSources.clear() #clear the list first since it might already be populated
        if currentcollector != None: #check to make sure that there is a collector selected
            self.destinationsourcesdict = {}
            # populate the list of sources
            self.destinationsources = self.sumodestination.sources(self.destinationcollectordict[str(currentcollector.text())])
            for source in self.destinationsources:
                self.destinationsourcesdict[source['name']]='' #this is sloppy but I just want a dict of names
            for source in self.destinationsourcesdict:
                self.listWidgetDestinationSources.addItem(source) #populate the display with sources

    def copysourcesfromsourcetodestinationdialog(self):

        sourcecollector = self.listWidgetSourceCollectors.selectedItems() #get the selected source collector
        if len (sourcecollector) == 1: #make sure there is a collector selected, otherwise bail
            sourcecollector = sourcecollector[0].text() #qstring to string conversion
            destinationcollector = self.listWidgetDestinationCollectors.selectedItems() #get the selected dest collector
            if len(destinationcollector) == 1: #make sure there is a collector selected, otherwise bail

                destinationcollectorqstring = destinationcollector[0]
                destinationcollector = str(destinationcollector[0].text()) #qstring to string conversion
                sourcesources = self.listWidgetSourceSources.selectedItems() #get the selected sources
                if len(sourcesources) > 0: #make sure at least one source is selected
                    sourcelist = []
                    for source in sourcesources: #iterate through source names to build a warning message
                        sourcelist.append(source.text())
                    message = "You are about to copy the following sources from collector \"" + sourcecollector + "\" to \"" + destinationcollector + "\". Is this correct? \n\n"
                    for source in sourcelist:
                        message = message + source + "\n"
                    self.collectorcopyUI.labelCollectorCopy.setText(message) #set the warning message in the copy dialog
                    self.collectorcopyUI.dateTimeEdit.setMaximumDate(QtCore.QDate.currentDate()) #make sure user doesn't pick time in future
                    self.collectorcopyUI.dateTimeEdit.setDate(QtCore.QDate.currentDate()) #set date to today
                    result = self.collectorcopyUI.exec_() #bring up the copy dialog
                    #process collection time override inputs
                    overridecollectiondate = self.collectorcopyUI.checkBoxOverrideCollectionStartTime.isChecked()
                    overridedate = self.collectorcopyUI.dateTimeEdit.dateTime()
                    #THIS IS BROKEN.... for some reason the API will not accept the following as valid Epoch time
                    #Maybe the longint isn't getting the "L" appended to it?
                    overridedatemillis = long(overridedate.currentMSecsSinceEpoch())
                    if result:  #If they clicked "OK" rather than cancel
                        for source in sourcelist: #iterate through the selected sources and copy them
                            for sumosource in self.sourcesources:
                                #
                                if sumosource['name'] == source:
                                    if 'id' in sumosource: #the API creates an ID so this must be deleted before sending
                                        del sumosource['id']
                                    if 'alive' in sumosource:
                                        del sumosource['alive'] #the API sets this itself so this must be deleted before sending
                                    if overridecollectiondate:
                                        sumosource['cutoffTimestamp'] = overridedatemillis
                                    template = {}
                                    template['source'] = sumosource #the API expects a dict with a key called 'source'
                                    notduplicate = True
                                    for sumodest in self.destinationsources:
                                        if sumodest['name'] == source: #make sure the source doesn't already exist in the destination
                                            notduplicate = False
                                    if notduplicate: #finally lets copy this thing
                                        self.sumodestination.create_source(self.destinationcollectordict[destinationcollector], template)
                                    else:
                                        self.errorbox(source + ' already exists, skipping.')
                        #call the update method for the dest sources since they have changed after the copy
                        self.updatedestinationlistsource(destinationcollectorqstring, destinationcollectorqstring)


                else:
                    self.errorbox('No Sources Selected.')
            else:
                self.errorbox('No Destination Collector Selected.')
        else:
            self.errorbox('No Source Collector Selected.')

    def backupcollector(self):
        sourcecollector = self.listWidgetSourceCollectors.selectedItems() #get which sources have been selected
        if len (sourcecollector) == 1: #make sure something was selected
            if self.sourcesources: #make sure there's something to write to the file
                sourcecollector = str(sourcecollector[0].text()) + r'.json'
                savefile = str(QtGui.QFileDialog.getSaveFileName(self, 'Save As...', sourcecollector))
                if savefile:
                    with open(savefile, 'w') as filepointer:
                        json.dump(self.sourcesources, filepointer)
                    self.infobox('Wrote file ' + savefile)
            else:
                self.errorbox('No sources to backup.')

        else:
            self.errorbox('No Source Collector Selected.')

    def restoresources(self):
        destinationcollector = self.listWidgetDestinationCollectors.selectedItems()
        if len(destinationcollector) == 1:
            destinationcollectorqstring = destinationcollector[0]
            destinationcollector = str(destinationcollector[0].text())
            restorefile = str(QtGui.QFileDialog.getOpenFileName(self, 'Open Backup..','',selectedFilter='*.json'))
            sources = None
            try:
                with open(restorefile) as data_file:
                    sources = json.load(data_file)
            except:
                self.errorbox('Failed to load JSON file.')
            if sources:
                self.restoresourcesUI.dateTimeEdit.setMaximumDate(QtCore.QDate.currentDate())
                self.restoresourcesUI.dateTimeEdit.setDate(QtCore.QDate.currentDate())
                self.restoresourcesUI.listWidgetRestoreSources.clear()
                sourcedict = {}
                for source in sources:
                    sourcedict[source['name']]=''
                for source in sourcedict:
                    self.restoresourcesUI.listWidgetRestoreSources.addItem(source)
                result = self.restoresourcesUI.exec_()
                overridecollectiondate = self.restoresourcesUI.checkBoxOverrideCollectionStartTime.isChecked()
                overridedate = self.restoresourcesUI.dateTimeEdit.dateTime()
                overridedatemillis = long(overridedate.currentMSecsSinceEpoch())
                if result:
                    selectedsources = self.restoresourcesUI.listWidgetRestoreSources.selectedItems()
                    if len(selectedsources) > 0:
                        for selectedsource in selectedsources:
                            for sumosource in sources:
                                    if sumosource['name'] == str(selectedsource.text()):
                                        if 'id' in sumosource:
                                            del sumosource['id']
                                        if 'alive' in sumosource:
                                            del sumosource['alive']
                                        if overridecollectiondate:
                                            sumosource['cutoffTimestamp'] = overridedatemillis
                                        template = {}
                                        template['source'] = sumosource
                                        notduplicate = True
                                        for sumodest in self.destinationsources:
                                            if sumodest['name'] == source:
                                                notduplicate = False
                                        if notduplicate:
                                            self.sumodestination.create_source(self.destinationcollectordict[destinationcollector], template)
                                        else:
                                            self.errorbox(source + ' already exists, skipping.')
                            self.updatedestinationlistsource(destinationcollectorqstring, destinationcollectorqstring)
                    else:
                        self.errorbox('No sources selected for import.')





        else:
            self.errorbox('No Destination Collector Selected.')

    def deletesource(self):
        sourcetodelete = self.listWidgetDestinationSources.selectedItems()
        if len(sourcetodelete) > 1:
            self.errorbox('Too many sources selected. There can be only one!')
        if len(sourcetodelete) == 1:
            message = "You are about to delete the following source:\n\n" + str(sourcetodelete[0].text()) + '\n\nIf you are sure type "DELETE" in the box below.'
            self.deletesourceUI.labelDeleteSources.setText(message)
            result = self.deletesourceUI.exec_()
            if result:
                if str(self.deletesourceUI.lineEditVerifyDelete.text()) == "DELETE":
                    destinationcollector = self.listWidgetDestinationCollectors.selectedItems()
                    destinationcollectorqstring = destinationcollector[0]
                    destinationcollector = str(destinationcollector[0].text())
                    destinationcollectorid = self.destinationcollectordict[destinationcollector]
                    for destinationsource in self.destinationsources:
                        if destinationsource['name'] == str(sourcetodelete[0].text()):
                            self.sumodestination.delete_source_by_id(destinationcollectorid, destinationsource['id'])
                            self.updatedestinationlistsource(destinationcollectorqstring, destinationcollectorqstring)
                else:
                    self.errorbox('You failed to type "DELETE". Crisis averted!')
        else:
            self.errorbox('No source selected.')

    def runsearch(self):

        self.tableWidgetSearchResults.clear()
        selectedtimezone = str(self.comboBoxTimeZone.currentText())
        starttime = str(self.dateTimeEditSearchStartTime.dateTime().toString(QtCore.Qt.ISODate))
        endtime = str(self.dateTimeEditSearchEndTime.dateTime().toString(QtCore.Qt.ISODate))
        sourceurl = self.loadedapiurls[str(self.comboBoxSourceRegion.currentText())]
        sourceusername = str(self.SourceUserName.text())
        sourcepassword = str(self.SourcePassword.text())
        searchstring = str(self.plainTextEditSearch.toPlainText())
        regexprog = re.compile(r'\S+')
        jobsubmitted = False
        savetofile = self.checkBoxSaveSearch.isChecked()
        converttimefromepoch = self.checkBoxConvertTimeFromEpoch.isChecked()
        self.jobmessages = []
        self.jobrecords = []

        if (re.match(regexprog,sourceusername) != None) and (re.match(regexprog,sourcepassword) != None):
            self.sumosource = SumoLogic(sourceusername, sourcepassword, endpoint=sourceurl)

            if (re.match(regexprog, searchstring)) != None:
                try:
                    searchjob = self.sumosource.search_job(searchstring, starttime, endtime, selectedtimezone)
                    jobsubmitted = True
                except:
                    self.errorbox('Incorrect Credentials.')
                if jobsubmitted:
                    self.labelSearchResultCount.setText('0')
                    jobstatus = self.sumosource.search_job_status(searchjob)
                    nummessages = jobstatus['messageCount']
                    numrecords = jobstatus['recordCount']
                    self.labelSearchResultCount.setText(str(nummessages))
                    while jobstatus['state'] == 'GATHERING RESULTS':
                        time.sleep(5)
                        jobstatus = self.sumosource.search_job_status(searchjob)
                        numrecords = jobstatus['recordCount']
                        nummessages = jobstatus['messageCount']
                        self.labelSearchResultCount.setText(str(nummessages))
                    if nummessages is not 0:

                        #return messages
                        if self.buttonGroupOutputType.checkedId() == -2:
                            iterations = nummessages // 10000 + 1
                            for iteration in range (1,iterations + 1):
                                messages = self.sumosource.search_job_messages(searchjob,limit=10000,offset=((iteration-1)*10000))
                                for message in messages['messages']:
                                    self.jobmessages.append(message)
                            self.tableWidgetSearchResults.setRowCount(len(self.jobmessages))
                            self.tableWidgetSearchResults.setColumnCount(2)
                            self.tableWidgetSearchResults.setHorizontalHeaderLabels(['time','_raw'])
                            index = 0
                            for message in self.jobmessages:
                                if converttimefromepoch:
                                    timezone = pytz.timezone(selectedtimezone)
                                    converteddatetime = datetime.fromtimestamp(float(message['map']['_messagetime']) / 1000, timezone)
                                    timestring = str(converteddatetime.strftime('%Y-%m-%d %H:%M:%S'))
                                    message['map']['_messagetime'] = timestring
                                self.tableWidgetSearchResults.setItem(index,0,QtGui.QTableWidgetItem(message['map']['_messagetime']))
                                self.tableWidgetSearchResults.setItem(index,1,QtGui.QTableWidgetItem(message['map']['_raw']))
                                index += 1
                            self.tableWidgetSearchResults.resizeRowsToContents()
                            self.tableWidgetSearchResults.resizeColumnsToContents()
                            if savetofile:
                                filename = QtGui.QFileDialog.getSaveFileName(self, 'Save CSV', '', selectedFilter='*.csv')
                                if filename:
                                    with open(filename,'wb') as csvfile:
                                        messagecsv = csv.DictWriter(csvfile,self.jobmessages[0]['map'].keys())
                                        messagecsv.writeheader()
                                        for entry in self.jobmessages:
                                            messagecsv.writerow(entry['map'])
                        #return records
                        if self.buttonGroupOutputType.checkedId() == -3:
                            iterations = numrecords // 10000 + 1
                            for iteration in range (1,iterations + 1):
                                records = self.sumosource.search_job_records(searchjob,limit=10000,offset=((iteration-1)*10000))
                                for record in records['records']:
                                    self.jobrecords.append(record)
                            self.tableWidgetSearchResults.setRowCount(len(self.jobrecords))
                            numfields = len(records['fields'])
                            self.tableWidgetSearchResults.setColumnCount(numfields)
                            fieldnames = []
                            for field in records['fields']:
                                fieldnames.append(field['name'])
                            self.tableWidgetSearchResults.setHorizontalHeaderLabels(fieldnames)
                            index = 0
                            for record in self.jobrecords:
                                columnnum = 0
                                for fieldname in fieldnames:
                                    if converttimefromepoch and (fieldname == '_timeslice'):
                                        timezone = pytz.timezone(selectedtimezone)
                                        converteddatetime = datetime.fromtimestamp(float(record['map'][fieldname]) / 1000, timezone)
                                        timestring = str(converteddatetime.strftime('%Y-%m-%d %H:%M:%S'))
                                        record['map']['_timeslice'] = timestring
                                    self.tableWidgetSearchResults.setItem(index, columnnum, QtGui.QTableWidgetItem(record['map'][fieldname]))
                                    columnnum += 1
                                index += 1
                            self.tableWidgetSearchResults.resizeRowsToContents()
                            self.tableWidgetSearchResults.resizeColumnsToContents()
                            if savetofile:
                                filename = QtGui.QFileDialog.getSaveFileName(self, 'Save CSV', '', selectedFilter='*.csv')
                                if filename:
                                    with open(filename,'wb') as csvfile:
                                        recordcsv = csv.DictWriter(csvfile,self.jobrecords[0]['map'].keys())
                                        recordcsv.writeheader()
                                        for entry in self.jobrecords:
                                            recordcsv.writerow(entry['map'])
                    else:
                        self.errorbox('Search did not return any messages.')
            else:
                self.errorbox('Please enter a search.')
        else:
            self.errorbox('No user and/or password.')

    def errorbox(self, message):
        msgBox = QtGui.QMessageBox()
        msgBox.setWindowTitle('Error')
        msgBox.setText(message)
        msgBox.addButton(QtGui.QPushButton('OK'), QtGui.QMessageBox.RejectRole)
        ret = msgBox.exec_()

    def infobox(self, message):
        msgBox = QtGui.QMessageBox()
        msgBox.setWindowTitle('Info')
        msgBox.setText(message)
        msgBox.addButton(QtGui.QPushButton('OK'), QtGui.QMessageBox.RejectRole)
        ret = msgBox.exec_()

    def initModels(self):
        # Load API Endpoint List from file and create model for the comboboxes
        with open(os.path.join(self.basedir,'data/apiurls.json'), 'r') as infile:
            self.loadedapiurls=json.load(infile)

        self.apiurlsmodel = QtGui.QStandardItemModel()
        for key in self.loadedapiurls:
            text_item = QtGui.QStandardItem(key)
            self.apiurlsmodel.appendRow(text_item)

        self.comboBoxSourceRegion.setModel(self.apiurlsmodel)
        self.comboBoxDestinationRegion.setModel(self.apiurlsmodel)

        #Load Timezones and create model for timezone combobox

        self.timezonemodel = QtGui.QStandardItemModel()
        for zone in pytz.common_timezones:
            text_item = QtGui.QStandardItem(zone)
            self.timezonemodel.appendRow(text_item)

        self.comboBoxTimeZone.setModel(self.timezonemodel)

        # set search start and endtimes to now-ish
        self.dateTimeEditSearchStartTime.setDateTime(QtCore.QDateTime.currentDateTime().addSecs(-900))
        self.dateTimeEditSearchEndTime.setDateTime(QtCore.QDateTime.currentDateTime())

        # set timezone combobox to local timezone
        localtimezone = str(get_localzone())
        index = self.comboBoxTimeZone.findText(localtimezone, QtCore.Qt.MatchFixedString)
        if index >= 0:
            self.comboBoxTimeZone.setCurrentIndex(index)
attr, val = args[3], args[4]
cs = sumo.collectors()
time.sleep(delay)
f = [
    {
        u"regexp": u"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.(\\d{1,3})",
        u"mask": u"255",
        u"filterType": u"Mask",
        u"name": u"last octet mask",
    }
]

for c in cs:
    if "category" not in c or "bwe" not in c["category"] and "bwm" not in c["category"]:
        print "collector: " + c["name"]
        ss = sumo.sources(c["id"])
        time.sleep(delay)
        for s in ss:
            sv, etag = sumo.source(c["id"], s["id"])
            time.sleep(delay)
            svi = sv["source"]
            if "category" not in svi or "bwe" not in svi["category"] and "bwm" not in svi["category"]:
                print "source: " + svi["name"]
                svi["filters"] = f
                r = sumo.update_source(c["id"], sv, etag)
                print r
                print r.text
                time.sleep(delay)
            # if svi['forceTimeZone'] == False:
            #    svi['forceTimeZone'] = True
            #    svi[u'timeZone'] = u'UTC'