Пример #1
0
class Geocode:
	def __init__(self, geocoderCache = True, printStatus = False):
		self.printStatus = printStatus
		self.geocoderCache = geocoderCache
		if self.geocoderCache:
			self.db = FileBackend('./geo-cache')
	def getGeo(self, lon, lat):
		if self.geocoderCache:
			try:
				nodeObj = self.db.get(GeoAssign,{'lat' : lat, 'lon' : lon})
				nodeObj['cached'] = True
				return nodeObj
			except GeoAssign.DoesNotExist:
				pass
		if self.printStatus:
			print('lon: '+str(lon)+', lat: '+str(lat)+' not in cache - start lookup at Nominatim-API')
		geolocator = Nominatim()
		location = geolocator.reverse([lat, lon], timeout=20)
		if 'address' in location.raw:
			location = location.raw['address']
			nodeObj = GeoAssign({
				'lat' : lat,
				'lon' : lon,
				'payload' : location
				})
			self.db.save(nodeObj)
			self.db.commit()
			nodeObj['cached'] = False
			return nodeObj
		else:
			# got no results (i.e. coordinates are incorrect)
			return None
Пример #2
0
    def on_data(self, data):
        dict_data = json.loads(data)
        db_data = {}
        try:
            # Ignore retweets
            if dict_data['retweeted'] == False:
                try:
                    pp("GET DATA")

                    # Get relevant key value pairs only
                    for KEY in DESIRED_KEYS:
                        db_data[KEY] = dict_data[KEY]
                    db_data['screen_name'] = dict_data['user']['screen_name']
                    cards = parse_text(db_data['text'])

                    # Add it to the db_data dict to import into BlitzDB
                    db_data['cards'] = cards
                    pp(db_data)
                except:
                    pp("GET DATA FAILED")
                try:
                    # Import into BlitzDB
                    backend = FileBackend("./test-db")
                    tweet = Tweet(db_data)
                    backend.save(tweet)
                    # backend.commit()
                    pp("DATABASE INSERTION SUCCESSFUL")
                except:
                    pp("DATA INSERTION FAILED")
        except:
            pp("Retweet detected, skipping...")

        return True
Пример #3
0
class ImageToTumblr(object):
    def __init__(self, bot):
        self.cfg = PluginConfig(self)
        self.image_filetypes = self.cfg.get("image_filetypes").split(",")
        self.db = FileBackend(self.cfg.get("main_db"))
        self.tumblr = pytumblr.TumblrRestClient(
            self.cfg.get("consumer_key"),
            self.cfg.get("consumer_secret"),
            self.cfg.get("oauth_token"),
            self.cfg.get("oauth_secret"),
        )
        irc3.base.logging.log(
            irc3.base.logging.WARN, "Tumblr poster ready! Posting all URLs with: %s" % self.image_filetypes
        )

    def post_image(self, text, poster):
        # Strip everything but the address
        m = re.match(r".*(?P<url>http.*)", text)
        url = m.group("url")
        # Make sure we didn't do this one already
        try:
            self.db.get(PostedImage, {"url": url})
        except PostedImage.DoesNotExist:
            try:
                # First we post it to tumblr
                p = self.tumblr.create_photo(
                    "mmerpimages", state="published", source=str(url), caption="Found by %s" % poster
                )
                irc3.base.logging.log(irc3.base.logging.WARN, "Posting image by %s: %s" % (poster, url))

                # And then record the fact that we did.
                self.db.save(PostedImage({"url": url}))
                self.db.commit()
            except:
                irc3.base.logging.log(irc3.base.logging.WARN, "Could not post to tumblr: %s" % url)
                return
        else:
            irc3.base.logging.log(irc3.base.logging.WARN, "Not posting duplicate image: %s" % url)
            return

    @irc3.event(irc3.rfc.PRIVMSG)  # Triggered on every message anywhere.
    def parse_image(self, target, mask, data, event):
        for extension in self.image_filetypes:
            if "." + extension.lower() in data:
                self.post_image(data, mask.nick)
Пример #4
0
class Database:

    def __init__(self):
        self.backend = FileBackend(settings['database_location'])

    def save(self, entry):
        self.backend.save(entry)
        self.backend.commit()

    def search(self, table, query):
        if table == 'user':
            doc = User
        elif table == 'subm':
            doc = Submission
        try:
            return self.backend.get(doc, query)
        except doc.DoesNotExist:
            return None
Пример #5
0
class Database(object):
    """Blitzdb database."""
    def __init__(self):
        """Load backend."""
        self.db = FileBackend(expanduser("~/.omesa/db"))

    def _query(self, f, q):
        try:
            out = f(*q)
        except KeyError:
            self.db = FileBackend(expanduser("~/.omesa/db"))
            f = self.db.filter
            out = f(*q)
        return out

    def save(self, doc):
        """Save document do db."""
        self.db.save(doc)
        self.db.commit()

    def fetch(self, doc, q):
        """Filter and return first entry."""
        try:
            return self._query(self.db.filter, (doc, q))[0]
        except IndexError:
            print(str(doc), str(q))
            print("File does not exist.")

    def get_component(self, doc, name):
        # FIXME: see if returning non-decoded is relevant for anything
        try:
            return sr.decode(
                dict(self._query(self.db.filter, (doc, {
                    'name': name
                }))[0]))
        except IndexError:
            print(str(doc), {'name': name})
            print("File does not exist.")

    def getall(self, doc):
        """Returns all entries in db."""
        return [d for d in self._query(self.db.filter, (doc, {}))]
Пример #6
0
class Database(object):
    """Blitzdb database."""

    def __init__(self):
        """Load backend."""
        self.db = FileBackend(expanduser("~/.omesa/db"))

    def _query(self, f, q):
        try:
            out = f(*q)
        except KeyError:
            self.db = FileBackend(expanduser("~/.omesa/db"))
            f = self.db.filter
            out = f(*q)
        return out

    def save(self, doc):
        """Save document do db."""
        self.db.save(doc)
        self.db.commit()

    def fetch(self, doc, q):
        """Filter and return first entry."""
        try:
            return self._query(self.db.filter, (doc, q))[0]
        except IndexError:
            print(str(doc), str(q))
            print("File does not exist.")

    def get_component(self, doc, name):
        # FIXME: see if returning non-decoded is relevant for anything
        try:
            return sr.decode(dict(self._query(
                self.db.filter, (doc, {'name': name}))[0]))
        except IndexError:
            print(str(doc), {'name': name})
            print("File does not exist.")

    def getall(self, doc):
        """Returns all entries in db."""
        return [d for d in self._query(self.db.filter, (doc, {}))]
Пример #7
0
class eventmanager:
    def __init__(self, trigger_id, jsonfilelist, triggerdir, datadir, real,
                 trigger_path):

        #os.system('kinit -k -t /var/keytab/desgw.keytab desgw/des/[email protected]')
        tstart = time.time()

        if real:
            self.backend = FileBackend("./realdb")
        else:
            self.backend = FileBackend("./testdb")

        try:
            thisevent = self.backend.get(Trigger, {'id': trigger_id})
            print 'Found this event in desgw database...'
        except Trigger.DoesNotExist:
            thisevent = Trigger({
                'id':
                trigger_id,
                'jsonfilelist':
                jsonfilelist,
                'triggerpath':
                triggerdir,
                'mapspath':
                datadir,
                'jobids': [
                    (0, 'jsonfile_corresponding_to_jobid.json'),
                ],
            })
            print 'Database entry created!'

        self.trigger_id = trigger_id
        self.trigger_path = trigger_path

        self.backend.save(thisevent)
        self.backend.commit()

        with open(os.path.join(triggerdir, "strategy.yaml"), "r") as f:
            self.config = yaml.safe_load(f)
        self.filterobslist = np.array(self.config['exposure_filter'],
                                      dtype='str')
        self.strategydict = {}

        for f in np.unique(self.filterobslist):
            self.strategydict[f] = len(
                self.filterobslist[self.filterobslist == f])

        self.connection = ea.connect(DATABASE)
        self.cursor = self.connection.cursor()

        self.jsonfilelist = jsonfilelist

        print self.jsonfilelist
        if hardjson:
            self.jsonfilelist = hj

        self.trigger_id = trigger_id
        self.datadir = datadir
        self.triggerdir = triggerdir
        self.processingdir = os.path.join(self.triggerdir, 'PROCESSING')
        if not os.path.exists(self.processingdir):
            os.makedirs(self.processingdir)

        dire = './processing/' + trigger_id + '/'
        if not os.path.exists(dire):
            os.makedirs(dire)

        with open(os.path.join(triggerdir, "strategy.yaml"), "r") as f:
            self.strategy = yaml.safe_load(f)

        with open("jobmanager.yaml", "r") as g:
            self.jmconfig = yaml.safe_load(g)


        q1 = "select expnum,nite,mjd_obs,telra,teldec,band,exptime,propid,obstype,object from exposure where " \
             "nite>20130828 and nite<20150101 and expnum<300000 and obstype='object' order by expnum"  # y1 images
        self.connection.query_and_save(q1, './processing/exposuresY1.tab')

        q2 = "select expnum,nite,mjd_obs,radeg,decdeg,band,exptime,propid,obstype,object from prod.exposure where " \
             "nite>20150901 and obstype='object' order by expnum"  # y2 and later
        self.connection.query_and_save(q2, './processing/exposuresCurrent.tab')

        os.system(
            'cat ./processing/exposuresY1.tab ./processing/exposuresCurrent.tab > ./processing/exposures.list'
        )

        self.submit_all_jsons_for_sejobs(
        )  #preps all DES images that already exist
        tfin = time.time()
        print 'TOTAL SE JOBS TIME', tfin - tstart
        #sys.exit()
        self.monitor_images_from_mountain(
        )  #A loop that waits for images off mountain and submits for processing

    def submit_all_jsons_for_sejobs(self):
        obsStartTime = self.getDatetimeOfFirstJson(
            self.jsonfilelist[0])  # THIS IS A DATETIME OBJ
        currentTime = dt.utcnow()
        print '***** The current time is UTC', currentTime, '*****'
        delt = obsStartTime - currentTime

        timedelta = td(days=delt.days,
                       seconds=delt.seconds).total_seconds() / 3600.
        print '***** The time delta is ', timedelta, 'hours *****'
        # if timedelta > np.pi:

        sejob_timecushion = self.jmconfig["sejob_timecushion"]

        if timedelta > sejob_timecushion:
            for jsonfile in self.jsonfilelist:
                print 'json', jsonfile
                try:  #check if this json file is already in the submitted preprocessing database
                    thisjson = self.backend.get(
                        preprocessing,
                        {'jsonfilename': os.path.join(self.datadir, jsonfile)})
                    print 'Found this json in desgw database...'
                except preprocessing.DoesNotExist:  #do submission and then add to database

                    print 'cd diffimg-proc; ./SEMaker_RADEC.sh ' + os.path.join(
                        self.datadir, jsonfile)
                    os.chdir("diffimg-proc")
                    out = os.popen(
                        './SEMaker_RADEC.sh ' +
                        os.path.join(self.datadir, jsonfile)).read()
                    #out = os.popen('ls').read()
                    os.chdir("..")
                    print out
                    if 'non-zero exit status' in out:
                        dt.sendEmailSubject(
                            self.trigger_id,
                            'Error in creating dag for .json: ' + out)
                    else:
                        for o in out.split('\n'):
                            if 'file://' in o:
                                dagfile = o.split('/')[-1]
                                self.dagfile = os.path.join(
                                    self.processingdir,
                                    jsonfile.split('/')[-1].split('.')[0] +
                                    '_' + dagfile)
                                os.system('cp diffimg-proc/' + dagfile + ' ' +
                                          self.dagfile)
                                jobsubmitline = copy(o)
                        print self.dagfile

                    print 'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; jobsub_submit_dag -G des --role=DESGW file://' + self.dagfile

                    out = os.popen(
                        'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; '
                        'jobsub_submit_dag -G des --role=DESGW file://' +
                        self.dagfile).read()
                    print out
                    if 'non-zero exit status' in out:
                        dt.sendEmailSubject(
                            self.trigger_id,
                            'Error in submitting .json for preprocessing: ' +
                            out)
                    else:
                        if doimmediateremove:
                            for o in out.split('\n'):
                                if 'Use job id' in o:
                                    jobid = o.split()[3]
                            out = os.popen(
                                'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; '
                                'jobsub_rm --jobid=' + jobid +
                                ' --group=des --role=DESGW').read()
                            print out

                    thisjson = preprocessing({
                        'jsonfilename':
                        os.path.join(self.datadir, jsonfile),
                        'jobid':
                        jobid,
                        'dagfile':
                        self.dagfile,
                        'status':
                        'Submitted'
                    })

                    self.backend.save(thisjson)
                    self.backend.commit()
                    print 'saved'
                #raw_input()
                #runProcessingIfNotAlready(image, self.backend)

                #sys.exit()

        print 'Finished submitting minidagmaker with all json files'
        #sys.exit()
        #raw_input()

    # Loop queries for images from mountain and submits them
    # Need to add complexity that monitors filter strategy and waits for entire groups of images to be co-added
    def monitor_images_from_mountain(self):
        #NEED TO ADD COADD LOGIC USING STRATEGY FROM CONFIG

        exposure_filter = np.array(self.strategy['exposure_filter'],
                                   dtype='str')
        uniquefilts = np.unique(self.strategy['exposure_filter'])

        filterstrategy = {}
        for f in uniquefilts:
            filterstrategy[f] = len(exposure_filter[exposure_filter == f])

        print 'filter strategy dictionary  ', filterstrategy

        starttime = time.time()
        pptime = time.time()
        keepgoing = True
        index = -1
        submission_counter = 0
        maxsub = 10000
        postprocessingtime = 10  #every half hour fire off Tim's code for post-processing
        while keepgoing:
            #os.system('kinit -k -t /var/keytab/desgw.keytab desgw/des/[email protected]')
            index += 1
            newfireds = []
            if time.time() - starttime > 50000:
                keepgoing = False
                continue

            ofile = open(os.path.join(self.triggerdir, 'latestquery.txt'), 'w')

            ofile.write(
                "--------------------------------------------------------------------------------------------------\n"
            )
            ofile.write(
                "EXPNUM\tNITE\tBAND\tEXPTIME\tRADEG\t DECDEG\tPROPID\tOBJECT\n"
            )
            ofile.write(
                "--------------------------------------------------------------------------------------------------\n"
            )

            print "--------------------------------------------------------------------------------------------------"
            print "EXPNUM\tNITE\tBAND\tEXPTIME\tRADEG\t DECDEG\tPROPID\tOBJECT"
            print "--------------------------------------------------------------------------------------------------"

            query = "SELECT expnum,nite,band,exptime,radeg,decdeg,propid,object FROM prod.exposure@desoper WHERE " \
                    "expnum > 475900 and propid=" + propid + "and obstype='object' ORDER BY expnum"  # latest

            self.cursor.execute(query)

            for s in self.cursor:
                ofile.write(
                    str(s[0]) + "\t" + str(s[1]) + "\t" + str(s[2]) + "\t" +
                    str(s[3]) + "\t" + str(s[4]) + "\t" + str(s[5]) + "\t" +
                    str(s[6]) + "\t" + str(s[7]) + '\n')
                print str(s[0]) + "\t" + str(s[1]) + "\t" + str(
                    s[2]) + "\t" + str(s[3]) + "\t" + str(s[4]) + "\t" + str(
                        s[5]) + "\t" + str(s[6]) + "\t" + str(s[7])

                if not 'DESGW' in str(s[7]): continue
                #print 'exptime',float(s[3])
                if not float(s[3]) > 29.:
                    continue  #exposure must be longer than 30 seconds

                expnum = str(s[0])
                nite = str(s[1])
                band = str(s[2])
                exptime = str(s[3])

                #FIRST CHECK HERE THAT THE EXPOSURE NUMBER ISNT ALREADY IN THE DATABASE
                try:
                    exposure = self.backend.get(exposures, {'expnum': expnum})
                    print 'Found this exposure in desgw database...'
                    # print exposure.attributes
                    # if expnum == 506432:
                    #     sys.exit()
                    # self.backend.delete(exposure)
                    # self.backend.commit()
                    # exposure = self.backend.get(exposures, {'expnum': expnum})

                except exposures.DoesNotExist:  # add to database
                    #runProcessingIfNotAlready(image,self.backend)

                    print './diffimg-proc/getTiling.sh ' + expnum

                    res = os.popen('./diffimg-proc/getTiling.sh ' +
                                   expnum).readlines()
                    print res
                    #sys.exit()
                    field, tiling = res[-2], res[-1]
                    #print 'field_tiling',field_tiling
                    hexnite = field.strip() + '_' + tiling.strip() + '_' + str(
                        nite)
                    #print hexnite
                    #sys.exit()
                    #print 'hexnite',hexnite
                    print 'Creating exposure in database...', hexnite
                    #raw_input()
                    if '--' in hexnite:
                        print 'found bad example'
                        #raw_input()

                    exposure = exposures({
                        'expnum': expnum,
                        'nite': nite,
                        'field': field,
                        'tiling': tiling,
                        'hexnite': hexnite,
                        'band': band,
                        'jobid': np.nan,
                        'exptime': exptime,
                        'status': 'Awaiting additional exposures',
                        'triggerid': self.trigger_id,
                        'object': str(s[7])
                    })

                    self.backend.save(exposure)
                    self.backend.commit()

                hexnite = exposure.hexnite
                print 'hexnite', hexnite
                if '--' in hexnite:
                    print exposure.attributes
                    #raw_input()
                #raw_input()
                #sys.exit()
                try:
                    hex = self.backend.get(hexes, {'hexnite': hexnite})
                    #self.backend.delete(hex)
                    #self.backend.commit()
                    #hex = self.backend.get(hexes, {'hexnite': hexnite})

                    #print 'Found this hex in desgw database...'
                except hexes.DoesNotExist:
                    hex = hexes({
                        'hexnite':
                        hexnite,
                        'strategy':
                        self.strategy['exposure_filter'],
                        'num_target_g':
                        len(exposure_filter[exposure_filter == 'g']),
                        'num_target_r':
                        len(exposure_filter[exposure_filter == 'r']),
                        'num_target_i':
                        len(exposure_filter[exposure_filter == 'i']),
                        'num_target_z':
                        len(exposure_filter[exposure_filter == 'z']),
                        'observed_g': [],
                        'observed_r': [],
                        'observed_i': [],
                        'observed_z': [],
                        'exposures': [],
                        'status':
                        'Awaiting additional exposures',
                        'dagfile':
                        None,
                    })

                    self.backend.save(hex)
                    self.backend.commit()
                    print hex.attributes
                    print 'created new hex'
                    #raw_input()

                if hex.status == 'Submitted for processing':
                    print 'This hex has already been submitted for processing'
                    continue

                # if '--' in hexnite:
                #     print hex.attributes
                #     raw_input()
                # if hex.status == 'Submitted for processing':
                #     print 'Hex ',hexnite,' band',band,'exposure',expnum,'has already been submitted for processing'
                #     #raw_input()
                #     continue

                if band == 'g':
                    if not expnum in hex.observed_g:
                        hex.observed_g.append(expnum)
                        hex.exposures.append(expnum)
                if band == 'r':
                    if not expnum in hex.observed_r:
                        hex.observed_r.append(expnum)
                        hex.exposures.append(expnum)
                if band == 'i':
                    if not expnum in hex.observed_i:
                        hex.observed_i.append(expnum)
                        hex.exposures.append(expnum)
                if band == 'z':
                    if not expnum in hex.observed_z:
                        hex.observed_z.append(expnum)
                        hex.exposures.append(expnum)

                self.backend.save(hex)
                self.backend.commit()

                print hex.attributes

                didwork = False
                if len(hex.observed_g) == hex.num_target_g:
                    if len(hex.observed_r) == hex.num_target_r:
                        if len(hex.observed_i) == hex.num_target_i:
                            if len(hex.observed_z) == hex.num_target_z:
                                print 'All exposures in strategy satisfied! '
                                #raw_input()
                                submissionPassed = True

                                for target, exps in zip([
                                        hex.num_target_g, hex.num_target_r,
                                        hex.num_target_i, hex.num_target_z
                                ], [
                                        hex.observed_g, hex.observed_r,
                                        hex.observed_i, hex.observed_z
                                ]):

                                    if target == 0: continue
                                    exposurestring = ''
                                    logstring = ''
                                    for ex in exps:
                                        exposurestring += ex + ' '
                                        logstring += ex + '_'

                                    print 'cd diffimg-proc; source DAGMaker.sh ' + exposurestring
                                    os.chdir("diffimg-proc")
                                    #out = os.popen('ls').read()
                                    out = os.popen('./DAGMaker.sh ' +
                                                   exposurestring).read()
                                    os.chdir("..")
                                    print out
                                    f = open(
                                        os.path.join(
                                            self.processingdir,
                                            logstring + hexnite + '.log'), 'w')
                                    f.write(out)
                                    f.close()
                                    tt = time.time()
                                    if not 'To submit this DAG do' in out:
                                        dt.sendEmailSubject(
                                            self.trigger_id,
                                            'Error in creating dag for desgw hex: '
                                            + out)
                                        submissionPassed = False
                                    else:
                                        for o in out.split('\n'):
                                            if 'file://' in o:
                                                dagfile = o.split('/')[-1]
                                                self.dagfile = os.path.join(
                                                    self.processingdir,
                                                    logstring + 'job.dag')
                                                os.system('cp diffimg-proc/' +
                                                          dagfile + ' ' +
                                                          self.dagfile)
                                        print self.dagfile

                                    print 'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; jobsub_submit_dag -G des --role=DESGW file://' + self.dagfile

                                    out = os.popen(
                                        'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; '
                                        'jobsub_submit_dag -G des --role=DESGW file://'
                                        + self.dagfile).read()
                                    print out
                                    if 'non-zero exit status' in out:
                                        dt.sendEmailSubject(
                                            self.trigger_id,
                                            'Error in submitting hex dag for processing: '
                                            + out)
                                        submissionPassed = False
                                    else:
                                        if doimmediateremove:
                                            for o in out.split('\n'):
                                                if 'Use job id' in o:
                                                    jobid = o.split()[3]
                                            out = os.popen(
                                                'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; '
                                                'jobsub_rm --jobid=' + jobid +
                                                ' --group=des --role=DESGW'
                                            ).read()
                                            print out
                                    ttt = time.time()
                                    #print 'TOTAL JOBSUB FOR A SINGLE DESGW IMAGE',ttt-tt

                                    #sys.exit()
                                #raw_input()
                                if submissionPassed:

                                    hex.status = 'Submitted for processing'
                                    hex.dagfile = self.dagfile
                                    self.backend.save(hex)
                                    self.backend.commit()

                                    for expn in hex.exposures:
                                        print expn, 'updated in database to Submitted For Processing'
                                        exp = self.backend.get(
                                            exposures, {'expnum': expn})
                                        exp.status = 'Submitted for processing'
                                        self.backend.save(exp)
                                        self.backend.commit()
                                    didwork = True
                                print 'didwork', didwork
                                print 'dagfile', self.dagfile
                                #raw_input()

                if not didwork:
                    print 'Could not find all images in strategy for this hex... Added hex', hexnite,' to database ' \
                        'and will continue waiting...'
                    #raw_input()

                #HERE YOU NEED TO ADD TO HEXSTRATEGYDICT DATABASE

                if time.time(
                ) - pptime > postprocessingtime:  #happens every 30 minutes or so...
                    pptime = time.time()
                    print '***** Firing post processing script *****'
                    #sys.exit()
                    self.submit_post_processing()
                #sys.exit()
                print 'Waiting 10s to check from mountain...'
                #sys.exit()
                time.sleep(10)  #looping over checking the mountain top

            # cfiles = os.listdir(os.path.join(trigger_path,trigger_id,'candidates'))
            # for f in cfiles:
            #     if f.split('.')[-1] == 'npz':
            #         cp.makeNewPage(f)

    def submit_post_processing(self):
        firedlist = open('./processing/firedlist.txt', 'r')
        fl = firedlist.readlines()
        firedlist.close()
        print fl
        fl = ['475914', '475915', '475916', '482859', '482860', '482861']
        expnumlist = ''
        for f in fl:
            expnumlist += f.strip() + ' '

        print 'FIRING TIMs CODE'

        gwpostdir = os.environ['GWPOST_DIR']
        print 'source ' + os.path.join(gwpostdir, 'diffimg_setup.sh') + '; \
                        python '                                +os.path.join(gwpostdir,'postproc.py')\
                         +' --expnums ' + expnumlist\
                         + ' --outputdir ' + os.path.join(self.trigger_path,self.trigger_id,'candidates')\
                         + ' --triggerid '+self.trigger_id+' --season 46 --ups True'
        # os.system('source ' + os.path.join(gwpostdir, 'diffimg_setup.sh') + '; \
        #                  python '+os.path.join(gwpostdir,'postproc.py')\
        #                  +' --expnums ' + expnumlist\
        #                  + ' --outputdir ' + os.path.join(trigger_path,trigger_id,'candidates')\
        #                  + ' --triggerid '+trigger_id+' --season 46 --ups True' )

        #pid = os.spawnlp(os.P_WAIT, "source", os.path.join(gwpostdir, 'diffimg_setup.sh'))
        args = ['ssh -t [email protected] "source '+ os.path.join(gwpostdir, 'mi_setup.sh')+'; '+
                        'yes | python '+os.path.join(gwpostdir,'postproc.py')\
                         +' --expnums ' + expnumlist\
                         + ' --outputdir ' + os.path.join(self.trigger_path,self.trigger_id,'candidates')\
                         + ' --triggerid '+self.trigger_id+' --season 46 --ups True"'
                ]
        print args

        #p = subprocess.Popen(args,stdout=PIPE, stderr=PIPE,shell=True)
        #print p.communicate()
        #p = subprocess.Popen(args,stdin=None, stdout=None, stderr=None, close_fds=True,shell=True)
        return

    def getDatetimeOfFirstJson(self, jsonstring):
        js = jsonstring.split('UTC')[1]  #-2015-12-27-3:2:00.json
        #date_object = dt.strptime(js, '-%Y-%m-%d-%H_%M_%S.json')
        date_object = dt.strptime(js, '-%Y-%m-%d-%H_%M_%S-test.json')
        print '***** Datetime of first observation UTC', date_object, '*****'
        return date_object

    def sortHexes(self):
        pass
Пример #8
0
class TwitchNotifier(commands.Cog):
    def __init__(self, bot):
        self.bot: 'PixlBot' = bot
        self.config = bot.config['TwitchNotifier']
        self.backend = FileBackend('db')
        self.backend.autocommit = True
        self.bot.logger.info("Twitch notifier plugin ready")
        self.uuids = []
        self.online_uuids = []
        self.sslcontext = ssl.SSLContext()
        self.sslcontext.load_cert_chain(self.config['cert_path'],
                                        self.config['key_path'])
        self._twitch_init_()

    def _twitch_init_(self):
        self.bot.logger.info("Registering with Twitch...")
        self.twitch = Twitch(self.config['id'], self.config['secret'])
        self.twitch.authenticate_app([])
        self.bot.logger.info(
            f"Registering webhook endpoint {self.config['myurl']} ...")
        self.hook = TwitchWebHook(self.config['myurl'],
                                  self.config['id'],
                                  self.config['port'],
                                  ssl_context=self.sslcontext)
        self.hook.authenticate(self.twitch)
        self.bot.logger.info("Clearing all hook subscriptions...")
        self.hook.unsubscribe_all(self.twitch)  # Clear all subs on startup
        self.hook.start()
        self._register_all()

    def _login_to_id(self, name: str) -> Optional[str]:
        """Returns the twitch ID for a given login name, or None if the name couldn't be resolved."""
        try:
            res: dict = self.twitch.get_users(logins=[name])
        except TwitchBackendException as e:
            self.bot.logger.error(f"Backend error fetching user! {e}")
            return None
        if len(res) == 0:
            return None
        else:
            return res['data'][0]['id']

    def _register_all(self):
        """Attempts to register stream_changed callbacks for all configured users."""
        self.bot.logger.info("Registering callbacks for all watched users..")
        users = self.backend.filter(TwitchWatchedUser,
                                    {'twitch_name': {
                                        "$exists": True
                                    }})
        if not users:
            self.bot.logger.info("No users to watch. No callbacks registered.")
        else:
            for u in users:
                self.bot.logger.info(f"Registering: {u['twitch_name']}")
                success, uuid = self.hook.subscribe_stream_changed(
                    u['twitch_id'], self._cb_stream_changed)
                if success and uuid:
                    self.uuids.append(uuid)
                    self.bot.logger.info(
                        f"{success}: registered subscription UUID: {uuid}")
                else:
                    self.bot.logger.error(
                        f"{success}: failed registering subscription: {uuid}")

    def _cb_stream_changed(self, uuid, data):
        """Callback for Twitch webhooks, fires on stream change event"""
        self.bot.logger.debug(f"Callback data for {uuid}: {data}")
        if data["type"] == "offline":
            if uuid in self.online_uuids:
                self.online_uuids.remove(
                    uuid
                )  # Stupid twitch sending the same damn webhook multiple times...
                return
            else:
                self.bot.logger.debug(
                    f"Ignoring duplicate offline callback for {uuid}")
                return
        elif data["type"] == "live":
            if uuid in self.online_uuids:
                self.bot.logger.debug(
                    f"Ignoring duplicate live callback for {uuid}")
                return
            else:
                self.online_uuids.append(uuid)
        else:
            self.bot.logger.error(
                f"Got a callback type we can't handle: {data['type']}")
            return

        if uuid not in self.uuids:
            self.bot.logger.error(
                f"Got a callback for a UUID we're not tracking: {uuid}, my UUIDs: {self.uuids}"
            )
            return

        try:
            item = self.backend.get(TwitchWatchedUser,
                                    {"twitch_id": data["user_id"]})
        except TwitchWatchedUser.DoesNotExist:
            self.bot.logger.error(
                f"Got a callback for a USER we're not tracking: {data['user_id']} -> {data['user_name']}"
            )
            return
        channel: discord.TextChannel = self.bot.get_channel(
            item['notify_channel'])

        width = 640
        height = 360
        url = data['thumbnail_url'].format(width=width, height=height)

        tu = self.twitch.get_users(data['user_id'])['data'][0]
        self.bot.logger.debug(tu)

        embed = discord.Embed(
            title=f"Now streaming {data['game_name']}",
            description=data['title'],
            color=discord.Color.green(),
        )
        embed.set_image(url=url)
        embed.set_thumbnail(url=tu["profile_image_url"])
        embed.set_author(name=item["twitch_name"],
                         url=f"https://twitch.tv/{data['user_name']}")
        embed.add_field(name="Watch live at",
                        value=f"https://twitch.tv/{data['user_name']}")
        self.bot.loop.create_task(
            channel.
            send(  # This isn't an async function, so enqueue it manually
                embed=embed))
        self.bot.logger.info(
            f"Successfully sent online notification for {data['user_id']}")

    @cog_ext.cog_subcommand(
        base="Twitchwatch",
        name="add_notification",
        description="Add a go live notification for Twitch",
        options=[twitch_name, notify_channel, notify_text],
        guild_ids=util.guilds)
    async def add_notification(self, ctx: SlashContext, twitch_name: str,
                               notify_channel: discord.TextChannel,
                               notify_text: str):
        twitch_id = self._login_to_id(twitch_name)
        try:
            self.backend.get(TwitchWatchedUser, {'twitch_name': twitch_name})
        except TwitchWatchedUser.DoesNotExist:
            pass
        except TwitchWatchedUser.MultipleDocumentsReturned:
            self.bot.logger.error(
                "Multiple users returned - database inconsistent???")
            return
        if not twitch_id:
            await ctx.send(embed=mkembed(
                'error',
                f"Unable to get the Twitch ID for the name {twitch_name}"))
            return
        await ctx.defer()  # This bit can take a minute.
        success, uuid = self.hook.subscribe_stream_changed(
            twitch_id, self._cb_stream_changed)
        if success and uuid:
            self.uuids.append(uuid)
            self.bot.logger.info(
                f"{success}: registered subscription UUID: {uuid}")
        else:
            self.bot.logger.error(
                f"{success}: failed registering subscription: {uuid}")
            await ctx.send("Bluh, couldn't register the webhook with twitch :("
                           )
            return
        item = TwitchWatchedUser({
            'twitch_name': twitch_name,
            'twitch_id': twitch_id,
            'discord_name': ctx.author.id,
            'notify_channel': notify_channel.id,
            'notify_text': notify_text,
            'uuid': str(uuid)
        })
        self.bot.logger.debug(f"DB object dump: {item.__dict__}")
        self.backend.save(item)
        await ctx.send(embed=mkembed("done",
                                     f"Notification added for {twitch_name}",
                                     channel=notify_channel.name))

    @cog_ext.cog_subcommand(
        base="Twitchwatch",
        name="del_notification",
        description="Remove a go live notification for Twitch",
        options=[twitch_name],
        guild_ids=util.guilds)
    async def del_notification(self, ctx: SlashContext, twitch_name: str):
        try:
            item = self.backend.get(TwitchWatchedUser,
                                    {'twitch_name': twitch_name})
        except TwitchWatchedUser.DoesNotExist:
            await ctx.send(embed=mkembed(
                "error", f"No notification exists for {twitch_name}"))
            return
        self.hook.unsubscribe(item['uuid'])
        self.bot.logger.info(f"Removing watch {item['uuid']}: {twitch_name}")
        self.backend.delete(item)
        if item['uuid'] in self.uuids:
            self.uuids.remove(item['uuid'])
        await ctx.send(
            embed=mkembed("done", f"Notification for {twitch_name} removed."))
Пример #9
0
class BlitzDB(Database):

    key = 'pk'
    _database_type = 'blitzdb'

    class Data(Document):
        pass

    class Cutout(Document):
        pass

    class Fingerprint(Document):
        pass

    class Similarity(Document):
        pass

    def _get_table(self, table_name):
        if table_name == 'data':
            return BlitzDB.Data
        elif table_name == 'cutout':
            return BlitzDB.Cutout
        elif table_name == 'fingerprint':
            return BlitzDB.Fingerprint
        elif table_name == 'similarity':
            return BlitzDB.Similarity
        else:
            log.error('BAD TABLE NAME {}'.format(table_name))

    def __init__(self, filename):
        self._filename = filename
        self._backend = FileBackend(self._filename)

    def save(self, table, data):

        # Convert to dict if not one already
        if not isinstance(data, dict):
            data = data.save()

        blitz_table = self._get_table(table)
        data.update({'pk': data['uuid']})
        save_id = self._backend.save(blitz_table(data))
        self._backend.commit()
        return save_id['pk']

    def find(self, table, key=None):
        blitz_table = self._get_table(table)
        factory = get_factory(table)

        if key is None:
            return [
                factory(dict(x), db=self)
                for x in self._backend.filter(blitz_table, {})
            ]
        elif isinstance(key, list):
            return [
                factory(dict(x), db=self) for x in self._backend.filter(
                    blitz_table, {'pk': {
                        '$in': key
                    }})
            ]
        else:
            return factory(dict(self._backend.get(blitz_table, {'pk': key})),
                           db=self)

    def count(self, table):
        blitz_table = self._get_table(table)
        return len(self._backend.filter(blitz_table, {}))

    def update(self, table, key, data):
        blitz_table = self._get_table(table)
        entry = self._backend.get(blitz_table, {'pk': key})
        for k, v in data.items():
            setattr(entry, k, v)
        entry.save()
        self._backend.commit()

    def close(self):
        pass

    def delete_database(self):
        shutil.rmtree(self._filename)
Пример #10
0
class Profiles(object):
    def __init__(self, bot):
        self.bot = bot

        self.cfg = PluginConfig(self)
        self.db = FileBackend(self.cfg.get('main_db'))

        mtt = MessageRetargeter(bot)
        self.msg = mtt.msg

        web = Flask(__name__, template_folder=tmpl_dir)
        mako = MakoTemplates()
        mako.init_app(web)

        # Add routes here
        web.add_url_rule('/edit_web/<args>', 'edit_web', self.edit_web, methods=['GET', 'POST'])

        _thread.start_new_thread(web.run, (), {'host': '0.0.0.0'})

    @command
    def learn(self, mask, target, args):
        """
        Stores information allowing for later retrieval. Names are downcased for sanity.


        Usage:
            %%learn <name> <information>...
        """
        name = args['<name>'].lower()
        info = ' '.join(args['<information>'])

        try:
            profile = self.db.get(Profile, {'name': name})
        except Profile.DoesNotExist:
            profile = Profile(
                {
                    'name': name,
                    'owner': mask.nick.lower(),
                    'lines': [info],
                    'random': False,
                    'public': False
                }
            )
            profile.save(self.db)
            self.db.commit()
            self.msg(mask, target, 'Your data "%s" has been stored.' % name)
            return

        except Profile.MultipleDocumentsReturned:
            self.msg(mask, target, "Found more than one %s. This is bad! Please notify the bot owner." % name)
            return

        if is_allowed_to(Action.edit, mask.nick, profile):
            lines_to_append = profile.lines
            lines_to_append.append(info)
            profile.save(self.db)
            self.db.commit()
            self.msg(mask, target, 'Your data "%s" has been updated.' % name)
            return
        else:
            self.msg(mask, target, 'You are not authorized to edit "%s". Ask %s instead.'
                     % (name, profile.owner))
            return

    @command
    def query(self, mask, target, args):
        """
        Retrieve the information associated with <name>. If the item is marked random, then one random item will be
        returned.

        Usage:
            %%query <name>
            ?? <name>
        """
        name = args['<name>'].lower()

        try:
            profile = self.db.get(Profile, {'name': name})
        except Profile.DoesNotExist:
            self.msg(mask, target, 'I cannot find "%s" in the records.' % name)
            return

        if profile.random:
            self.msg(mask, target, get_flags(profile) + random.choice(profile.lines))
        else:
            for line in profile.lines:
                self.msg(mask, target, get_flags(profile) + line)
                if len(profile.lines) >= int(self.cfg.get('throttle_max')):
                    sleep(int(self.cfg.get('throttle_time')))

    @command
    def forget(self, mask, target, args):
        """
        Delete <name> from the records. Only the person who created the item can remove it.

        Usage:
            %%forget <name>
        """
        name = args['<name>'].lower()
        try:
            profile = self.db.get(Profile, {'name': name})
        except Profile.DoesNotExist:
            self.msg(mask, target, 'I cannot find "%s" in the records.' % name)
            return

        if is_allowed_to(Action.delete, mask.nick, profile):
            self.db.delete(profile)
            self.db.commit()
            self.msg(mask, target, "%s has been deleted." % name)
        else:
            self.msg(mask, target, 'You are not authorized to delete "%s". Ask %s instead.'
                     % (name, profile.owner))

    @command(permission='admin', show_in_help_list=False)
    def rmf(self, mask, target, args):
        """
        Delete <name> from the records without checking permissions.

        Usage:
            %%rmf <name>
        """
        name = args['<name>'].lower()
        try:
            profile = self.db.get(Profile, {'name': name})
        except Profile.DoesNotExist:
            self.msg(mask, target, 'I cannot find "%s" in the records.' % name)
            return

        self.db.delete(profile)
        self.db.commit()
        self.msg(mask, target, "%s has been deleted." % name)

    @command(permission='admin', show_in_help_list=False)
    def chown(self, mask, target, args):
        """
        Change the owner of <name> to <newowner>.

        Usage:
            %%chown <name> <newowner>
        """
        name = args['<name>'].lower()
        newowner = args['<newowner>'].lower()
        try:
            profile = self.db.get(Profile, {'name': name})
        except Profile.DoesNotExist:
            self.msg(mask, target, 'I cannot find "%s" in the records.' % name)
            return
        profile.owner = newowner
        self.db.save(profile)
        self.db.commit()
        self.msg(mask, target, "%s is now owned by %s." % (name, newowner))

    @command
    def toggle_public(self, mask, target, args):
        """
        Changes whether <name> is publicly editable or not

        Usage:
            %%toggle_public <name>
        """
        profile = args['<name>'].lower()
        try:
            profile = self.db.get(Profile, {'name': profile})
        except Profile.DoesNotExist:
            self.msg(mask, target, 'I cannot find "%s" in the records.' % profile)
            return

        if is_allowed_to(Action.edit, mask.nick, profile):
            if profile.public:
                profile.public = False
                self.msg(mask, target, '"%s" is no longer publicly editable.' % profile)
            else:
                profile.public = True
                self.msg(mask, target, '"%s" is now publicly editable.' % profile)
            self.db.save(profile)
            self.db.commit()
            return
        else:
            self.msg(mask, target, 'You are not authorized to edit "%s". Ask %s instead.'
                     % (profile, profile.owner))
            return

    @command
    def toggle_random(self, mask, target, args):
        """
        Toggle the randomness of an item, so that it shows a single random line instead of all lines when queried.

        Usage:
            %%toggle_random <name>
        """
        name = args['<name>'].lower()
        try:
            profile = self.db.get(Profile, {'name': name})
        except Profile.DoesNotExist:
            self.msg(mask, target, 'I cannot find "%s" in the records.' % name)
            return

        if is_allowed_to(Action.edit, mask.nick, profile):
            profile.random = not profile.random
            self.msg(mask, target, 'Random mode for %s is set to: %s' % (profile.name, profile.random))
            profile.save(self.db)
            self.db.commit()
        else:
            self.msg(mask, target, 'You are not authorized to edit "%s". Ask %s instead.'
                     % (name, profile.owner))
            irc3.base.logging.log(irc3.base.logging.WARN,
                                  "%s tried to edit %s, but can't since it's owned by %s" %
                                  (mask.nick, profile.name, profile.owner)
                                  )

    @event("(@(?P<tags>\S+) )?:(?P<mask>\S+) PRIVMSG (?P<target>\S+) :\?\? (?P<data>.*)")
    def easy_query(self, mask, target, data):
        self.bot.get_plugin(Commands).on_command(cmd='query', mask=mask, target=target, data=data)

    ####
    # All web stuff below this point
    #

    @command
    def edit(self, mask, target, args):
        """
        Sends you a webpage link to edit <name>. Great for longer profiles. Make sure to keep the URL you are given
        secure, as with it, anyone can edit your profiles.

        Usage:
            %%edit <name>
        """
        # TODO: Clear any existing sessions the user has

        data = {
            'id': str(uuid.uuid4()),
            'name': mask.nick,
            'profile': args['<name>']
        }

        name = args['<name>'].lower()

        try:
            profile = self.db.get(Profile, {'name': name})
        except Profile.DoesNotExist:
            self.msg(mask, target, 'I cannot find "%s" in the records.' % name)
            return

        if is_allowed_to(Action.fulledit, mask.nick, profile):
            newses = Session(data)
            self.db.save(newses)
            self.db.commit()
            self.bot.privmsg(mask.nick,
                             "An editor has been set up for you at http://skaianet.tkware.us:5000/edit_web/%s" % str(
                                 data['id']))
            self.bot.privmsg(mask.nick,
                             "Be very careful not to expose this address - with it, anyone can edit your stuff")
        else:
            self.msg(mask, target, 'You are not authorized to webedit "%s". Ask %s instead.'
                     % (name, profile.owner))

    def edit_web(self, args):
        # Web endpoint: /edit_web/<args>

        if request.method == 'GET':
            # Does the session exist?
            try:
                edit_session = self.db.get(Session, {'id': args})
            except Session.DoesNotExist:
                return render_template('youfail.html',
                                       bot=self.bot,
                                       failreason='Invalid Session',
                                       userfail=True)
            # Does the profile exist?
            name = edit_session.profile
            try:
                profile = self.db.get(Profile, {'name': name.lower()})
            except Profile.DoesNotExist:
                return render_template('youfail.html',
                                       bot=self.bot,
                                       failreason='I cannot find "%s" in the records.' % name
                                       )
            # Kick off to the edit page!
            return render_template('edit.html',
                                   bot=self.bot,
                                   profile=profile,
                                   username=edit_session.name,
                                   sessionid=edit_session.id
                                   )
        elif request.method == 'POST':
            # We have to look up the session ID one more time. Something could have happened to the profile
            # since we created the session.
            try:
                edit_session = self.db.get(Session, {'id': request.form['ID']})
            except Session.DoesNotExist:
                return render_template('youfail.html',
                                       bot=self.bot,
                                       failreason='Invalid Session',
                                       userfail=True)
            name = request.form['profile']
            try:
                profile = self.db.get(Profile, {'name': request.form['profile']})
            except Profile.DoesNotExist:
                return render_template('youfail.html',
                                       bot=self.bot,
                                       failreason='I cannot find "%s" in the records.' % name,
                                       userfail=True
                                       )

            # Now with the profile in hand, blank the lines field and rebuild it from the form.
            # Here we grab all numeric items from the submission, sort it, and one by one refill the DB object.
            lines = [item for item in request.form if item.isdigit()]
            lines.sort()
            profile.lines = []
            for item in lines:
                profile.lines.append(request.form[item])
            self.db.save(profile)
            self.db.delete(edit_session)
            self.db.commit()
            return render_template('done.html',
                                   bot=self.bot,
                                   profile=profile.name
                                   )
Пример #11
0
class Responder(commands.Cog):
    def __init__(self, bot):
        self.bot = bot
        self.backend = FileBackend('db')
        self.backend.autocommit = True
        bot.logger.info("Responder plugin ready")

    def _find_one(self, name: str) -> Union[ResponseCommand, None]:
        """Searches for a response in the DB, returning it if found, or None if it doesn't exist or there are multiples.
        This exists to tie up the Blitzdb boilerplate in one place."""
        try:
            comm = self.backend.get(ResponseCommand, {'command': name})
        except ResponseCommand.DoesNotExist:
            return None
        except ResponseCommand.MultipleDocumentsReturned:
            self.bot.logger.error(f"_find_one discarding multiple results returned for '{name}'")
            return None
        else:
            return comm

    def _reply_allowed(self, comm: ResponseCommand, message: discord.Message) -> bool:
        """Determine whether a message can be replied to based on its attributes
        In general, if a user or channel restriction is set on a command, it can only be used when called in the
        listed channel or by the listed user.
        """
        self.bot.logger.debug(f"Restriction dump: {comm.get('restrictions')}")
        if not comm.get("restrictions"):
            # No restrictions on this command, we can respond without doing anything else.
            return True
        else:
            if comm["restrictions"].get("channels"):
                channels = comm["restrictions"]["channels"]
                if message.channel.id in channels:
                    return True
                else:
                    return False
            elif comm["restrictions"].get("users"):
                users = comm["restrictions"]["users"]
                if message.author.id in users:
                    return True
                else:
                    return False
            else:
                return True

    @cog_ext.cog_subcommand(base="Autoresponder", name="addresponse",
                            description="Adds an automatic response to certain text",
                            options=[respond_to, response], guild_ids=util.guilds)
    async def addresponse(self, ctx: SlashContext, respond_to: str, response: str):
        """Adds an automatic response to (name) as (response)
        The first word (name) is the text that will be replied to. Everything else is what it will be replied to with.
        If you want to reply to an entire phrase, enclose name in quotes."""
        if self._find_one(respond_to):
            await ctx.send(embed=mkembed('error', f"'{respond_to}' already exists."))
            return
        else:
            comm = ResponseCommand(
                {'command': respond_to, 'reply': response, 'creator_str': str(ctx.author), 'creator_id': ctx.author.id}
            )
            self.backend.save(comm)
            self.bot.logger.info(f"'{response}' was added by {ctx.author.display_name}")
            await ctx.send(embed=mkembed('done', "Autoresponse saved.", reply_to=respond_to, reply_with=response))

    @cog_ext.cog_subcommand(base="Autoresponder", name="delresponse",
                            description="Removes an automatic reponse from certain text",
                            options=[respond_to], guild_ids=util.guilds)
    async def delresponse(self, ctx: SlashContext, respond_to: str):
        """Removes an autoresponse.
        Only the initial creator of a response can remove it."""
        comm = self._find_one(respond_to)
        if not comm:
            await ctx.send(embed=mkembed('error', f"{respond_to} is not defined."))
            return
        elif not ctx.author.id == comm['creator_id']:
            await ctx.send(
                embed=mkembed('error', f"You are not the creator of {respond_to}. Ask {comm['creator_str']}"))
        else:
            self.backend.delete(comm)
            self.bot.logger.info(f"'{respond_to}' was deleted by {ctx.author.display_name}")
            await ctx.send(embed=mkembed('info', f"{respond_to} has been removed."))

    # @commands.command()
    # @cog_ext.cog_subcommand(base="Autoresponder", name="limit_user",
    #                         description="Limit a response to triggering on a certain user. Leave users blank to remove.",
    #                         options=[respond_to, restrict_user],
    #                         guild_ids=util.guilds)
    async def limitchannel(self, ctx: SlashContext, respond_to: str, **kwargs):
        comm = self._find_one(respond_to)
        if not comm:
            await ctx.send(embed=mkembed('error', f"'{respond_to}' does not exist."))
            return
        if not ctx.author.id == comm['creator_id']:
            await ctx.send(
                embed=mkembed('error', f"You are not the creator of '{respond_to}'. Ask {comm['creator_str']}"))
            return
        if len(kwargs) == 0:
            comm["restrictions"] = {}
            self.backend.save(comm)
            await ctx.send(embed=mkembed('done', f"All restrictions removed from {respond_to}"))
            return
        if kwargs['restrict_user']:
            if not comm.get("restrictions"):
                comm["restrictions"] = {}
            elif not comm["restrictions"].get("users"):
                comm["restrictions"]["users"] = []
            comm["restrictions"]["users"] = list(set(
                comm["restrictions"]["users"] + [u.id for u in restrict_user]
            ))
            self.backend.save(comm)
            display_users = [self.bot.get_user(u).display_name for u in comm["restrictions"]["users"]]
            await ctx.send(
                embed=mkembed('done', 'User restriction updated:', command=comm['command'], users=display_users)
            )
        if kwargs['restrict_channel']:
            if not comm.get("restrictions"):
                comm["restrictions"] = {}
            if not comm["restrictions"].get("channels"):
                comm["restrictions"]["channels"] = []
            comm["restrictions"]["channels"] = list(set(
                comm["restrictions"]["channels"] + [c.id for c in ctx.message.channel_mentions]
            ))
            display_channels = [self.bot.get_channel(c).name for c in comm["restrictions"]["channels"]]
            self.backend.save(comm)
            await ctx.send(
                embed=mkembed('done', 'Channel restriction updated:',
                              Command=comm['command'],
                              Channels=display_channels
                              )
            )

    @commands.command()
    async def responserestrictions(self, ctx: context, name: str):
        """Show the restriction list for a given command"""
        comm = self._find_one(name)
        if not comm:
            await ctx.send(embed=mkembed('error', f"{name} does not exist."))
            return
        await ctx.send(
            embed=mkembed('info', f"Information for `{name}`",
                          Reply=comm['reply'],
                          Restrictions=comm.get('restrictions', 'None'),
                          Creator=comm['creator_str']
                          )
        )

    @commands.Cog.listener()
    async def on_message(self, message: discord.message):
        comm = self._find_one(message.content)
        if comm and self._reply_allowed(comm, message):
            await message.channel.send(comm['reply'])
Пример #12
0
def create_app(configfile=None):
    app = Flask(__name__)
    AppConfig(app, configfile)  # Flask-Appconfig is not necessary, but
    # highly recommend =)
    # https://github.com/mbr/flask-appconfig
    Bootstrap(app)

    login_manager = LoginManager()
    login_manager.init_app(app)
    login_manager.login_view = 'login'

    #NoSQL Backend
    backend = FileBackend("/tmp/wakeonlan.db")
    backend.create_index(Device, fields={'id': 1}, unique=True)

    #TEST Devices
    alldevices = backend.filter(Device, {})
    if len(alldevices) == 0:
        try:
            pc1 = Device({
                "id": "001122334411",
                "name": "PC 1",
                "mac": "00:11:22:33:44:11",
                "ip": "192.168.222.111",
                'status': ''
            })
            backend.save(pc1)
            pc2 = Device({
                "id": "001122334422",
                "name": "PC 2",
                "mac": "00:11:22:33:44:22",
                "ip": "192.168.222.112",
                'status': ''
            })
            backend.save(pc2)
            pc3 = Device({
                "id": "001122334433",
                "name": "Router",
                "mac": "00:11:22:33:44:33",
                "ip": "192.168.222.1",
                'status': ''
            })
            backend.save(pc3)
            backend.commit()
        except:
            backend.revert()
            pass

    # in a real app, these should be configured through Flask-Appconfig
    app.config['SECRET_KEY'] = 'devkey'

    # app.config['RECAPTCHA_PUBLIC_KEY'] = \
    #     '6Lfol9cSAAAAADAkodaYl9wvQCwBMr3qGR_PPHcw'

    def getDeviceById(id):
        device = None
        try:
            device = backend.get(Device, {'id': id})
        except:
            pass

        return device

    def pingDeviceById(id):
        #Get Device
        device = backend.get(Device, {'id': id})

        if device:
            #Get Device's IP
            ip = device['ip']
            result = pingDeviceByIp(ip)

            #Update Status   UP/Down/''
            if result == 0:
                device['status'] = 'UP'
            else:
                device['status'] = 'DOWN'

            backend.save(device)
            return result

        return None

    def wolDeviceById(id):
        #Get Device
        device = backend.get(Device, {'id': id})

        if device:
            #WoL for Device MAC
            mac = device['mac']
            wolDeviceByMac(mac)

        return None

    @login_manager.user_loader
    def user_loader(user_id):
        """Given *user_id*, return the associated User object.
        :param unicode user_id: user_id (email) user to retrieve
        """
        user_entry = User.getById(user_id)
        if user_entry is not None:
            user = User(user_entry[0], user_entry[1])
            return user
        else:
            return None

    @app.route('/', methods=('GET', 'POST'))
    @login_required
    def index():
        form = ExampleForm()
        form.validate_on_submit()  # to get error messages to the browser
        # flash('critical message', 'critical')
        # flash('error message', 'error')
        # flash('warning message', 'warning')
        # flash('info message', 'info')
        # flash('debug message', 'debug')
        # flash('different message', 'different')
        # flash('uncategorized message')
        alldevices = None
        alldevices = backend.filter(Device, {}).sort('name')

        #app.logger.info('Devices: %s' % (len(alldevices) ) )

        return render_template('index.html', form=form, devices=alldevices)

    @app.route('/login', methods=('GET', 'POST'))
    def login():
        if request.method == 'GET':
            form = LoginForm()
            form.validate_on_submit()  # to get error messages to the browser
            return render_template('login.html', form=form)

        username = request.form['username']
        password = request.form['password']

        user_entry = User.get(username, password)
        if user_entry is None:
            flash('Username or Passord is invalid', 'error')
            return redirect(url_for('login'))

        user = User(user_entry[0], user_entry[1])
        login_user(user, remember=True)
        return redirect(request.args.get('next') or url_for('index'))

    @app.route("/logout", methods=["GET"])
    @login_required
    def logout():
        """Logout the current user."""
        user = current_user
        user.authenticated = False
        logout_user()
        return redirect(url_for('login'))

    @app.route('/addDevice', methods=('GET', 'POST'))
    @login_required
    def addDevice():
        if request.method == 'GET':
            form = AddDeviceForm()
            form.validate_on_submit()  # to get error messages to the browser
            return render_template('add_device.html', form=form)

        name = request.form['name']
        mac = request.form['mac']
        ip = request.form['ip']
        id = mac.replace(':', '')

        try:
            newDevice = Device({
                "id": id,
                "name": name,
                "mac": mac,
                "ip": ip,
                'status': ''
            })
            backend.save(newDevice)
            backend.commit()
        except:
            flash('Error creating new Device', 'error')
            pass

        return redirect(url_for('index'))

    @app.route('/editListDevice', methods=('GET', 'POST'))
    @login_required
    def editListDevice():
        alldevices = None
        alldevices = backend.filter(Device, {}).sort('name')

        return render_template('list_device.html', devices=alldevices)

    @app.route('/pingDevice/<deviceId>', methods=('GET', 'POST'))
    @login_required
    def pingDevice(deviceId):
        app.logger.info('pingDevice: %s' % (deviceId))
        device = getDeviceById(deviceId)
        result = pingDeviceById(deviceId)

        app.logger.info('pingDevice: %s' % (result))

        if result is None:
            flash('Ping - Error on device %s' % (device['name']), 'error')
        elif result == 0:
            flash('Device %s is UP' % (device['name']), 'info')
        else:
            flash('Device %s is DOWN' % (device['name']), 'error')

        return redirect(url_for('index'))

    @app.route('/wolDevice/<deviceId>', methods=('GET', 'POST'))
    @login_required
    def wolDevice(deviceId):
        app.logger.info('wolDevice: %s' % (deviceId))
        device = getDeviceById(deviceId)
        result = wolDeviceById(deviceId)

        if device:
            flash('WoL sent to %s' % (device['name']), 'info')
        else:
            flash('WoL error', 'error')

        return redirect(url_for('index'))

    @app.route('/deleteDevice/<deviceId>', methods=('GET', 'POST'))
    @login_required
    def deleteDevice(deviceId):
        app.logger.info('wolDevice: %s' % (deviceId))
        device = getDeviceById(deviceId)

        try:
            backend.delete(device)
            backend.commit()
            flash('%s Deleted' % (device['name']), 'info')
        except:
            flash('Delete error', 'error')
            pass

        return redirect(url_for('editListDevice'))

    return app
Пример #13
0
def create_app(configfile=None):
    app = Flask(__name__)
    AppConfig(app, configfile)  # Flask-Appconfig is not necessary, but
                                # highly recommend =)
                                # https://github.com/mbr/flask-appconfig
    Bootstrap(app)

    login_manager = LoginManager()
    login_manager.init_app(app)
    login_manager.login_view='login'


    #NoSQL Backend
    backend = FileBackend("/tmp/wakeonlan.db")
    backend.create_index(Device, fields={'id':1}, unique=True)
    
    #TEST Devices
    alldevices = backend.filter(Device, {})
    if len(alldevices) == 0 :
        try:
            pc1 = Device({"id" : "001122334411", "name" : "PC 1", "mac" : "00:11:22:33:44:11", "ip":"192.168.222.111", 'status' : ''})
            backend.save(pc1)
            pc2 = Device({"id" : "001122334422","name" : "PC 2", "mac" : "00:11:22:33:44:22", "ip":"192.168.222.112", 'status' : ''})
            backend.save(pc2)
            pc3 = Device({"id" : "001122334433","name" : "Router", "mac" : "00:11:22:33:44:33", "ip":"192.168.222.1", 'status' : ''})
            backend.save(pc3)
            backend.commit()
        except: 
            backend.revert()
            pass

    # in a real app, these should be configured through Flask-Appconfig
    app.config['SECRET_KEY'] = 'devkey'
    # app.config['RECAPTCHA_PUBLIC_KEY'] = \
    #     '6Lfol9cSAAAAADAkodaYl9wvQCwBMr3qGR_PPHcw'

    def getDeviceById(id):
        device = None
        try:
            device = backend.get(Device, {'id':id})
        except:
            pass

        return device

    def pingDeviceById(id):
        #Get Device
        device = backend.get(Device, {'id':id})

        if device:
            #Get Device's IP
            ip = device['ip']
            result = pingDeviceByIp(ip)

            #Update Status   UP/Down/''
            if result==0:
                device['status'] = 'UP'
            else:
                device['status'] = 'DOWN'

            backend.save(device)
            return result

        return None

    def wolDeviceById(id):
        #Get Device
        device = backend.get(Device, {'id':id})

        if device:
            #WoL for Device MAC
            mac = device['mac']
            wolDeviceByMac(mac)

        return None

    @login_manager.user_loader
    def user_loader(user_id):
        """Given *user_id*, return the associated User object.
        :param unicode user_id: user_id (email) user to retrieve
        """
        user_entry = User.getById(user_id)
        if user_entry is not None:
            user = User(user_entry[0], user_entry[1])
            return user
        else:
            return None

    @app.route('/', methods=('GET', 'POST'))
    @login_required
    def index():
        form = ExampleForm()
        form.validate_on_submit()  # to get error messages to the browser
        # flash('critical message', 'critical')
        # flash('error message', 'error')
        # flash('warning message', 'warning')
        # flash('info message', 'info')
        # flash('debug message', 'debug')
        # flash('different message', 'different')
        # flash('uncategorized message')
        alldevices = None
        alldevices = backend.filter(Device, {}).sort('name')

        #app.logger.info('Devices: %s' % (len(alldevices) ) )

        return render_template('index.html', form=form, devices = alldevices)

    @app.route('/login', methods=('GET', 'POST'))
    def login():
        if request.method == 'GET':
            form = LoginForm()
            form.validate_on_submit()  # to get error messages to the browser
            return render_template('login.html', form=form)

        username = request.form['username']
        password = request.form['password']

        user_entry = User.get(username, password)
        if user_entry is None:
            flash('Username or Passord is invalid', 'error')
            return redirect(url_for('login'))

        user = User(user_entry[0], user_entry[1])
        login_user(user, remember=True)
        return redirect(request.args.get('next') or url_for('index'))


    @app.route("/logout", methods=["GET"])
    @login_required
    def logout():
        """Logout the current user."""
        user = current_user
        user.authenticated = False
        logout_user()
        return redirect(url_for('login'))



    @app.route('/addDevice', methods=('GET', 'POST'))
    @login_required
    def addDevice():
        if request.method == 'GET':
            form = AddDeviceForm()
            form.validate_on_submit()  # to get error messages to the browser
            return render_template('add_device.html', form=form)

        name = request.form['name']
        mac = request.form['mac']
        ip = request.form['ip']
        id = mac.replace(':','')

        try:
            newDevice = Device({"id" : id, "name" : name, "mac" : mac, "ip":ip, 'status' : ''})
            backend.save(newDevice)
            backend.commit()
        except:
            flash('Error creating new Device', 'error')
            pass

        return redirect(url_for('index'))


    @app.route('/editListDevice', methods=('GET', 'POST'))
    @login_required
    def editListDevice():
        alldevices = None
        alldevices = backend.filter(Device, {}).sort('name')

        return render_template('list_device.html', devices = alldevices)



    @app.route('/pingDevice/<deviceId>', methods=('GET', 'POST'))
    @login_required
    def pingDevice(deviceId):
        app.logger.info('pingDevice: %s' % (deviceId ) )
        device = getDeviceById(deviceId)
        result = pingDeviceById(deviceId)

        app.logger.info('pingDevice: %s' % (result ) )

        if result is None:
            flash('Ping - Error on device %s' % (device['name']), 'error')
        elif result == 0:
            flash('Device %s is UP' % (device['name']), 'info')
        else:
            flash('Device %s is DOWN' % (device['name']), 'error')

        return redirect(url_for('index'))


    @app.route('/wolDevice/<deviceId>', methods=('GET', 'POST'))
    @login_required
    def wolDevice(deviceId):
        app.logger.info('wolDevice: %s' % (deviceId ) )
        device = getDeviceById(deviceId)
        result = wolDeviceById(deviceId)

        if device:
            flash('WoL sent to %s' % (device['name']), 'info')
        else:
            flash('WoL error', 'error')

        return redirect(url_for('index'))


    @app.route('/deleteDevice/<deviceId>', methods=('GET', 'POST'))
    @login_required
    def deleteDevice(deviceId):
        app.logger.info('wolDevice: %s' % (deviceId ) )
        device = getDeviceById(deviceId)

        try:
            backend.delete(device)
            backend.commit()
            flash('%s Deleted' % (device['name']), 'info')
        except:
            flash('Delete error', 'error')
            pass

        return redirect(url_for('editListDevice'))


    return app
Пример #14
0
class Responder(commands.Cog):
    def __init__(self, bot):
        self.bot = bot
        self.backend = FileBackend('./responder-db')
        self.backend.autocommit = True
        bot.logger.info("Responder plugin ready")

    def _find_one(self, name: str) -> Union[ResponseCommand, None]:
        """Searches for a response in the DB, returning it if found, or None if it doesn't exist or there are multiples.
        This exists to tie up the Blitzdb boilerplate in one place."""
        try:
            comm = self.backend.get(ResponseCommand, {'command': name})
        except ResponseCommand.DoesNotExist:
            return None
        except ResponseCommand.MultipleDocumentsReturned:
            self.bot.logger.error(
                f"_find_one discarding multiple results returned for '{name}'")
            return None
        else:
            return comm

    def _reply_allowed(self, comm: ResponseCommand,
                       message: discord.Message) -> bool:
        """Determine whether a message can be replied to based on its attributes
        In general, if a user or channel restriction is set on a command, it can only be used when called in the
        listed channel or by the listed user.
        """
        self.bot.logger.debug(f"Restriction dump: {comm.get('restrictions')}")
        if not comm.get("restrictions"):
            # No restrictions on this command, we can respond without doing anything else.
            return True
        else:
            if comm["restrictions"].get("channels"):
                channels = comm["restrictions"]["channels"]
                if message.channel.id in channels:
                    return True
                else:
                    return False
            elif comm["restrictions"].get("users"):
                users = comm["restrictions"]["users"]
                if message.author.id in users:
                    return True
                else:
                    return False
            else:
                return True

    @commands.command()
    async def addresponse(self, ctx: context, name: str, *responsen):
        """Adds an automatic response to (name) as (response)
        The first word (name) is the text that will be replied to. Everything else is what it will be replied to with.
        If you want to reply to an entire phrase, enclose name in quotes."""
        arg2 = " ".join(responsen)
        if self._find_one(name):
            await ctx.send(embed=mkembed('error', f"'{name}' already exists."))
            return
        else:
            comm = ResponseCommand({
                'command': name,
                'reply': arg2,
                'creator_str': str(ctx.author),
                'creator_id': ctx.author.id
            })
            self.backend.save(comm)
            self.bot.logger.info(
                f"'{name}' was added by {ctx.author.display_name}")
            await ctx.send(embed=mkembed('done', f"Saved: {name} => {arg2}"))

    @commands.command()
    async def delresponse(self, ctx: context, name: str):
        """Removes an autoresponse.
        Only the initial creator of a response can remove it."""
        comm = self._find_one(name)
        if not comm:
            ctx.send(embed=mkembed('error', f"{name} is not defined."))
            return
        elif not ctx.author.id == comm['creator_id']:
            await ctx.send(embed=mkembed(
                'error',
                f"You are not the creator of {name}. Ask {comm['creator_str']}"
            ))
        else:
            self.backend.delete(comm)
            self.bot.logger.info(
                f"'{name}' was deleted by {ctx.author.display_name}")
            await ctx.send(embed=mkembed('error', f"{name} has been removed."))

    @commands.command()
    async def responselimit(self, ctx: context, name: str, *tags: str):
        """Adds a restriction mode on a response so it only triggers in certain circumstances.
        Tags is a space-separated list of users and channels that this responder will apply to. These must be
        discord tags or pings, i.e. discord must show them as links, not just text.
        If the word NONE is used, all restrictions are removed.
        """
        comm = self._find_one(name)
        if not comm:
            await ctx.send(embed=mkembed('error', f"{name} does not exist."))
            return
        if not ctx.author.id == comm['creator_id']:
            await ctx.send(embed=mkembed(
                'error',
                f"You are not the creator of {name}. Ask {comm['creator_str']}"
            ))
            return
        if tags[0] == "NONE":
            comm["restrictions"] = {}
            self.backend.save(comm)
            await ctx.send(
                embed=mkembed('done', f"All restrictions removed from {name}"))
            return
        if ctx.message.mentions:
            if not comm.get("restrictions"):
                comm["restrictions"] = {}
            elif not comm["restrictions"].get("users"):
                comm["restrictions"]["users"] = []
            comm["restrictions"]["users"] = list(
                set(comm["restrictions"]["users"] +
                    [u.id for u in ctx.message.mentions]))
            self.backend.save(comm)
            display_users = [
                self.bot.get_user(u).display_name
                for u in comm["restrictions"]["users"]
            ]
            await ctx.send(embed=mkembed('done',
                                         'User restriction updated:',
                                         command=comm['command'],
                                         users=display_users))
        if ctx.message.channel_mentions:
            if not comm.get("restrictions"):
                comm["restrictions"] = {}
            if not comm["restrictions"].get("channels"):
                comm["restrictions"]["channels"] = []
            comm["restrictions"]["channels"] = list(
                set(comm["restrictions"]["channels"] +
                    [c.id for c in ctx.message.channel_mentions]))
            display_channels = [
                self.bot.get_channel(c).name
                for c in comm["restrictions"]["channels"]
            ]
            self.backend.save(comm)
            await ctx.send(embed=mkembed('done',
                                         'Channel restriction updated:',
                                         Command=comm['command'],
                                         Channels=display_channels))

    @commands.command()
    async def responserestrictions(self, ctx: context, name: str):
        """Show the restriction list for a given command"""
        comm = self._find_one(name)
        if not comm:
            await ctx.send(embed=mkembed('error', f"{name} does not exist."))
            return
        await ctx.send(
            embed=mkembed('info',
                          f"Information for `{name}`",
                          Reply=comm['reply'],
                          Restrictions=comm.get('restrictions', 'None'),
                          Creator=comm['creator_str']))

    @commands.Cog.listener()
    async def on_message(self, message: discord.message):
        comm = self._find_one(message.content)
        if comm and self._reply_allowed(comm, message):
            await message.channel.send(comm['reply'])
Пример #15
0
class eventmanager:
    def __init__(self, trigger_id, jsonfilelist, triggerdir, datadir, real, trigger_path):

        #os.system('kinit -k -t /var/keytab/desgw.keytab desgw/des/[email protected]')
        tstart = time.time()

        if real:
            self.backend = FileBackend("./realdb")
        else:
            self.backend = FileBackend("./testdb")

        try:
            self.thisevent = self.backend.get(Trigger, {'id': trigger_id})
            print 'Found this event in desgw database...'
        except Trigger.DoesNotExist:
            self.thisevent = Trigger({
                'id':trigger_id,
                'jsonfilelist':jsonfilelist,
                'triggerpath':triggerdir,
                'mapspath':datadir,
                'jobids':[
                    (0,'jsonfile_corresponding_to_jobid.json'),
                ],
                'postprocint': 0
            })
            print 'Database entry created!'


        self.trigger_id = trigger_id
        self.trigger_path = trigger_path

        self.backend.save(self.thisevent)
        self.backend.commit()

        with open(os.path.join(triggerdir,"strategy.yaml"), "r") as f:
            self.config = yaml.safe_load(f);
        self.filterobslist = np.array(self.config['exposure_filter'],dtype='str')
        self.strategydict = {}

        for f in np.unique(self.filterobslist):
            self.strategydict[f] = len(self.filterobslist[self.filterobslist == f])

        self.connection = ea.connect(DATABASE)
        self.cursor = self.connection.cursor()

        self.jsonfilelist = jsonfilelist

        print self.jsonfilelist
        if hardjson:
            self.jsonfilelist = hj

        #self.pp = subprocess.Popen('echo starting',stdout=PIPE, stderr=PIPE,shell=True)

        self.trigger_id = trigger_id
        self.datadir = datadir
        self.triggerdir = triggerdir
        self.processingdir = os.path.join(self.triggerdir,'PROCESSING')
        if not os.path.exists(self.processingdir):
            os.makedirs(self.processingdir)

        dire = './processing/' + trigger_id + '/'
        if not os.path.exists(dire):
            os.makedirs(dire)

        with open(os.path.join(triggerdir, "strategy.yaml"), "r") as f:
            self.strategy = yaml.safe_load(f)

        with open("jobmanager.yaml", "r") as g:
            self.jmconfig = yaml.safe_load(g);


        q1 = "select expnum,nite,mjd_obs,telra,teldec,band,exptime,propid,obstype,object from exposure where " \
             "nite>20130828 and nite<20150101 and expnum<300000 and obstype='object' order by expnum"  # y1 images
        self.connection.query_and_save(q1, './processing/exposuresY1.tab')

        q2 = "select expnum,nite,mjd_obs,radeg,decdeg,band,exptime,propid,obstype,object from prod.exposure where " \
             "nite>20150901 and obstype='object' order by expnum"  # y2 and later
        self.connection.query_and_save(q2, './processing/exposuresCurrent.tab')

        os.system('cat ./processing/exposuresY1.tab ./processing/exposuresCurrent.tab > ./processing/exposures.list')

        #self.submit_post_processing()

        self.submit_all_jsons_for_sejobs()#preps all DES images that already exist
        tfin = time.time()
        print 'TOTAL SE JOBS TIME', tfin - tstart
        #sys.exit()
        self.monitor_images_from_mountain()#A loop that waits for images off mountain and submits for processing

    def submit_all_jsons_for_sejobs(self):
        obsStartTime = self.getDatetimeOfFirstJson(self.jsonfilelist[0])  # THIS IS A DATETIME OBJ
        currentTime = dt.utcnow()
        print '***** The current time is UTC', currentTime, '*****'
        delt = obsStartTime - currentTime

        timedelta = td(days=delt.days, seconds=delt.seconds).total_seconds() / 3600.
        print '***** The time delta is ', timedelta, 'hours *****'
        # if timedelta > np.pi:

        sejob_timecushion = self.jmconfig["sejob_timecushion"]

        if timedelta > sejob_timecushion:
            for jsonfile in self.jsonfilelist:
                print 'json',jsonfile
                try: #check if this json file is already in the submitted preprocessing database
                    thisjson = self.backend.get(preprocessing, {'jsonfilename': os.path.join(self.datadir, jsonfile)})
                    print 'Found this json in desgw database...'
                except preprocessing.DoesNotExist: #do submission and then add to database

                    print 'cd diffimg-proc; ./SEMaker_RADEC.sh '+os.path.join(self.datadir, jsonfile)
                    os.chdir("diffimg-proc")
                    out = os.popen('./SEMaker_RADEC.sh '+os.path.join(self.datadir, jsonfile)).read()
                    of = open(os.path.join(self.processingdir,jsonfile.split('/')[-1].split('.')[0]+'.SEMakerlog'),'w')
                    of.write(out)
                    of.close()
                    #out = os.popen('ls').read()
                    os.chdir("..")
                    print out
                    if 'non-zero exit status' in out:
                        dt.sendEmailSubject(self.trigger_id,'Error in creating SEMaker dag for .json: '+out)
                    else:
                        for o in out.split('\n'):
                            if 'file://' in o:
                                dagfile = o.split('/')[-1]
                                self.dagfile = os.path.join(self.processingdir,jsonfile.split('/')[-1].split('.')[0]+'_'+dagfile)
                                os.system('cp diffimg-proc/'+dagfile+' '+self.dagfile)
                                jobsubmitline = copy(o)
                        print self.dagfile

                    print 'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; jobsub_submit_dag -G des --role=DESGW file://'+self.dagfile

                    out = os.popen(
                        'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; '
                        'jobsub_submit_dag -G des --role=DESGW file://'+self.dagfile).read()
                    print out

                    of = open(os.path.join(self.processingdir,
                                           jsonfile.split('/')[-1].split('.')[0] + '.SEdagsubmitlog'), 'w')
                    of.write(out)
                    of.close()

                    if 'non-zero exit status' in out:
                        dt.sendEmailSubject(self.trigger_id, 'Error in submitting .json for preprocessing: ' + out)
                    else:
                        for o in out.split('\n'):
                            if 'Use job id' in o:
                                jobid = o.split()[3]
                        if doimmediateremove:
                            out = os.popen(
                                'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; '
                                'jobsub_rm --jobid=' + jobid + ' --group=des --role=DESGW').read()
                            print out

                    thisjson = preprocessing({
                        'jsonfilename': os.path.join(self.datadir, jsonfile),
                        'jobid': jobid,
                        'dagfile': self.dagfile,
                        'status' : 'Submitted'
                    })

                    self.backend.save(thisjson)
                    self.backend.commit()
                    print 'saved'
                    #sys.exit()
                #raw_input()
                #runProcessingIfNotAlready(image, self.backend)

                #sys.exit()

        print 'Finished submitting minidagmaker with all json files'
        #sys.exit()
        #raw_input()

    # Loop queries for images from mountain and submits them
    # Need to add complexity that monitors filter strategy and waits for entire groups of images to be co-added
    def monitor_images_from_mountain(self):
        #NEED TO ADD COADD LOGIC USING STRATEGY FROM CONFIG

        exposure_filter = np.array(self.strategy['exposure_filter'],dtype='str')
        uniquefilts = np.unique(self.strategy['exposure_filter'])

        filterstrategy = {}
        for f in uniquefilts:
            filterstrategy[f] = len(exposure_filter[exposure_filter == f])

        print 'filter strategy dictionary  ', filterstrategy

        starttime = time.time()
        pptime = time.time()
        keepgoing = True
        index = -1
        submission_counter = 0
        maxsub = 10000
        postprocessingtime = 2000 #every half hour fire off Tim's code for post-processing
        while keepgoing:
            #os.system('kinit -k -t /var/keytab/desgw.keytab desgw/des/[email protected]')
            index += 1
            newfireds = []
            if time.time() - starttime > 50000:
                keepgoing = False
                continue

            ofile = open(os.path.join(self.triggerdir , 'latestquery.txt'), 'w')

            ofile.write(
                "--------------------------------------------------------------------------------------------------\n")
            ofile.write("EXPNUM\tNITE\tBAND\tEXPTIME\tRADEG\t DECDEG\tPROPID\tOBJECT\n")
            ofile.write(
                "--------------------------------------------------------------------------------------------------\n")

            print "--------------------------------------------------------------------------------------------------"
            print "EXPNUM\tNITE\tBAND\tEXPTIME\tRADEG\t DECDEG\tPROPID\tOBJECT"
            print "--------------------------------------------------------------------------------------------------"

            query = "SELECT expnum,nite,band,exptime,radeg,decdeg,propid,object FROM prod.exposure@desoper WHERE " \
                    "expnum > 475900 and propid=" + propid + "and obstype='object' ORDER BY expnum"  # latest


            self.cursor.execute(query)

            for s in self.cursor:
                ofile.write(
                    str(s[0]) + "\t" + str(s[1]) + "\t" + str(s[2]) + "\t" + str(s[3]) + "\t" + str(s[4]) + "\t" + str(
                        s[5]) + "\t" + str(s[6]) + "\t" + str(s[7]) + '\n')
                print str(s[0]) + "\t" + str(s[1]) + "\t" + str(s[2]) + "\t" + str(s[3]) + "\t" + str(
                    s[4]) + "\t" + str(s[5]) + "\t" + str(s[6]) + "\t" + str(s[7])

                if not 'DESGW' in str(s[7]): continue
                #print 'exptime',float(s[3])
                if not float(s[3]) > 29.: continue #exposure must be longer than 30 seconds

                expnum = str(s[0])
                nite = str(s[1])
                band = str(s[2])
                exptime = str(s[3])


                #FIRST CHECK HERE THAT THE EXPOSURE NUMBER ISNT ALREADY IN THE DATABASE
                try:
                    exposure = self.backend.get(exposures, {'expnum': expnum})
                    print 'Found this exposure in desgw database...'
                    # print exposure.attributes
                    # if expnum == 506432:
                    #     sys.exit()
                    # self.backend.delete(exposure)
                    # self.backend.commit()
                    # exposure = self.backend.get(exposures, {'expnum': expnum})

                except exposures.DoesNotExist:  # add to database
                    #runProcessingIfNotAlready(image,self.backend)

                    print './diffimg-proc/getTiling.sh '+expnum

                    res = os.popen('./diffimg-proc/getTiling.sh '+expnum).readlines()
                    print res
                    #sys.exit()
                    field,tiling =res[-2],res[-1]
                    #print 'field_tiling',field_tiling
                    hexnite = field.strip()+'_'+tiling.strip()+'_'+str(nite)
                    #print hexnite
                    #sys.exit()
                    #print 'hexnite',hexnite
                    print 'Creating exposure in database...',hexnite
                    #raw_input()
                    if '--' in hexnite:
                        print 'found bad example'
                        #raw_input()

                    exposure = exposures({
                        'expnum':expnum,
                        'nite':nite,
                        'field':field,
                        'tiling':tiling,
                        'hexnite':hexnite,
                        'band':band,
                        'jobid':np.nan,
                        'exptime':exptime,
                        'status':'Awaiting additional exposures',
                        'triggerid': self.trigger_id,
                        'object':str(s[7])
                    })

                    self.backend.save(exposure)
                    self.backend.commit()

                hexnite = exposure.hexnite
                print 'hexnite',hexnite
                if '--' in hexnite:
                    print exposure.attributes
                    #raw_input()
                #raw_input()
                #sys.exit()
                try:
                    hex = self.backend.get(hexes, {'hexnite': hexnite})
                    #self.backend.delete(hex)
                    #self.backend.commit()
                    #hex = self.backend.get(hexes, {'hexnite': hexnite})

                    #print 'Found this hex in desgw database...'
                except hexes.DoesNotExist:
                    hex = hexes({
                        'hexnite': hexnite,
                        'strategy': self.strategy['exposure_filter'],
                        'num_target_g': len(exposure_filter[exposure_filter == 'g']),
                        'num_target_r': len(exposure_filter[exposure_filter == 'r']),
                        'num_target_i': len(exposure_filter[exposure_filter == 'i']),
                        'num_target_z': len(exposure_filter[exposure_filter == 'z']),
                        'observed_g': [],
                        'observed_r': [],
                        'observed_i': [],
                        'observed_z': [],
                        'exposures': [],
                        'status': 'Awaiting additional exposures',
                        'dagfile' : None,
                    })

                    self.backend.save(hex)
                    self.backend.commit()
                    print hex.attributes
                    print 'created new hex'
                    #raw_input()

                if hex.status == 'Submitted for processing':
                    print 'This hex has already been submitted for processing'
                    continue

                # if '--' in hexnite:
                #     print hex.attributes
                #     raw_input()
                # if hex.status == 'Submitted for processing':
                #     print 'Hex ',hexnite,' band',band,'exposure',expnum,'has already been submitted for processing'
                #     #raw_input()
                #     continue

                if band == 'g':
                    if not expnum in hex.observed_g:
                        hex.observed_g.append(expnum)
                        hex.exposures.append(expnum)
                if band == 'r':
                    if not expnum in hex.observed_r:
                        hex.observed_r.append(expnum)
                        hex.exposures.append(expnum)
                if band == 'i':
                    if not expnum in hex.observed_i:
                        hex.observed_i.append(expnum)
                        hex.exposures.append(expnum)
                if band == 'z':
                    if not expnum in hex.observed_z:
                        hex.observed_z.append(expnum)
                        hex.exposures.append(expnum)

                self.backend.save(hex)
                self.backend.commit()

                print hex.attributes

                didwork = False
                if len(hex.observed_g) == hex.num_target_g:
                    if len(hex.observed_r) == hex.num_target_r:
                        if len(hex.observed_i) == hex.num_target_i:
                            if len(hex.observed_z) == hex.num_target_z:
                                print 'All exposures in strategy satisfied! '
                                #raw_input()
                                submissionPassed = True

                                for target, exps in zip([hex.num_target_g,hex.num_target_r,hex.num_target_i,hex.num_target_z],
                                                        [hex.observed_g,hex.observed_r,hex.observed_i,hex.observed_z]):

                                    if target == 0: continue
                                    exposurestring = ''
                                    logstring = ''
                                    for ex in exps:
                                        exposurestring += ex+' '
                                        logstring += ex+'_'

                                    print 'cd diffimg-proc; source DAGMaker.sh ' + exposurestring
                                    os.chdir("diffimg-proc")
                                    #out = os.popen('ls').read()
                                    out = os.popen('./DAGMaker.sh ' + exposurestring ).read()


                                    os.chdir("..")
                                    print out
                                    f = open(os.path.join(self.processingdir,logstring+hexnite+'.dagmakerlog'),'w')
                                    f.write(out)
                                    f.close()
                                    tt = time.time()
                                    if not 'To submit this DAG do' in out:
                                        dt.sendEmailSubject(self.trigger_id, 'Error in creating dag for desgw hex: ' + out)
                                        submissionPassed = False
                                    else:
                                        for o in out.split('\n'):
                                            if 'file://' in o:
                                                dagfile = o.split('/')[-1]
                                                self.dagfile = os.path.join(self.processingdir,logstring+'job.dag')
                                                os.system('cp diffimg-proc/' + dagfile + ' ' + self.dagfile)
                                        print self.dagfile

                                    print 'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; jobsub_submit_dag -G des --role=DESGW file://' + self.dagfile

                                    out = os.popen(
                                        'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; '
                                        'jobsub_submit_dag -G des --role=DESGW file://' + self.dagfile).read()
                                    print out
                                    f = open(os.path.join(self.processingdir, logstring + hexnite + '.dagsumbitlog'), 'w')
                                    f.write(out)
                                    f.close()

                                    if 'non-zero exit status' in out:
                                        dt.sendEmailSubject(self.trigger_id,
                                                            'Error in submitting hex dag for processing: ' + out)
                                        submissionPassed = False
                                    else:
                                        if doimmediateremove:
                                            for o in out.split('\n'):
                                                if 'Use job id' in o:
                                                    jobid = o.split()[3]
                                            out = os.popen(
                                                'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; '
                                                'jobsub_rm --jobid=' + jobid + ' --group=des --role=DESGW').read()
                                            print out
                                    ttt = time.time()
                                    #print 'TOTAL JOBSUB FOR A SINGLE DESGW IMAGE',ttt-tt

                                    #sys.exit()
                                #raw_input()
                                if submissionPassed:

                                    hex.status = 'Submitted for processing'
                                    hex.dagfile = self.dagfile
                                    self.backend.save(hex)
                                    self.backend.commit()

                                    for expn in hex.exposures:
                                        print expn, 'updated in database to Submitted For Processing'
                                        exp = self.backend.get(exposures, {'expnum': expn})
                                        exp.status = 'Submitted for processing'
                                        self.backend.save(exp)
                                        self.backend.commit()
                                    didwork = True
                                print 'didwork',didwork
                                print 'dagfile',self.dagfile
                                #sys.exit()
                                #raw_input()

                if not didwork:
                    print 'Could not find all images in strategy for this hex... Added hex', hexnite,' to database ' \
                        'and will continue waiting...'
                    #raw_input()


                if time.time() - pptime > postprocessingtime: #happens every 30 minutes or so...
                    pptime = time.time()
                    print '***** Firing post processing script *****'
                    #sys.exit()

                    #ppout = self.pp.communicate()

                    # if self.thisevent.postprocint > 0:
                    #     print ppout
                    #     f = open(os.path.join(self.processingdir,'postproc_attempt'+str(int(self.thisevent.postprocint))+'.log'),'w')
                    #     f.write(ppout)
                    #     f.close()
                    self.thisevent.postprocint += 1
                    self.backend.save(self.thisevent)
                    self.backend.commit()

                    self.submit_post_processing()
                #sys.exit()
                #print 'Waiting 10s to check from mountain...'
                #sys.exit()
                time.sleep(10)#looping over checking the mountain top

            # cfiles = os.listdir(os.path.join(trigger_path,trigger_id,'candidates'))
            # for f in cfiles:
            #     if f.split('.')[-1] == 'npz':
            #         cp.makeNewPage(f)

    def submit_post_processing(self):
        #firedlist = open('./processing/firedlist.txt', 'r')
        #fl = firedlist.readlines()
        #firedlist.close()
        #print fl
        #fl = ['475914','475915','475916','482859','482860','482861']

        fl = self.backend.filter(exposures, {'triggerid': self.trigger_id})

        expnumlist = ''
        for f in fl:
            expnumlist += f.expnum.strip()+' '
        print expnumlist
        #expnumlist = '475905  475909  475913  475917  475921  475925  475929  475933  475937  475941  475945  475949  475953  475957  475961'
        print 'FIRING TIMs CODE'
        try:
            os.mkdir(os.path.join(self.trigger_path,self.trigger_id,'candidates'))
        except:
            print 'Candidates directory exists,',os.path.join(self.trigger_path,self.trigger_id,'candidates')
            pass
        #sys.exit()
        gwpostdir = os.environ['GWPOST_DIR']
        print 'source ' + os.path.join(gwpostdir, 'diffimg_setup.sh') + '; \
                        python '+os.path.join(gwpostdir,'postproc.py')\
                         +' --expnums ' + expnumlist\
                         + ' --outputdir ' + os.path.join(self.trigger_path,self.trigger_id,'candidates')\
                         + ' --triggerid '+self.trigger_id+' --season 70 --ups True'
        # os.system('source ' + os.path.join(gwpostdir, 'diffimg_setup.sh') + '; \
        #                  python '+os.path.join(gwpostdir,'postproc.py')\
        #                  +' --expnums ' + expnumlist\
        #                  + ' --outputdir ' + os.path.join(trigger_path,trigger_id,'candidates')\
        #                  + ' --triggerid '+trigger_id+' --season 46 --ups True' )

        #pid = os.spawnlp(os.P_WAIT, "source", os.path.join(gwpostdir, 'diffimg_setup.sh'))
        # args = ['yes | ssh -t [email protected] "source '+ os.path.join(gwpostdir, 'mi_setup.sh')+'; '+
        #                 'yes | python '+os.path.join(gwpostdir,'postproc.py')\
        #                  +' --expnums ' + expnumlist\
        #                  + ' --outputdir ' + os.path.join(self.trigger_path,self.trigger_id,'candidates')\
        #                  + ' --triggerid '+self.trigger_id+' --season 46 --ups True"'
        #         ]

        args = ['yes | python ' + os.path.join(gwpostdir, 'postproc.py') \
                + ' --expnums ' + expnumlist \
                + ' --outputdir ' + os.path.join(self.trigger_path, self.trigger_id, 'candidates') \
                + ' --triggerid ' + self.trigger_id + ' --season 70 --ups True']
        print args

        #sys.exit()
        f = open(os.path.join(self.processingdir,'postproc_attempt'+str(int(self.thisevent.postprocint)))+'.log','w')
        self.pp = subprocess.Popen(args,stdout=f, stderr=f,shell=True)
        #p = subprocess.Popen(args, stdin=None, stdout=None, stderr=None, close_fds=True, shell=True)
        #print 'going'*1000
        #print self.pp.communicate()
        #print 'gone'*1000
        #p = subprocess.Popen(args,stdin=None, stdout=None, stderr=None, close_fds=True,shell=True)
        #p.communicate()
        #sys.exit()
        return



    def getDatetimeOfFirstJson(self,jsonstring):
        #'M263920-30-UTC-2016-12-1-0:44:00.json'
        js = jsonstring.split('UTC')[1]#-2015-12-27-3:2:00.json
        #date_object = dt.strptime(js, '-%Y-%m-%d-%H_%M_%S.json')
        date_object = dt.strptime(js, '-%Y-%m-%d-%H:%M:%S.json')
        print '***** Datetime of first observation UTC',date_object,'*****'
        return date_object

    def sortHexes(self):
        pass