class DataProvider:
def __init__(self, db_path):
self.db = FileBackend(db_path)
def last(self, user):
return get_first_func(self.db.filter(WeightRecord, {'user': user, 'last': True}))
def all_mornings(self, user):
db_filter = self.db.filter(WeightRecord, {'user': user, 'morning': True})
#if db_filter:
# for i in db_filter:
# logging.debug("T0 {}".format(i.time))
sor = sorted(db_filter, key=lambda x: x.time, reverse=False)
#if sor:
# for i in sor:
# logging.debug("T1 {}".format(i.time))
return sor
def last_morning(self, data):
return get_first_func(self.db.filter(WeightRecord, {
'last': True, 'morning': True, 'user': data.user}))
def today_morning(self, data):
return get_first_func(self.db.filter(WeightRecord, {
'year': data.year, 'month': data.month, 'day': data.day, 'user': data.user, 'morning': True}))
def save(self, record):
record.save(self.db)
def commit(self):
self.db.commit()
class Geocode:
def __init__(self, geocoderCache = True, printStatus = False):
self.printStatus = printStatus
self.geocoderCache = geocoderCache
if self.geocoderCache:
self.db = FileBackend('./geo-cache')
def getGeo(self, lon, lat):
if self.geocoderCache:
try:
nodeObj = self.db.get(GeoAssign,{'lat' : lat, 'lon' : lon})
nodeObj['cached'] = True
return nodeObj
except GeoAssign.DoesNotExist:
pass
if self.printStatus:
print('lon: '+str(lon)+', lat: '+str(lat)+' not in cache - start lookup at Nominatim-API')
geolocator = Nominatim()
location = geolocator.reverse([lat, lon], timeout=20)
if 'address' in location.raw:
location = location.raw['address']
nodeObj = GeoAssign({
'lat' : lat,
'lon' : lon,
'payload' : location
})
self.db.save(nodeObj)
self.db.commit()
nodeObj['cached'] = False
return nodeObj
else:
# got no results (i.e. coordinates are incorrect)
return None
class JavManagerDB:
def __init__(self):
self.jav_db = FileBackend('jav_manager.db')
def upcreate_jav(self, jav_obj: dict):
# uniform car to upper case
jav_obj['car'] = str(jav_obj['car']).upper()
# set pk to car
jav_obj['pk'] = jav_obj['car']
# set default to no opinion
#0-want, 1-viewed, 2-no opinion 3-local 4-downloading
jav_obj.setdefault('stat', 2)
_jav_doc = JavObj(jav_obj)
_jav_doc.save(self.jav_db)
self.jav_db.commit()
def get_by_pk(self, pk: str):
return self.jav_db.get(JavObj, {'pk': pk.upper()})
def pk_exist(self, pk: str):
try:
self.jav_db.get(JavObj, {'pk': pk})
return True
except DoesNotExist:
return False
class Geocode:
def __init__(self, geocoderCache=True, printStatus=False):
self.printStatus = printStatus
self.geocoderCache = geocoderCache
if self.geocoderCache:
self.db = FileBackend('./geo-cache')
def getGeo(self, lon, lat):
if self.geocoderCache:
try:
nodeObj = self.db.get(GeoAssign, {'lat': lat, 'lon': lon})
nodeObj['cached'] = True
return nodeObj
except GeoAssign.DoesNotExist:
pass
if self.printStatus:
print('lon: ' + str(lon) + ', lat: ' + str(lat) +
' not in cache - start lookup at Nominatim-API')
geolocator = Nominatim()
location = geolocator.reverse([lat, lon], timeout=20)
if 'address' in location.raw:
location = location.raw['address']
nodeObj = GeoAssign({'lat': lat, 'lon': lon, 'payload': location})
self.db.save(nodeObj)
self.db.commit()
nodeObj['cached'] = False
return nodeObj
else:
# got no results (i.e. coordinates are incorrect)
return None
class JavManagerDB:
def __init__(self):
self.jav_db = FileBackend('jav_manager.db')
def create_indexes(self):
print('creating index for stat')
self.jav_db.create_index(JavObj, 'stat')
def rebuild_index(self):
self.jav_db.rebuild_index(self.jav_db.get_collection_for_cls(JavObj),
'stat')
def bulk_list(self):
return self.jav_db.filter(JavObj, {})
def partial_search(self, search_string: str):
rt = self.jav_db.filter(JavObj,
{'pk': {
'$regex': search_string.upper()
}})[:20]
return rt
def query_on_filter(self, filter_on: dict, page=1, limit=8):
rt = self.jav_db.filter(JavObj, filter_on)
rt_max_page = ceil(len(rt) / limit)
rt_list = rt[(page - 1) * limit:(page) * limit]
return [dict(x) for x in rt_list], rt_max_page
def upcreate_jav(self, jav_obj: dict):
# uniform car to upper case
jav_obj['car'] = str(jav_obj['car']).upper()
# set pk to car
jav_obj['pk'] = jav_obj['car']
# pull existing data since this is update function
try:
current_jav_obj = dict(self.get_by_pk(jav_obj['car']))
# overwrite current db dict with input dict
current_jav_obj.update(jav_obj)
except DoesNotExist:
# set default to no opinion
#0-want, 1-viewed, 2-no opinion 3-local 4-downloading
jav_obj.setdefault('stat', 2)
_jav_doc = JavObj(jav_obj)
_jav_doc.save(self.jav_db)
self.jav_db.commit()
print('writed ', jav_obj)
def get_by_pk(self, pk: str):
return self.jav_db.get(JavObj, {'pk': pk.upper()})
def pk_exist(self, pk: str):
try:
self.jav_db.get(JavObj, {'pk': pk.upper()})
return True
except DoesNotExist:
return False
def wrapped_f(bot, update, *args, **kwargs):
msg_entry = Message(json.loads(update.to_json()))
backend = FileBackend(settings.ACCESS_LOG_DB_PATH)
backend.save(msg_entry)
backend.commit()
return f(bot, update, *args, **kwargs)
class DataProvider:
def __init__(self, db_path):
self.db = FileBackend(db_path)
def last(self, user):
return get_first_func(
self.db.filter(WeightRecord, {
'user': user,
'last': True
}))
def all_mornings(self, user):
db_filter = self.db.filter(WeightRecord, {
'user': user,
'morning': True
})
#if db_filter:
# for i in db_filter:
# logging.debug("T0 {}".format(i.time))
sor = sorted(db_filter, key=lambda x: x.time, reverse=False)
#if sor:
# for i in sor:
# logging.debug("T1 {}".format(i.time))
return sor
def last_morning(self, data):
return get_first_func(
self.db.filter(WeightRecord, {
'last': True,
'morning': True,
'user': data.user
}))
def today_morning(self, data):
return get_first_func(
self.db.filter(
WeightRecord, {
'year': data.year,
'month': data.month,
'day': data.day,
'user': data.user,
'morning': True
}))
def save(self, record):
record.save(self.db)
def commit(self):
self.db.commit()
class ImageToTumblr(object):
def __init__(self, bot):
self.cfg = PluginConfig(self)
self.image_filetypes = self.cfg.get("image_filetypes").split(",")
self.db = FileBackend(self.cfg.get("main_db"))
self.tumblr = pytumblr.TumblrRestClient(
self.cfg.get("consumer_key"),
self.cfg.get("consumer_secret"),
self.cfg.get("oauth_token"),
self.cfg.get("oauth_secret"),
)
irc3.base.logging.log(
irc3.base.logging.WARN, "Tumblr poster ready! Posting all URLs with: %s" % self.image_filetypes
)
def post_image(self, text, poster):
# Strip everything but the address
m = re.match(r".*(?P<url>http.*)", text)
url = m.group("url")
# Make sure we didn't do this one already
try:
self.db.get(PostedImage, {"url": url})
except PostedImage.DoesNotExist:
try:
# First we post it to tumblr
p = self.tumblr.create_photo(
"mmerpimages", state="published", source=str(url), caption="Found by %s" % poster
)
irc3.base.logging.log(irc3.base.logging.WARN, "Posting image by %s: %s" % (poster, url))
# And then record the fact that we did.
self.db.save(PostedImage({"url": url}))
self.db.commit()
except:
irc3.base.logging.log(irc3.base.logging.WARN, "Could not post to tumblr: %s" % url)
return
else:
irc3.base.logging.log(irc3.base.logging.WARN, "Not posting duplicate image: %s" % url)
return
@irc3.event(irc3.rfc.PRIVMSG) # Triggered on every message anywhere.
def parse_image(self, target, mask, data, event):
for extension in self.image_filetypes:
if "." + extension.lower() in data:
self.post_image(data, mask.nick)
class Database:
def __init__(self):
self.backend = FileBackend(settings['database_location'])
def save(self, entry):
self.backend.save(entry)
self.backend.commit()
def search(self, table, query):
if table == 'user':
doc = User
elif table == 'subm':
doc = Submission
try:
return self.backend.get(doc, query)
except doc.DoesNotExist:
return None
def cleardb(real=False):
password = '******'
user_input = raw_input('Please Enter Password: '******'Incorrect Password, terminating... \n')
if real:
backend = FileBackend("./realdb")
else:
backend = FileBackend("./testdb")
images = backend.filter(preprocessing, {'status': 'Submitted'})
images.delete()
hexen = backend.filter(hexes,{'num_target_g':0})
hexen.delete()
exposuren = backend.filter(exposures,{'status':'Awaiting additional exposures'})
exposuren.delete()
backend.commit()
def main():
backend = FileBackend("./my-db") #locate the backend
fin = file("Cu_material_db.yaml","r") #read the yaml file
in_param = yaml.load(fin) #load it
name = in_param.get("Name") #extract out the parameters
cond = in_param.get("Conductivity")
n = in_param.get("Carrier_Density")
supercond = in_param.get("Superconductivity")
res = in_param.get("Resistance")
right_input_flag = True #check the superconducting type matches the number of critical fields
if supercond.get("Superconducting"): #create a new entry for a superconductor
right_input_flag = False if len(supercond.get("Critical_Fields")) != supercond.get("Type") else True
if right_input_flag:
new = Material({"name": name, "cond": cond, "n": n,
"sc": supercond.get("Superconducting"),
"sctype" : supercond.get("Type"),
"Tc" : supercond.get("Critical_Temperature"),
"nc" : supercond.get("Critical_Current_Density"),
"Hc" : supercond.get("Critical_Fields"),
"L" : supercond.get("Penetration_Depth"),
"Tref" : res.get("Ref_Temperature"),
"res" : res.get("Resistivity"),
"alpha" : res.get("Temperature_Coefficient")
})
else:
print "Superconducting type and number of critical fields don't match!"
else: #create a new entry for a non-superconductor
new = Material({"name": name, "cond": cond, "n": n,
"sc": supercond.get("Superconducting"),
"Tref" : res.get("Ref_Temperature"),
"res" : res.get("Resistivity"),
"alpha" : res.get("Temperature_Coefficient")
})
if right_input_flag:
new.save(backend) #save the entry in the database
#always update the database when there's an old entry of this material in the database
mtrl = backend.filter(Material, {"name":name})
mtrl.delete()
backend.commit()
print "Input succeeded!"
class Database(object):
"""Blitzdb database."""
def __init__(self):
"""Load backend."""
self.db = FileBackend(expanduser("~/.omesa/db"))
def _query(self, f, q):
try:
out = f(*q)
except KeyError:
self.db = FileBackend(expanduser("~/.omesa/db"))
f = self.db.filter
out = f(*q)
return out
def save(self, doc):
"""Save document do db."""
self.db.save(doc)
self.db.commit()
def fetch(self, doc, q):
"""Filter and return first entry."""
try:
return self._query(self.db.filter, (doc, q))[0]
except IndexError:
print(str(doc), str(q))
print("File does not exist.")
def get_component(self, doc, name):
# FIXME: see if returning non-decoded is relevant for anything
try:
return sr.decode(
dict(self._query(self.db.filter, (doc, {
'name': name
}))[0]))
except IndexError:
print(str(doc), {'name': name})
print("File does not exist.")
def getall(self, doc):
"""Returns all entries in db."""
return [d for d in self._query(self.db.filter, (doc, {}))]
def cleardb(real=False):
password = '******'
user_input = raw_input('Please Enter Password: '******'Incorrect Password, terminating... \n')
if real:
backend = FileBackend("./realdb")
else:
backend = FileBackend("./testdb")
images = backend.filter(preprocessing, {'status': 'Submitted'})
images.delete()
hexen = backend.filter(hexes, {'num_target_g': 0})
hexen.delete()
exposuren = backend.filter(exposures,
{'status': 'Awaiting additional exposures'})
exposuren.delete()
backend.commit()
class Database(object):
"""Blitzdb database."""
def __init__(self):
"""Load backend."""
self.db = FileBackend(expanduser("~/.omesa/db"))
def _query(self, f, q):
try:
out = f(*q)
except KeyError:
self.db = FileBackend(expanduser("~/.omesa/db"))
f = self.db.filter
out = f(*q)
return out
def save(self, doc):
"""Save document do db."""
self.db.save(doc)
self.db.commit()
def fetch(self, doc, q):
"""Filter and return first entry."""
try:
return self._query(self.db.filter, (doc, q))[0]
except IndexError:
print(str(doc), str(q))
print("File does not exist.")
def get_component(self, doc, name):
# FIXME: see if returning non-decoded is relevant for anything
try:
return sr.decode(dict(self._query(
self.db.filter, (doc, {'name': name}))[0]))
except IndexError:
print(str(doc), {'name': name})
print("File does not exist.")
def getall(self, doc):
"""Returns all entries in db."""
return [d for d in self._query(self.db.filter, (doc, {}))]
class eventmanager:
def __init__(self, trigger_id, jsonfilelist, triggerdir, datadir, real,
trigger_path):
#os.system('kinit -k -t /var/keytab/desgw.keytab desgw/des/[email protected]')
tstart = time.time()
if real:
self.backend = FileBackend("./realdb")
else:
self.backend = FileBackend("./testdb")
try:
thisevent = self.backend.get(Trigger, {'id': trigger_id})
print 'Found this event in desgw database...'
except Trigger.DoesNotExist:
thisevent = Trigger({
'id':
trigger_id,
'jsonfilelist':
jsonfilelist,
'triggerpath':
triggerdir,
'mapspath':
datadir,
'jobids': [
(0, 'jsonfile_corresponding_to_jobid.json'),
],
})
print 'Database entry created!'
self.trigger_id = trigger_id
self.trigger_path = trigger_path
self.backend.save(thisevent)
self.backend.commit()
with open(os.path.join(triggerdir, "strategy.yaml"), "r") as f:
self.config = yaml.safe_load(f)
self.filterobslist = np.array(self.config['exposure_filter'],
dtype='str')
self.strategydict = {}
for f in np.unique(self.filterobslist):
self.strategydict[f] = len(
self.filterobslist[self.filterobslist == f])
self.connection = ea.connect(DATABASE)
self.cursor = self.connection.cursor()
self.jsonfilelist = jsonfilelist
print self.jsonfilelist
if hardjson:
self.jsonfilelist = hj
self.trigger_id = trigger_id
self.datadir = datadir
self.triggerdir = triggerdir
self.processingdir = os.path.join(self.triggerdir, 'PROCESSING')
if not os.path.exists(self.processingdir):
os.makedirs(self.processingdir)
dire = './processing/' + trigger_id + '/'
if not os.path.exists(dire):
os.makedirs(dire)
with open(os.path.join(triggerdir, "strategy.yaml"), "r") as f:
self.strategy = yaml.safe_load(f)
with open("jobmanager.yaml", "r") as g:
self.jmconfig = yaml.safe_load(g)
q1 = "select expnum,nite,mjd_obs,telra,teldec,band,exptime,propid,obstype,object from exposure where " \
"nite>20130828 and nite<20150101 and expnum<300000 and obstype='object' order by expnum" # y1 images
self.connection.query_and_save(q1, './processing/exposuresY1.tab')
q2 = "select expnum,nite,mjd_obs,radeg,decdeg,band,exptime,propid,obstype,object from prod.exposure where " \
"nite>20150901 and obstype='object' order by expnum" # y2 and later
self.connection.query_and_save(q2, './processing/exposuresCurrent.tab')
os.system(
'cat ./processing/exposuresY1.tab ./processing/exposuresCurrent.tab > ./processing/exposures.list'
)
self.submit_all_jsons_for_sejobs(
) #preps all DES images that already exist
tfin = time.time()
print 'TOTAL SE JOBS TIME', tfin - tstart
#sys.exit()
self.monitor_images_from_mountain(
) #A loop that waits for images off mountain and submits for processing
def submit_all_jsons_for_sejobs(self):
obsStartTime = self.getDatetimeOfFirstJson(
self.jsonfilelist[0]) # THIS IS A DATETIME OBJ
currentTime = dt.utcnow()
print '***** The current time is UTC', currentTime, '*****'
delt = obsStartTime - currentTime
timedelta = td(days=delt.days,
seconds=delt.seconds).total_seconds() / 3600.
print '***** The time delta is ', timedelta, 'hours *****'
# if timedelta > np.pi:
sejob_timecushion = self.jmconfig["sejob_timecushion"]
if timedelta > sejob_timecushion:
for jsonfile in self.jsonfilelist:
print 'json', jsonfile
try: #check if this json file is already in the submitted preprocessing database
thisjson = self.backend.get(
preprocessing,
{'jsonfilename': os.path.join(self.datadir, jsonfile)})
print 'Found this json in desgw database...'
except preprocessing.DoesNotExist: #do submission and then add to database
print 'cd diffimg-proc; ./SEMaker_RADEC.sh ' + os.path.join(
self.datadir, jsonfile)
os.chdir("diffimg-proc")
out = os.popen(
'./SEMaker_RADEC.sh ' +
os.path.join(self.datadir, jsonfile)).read()
#out = os.popen('ls').read()
os.chdir("..")
print out
if 'non-zero exit status' in out:
dt.sendEmailSubject(
self.trigger_id,
'Error in creating dag for .json: ' + out)
else:
for o in out.split('\n'):
if 'file://' in o:
dagfile = o.split('/')[-1]
self.dagfile = os.path.join(
self.processingdir,
jsonfile.split('/')[-1].split('.')[0] +
'_' + dagfile)
os.system('cp diffimg-proc/' + dagfile + ' ' +
self.dagfile)
jobsubmitline = copy(o)
print self.dagfile
print 'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; jobsub_submit_dag -G des --role=DESGW file://' + self.dagfile
out = os.popen(
'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; '
'jobsub_submit_dag -G des --role=DESGW file://' +
self.dagfile).read()
print out
if 'non-zero exit status' in out:
dt.sendEmailSubject(
self.trigger_id,
'Error in submitting .json for preprocessing: ' +
out)
else:
if doimmediateremove:
for o in out.split('\n'):
if 'Use job id' in o:
jobid = o.split()[3]
out = os.popen(
'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; '
'jobsub_rm --jobid=' + jobid +
' --group=des --role=DESGW').read()
print out
thisjson = preprocessing({
'jsonfilename':
os.path.join(self.datadir, jsonfile),
'jobid':
jobid,
'dagfile':
self.dagfile,
'status':
'Submitted'
})
self.backend.save(thisjson)
self.backend.commit()
print 'saved'
#raw_input()
#runProcessingIfNotAlready(image, self.backend)
#sys.exit()
print 'Finished submitting minidagmaker with all json files'
#sys.exit()
#raw_input()
# Loop queries for images from mountain and submits them
# Need to add complexity that monitors filter strategy and waits for entire groups of images to be co-added
def monitor_images_from_mountain(self):
#NEED TO ADD COADD LOGIC USING STRATEGY FROM CONFIG
exposure_filter = np.array(self.strategy['exposure_filter'],
dtype='str')
uniquefilts = np.unique(self.strategy['exposure_filter'])
filterstrategy = {}
for f in uniquefilts:
filterstrategy[f] = len(exposure_filter[exposure_filter == f])
print 'filter strategy dictionary ', filterstrategy
starttime = time.time()
pptime = time.time()
keepgoing = True
index = -1
submission_counter = 0
maxsub = 10000
postprocessingtime = 10 #every half hour fire off Tim's code for post-processing
while keepgoing:
#os.system('kinit -k -t /var/keytab/desgw.keytab desgw/des/[email protected]')
index += 1
newfireds = []
if time.time() - starttime > 50000:
keepgoing = False
continue
ofile = open(os.path.join(self.triggerdir, 'latestquery.txt'), 'w')
ofile.write(
"--------------------------------------------------------------------------------------------------\n"
)
ofile.write(
"EXPNUM\tNITE\tBAND\tEXPTIME\tRADEG\t DECDEG\tPROPID\tOBJECT\n"
)
ofile.write(
"--------------------------------------------------------------------------------------------------\n"
)
print "--------------------------------------------------------------------------------------------------"
print "EXPNUM\tNITE\tBAND\tEXPTIME\tRADEG\t DECDEG\tPROPID\tOBJECT"
print "--------------------------------------------------------------------------------------------------"
query = "SELECT expnum,nite,band,exptime,radeg,decdeg,propid,object FROM prod.exposure@desoper WHERE " \
"expnum > 475900 and propid=" + propid + "and obstype='object' ORDER BY expnum" # latest
self.cursor.execute(query)
for s in self.cursor:
ofile.write(
str(s[0]) + "\t" + str(s[1]) + "\t" + str(s[2]) + "\t" +
str(s[3]) + "\t" + str(s[4]) + "\t" + str(s[5]) + "\t" +
str(s[6]) + "\t" + str(s[7]) + '\n')
print str(s[0]) + "\t" + str(s[1]) + "\t" + str(
s[2]) + "\t" + str(s[3]) + "\t" + str(s[4]) + "\t" + str(
s[5]) + "\t" + str(s[6]) + "\t" + str(s[7])
if not 'DESGW' in str(s[7]): continue
#print 'exptime',float(s[3])
if not float(s[3]) > 29.:
continue #exposure must be longer than 30 seconds
expnum = str(s[0])
nite = str(s[1])
band = str(s[2])
exptime = str(s[3])
#FIRST CHECK HERE THAT THE EXPOSURE NUMBER ISNT ALREADY IN THE DATABASE
try:
exposure = self.backend.get(exposures, {'expnum': expnum})
print 'Found this exposure in desgw database...'
# print exposure.attributes
# if expnum == 506432:
# sys.exit()
# self.backend.delete(exposure)
# self.backend.commit()
# exposure = self.backend.get(exposures, {'expnum': expnum})
except exposures.DoesNotExist: # add to database
#runProcessingIfNotAlready(image,self.backend)
print './diffimg-proc/getTiling.sh ' + expnum
res = os.popen('./diffimg-proc/getTiling.sh ' +
expnum).readlines()
print res
#sys.exit()
field, tiling = res[-2], res[-1]
#print 'field_tiling',field_tiling
hexnite = field.strip() + '_' + tiling.strip() + '_' + str(
nite)
#print hexnite
#sys.exit()
#print 'hexnite',hexnite
print 'Creating exposure in database...', hexnite
#raw_input()
if '--' in hexnite:
print 'found bad example'
#raw_input()
exposure = exposures({
'expnum': expnum,
'nite': nite,
'field': field,
'tiling': tiling,
'hexnite': hexnite,
'band': band,
'jobid': np.nan,
'exptime': exptime,
'status': 'Awaiting additional exposures',
'triggerid': self.trigger_id,
'object': str(s[7])
})
self.backend.save(exposure)
self.backend.commit()
hexnite = exposure.hexnite
print 'hexnite', hexnite
if '--' in hexnite:
print exposure.attributes
#raw_input()
#raw_input()
#sys.exit()
try:
hex = self.backend.get(hexes, {'hexnite': hexnite})
#self.backend.delete(hex)
#self.backend.commit()
#hex = self.backend.get(hexes, {'hexnite': hexnite})
#print 'Found this hex in desgw database...'
except hexes.DoesNotExist:
hex = hexes({
'hexnite':
hexnite,
'strategy':
self.strategy['exposure_filter'],
'num_target_g':
len(exposure_filter[exposure_filter == 'g']),
'num_target_r':
len(exposure_filter[exposure_filter == 'r']),
'num_target_i':
len(exposure_filter[exposure_filter == 'i']),
'num_target_z':
len(exposure_filter[exposure_filter == 'z']),
'observed_g': [],
'observed_r': [],
'observed_i': [],
'observed_z': [],
'exposures': [],
'status':
'Awaiting additional exposures',
'dagfile':
None,
})
self.backend.save(hex)
self.backend.commit()
print hex.attributes
print 'created new hex'
#raw_input()
if hex.status == 'Submitted for processing':
print 'This hex has already been submitted for processing'
continue
# if '--' in hexnite:
# print hex.attributes
# raw_input()
# if hex.status == 'Submitted for processing':
# print 'Hex ',hexnite,' band',band,'exposure',expnum,'has already been submitted for processing'
# #raw_input()
# continue
if band == 'g':
if not expnum in hex.observed_g:
hex.observed_g.append(expnum)
hex.exposures.append(expnum)
if band == 'r':
if not expnum in hex.observed_r:
hex.observed_r.append(expnum)
hex.exposures.append(expnum)
if band == 'i':
if not expnum in hex.observed_i:
hex.observed_i.append(expnum)
hex.exposures.append(expnum)
if band == 'z':
if not expnum in hex.observed_z:
hex.observed_z.append(expnum)
hex.exposures.append(expnum)
self.backend.save(hex)
self.backend.commit()
print hex.attributes
didwork = False
if len(hex.observed_g) == hex.num_target_g:
if len(hex.observed_r) == hex.num_target_r:
if len(hex.observed_i) == hex.num_target_i:
if len(hex.observed_z) == hex.num_target_z:
print 'All exposures in strategy satisfied! '
#raw_input()
submissionPassed = True
for target, exps in zip([
hex.num_target_g, hex.num_target_r,
hex.num_target_i, hex.num_target_z
], [
hex.observed_g, hex.observed_r,
hex.observed_i, hex.observed_z
]):
if target == 0: continue
exposurestring = ''
logstring = ''
for ex in exps:
exposurestring += ex + ' '
logstring += ex + '_'
print 'cd diffimg-proc; source DAGMaker.sh ' + exposurestring
os.chdir("diffimg-proc")
#out = os.popen('ls').read()
out = os.popen('./DAGMaker.sh ' +
exposurestring).read()
os.chdir("..")
print out
f = open(
os.path.join(
self.processingdir,
logstring + hexnite + '.log'), 'w')
f.write(out)
f.close()
tt = time.time()
if not 'To submit this DAG do' in out:
dt.sendEmailSubject(
self.trigger_id,
'Error in creating dag for desgw hex: '
+ out)
submissionPassed = False
else:
for o in out.split('\n'):
if 'file://' in o:
dagfile = o.split('/')[-1]
self.dagfile = os.path.join(
self.processingdir,
logstring + 'job.dag')
os.system('cp diffimg-proc/' +
dagfile + ' ' +
self.dagfile)
print self.dagfile
print 'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; jobsub_submit_dag -G des --role=DESGW file://' + self.dagfile
out = os.popen(
'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; '
'jobsub_submit_dag -G des --role=DESGW file://'
+ self.dagfile).read()
print out
if 'non-zero exit status' in out:
dt.sendEmailSubject(
self.trigger_id,
'Error in submitting hex dag for processing: '
+ out)
submissionPassed = False
else:
if doimmediateremove:
for o in out.split('\n'):
if 'Use job id' in o:
jobid = o.split()[3]
out = os.popen(
'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; '
'jobsub_rm --jobid=' + jobid +
' --group=des --role=DESGW'
).read()
print out
ttt = time.time()
#print 'TOTAL JOBSUB FOR A SINGLE DESGW IMAGE',ttt-tt
#sys.exit()
#raw_input()
if submissionPassed:
hex.status = 'Submitted for processing'
hex.dagfile = self.dagfile
self.backend.save(hex)
self.backend.commit()
for expn in hex.exposures:
print expn, 'updated in database to Submitted For Processing'
exp = self.backend.get(
exposures, {'expnum': expn})
exp.status = 'Submitted for processing'
self.backend.save(exp)
self.backend.commit()
didwork = True
print 'didwork', didwork
print 'dagfile', self.dagfile
#raw_input()
if not didwork:
print 'Could not find all images in strategy for this hex... Added hex', hexnite,' to database ' \
'and will continue waiting...'
#raw_input()
#HERE YOU NEED TO ADD TO HEXSTRATEGYDICT DATABASE
if time.time(
) - pptime > postprocessingtime: #happens every 30 minutes or so...
pptime = time.time()
print '***** Firing post processing script *****'
#sys.exit()
self.submit_post_processing()
#sys.exit()
print 'Waiting 10s to check from mountain...'
#sys.exit()
time.sleep(10) #looping over checking the mountain top
# cfiles = os.listdir(os.path.join(trigger_path,trigger_id,'candidates'))
# for f in cfiles:
# if f.split('.')[-1] == 'npz':
# cp.makeNewPage(f)
def submit_post_processing(self):
firedlist = open('./processing/firedlist.txt', 'r')
fl = firedlist.readlines()
firedlist.close()
print fl
fl = ['475914', '475915', '475916', '482859', '482860', '482861']
expnumlist = ''
for f in fl:
expnumlist += f.strip() + ' '
print 'FIRING TIMs CODE'
gwpostdir = os.environ['GWPOST_DIR']
print 'source ' + os.path.join(gwpostdir, 'diffimg_setup.sh') + '; \
python ' +os.path.join(gwpostdir,'postproc.py')\
+' --expnums ' + expnumlist\
+ ' --outputdir ' + os.path.join(self.trigger_path,self.trigger_id,'candidates')\
+ ' --triggerid '+self.trigger_id+' --season 46 --ups True'
# os.system('source ' + os.path.join(gwpostdir, 'diffimg_setup.sh') + '; \
# python '+os.path.join(gwpostdir,'postproc.py')\
# +' --expnums ' + expnumlist\
# + ' --outputdir ' + os.path.join(trigger_path,trigger_id,'candidates')\
# + ' --triggerid '+trigger_id+' --season 46 --ups True' )
#pid = os.spawnlp(os.P_WAIT, "source", os.path.join(gwpostdir, 'diffimg_setup.sh'))
args = ['ssh -t [email protected] "source '+ os.path.join(gwpostdir, 'mi_setup.sh')+'; '+
'yes | python '+os.path.join(gwpostdir,'postproc.py')\
+' --expnums ' + expnumlist\
+ ' --outputdir ' + os.path.join(self.trigger_path,self.trigger_id,'candidates')\
+ ' --triggerid '+self.trigger_id+' --season 46 --ups True"'
]
print args
#p = subprocess.Popen(args,stdout=PIPE, stderr=PIPE,shell=True)
#print p.communicate()
#p = subprocess.Popen(args,stdin=None, stdout=None, stderr=None, close_fds=True,shell=True)
return
def getDatetimeOfFirstJson(self, jsonstring):
js = jsonstring.split('UTC')[1] #-2015-12-27-3:2:00.json
#date_object = dt.strptime(js, '-%Y-%m-%d-%H_%M_%S.json')
date_object = dt.strptime(js, '-%Y-%m-%d-%H_%M_%S-test.json')
print '***** Datetime of first observation UTC', date_object, '*****'
return date_object
def sortHexes(self):
pass
class VisinumDatabase:
def __init__(self,
dbpath,
name=VisinumDB_name,
DbDocClass=DbDocFileMetadata): # , attrs={}, **kwargs
self.DbDocClass = DbDocClass
self.name = name
self.open_database(
dbpath) # sets self.dbpath and self.db as reference to database
def open_database(self, dbpath): # (self, dbpath, attrs={}, **kwargs)
if os.path.basename(dbpath) == self.name:
self.dbpath = dbpath # opening existing Visinum database
elif os.path.isdir(os.path.join(dbpath, self.name)):
self.dbpath = os.path.join(dbpath,
self.name) # opening existing database
logger.info("Found Visinum database %s in directory %s" %
(self.name, dbpath))
elif os.path.isdir(dbpath):
self.dbpath = os.path.join(dbpath, self.name)
logger.info("Creating new Visinum database %s in directory %s" %
(self.name, dbpath))
else:
logger.error("Database path (dbpath) incorrectly specified %s" %
(dbpath, ))
raise ValueError(
"Database path (dbpath) incorrectly specified %s" % (dbpath, ))
self.db = FileBackend(self.dbpath, {'serializer_class': 'json'})
logger.info("Opening Visinum database %s" % (self.dbpath, ))
config_attrs = self.get_config()
"""try:
#config_attrs = self.db.get(self.DbDocClass, {'visinum_type' : 'db_config'})
config_attrs = self.get_config()
except self.DbDocClass.DoesNotExist:
self.set_config( {'visinum_type' : 'db_config',
'title' : os.path.basename( os.path.dirname(self.dbpath) ),
'path_orig' : self.dbpath,
'UUID': make_uuid(version=3, namespace='uuid.NAMESPACE_URL', name=self.dbpath)
} )
#config_attrs.update(attrs)
#self.create_new_item(config_attrs, config_attrs, **kwargs )"""
self.config_attrs = config_attrs
def get_config(self, attr=None):
try:
dbitem = self.db.get(self.DbDocClass,
{'visinum_type': 'db_config'})
config_attrs = dbitem.attributes
except self.DbDocClass.DoesNotExist:
try:
config_UUID = make_uuid(version=3,
namespace='uuid.NAMESPACE_URL',
name=self.dbpath)
dbitem = self.db.get(self.DbDocClass, {'UUID': config_UUID})
config_attrs = dbitem.attributes
except self.DbDocClass.DoesNotExist:
# cannot find db configuration, setup (reset) new configuration
config_attrs = {
'visinum_type':
'db_config',
'title':
os.path.basename(os.path.dirname(self.dbpath)),
'UUID':
make_uuid(version=3,
namespace='uuid.NAMESPACE_URL',
name=self.dbpath),
'path_orig':
self.dbpath
}
self.set_config(config_attrs, reset=True)
logger.warning(
"Cannot find db configuration, re-setting configuration %s"
% (config_attrs['UUID'], ))
if attr: # if attr and attr in config_attrs:
return config_attrs[attr]
return config_attrs
def set_config(self, attrs={}, reset=False, **kwargs):
if reset:
config_attrs = {}
else:
config_attrs = self.get_config()
config_attrs.update(attrs)
for k, v in kwargs.items():
config_attrs[k] = v
UUID = self.set_dbitem(config_attrs)
self.config_attrs = config_attrs
return UUID
def extract_file_metadata(self, dirpath, update=True):
rootNode = make_file_system_tree(dirpath, excludedirs=[self.name])
"""UUID = self.save_item(rootNode.to_vndict(), {'visinum_type':'datatree',
'name':'datatree ' + rootNode.name,
'visinum_datatree':'file_system',
'visinum_node':rootNode.__class__.__name__,
'fpath':dirpath})"""
rootNode.__class__.metadata_to_dict = node_metadata_to_dict
for node in list(rootNode):
mdata = {}
if update:
try:
dbdoc = self.get_dbitem(node._UUID)
mdata.update(dbdoc.attributes)
del dbdoc
except:
pass
mdata.update(node.metadata_to_dict(self.db, dirpath))
dbdoc = self.DbDocClass(mdata)
dbdoc.save(self.db)
logger.info('Metadata extracted from %s' %
(node.get_path(dirpath), ))
logger.info("Completed processing data tree %s " % (rootNode.name, ))
self.db.commit()
for node in list(rootNode):
dbdoc = self.db.get(self.DbDocClass, {'UUID': node._UUID})
if node is rootNode:
dbdoc.parent = None
#dbdoc.visinum_type = 'datatree'
dbdoc.visinum_datatree = 'file_system'
#self.set_config(filesystemtreeroot=rootNode._UUID)
self.set_config(filesystemtreeroot=rootNode._UUID
) # filesystemtreeroot=dbdoc
##self.datatreelist.append( ('file system', rootNode._UUID) )
else:
#dbdoc.parent = node.parent.UUID
dbdoc.visinum_parent = self.db.get(self.DbDocClass,
{'UUID': node.parent._UUID})
dbdoc.visinum_node = node.__class__.__name__
dbdoc.visinum_childs = [
] # database will not accept _childs as a doc attribute
for child in node._childs:
#_childs.append(child.UUID)
dbdoc.visinum_childs.append(
self.db.get(self.DbDocClass, {'UUID': child._UUID}))
dbdoc.visinum_nchilds = len(node._childs)
dbdoc.save(self.db)
self.db.commit()
logger.info("Metadata committed to DB %s" % (rootNode.name, ))
def set_dbitem(self, attrs={}, commit=True, **kwargs):
for k, v in kwargs.items():
attrs[k] = v
if not 'UUID' in attrs:
attrs['UUID'] = make_uuid()
dbItem = self.DbDocClass(attrs)
dbItem.save(self.db)
if commit:
self.db.commit()
return attrs['UUID']
def get_dbitem(self, attrs={}, DbDocClass=None):
if not DbDocClass:
DbDocClass = self.DbDocClass
if isinstance(attrs, str):
attrs = {'UUID': attrs}
return self.db.get(DbDocClass, attrs)
def filter(self, query, DbDocClass=None):
if not DbDocClass:
DbDocClass = self.DbDocClass
return self.db.filter(DbDocClass, query)
def tree_from_db(self, dbitem, root_node=None, parent=None):
#dbitem = self.get_dbitem(rootUUID)
metadata = dbitem.attributes # {k:v for k, v in dict_.items() if k != "_childs"}
node = FsNode(metadata['path'], parent=parent, metadata=metadata)
if not root_node:
root_node = node
if 'visinum_childs' in metadata:
for child in metadata['visinum_childs']:
self.tree_from_db(child, root_node, node)
return root_node
backend = FileBackend(".db")
class User(Document):
pass
with open('phrases.json') as f:
phrases = json.load(f)
users = backend.filter(User,{'twitted':False})
auth = tweepy.OAuthHandler(secrets.CONSUMER_KEY, secrets.CONSUMER_SECRET)
auth.set_access_token(secrets.ACCES_TOKEN, secrets.ACCES_TOKEN_SECRET)
api = tweepy.API(auth)
for startup in users:
user = startup.get('user').split('?')[0]
print user
if len(user) > 1:
phrase = random.choice(phrases)
print phrase % user
api.update_status(phrase % user)
startup.twitted = True
startup.save(backend)
else:
startup.twitted = True
startup.save(backend)
backend.commit()
print 'Tweeted, now sleeping'
time.sleep(random.randint(60, 600))
class BlitzDB(Database):
key = 'pk'
_database_type = 'blitzdb'
class Data(Document):
pass
class Cutout(Document):
pass
class Fingerprint(Document):
pass
class Similarity(Document):
pass
def _get_table(self, table_name):
if table_name == 'data':
return BlitzDB.Data
elif table_name == 'cutout':
return BlitzDB.Cutout
elif table_name == 'fingerprint':
return BlitzDB.Fingerprint
elif table_name == 'similarity':
return BlitzDB.Similarity
else:
log.error('BAD TABLE NAME {}'.format(table_name))
def __init__(self, filename):
self._filename = filename
self._backend = FileBackend(self._filename)
def save(self, table, data):
# Convert to dict if not one already
if not isinstance(data, dict):
data = data.save()
blitz_table = self._get_table(table)
data.update({'pk': data['uuid']})
save_id = self._backend.save(blitz_table(data))
self._backend.commit()
return save_id['pk']
def find(self, table, key=None):
blitz_table = self._get_table(table)
factory = get_factory(table)
if key is None:
return [
factory(dict(x), db=self)
for x in self._backend.filter(blitz_table, {})
]
elif isinstance(key, list):
return [
factory(dict(x), db=self) for x in self._backend.filter(
blitz_table, {'pk': {
'$in': key
}})
]
else:
return factory(dict(self._backend.get(blitz_table, {'pk': key})),
db=self)
def count(self, table):
blitz_table = self._get_table(table)
return len(self._backend.filter(blitz_table, {}))
def update(self, table, key, data):
blitz_table = self._get_table(table)
entry = self._backend.get(blitz_table, {'pk': key})
for k, v in data.items():
setattr(entry, k, v)
entry.save()
self._backend.commit()
def close(self):
pass
def delete_database(self):
shutil.rmtree(self._filename)
def create_app(configfile=None):
app = Flask(__name__)
AppConfig(app, configfile) # Flask-Appconfig is not necessary, but
# highly recommend =)
# https://github.com/mbr/flask-appconfig
Bootstrap(app)
login_manager = LoginManager()
login_manager.init_app(app)
login_manager.login_view='login'
#NoSQL Backend
backend = FileBackend("/tmp/wakeonlan.db")
backend.create_index(Device, fields={'id':1}, unique=True)
#TEST Devices
alldevices = backend.filter(Device, {})
if len(alldevices) == 0 :
try:
pc1 = Device({"id" : "001122334411", "name" : "PC 1", "mac" : "00:11:22:33:44:11", "ip":"192.168.222.111", 'status' : ''})
backend.save(pc1)
pc2 = Device({"id" : "001122334422","name" : "PC 2", "mac" : "00:11:22:33:44:22", "ip":"192.168.222.112", 'status' : ''})
backend.save(pc2)
pc3 = Device({"id" : "001122334433","name" : "Router", "mac" : "00:11:22:33:44:33", "ip":"192.168.222.1", 'status' : ''})
backend.save(pc3)
backend.commit()
except:
backend.revert()
pass
# in a real app, these should be configured through Flask-Appconfig
app.config['SECRET_KEY'] = 'devkey'
# app.config['RECAPTCHA_PUBLIC_KEY'] = \
# '6Lfol9cSAAAAADAkodaYl9wvQCwBMr3qGR_PPHcw'
def getDeviceById(id):
device = None
try:
device = backend.get(Device, {'id':id})
except:
pass
return device
def pingDeviceById(id):
#Get Device
device = backend.get(Device, {'id':id})
if device:
#Get Device's IP
ip = device['ip']
result = pingDeviceByIp(ip)
#Update Status UP/Down/''
if result==0:
device['status'] = 'UP'
else:
device['status'] = 'DOWN'
backend.save(device)
return result
return None
def wolDeviceById(id):
#Get Device
device = backend.get(Device, {'id':id})
if device:
#WoL for Device MAC
mac = device['mac']
wolDeviceByMac(mac)
return None
@login_manager.user_loader
def user_loader(user_id):
"""Given *user_id*, return the associated User object.
:param unicode user_id: user_id (email) user to retrieve
"""
user_entry = User.getById(user_id)
if user_entry is not None:
user = User(user_entry[0], user_entry[1])
return user
else:
return None
@app.route('/', methods=('GET', 'POST'))
@login_required
def index():
form = ExampleForm()
form.validate_on_submit() # to get error messages to the browser
# flash('critical message', 'critical')
# flash('error message', 'error')
# flash('warning message', 'warning')
# flash('info message', 'info')
# flash('debug message', 'debug')
# flash('different message', 'different')
# flash('uncategorized message')
alldevices = None
alldevices = backend.filter(Device, {}).sort('name')
#app.logger.info('Devices: %s' % (len(alldevices) ) )
return render_template('index.html', form=form, devices = alldevices)
@app.route('/login', methods=('GET', 'POST'))
def login():
if request.method == 'GET':
form = LoginForm()
form.validate_on_submit() # to get error messages to the browser
return render_template('login.html', form=form)
username = request.form['username']
password = request.form['password']
user_entry = User.get(username, password)
if user_entry is None:
flash('Username or Passord is invalid', 'error')
return redirect(url_for('login'))
user = User(user_entry[0], user_entry[1])
login_user(user, remember=True)
return redirect(request.args.get('next') or url_for('index'))
@app.route("/logout", methods=["GET"])
@login_required
def logout():
"""Logout the current user."""
user = current_user
user.authenticated = False
logout_user()
return redirect(url_for('login'))
@app.route('/addDevice', methods=('GET', 'POST'))
@login_required
def addDevice():
if request.method == 'GET':
form = AddDeviceForm()
form.validate_on_submit() # to get error messages to the browser
return render_template('add_device.html', form=form)
name = request.form['name']
mac = request.form['mac']
ip = request.form['ip']
id = mac.replace(':','')
try:
newDevice = Device({"id" : id, "name" : name, "mac" : mac, "ip":ip, 'status' : ''})
backend.save(newDevice)
backend.commit()
except:
flash('Error creating new Device', 'error')
pass
return redirect(url_for('index'))
@app.route('/editListDevice', methods=('GET', 'POST'))
@login_required
def editListDevice():
alldevices = None
alldevices = backend.filter(Device, {}).sort('name')
return render_template('list_device.html', devices = alldevices)
@app.route('/pingDevice/<deviceId>', methods=('GET', 'POST'))
@login_required
def pingDevice(deviceId):
app.logger.info('pingDevice: %s' % (deviceId ) )
device = getDeviceById(deviceId)
result = pingDeviceById(deviceId)
app.logger.info('pingDevice: %s' % (result ) )
if result is None:
flash('Ping - Error on device %s' % (device['name']), 'error')
elif result == 0:
flash('Device %s is UP' % (device['name']), 'info')
else:
flash('Device %s is DOWN' % (device['name']), 'error')
return redirect(url_for('index'))
@app.route('/wolDevice/<deviceId>', methods=('GET', 'POST'))
@login_required
def wolDevice(deviceId):
app.logger.info('wolDevice: %s' % (deviceId ) )
device = getDeviceById(deviceId)
result = wolDeviceById(deviceId)
if device:
flash('WoL sent to %s' % (device['name']), 'info')
else:
flash('WoL error', 'error')
return redirect(url_for('index'))
@app.route('/deleteDevice/<deviceId>', methods=('GET', 'POST'))
@login_required
def deleteDevice(deviceId):
app.logger.info('wolDevice: %s' % (deviceId ) )
device = getDeviceById(deviceId)
try:
backend.delete(device)
backend.commit()
flash('%s Deleted' % (device['name']), 'info')
except:
flash('Delete error', 'error')
pass
return redirect(url_for('editListDevice'))
return app
class Profiles(object):
def __init__(self, bot):
self.bot = bot
self.cfg = PluginConfig(self)
self.db = FileBackend(self.cfg.get('main_db'))
mtt = MessageRetargeter(bot)
self.msg = mtt.msg
web = Flask(__name__, template_folder=tmpl_dir)
mako = MakoTemplates()
mako.init_app(web)
# Add routes here
web.add_url_rule('/edit_web/<args>', 'edit_web', self.edit_web, methods=['GET', 'POST'])
_thread.start_new_thread(web.run, (), {'host': '0.0.0.0'})
@command
def learn(self, mask, target, args):
"""
Stores information allowing for later retrieval. Names are downcased for sanity.
Usage:
%%learn <name> <information>...
"""
name = args['<name>'].lower()
info = ' '.join(args['<information>'])
try:
profile = self.db.get(Profile, {'name': name})
except Profile.DoesNotExist:
profile = Profile(
{
'name': name,
'owner': mask.nick.lower(),
'lines': [info],
'random': False,
'public': False
}
)
profile.save(self.db)
self.db.commit()
self.msg(mask, target, 'Your data "%s" has been stored.' % name)
return
except Profile.MultipleDocumentsReturned:
self.msg(mask, target, "Found more than one %s. This is bad! Please notify the bot owner." % name)
return
if is_allowed_to(Action.edit, mask.nick, profile):
lines_to_append = profile.lines
lines_to_append.append(info)
profile.save(self.db)
self.db.commit()
self.msg(mask, target, 'Your data "%s" has been updated.' % name)
return
else:
self.msg(mask, target, 'You are not authorized to edit "%s". Ask %s instead.'
% (name, profile.owner))
return
@command
def query(self, mask, target, args):
"""
Retrieve the information associated with <name>. If the item is marked random, then one random item will be
returned.
Usage:
%%query <name>
?? <name>
"""
name = args['<name>'].lower()
try:
profile = self.db.get(Profile, {'name': name})
except Profile.DoesNotExist:
self.msg(mask, target, 'I cannot find "%s" in the records.' % name)
return
if profile.random:
self.msg(mask, target, get_flags(profile) + random.choice(profile.lines))
else:
for line in profile.lines:
self.msg(mask, target, get_flags(profile) + line)
if len(profile.lines) >= int(self.cfg.get('throttle_max')):
sleep(int(self.cfg.get('throttle_time')))
@command
def forget(self, mask, target, args):
"""
Delete <name> from the records. Only the person who created the item can remove it.
Usage:
%%forget <name>
"""
name = args['<name>'].lower()
try:
profile = self.db.get(Profile, {'name': name})
except Profile.DoesNotExist:
self.msg(mask, target, 'I cannot find "%s" in the records.' % name)
return
if is_allowed_to(Action.delete, mask.nick, profile):
self.db.delete(profile)
self.db.commit()
self.msg(mask, target, "%s has been deleted." % name)
else:
self.msg(mask, target, 'You are not authorized to delete "%s". Ask %s instead.'
% (name, profile.owner))
@command(permission='admin', show_in_help_list=False)
def rmf(self, mask, target, args):
"""
Delete <name> from the records without checking permissions.
Usage:
%%rmf <name>
"""
name = args['<name>'].lower()
try:
profile = self.db.get(Profile, {'name': name})
except Profile.DoesNotExist:
self.msg(mask, target, 'I cannot find "%s" in the records.' % name)
return
self.db.delete(profile)
self.db.commit()
self.msg(mask, target, "%s has been deleted." % name)
@command(permission='admin', show_in_help_list=False)
def chown(self, mask, target, args):
"""
Change the owner of <name> to <newowner>.
Usage:
%%chown <name> <newowner>
"""
name = args['<name>'].lower()
newowner = args['<newowner>'].lower()
try:
profile = self.db.get(Profile, {'name': name})
except Profile.DoesNotExist:
self.msg(mask, target, 'I cannot find "%s" in the records.' % name)
return
profile.owner = newowner
self.db.save(profile)
self.db.commit()
self.msg(mask, target, "%s is now owned by %s." % (name, newowner))
@command
def toggle_public(self, mask, target, args):
"""
Changes whether <name> is publicly editable or not
Usage:
%%toggle_public <name>
"""
profile = args['<name>'].lower()
try:
profile = self.db.get(Profile, {'name': profile})
except Profile.DoesNotExist:
self.msg(mask, target, 'I cannot find "%s" in the records.' % profile)
return
if is_allowed_to(Action.edit, mask.nick, profile):
if profile.public:
profile.public = False
self.msg(mask, target, '"%s" is no longer publicly editable.' % profile)
else:
profile.public = True
self.msg(mask, target, '"%s" is now publicly editable.' % profile)
self.db.save(profile)
self.db.commit()
return
else:
self.msg(mask, target, 'You are not authorized to edit "%s". Ask %s instead.'
% (profile, profile.owner))
return
@command
def toggle_random(self, mask, target, args):
"""
Toggle the randomness of an item, so that it shows a single random line instead of all lines when queried.
Usage:
%%toggle_random <name>
"""
name = args['<name>'].lower()
try:
profile = self.db.get(Profile, {'name': name})
except Profile.DoesNotExist:
self.msg(mask, target, 'I cannot find "%s" in the records.' % name)
return
if is_allowed_to(Action.edit, mask.nick, profile):
profile.random = not profile.random
self.msg(mask, target, 'Random mode for %s is set to: %s' % (profile.name, profile.random))
profile.save(self.db)
self.db.commit()
else:
self.msg(mask, target, 'You are not authorized to edit "%s". Ask %s instead.'
% (name, profile.owner))
irc3.base.logging.log(irc3.base.logging.WARN,
"%s tried to edit %s, but can't since it's owned by %s" %
(mask.nick, profile.name, profile.owner)
)
@event("(@(?P<tags>\S+) )?:(?P<mask>\S+) PRIVMSG (?P<target>\S+) :\?\? (?P<data>.*)")
def easy_query(self, mask, target, data):
self.bot.get_plugin(Commands).on_command(cmd='query', mask=mask, target=target, data=data)
####
# All web stuff below this point
#
@command
def edit(self, mask, target, args):
"""
Sends you a webpage link to edit <name>. Great for longer profiles. Make sure to keep the URL you are given
secure, as with it, anyone can edit your profiles.
Usage:
%%edit <name>
"""
# TODO: Clear any existing sessions the user has
data = {
'id': str(uuid.uuid4()),
'name': mask.nick,
'profile': args['<name>']
}
name = args['<name>'].lower()
try:
profile = self.db.get(Profile, {'name': name})
except Profile.DoesNotExist:
self.msg(mask, target, 'I cannot find "%s" in the records.' % name)
return
if is_allowed_to(Action.fulledit, mask.nick, profile):
newses = Session(data)
self.db.save(newses)
self.db.commit()
self.bot.privmsg(mask.nick,
"An editor has been set up for you at http://skaianet.tkware.us:5000/edit_web/%s" % str(
data['id']))
self.bot.privmsg(mask.nick,
"Be very careful not to expose this address - with it, anyone can edit your stuff")
else:
self.msg(mask, target, 'You are not authorized to webedit "%s". Ask %s instead.'
% (name, profile.owner))
def edit_web(self, args):
# Web endpoint: /edit_web/<args>
if request.method == 'GET':
# Does the session exist?
try:
edit_session = self.db.get(Session, {'id': args})
except Session.DoesNotExist:
return render_template('youfail.html',
bot=self.bot,
failreason='Invalid Session',
userfail=True)
# Does the profile exist?
name = edit_session.profile
try:
profile = self.db.get(Profile, {'name': name.lower()})
except Profile.DoesNotExist:
return render_template('youfail.html',
bot=self.bot,
failreason='I cannot find "%s" in the records.' % name
)
# Kick off to the edit page!
return render_template('edit.html',
bot=self.bot,
profile=profile,
username=edit_session.name,
sessionid=edit_session.id
)
elif request.method == 'POST':
# We have to look up the session ID one more time. Something could have happened to the profile
# since we created the session.
try:
edit_session = self.db.get(Session, {'id': request.form['ID']})
except Session.DoesNotExist:
return render_template('youfail.html',
bot=self.bot,
failreason='Invalid Session',
userfail=True)
name = request.form['profile']
try:
profile = self.db.get(Profile, {'name': request.form['profile']})
except Profile.DoesNotExist:
return render_template('youfail.html',
bot=self.bot,
failreason='I cannot find "%s" in the records.' % name,
userfail=True
)
# Now with the profile in hand, blank the lines field and rebuild it from the form.
# Here we grab all numeric items from the submission, sort it, and one by one refill the DB object.
lines = [item for item in request.form if item.isdigit()]
lines.sort()
profile.lines = []
for item in lines:
profile.lines.append(request.form[item])
self.db.save(profile)
self.db.delete(edit_session)
self.db.commit()
return render_template('done.html',
bot=self.bot,
profile=profile.name
)
def create_app(configfile=None):
app = Flask(__name__)
AppConfig(app, configfile) # Flask-Appconfig is not necessary, but
# highly recommend =)
# https://github.com/mbr/flask-appconfig
Bootstrap(app)
login_manager = LoginManager()
login_manager.init_app(app)
login_manager.login_view = 'login'
#NoSQL Backend
backend = FileBackend("/tmp/wakeonlan.db")
backend.create_index(Device, fields={'id': 1}, unique=True)
#TEST Devices
alldevices = backend.filter(Device, {})
if len(alldevices) == 0:
try:
pc1 = Device({
"id": "001122334411",
"name": "PC 1",
"mac": "00:11:22:33:44:11",
"ip": "192.168.222.111",
'status': ''
})
backend.save(pc1)
pc2 = Device({
"id": "001122334422",
"name": "PC 2",
"mac": "00:11:22:33:44:22",
"ip": "192.168.222.112",
'status': ''
})
backend.save(pc2)
pc3 = Device({
"id": "001122334433",
"name": "Router",
"mac": "00:11:22:33:44:33",
"ip": "192.168.222.1",
'status': ''
})
backend.save(pc3)
backend.commit()
except:
backend.revert()
pass
# in a real app, these should be configured through Flask-Appconfig
app.config['SECRET_KEY'] = 'devkey'
# app.config['RECAPTCHA_PUBLIC_KEY'] = \
# '6Lfol9cSAAAAADAkodaYl9wvQCwBMr3qGR_PPHcw'
def getDeviceById(id):
device = None
try:
device = backend.get(Device, {'id': id})
except:
pass
return device
def pingDeviceById(id):
#Get Device
device = backend.get(Device, {'id': id})
if device:
#Get Device's IP
ip = device['ip']
result = pingDeviceByIp(ip)
#Update Status UP/Down/''
if result == 0:
device['status'] = 'UP'
else:
device['status'] = 'DOWN'
backend.save(device)
return result
return None
def wolDeviceById(id):
#Get Device
device = backend.get(Device, {'id': id})
if device:
#WoL for Device MAC
mac = device['mac']
wolDeviceByMac(mac)
return None
@login_manager.user_loader
def user_loader(user_id):
"""Given *user_id*, return the associated User object.
:param unicode user_id: user_id (email) user to retrieve
"""
user_entry = User.getById(user_id)
if user_entry is not None:
user = User(user_entry[0], user_entry[1])
return user
else:
return None
@app.route('/', methods=('GET', 'POST'))
@login_required
def index():
form = ExampleForm()
form.validate_on_submit() # to get error messages to the browser
# flash('critical message', 'critical')
# flash('error message', 'error')
# flash('warning message', 'warning')
# flash('info message', 'info')
# flash('debug message', 'debug')
# flash('different message', 'different')
# flash('uncategorized message')
alldevices = None
alldevices = backend.filter(Device, {}).sort('name')
#app.logger.info('Devices: %s' % (len(alldevices) ) )
return render_template('index.html', form=form, devices=alldevices)
@app.route('/login', methods=('GET', 'POST'))
def login():
if request.method == 'GET':
form = LoginForm()
form.validate_on_submit() # to get error messages to the browser
return render_template('login.html', form=form)
username = request.form['username']
password = request.form['password']
user_entry = User.get(username, password)
if user_entry is None:
flash('Username or Passord is invalid', 'error')
return redirect(url_for('login'))
user = User(user_entry[0], user_entry[1])
login_user(user, remember=True)
return redirect(request.args.get('next') or url_for('index'))
@app.route("/logout", methods=["GET"])
@login_required
def logout():
"""Logout the current user."""
user = current_user
user.authenticated = False
logout_user()
return redirect(url_for('login'))
@app.route('/addDevice', methods=('GET', 'POST'))
@login_required
def addDevice():
if request.method == 'GET':
form = AddDeviceForm()
form.validate_on_submit() # to get error messages to the browser
return render_template('add_device.html', form=form)
name = request.form['name']
mac = request.form['mac']
ip = request.form['ip']
id = mac.replace(':', '')
try:
newDevice = Device({
"id": id,
"name": name,
"mac": mac,
"ip": ip,
'status': ''
})
backend.save(newDevice)
backend.commit()
except:
flash('Error creating new Device', 'error')
pass
return redirect(url_for('index'))
@app.route('/editListDevice', methods=('GET', 'POST'))
@login_required
def editListDevice():
alldevices = None
alldevices = backend.filter(Device, {}).sort('name')
return render_template('list_device.html', devices=alldevices)
@app.route('/pingDevice/<deviceId>', methods=('GET', 'POST'))
@login_required
def pingDevice(deviceId):
app.logger.info('pingDevice: %s' % (deviceId))
device = getDeviceById(deviceId)
result = pingDeviceById(deviceId)
app.logger.info('pingDevice: %s' % (result))
if result is None:
flash('Ping - Error on device %s' % (device['name']), 'error')
elif result == 0:
flash('Device %s is UP' % (device['name']), 'info')
else:
flash('Device %s is DOWN' % (device['name']), 'error')
return redirect(url_for('index'))
@app.route('/wolDevice/<deviceId>', methods=('GET', 'POST'))
@login_required
def wolDevice(deviceId):
app.logger.info('wolDevice: %s' % (deviceId))
device = getDeviceById(deviceId)
result = wolDeviceById(deviceId)
if device:
flash('WoL sent to %s' % (device['name']), 'info')
else:
flash('WoL error', 'error')
return redirect(url_for('index'))
@app.route('/deleteDevice/<deviceId>', methods=('GET', 'POST'))
@login_required
def deleteDevice(deviceId):
app.logger.info('wolDevice: %s' % (deviceId))
device = getDeviceById(deviceId)
try:
backend.delete(device)
backend.commit()
flash('%s Deleted' % (device['name']), 'info')
except:
flash('Delete error', 'error')
pass
return redirect(url_for('editListDevice'))
return app
class Memos(object):
def __init__(self, bot):
self.bot = bot
self.cfg = PluginConfig(self)
self.db = FileBackend(self.cfg.get('main_db'))
mtt = MessageRetargeter(bot)
self.msg = mtt.msg
@command
def note(self, target, mask, args):
"""
Leaves a note for <name>, containing <text>. The next time I see <name> speak, I will deliver any notes they
have waiting.
Notes taken in private are delivered in private, and vice versa.
Usage:
%%note <name> <text>...
"""
if mask.is_channel:
pubmsg = True
else:
pubmsg = False
if args['<name>'] == self.bot.nick:
self.msg(mask, target, "You can't leave notes for me, silly :)")
return
newmemo = Memo(
{
'sender': target.nick.lower(),
'recipient': args['<name>'].lower(),
'public': pubmsg,
'timestamp': ctime(),
'text': ' '.join(args['<text>'])
}
)
newmemo.save(self.db)
self.db.commit()
confirmation_msg = "Your note for %s has been queued for delivery." % args['<name>']
self.msg(mask, target, confirmation_msg)
@irc3.event(irc3.rfc.PRIVMSG) # Triggered on every message anywhere.
def check_notes(self, target, mask, data, event):
del data, event
try:
msgs = self.db.filter(Memo, {'recipient': mask.nick.lower()})
msgword = "message" if len(msgs) < 2 else "messages" # Fix: I have 1 messages for you!
except Memo.DoesNotExist:
return
if len(msgs) == 0:
return
# Avoid telling people they have messages in public, if any of them are set public=False
if contains_private_messages(msgs):
self.msg(mask, mask.nick, "I have %s %s for you, %s!" % (len(msgs), msgword, mask.nick))
else:
self.msg(mask, target, "I have %s %s for you, %s!" % (len(msgs), msgword, mask.nick))
# Actually deliver the memos
for msg in msgs:
# This looks ridiculous but we don't care about the timezone really, only the relative time
# from the local system clock.
now = datetime.datetime.strptime(ctime(), "%a %b %d %H:%M:%S %Y")
reltime = humanize.naturaltime(now - datetime.datetime.strptime(msg.timestamp, "%a %b %d %H:%M:%S %Y"))
message_text = "%s // %s // %s" % (msg.sender, reltime, msg.text)
if msg.public:
self.msg(mask, target, message_text)
self.db.delete(msg)
else:
self.bot.privmsg(mask.nick, message_text)
self.db.delete(msg)
self.db.commit()
class eventmanager:
def __init__(self, trigger_id, jsonfilelist, triggerdir, datadir, real, trigger_path):
#os.system('kinit -k -t /var/keytab/desgw.keytab desgw/des/[email protected]')
tstart = time.time()
if real:
self.backend = FileBackend("./realdb")
else:
self.backend = FileBackend("./testdb")
try:
self.thisevent = self.backend.get(Trigger, {'id': trigger_id})
print 'Found this event in desgw database...'
except Trigger.DoesNotExist:
self.thisevent = Trigger({
'id':trigger_id,
'jsonfilelist':jsonfilelist,
'triggerpath':triggerdir,
'mapspath':datadir,
'jobids':[
(0,'jsonfile_corresponding_to_jobid.json'),
],
'postprocint': 0
})
print 'Database entry created!'
self.trigger_id = trigger_id
self.trigger_path = trigger_path
self.backend.save(self.thisevent)
self.backend.commit()
with open(os.path.join(triggerdir,"strategy.yaml"), "r") as f:
self.config = yaml.safe_load(f);
self.filterobslist = np.array(self.config['exposure_filter'],dtype='str')
self.strategydict = {}
for f in np.unique(self.filterobslist):
self.strategydict[f] = len(self.filterobslist[self.filterobslist == f])
self.connection = ea.connect(DATABASE)
self.cursor = self.connection.cursor()
self.jsonfilelist = jsonfilelist
print self.jsonfilelist
if hardjson:
self.jsonfilelist = hj
#self.pp = subprocess.Popen('echo starting',stdout=PIPE, stderr=PIPE,shell=True)
self.trigger_id = trigger_id
self.datadir = datadir
self.triggerdir = triggerdir
self.processingdir = os.path.join(self.triggerdir,'PROCESSING')
if not os.path.exists(self.processingdir):
os.makedirs(self.processingdir)
dire = './processing/' + trigger_id + '/'
if not os.path.exists(dire):
os.makedirs(dire)
with open(os.path.join(triggerdir, "strategy.yaml"), "r") as f:
self.strategy = yaml.safe_load(f)
with open("jobmanager.yaml", "r") as g:
self.jmconfig = yaml.safe_load(g);
q1 = "select expnum,nite,mjd_obs,telra,teldec,band,exptime,propid,obstype,object from exposure where " \
"nite>20130828 and nite<20150101 and expnum<300000 and obstype='object' order by expnum" # y1 images
self.connection.query_and_save(q1, './processing/exposuresY1.tab')
q2 = "select expnum,nite,mjd_obs,radeg,decdeg,band,exptime,propid,obstype,object from prod.exposure where " \
"nite>20150901 and obstype='object' order by expnum" # y2 and later
self.connection.query_and_save(q2, './processing/exposuresCurrent.tab')
os.system('cat ./processing/exposuresY1.tab ./processing/exposuresCurrent.tab > ./processing/exposures.list')
#self.submit_post_processing()
self.submit_all_jsons_for_sejobs()#preps all DES images that already exist
tfin = time.time()
print 'TOTAL SE JOBS TIME', tfin - tstart
#sys.exit()
self.monitor_images_from_mountain()#A loop that waits for images off mountain and submits for processing
def submit_all_jsons_for_sejobs(self):
obsStartTime = self.getDatetimeOfFirstJson(self.jsonfilelist[0]) # THIS IS A DATETIME OBJ
currentTime = dt.utcnow()
print '***** The current time is UTC', currentTime, '*****'
delt = obsStartTime - currentTime
timedelta = td(days=delt.days, seconds=delt.seconds).total_seconds() / 3600.
print '***** The time delta is ', timedelta, 'hours *****'
# if timedelta > np.pi:
sejob_timecushion = self.jmconfig["sejob_timecushion"]
if timedelta > sejob_timecushion:
for jsonfile in self.jsonfilelist:
print 'json',jsonfile
try: #check if this json file is already in the submitted preprocessing database
thisjson = self.backend.get(preprocessing, {'jsonfilename': os.path.join(self.datadir, jsonfile)})
print 'Found this json in desgw database...'
except preprocessing.DoesNotExist: #do submission and then add to database
print 'cd diffimg-proc; ./SEMaker_RADEC.sh '+os.path.join(self.datadir, jsonfile)
os.chdir("diffimg-proc")
out = os.popen('./SEMaker_RADEC.sh '+os.path.join(self.datadir, jsonfile)).read()
of = open(os.path.join(self.processingdir,jsonfile.split('/')[-1].split('.')[0]+'.SEMakerlog'),'w')
of.write(out)
of.close()
#out = os.popen('ls').read()
os.chdir("..")
print out
if 'non-zero exit status' in out:
dt.sendEmailSubject(self.trigger_id,'Error in creating SEMaker dag for .json: '+out)
else:
for o in out.split('\n'):
if 'file://' in o:
dagfile = o.split('/')[-1]
self.dagfile = os.path.join(self.processingdir,jsonfile.split('/')[-1].split('.')[0]+'_'+dagfile)
os.system('cp diffimg-proc/'+dagfile+' '+self.dagfile)
jobsubmitline = copy(o)
print self.dagfile
print 'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; jobsub_submit_dag -G des --role=DESGW file://'+self.dagfile
out = os.popen(
'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; '
'jobsub_submit_dag -G des --role=DESGW file://'+self.dagfile).read()
print out
of = open(os.path.join(self.processingdir,
jsonfile.split('/')[-1].split('.')[0] + '.SEdagsubmitlog'), 'w')
of.write(out)
of.close()
if 'non-zero exit status' in out:
dt.sendEmailSubject(self.trigger_id, 'Error in submitting .json for preprocessing: ' + out)
else:
for o in out.split('\n'):
if 'Use job id' in o:
jobid = o.split()[3]
if doimmediateremove:
out = os.popen(
'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; '
'jobsub_rm --jobid=' + jobid + ' --group=des --role=DESGW').read()
print out
thisjson = preprocessing({
'jsonfilename': os.path.join(self.datadir, jsonfile),
'jobid': jobid,
'dagfile': self.dagfile,
'status' : 'Submitted'
})
self.backend.save(thisjson)
self.backend.commit()
print 'saved'
#sys.exit()
#raw_input()
#runProcessingIfNotAlready(image, self.backend)
#sys.exit()
print 'Finished submitting minidagmaker with all json files'
#sys.exit()
#raw_input()
# Loop queries for images from mountain and submits them
# Need to add complexity that monitors filter strategy and waits for entire groups of images to be co-added
def monitor_images_from_mountain(self):
#NEED TO ADD COADD LOGIC USING STRATEGY FROM CONFIG
exposure_filter = np.array(self.strategy['exposure_filter'],dtype='str')
uniquefilts = np.unique(self.strategy['exposure_filter'])
filterstrategy = {}
for f in uniquefilts:
filterstrategy[f] = len(exposure_filter[exposure_filter == f])
print 'filter strategy dictionary ', filterstrategy
starttime = time.time()
pptime = time.time()
keepgoing = True
index = -1
submission_counter = 0
maxsub = 10000
postprocessingtime = 2000 #every half hour fire off Tim's code for post-processing
while keepgoing:
#os.system('kinit -k -t /var/keytab/desgw.keytab desgw/des/[email protected]')
index += 1
newfireds = []
if time.time() - starttime > 50000:
keepgoing = False
continue
ofile = open(os.path.join(self.triggerdir , 'latestquery.txt'), 'w')
ofile.write(
"--------------------------------------------------------------------------------------------------\n")
ofile.write("EXPNUM\tNITE\tBAND\tEXPTIME\tRADEG\t DECDEG\tPROPID\tOBJECT\n")
ofile.write(
"--------------------------------------------------------------------------------------------------\n")
print "--------------------------------------------------------------------------------------------------"
print "EXPNUM\tNITE\tBAND\tEXPTIME\tRADEG\t DECDEG\tPROPID\tOBJECT"
print "--------------------------------------------------------------------------------------------------"
query = "SELECT expnum,nite,band,exptime,radeg,decdeg,propid,object FROM prod.exposure@desoper WHERE " \
"expnum > 475900 and propid=" + propid + "and obstype='object' ORDER BY expnum" # latest
self.cursor.execute(query)
for s in self.cursor:
ofile.write(
str(s[0]) + "\t" + str(s[1]) + "\t" + str(s[2]) + "\t" + str(s[3]) + "\t" + str(s[4]) + "\t" + str(
s[5]) + "\t" + str(s[6]) + "\t" + str(s[7]) + '\n')
print str(s[0]) + "\t" + str(s[1]) + "\t" + str(s[2]) + "\t" + str(s[3]) + "\t" + str(
s[4]) + "\t" + str(s[5]) + "\t" + str(s[6]) + "\t" + str(s[7])
if not 'DESGW' in str(s[7]): continue
#print 'exptime',float(s[3])
if not float(s[3]) > 29.: continue #exposure must be longer than 30 seconds
expnum = str(s[0])
nite = str(s[1])
band = str(s[2])
exptime = str(s[3])
#FIRST CHECK HERE THAT THE EXPOSURE NUMBER ISNT ALREADY IN THE DATABASE
try:
exposure = self.backend.get(exposures, {'expnum': expnum})
print 'Found this exposure in desgw database...'
# print exposure.attributes
# if expnum == 506432:
# sys.exit()
# self.backend.delete(exposure)
# self.backend.commit()
# exposure = self.backend.get(exposures, {'expnum': expnum})
except exposures.DoesNotExist: # add to database
#runProcessingIfNotAlready(image,self.backend)
print './diffimg-proc/getTiling.sh '+expnum
res = os.popen('./diffimg-proc/getTiling.sh '+expnum).readlines()
print res
#sys.exit()
field,tiling =res[-2],res[-1]
#print 'field_tiling',field_tiling
hexnite = field.strip()+'_'+tiling.strip()+'_'+str(nite)
#print hexnite
#sys.exit()
#print 'hexnite',hexnite
print 'Creating exposure in database...',hexnite
#raw_input()
if '--' in hexnite:
print 'found bad example'
#raw_input()
exposure = exposures({
'expnum':expnum,
'nite':nite,
'field':field,
'tiling':tiling,
'hexnite':hexnite,
'band':band,
'jobid':np.nan,
'exptime':exptime,
'status':'Awaiting additional exposures',
'triggerid': self.trigger_id,
'object':str(s[7])
})
self.backend.save(exposure)
self.backend.commit()
hexnite = exposure.hexnite
print 'hexnite',hexnite
if '--' in hexnite:
print exposure.attributes
#raw_input()
#raw_input()
#sys.exit()
try:
hex = self.backend.get(hexes, {'hexnite': hexnite})
#self.backend.delete(hex)
#self.backend.commit()
#hex = self.backend.get(hexes, {'hexnite': hexnite})
#print 'Found this hex in desgw database...'
except hexes.DoesNotExist:
hex = hexes({
'hexnite': hexnite,
'strategy': self.strategy['exposure_filter'],
'num_target_g': len(exposure_filter[exposure_filter == 'g']),
'num_target_r': len(exposure_filter[exposure_filter == 'r']),
'num_target_i': len(exposure_filter[exposure_filter == 'i']),
'num_target_z': len(exposure_filter[exposure_filter == 'z']),
'observed_g': [],
'observed_r': [],
'observed_i': [],
'observed_z': [],
'exposures': [],
'status': 'Awaiting additional exposures',
'dagfile' : None,
})
self.backend.save(hex)
self.backend.commit()
print hex.attributes
print 'created new hex'
#raw_input()
if hex.status == 'Submitted for processing':
print 'This hex has already been submitted for processing'
continue
# if '--' in hexnite:
# print hex.attributes
# raw_input()
# if hex.status == 'Submitted for processing':
# print 'Hex ',hexnite,' band',band,'exposure',expnum,'has already been submitted for processing'
# #raw_input()
# continue
if band == 'g':
if not expnum in hex.observed_g:
hex.observed_g.append(expnum)
hex.exposures.append(expnum)
if band == 'r':
if not expnum in hex.observed_r:
hex.observed_r.append(expnum)
hex.exposures.append(expnum)
if band == 'i':
if not expnum in hex.observed_i:
hex.observed_i.append(expnum)
hex.exposures.append(expnum)
if band == 'z':
if not expnum in hex.observed_z:
hex.observed_z.append(expnum)
hex.exposures.append(expnum)
self.backend.save(hex)
self.backend.commit()
print hex.attributes
didwork = False
if len(hex.observed_g) == hex.num_target_g:
if len(hex.observed_r) == hex.num_target_r:
if len(hex.observed_i) == hex.num_target_i:
if len(hex.observed_z) == hex.num_target_z:
print 'All exposures in strategy satisfied! '
#raw_input()
submissionPassed = True
for target, exps in zip([hex.num_target_g,hex.num_target_r,hex.num_target_i,hex.num_target_z],
[hex.observed_g,hex.observed_r,hex.observed_i,hex.observed_z]):
if target == 0: continue
exposurestring = ''
logstring = ''
for ex in exps:
exposurestring += ex+' '
logstring += ex+'_'
print 'cd diffimg-proc; source DAGMaker.sh ' + exposurestring
os.chdir("diffimg-proc")
#out = os.popen('ls').read()
out = os.popen('./DAGMaker.sh ' + exposurestring ).read()
os.chdir("..")
print out
f = open(os.path.join(self.processingdir,logstring+hexnite+'.dagmakerlog'),'w')
f.write(out)
f.close()
tt = time.time()
if not 'To submit this DAG do' in out:
dt.sendEmailSubject(self.trigger_id, 'Error in creating dag for desgw hex: ' + out)
submissionPassed = False
else:
for o in out.split('\n'):
if 'file://' in o:
dagfile = o.split('/')[-1]
self.dagfile = os.path.join(self.processingdir,logstring+'job.dag')
os.system('cp diffimg-proc/' + dagfile + ' ' + self.dagfile)
print self.dagfile
print 'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; jobsub_submit_dag -G des --role=DESGW file://' + self.dagfile
out = os.popen(
'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; '
'jobsub_submit_dag -G des --role=DESGW file://' + self.dagfile).read()
print out
f = open(os.path.join(self.processingdir, logstring + hexnite + '.dagsumbitlog'), 'w')
f.write(out)
f.close()
if 'non-zero exit status' in out:
dt.sendEmailSubject(self.trigger_id,
'Error in submitting hex dag for processing: ' + out)
submissionPassed = False
else:
if doimmediateremove:
for o in out.split('\n'):
if 'Use job id' in o:
jobid = o.split()[3]
out = os.popen(
'source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setup; setup jobsub_client; '
'jobsub_rm --jobid=' + jobid + ' --group=des --role=DESGW').read()
print out
ttt = time.time()
#print 'TOTAL JOBSUB FOR A SINGLE DESGW IMAGE',ttt-tt
#sys.exit()
#raw_input()
if submissionPassed:
hex.status = 'Submitted for processing'
hex.dagfile = self.dagfile
self.backend.save(hex)
self.backend.commit()
for expn in hex.exposures:
print expn, 'updated in database to Submitted For Processing'
exp = self.backend.get(exposures, {'expnum': expn})
exp.status = 'Submitted for processing'
self.backend.save(exp)
self.backend.commit()
didwork = True
print 'didwork',didwork
print 'dagfile',self.dagfile
#sys.exit()
#raw_input()
if not didwork:
print 'Could not find all images in strategy for this hex... Added hex', hexnite,' to database ' \
'and will continue waiting...'
#raw_input()
if time.time() - pptime > postprocessingtime: #happens every 30 minutes or so...
pptime = time.time()
print '***** Firing post processing script *****'
#sys.exit()
#ppout = self.pp.communicate()
# if self.thisevent.postprocint > 0:
# print ppout
# f = open(os.path.join(self.processingdir,'postproc_attempt'+str(int(self.thisevent.postprocint))+'.log'),'w')
# f.write(ppout)
# f.close()
self.thisevent.postprocint += 1
self.backend.save(self.thisevent)
self.backend.commit()
self.submit_post_processing()
#sys.exit()
#print 'Waiting 10s to check from mountain...'
#sys.exit()
time.sleep(10)#looping over checking the mountain top
# cfiles = os.listdir(os.path.join(trigger_path,trigger_id,'candidates'))
# for f in cfiles:
# if f.split('.')[-1] == 'npz':
# cp.makeNewPage(f)
def submit_post_processing(self):
#firedlist = open('./processing/firedlist.txt', 'r')
#fl = firedlist.readlines()
#firedlist.close()
#print fl
#fl = ['475914','475915','475916','482859','482860','482861']
fl = self.backend.filter(exposures, {'triggerid': self.trigger_id})
expnumlist = ''
for f in fl:
expnumlist += f.expnum.strip()+' '
print expnumlist
#expnumlist = '475905 475909 475913 475917 475921 475925 475929 475933 475937 475941 475945 475949 475953 475957 475961'
print 'FIRING TIMs CODE'
try:
os.mkdir(os.path.join(self.trigger_path,self.trigger_id,'candidates'))
except:
print 'Candidates directory exists,',os.path.join(self.trigger_path,self.trigger_id,'candidates')
pass
#sys.exit()
gwpostdir = os.environ['GWPOST_DIR']
print 'source ' + os.path.join(gwpostdir, 'diffimg_setup.sh') + '; \
python '+os.path.join(gwpostdir,'postproc.py')\
+' --expnums ' + expnumlist\
+ ' --outputdir ' + os.path.join(self.trigger_path,self.trigger_id,'candidates')\
+ ' --triggerid '+self.trigger_id+' --season 70 --ups True'
# os.system('source ' + os.path.join(gwpostdir, 'diffimg_setup.sh') + '; \
# python '+os.path.join(gwpostdir,'postproc.py')\
# +' --expnums ' + expnumlist\
# + ' --outputdir ' + os.path.join(trigger_path,trigger_id,'candidates')\
# + ' --triggerid '+trigger_id+' --season 46 --ups True' )
#pid = os.spawnlp(os.P_WAIT, "source", os.path.join(gwpostdir, 'diffimg_setup.sh'))
# args = ['yes | ssh -t [email protected] "source '+ os.path.join(gwpostdir, 'mi_setup.sh')+'; '+
# 'yes | python '+os.path.join(gwpostdir,'postproc.py')\
# +' --expnums ' + expnumlist\
# + ' --outputdir ' + os.path.join(self.trigger_path,self.trigger_id,'candidates')\
# + ' --triggerid '+self.trigger_id+' --season 46 --ups True"'
# ]
args = ['yes | python ' + os.path.join(gwpostdir, 'postproc.py') \
+ ' --expnums ' + expnumlist \
+ ' --outputdir ' + os.path.join(self.trigger_path, self.trigger_id, 'candidates') \
+ ' --triggerid ' + self.trigger_id + ' --season 70 --ups True']
print args
#sys.exit()
f = open(os.path.join(self.processingdir,'postproc_attempt'+str(int(self.thisevent.postprocint)))+'.log','w')
self.pp = subprocess.Popen(args,stdout=f, stderr=f,shell=True)
#p = subprocess.Popen(args, stdin=None, stdout=None, stderr=None, close_fds=True, shell=True)
#print 'going'*1000
#print self.pp.communicate()
#print 'gone'*1000
#p = subprocess.Popen(args,stdin=None, stdout=None, stderr=None, close_fds=True,shell=True)
#p.communicate()
#sys.exit()
return
def getDatetimeOfFirstJson(self,jsonstring):
#'M263920-30-UTC-2016-12-1-0:44:00.json'
js = jsonstring.split('UTC')[1]#-2015-12-27-3:2:00.json
#date_object = dt.strptime(js, '-%Y-%m-%d-%H_%M_%S.json')
date_object = dt.strptime(js, '-%Y-%m-%d-%H:%M:%S.json')
print '***** Datetime of first observation UTC',date_object,'*****'
return date_object
def sortHexes(self):
pass
