def main(argv): try: version = __version__ except: version = "1.0.0" configpath = '' statusmsg = {} debug = False starttime = None endtime = None source = 'database' try: opts, args = getopt.getopt(argv, "hc:e:s:D", [ "config=", "endtime=", "starttime=", "debug=", ]) except getopt.GetoptError: print('current_weatherchanges.py -c <config>') sys.exit(2) for opt, arg in opts: if opt == '-h': print('-------------------------------------') print('Description:') print( '-- current_weatherchanges.py will determine the primary instruments --' ) print( '-----------------------------------------------------------------' ) print('detailed description ..') print('...') print('...') print('-------------------------------------') print('Usage:') print('python weather_products.py -c <config>') print('-------------------------------------') print('Options:') print('-c (required) : configuration data path') print('-e : endtime - default is now') print( '-s : starttime - default is three days from now') print('-------------------------------------') print('Application:') print( 'python current_weatherchanges.py -c /etc/marcos/analysis.cfg') sys.exit() elif opt in ("-c", "--config"): # delete any / at the end of the string configpath = os.path.abspath(arg) elif opt in ("-s", "--starttime"): # get an endtime starttime = arg elif opt in ("-e", "--endtime"): # get an endtime endtime = arg elif opt in ("-D", "--debug"): # delete any / at the end of the string debug = True print("Running current_weather version {}".format(version)) print("--------------------------------") if endtime: try: endtime = DataStream()._testtime(endtime) except: print(" Could not interprete provided endtime. Please Check !") sys.exit(1) else: endtime = datetime.utcnow() if starttime: try: starttime = DataStream()._testtime(starttime) except: print(" Could not interprete provided starttime. Please Check !") sys.exit(1) else: starttime = datetime.utcnow() - timedelta(days=3) if starttime >= endtime: print(" Starttime is larger than endtime. Please correct !") sys.exit(1) if not os.path.exists(configpath): print('Specify a valid path to configuration information') print( '-- check magnetism_products.py -h for more options and requirements' ) sys.exit() print("1. Read and check validity of configuration data") config = GetConf(configpath) print("2. Activate logging scheme as selected in config") config = DefineLogger(config=config, category="PeriodicGraphs", job=os.path.basename(__file__), newname='mm-pg-currentweatherchanges.log', debug=debug) name1 = "{}-weatherchange".format(config.get('logname')) statusmsg[name1] = 'weather change analysis successful' print("3. Connect databases and select first available") try: config = ConnectDatabases(config=config, debug=debug) db = config.get('primaryDB') except: statusmsg[name1] = 'database failed' # it is possible to save data also directly to the brokers database - better do it elsewhere print("4. Weather change analysis") success = weather_change(db, config=config, starttime=starttime, endtime=endtime, debug=debug) if not success: statusmsg[name1] = 'weather change analysis failed' if not debug: martaslog = ml(logfile=config.get('logfile'), receiver=config.get('notification')) martaslog.telegram['config'] = config.get('notificationconfig') martaslog.msg(statusmsg) else: print("Debug selected - statusmsg looks like:") print(statusmsg)
def main(argv): version = '1.0.0' configpath = '' statusmsg = {} debug=False endtime = datetime.utcnow() varioinst = '' scalainst = '' try: opts, args = getopt.getopt(argv,"hc:e:D",["config=","endtime=","debug=",]) except getopt.GetoptError: print ('getprimary.py -c <config>') sys.exit(2) for opt, arg in opts: if opt == '-h': print ('-------------------------------------') print ('Description:') print ('-- getprimary.py will determine the primary instruments --') print ('-----------------------------------------------------------------') print ('detailed description ..') print ('...') print ('...') print ('-------------------------------------') print ('Usage:') print ('python getprimary.py -c <config>') print ('-------------------------------------') print ('Options:') print ('-c (required) : configuration data path') print ('-------------------------------------') print ('Application:') print ('python getprimary.py -c /etc/marcos/analysis.cfg') sys.exit() elif opt in ("-c", "--config"): # delete any / at the end of the string configpath = os.path.abspath(arg) elif opt in ("-e", "--endtime"): # get an endtime endtime = arg.split(',') elif opt in ("-D", "--debug"): # delete any / at the end of the string debug = True print ("Running getprimary version {}".format(version)) print ("--------------------------------") if not os.path.exists(configpath): print ('Specify a valid path to configuration information') print ('-- check magnetism_products.py -h for more options and requirements') sys.exit() print ("1. Read and check validity of configuration data") config = GetConf(configpath) print ("2. Activate logging scheme as selected in config") config = DefineLogger(config=config, category="DataProducts", job=os.path.basename(__file__), newname='mm-dp-getprimary.log') name1 = "{}-process".format(config.get('logname')) name2 = "{}-vario".format(config.get('logname')) name3 = "{}-scalar".format(config.get('logname')) name4 = "{}-write".format(config.get('logname')) statusmsg[name1] = 'successful' print ("3. Connect databases and select first available") try: config = ConnectDatabases(config=config, debug=debug) db = config.get('primaryDB') if debug: print (" -- success") except: if debug: print (" -- database failed") statusmsg[name1] = 'database failed' print ("4. Checking variometer instruments") try: varioinst, statusmsg = PrimaryVario(db, config.get('variometerinstruments'), endtime=endtime, logname=name2, statusmsg=statusmsg, debug=debug) print (" -> Using {}".format(varioinst)) except: if debug: print (" -- vario failed") statusmsg[name1] = 'vario failed' print ("5. Checking scalar instruments") try: scalainst, statusmsg = PrimaryScalar(db, config.get('scalarinstruments'), endtime=endtime, logname=name3, statusmsg=statusmsg, debug=debug) print (" -> Using {}".format(scalainst)) except: if debug: print (" -- scalar failed") statusmsg[name1] = 'scalar failed' print ("6. Updating Current values") try: if not debug: statusmsg = UpdateCurrentValuePath(config.get('currentvaluepath'), varioinst=varioinst, scalainst=scalainst, logname=name4, statusmsg=statusmsg, debug=debug) except: if debug: print (" -- update current failed") statusmsg[name1] = 'current value update failed' if not debug: martaslog = ml(logfile=config.get('logfile'),receiver=config.get('notification')) martaslog.telegram['config'] = config.get('notificationconfig') martaslog.msg(statusmsg) else: print ("Debug selected - statusmsg looks like:") print (statusmsg)
def main(argv): version = '1.0.0' configpath = '' statusmsg = {} debug = False endtime = datetime.utcnow() joblist = [] varioinst = '' scalainst = '' #joblist = ['flag','clean','archive','update','delete'] joblist = ['flag'] flagfilearchivepath = '' # default: flagarchive : /srv/archive/flags flagfilepath = '' consecutivethreshold = 100000 delsensor = 'RCST7_20160114_0001' delcomment = 'aof - threshold 5.0 window 43200.0 sec' # each input looks like: # { SensorNamePart : [timerange, keys, threshold, window, markall, lowlimit, highlimit] flagdict = { 'LEMI036': [7200, 'x,y,z', 6, 'Default', True, 'None', 'None'], 'LEMI025': [7200, 'x,y,z', 6, 'Default', True, 'None', 'None'], 'FGE': [7200, 'x,y,z', 5, 'Default', True, 'None', 'None'], 'GSM90_14245': [7200, 'f', 5, 'default', False, 'None', 'None'], 'GSM90_6': [7200, 'f', 5, 300, False, 'None', 'None'], 'GSM90_3': [7200, 'f', 5, 300, False, 'None', 'None'], 'GP20S3NSS2': [7200, 'f', 5, 'Default', False, 'None', 'None'], 'POS1': [7200, 'f', 4, 100, False, 'None', 'None'], 'BM35': [7200, 'var3', 'None', 'None', False, 750, 1000] } try: opts, args = getopt.getopt(argv, "hc:e:j:p:s:o:D", [ "config=", "endtime=", "joblist=", "path=", "sensor=", "comment=", "debug=" ]) except getopt.GetoptError: print('flagging.py -c <config>') sys.exit(2) for opt, arg in opts: if opt == '-h': print('-------------------------------------') print('Description:') print('-- flagging.py will determine the primary instruments --') print( '-----------------------------------------------------------------' ) print('detailed description ..') print('...') print('...') print('-------------------------------------') print('Usage:') print('python flagging.py -c <config>') print('-------------------------------------') print('Options:') print('-c (required) : configuration data path') print('-e : endtime, default is now') print( '-j : joblist: flag,clean,archive,update,delete; default is flag,clean' ) print( '-p : update - path to json files which end with flags.json' ) print('-s : delete - sensor') print('-o : delete - comment') print('-------------------------------------') print('Application:') print('python flagging.py -c /etc/marcos/analysis.cfg') print('Once per year:') print(' python flagging.py -c /etc/marcos/analysis.cfg -j archive') print('Eventually always:') print( ' python flagging.py -c /etc/marcos/analysis.cfg -j upload -p /srv/archive/flags/uploads/' ) print('Once a day/week:') print(' python flagging.py -c /etc/marcos/analysis.cfg -j clean') sys.exit() elif opt in ("-c", "--config"): # delete any / at the end of the string configpath = os.path.abspath(arg) elif opt in ("-e", "--endtime"): # get an endtime endtime = arg.split(',') elif opt in ("-j", "--joblist"): # get an endtime joblist = arg.split(',') elif opt in ("-p", "--path"): # delete any / at the end of the string flagfilepath = os.path.abspath(arg) elif opt in ("-s", "--sensor"): # hidden: delete sensor data delsensor = arg elif opt in ("-o", "--comment"): delcomment = arg elif opt in ("-D", "--debug"): # delete any / at the end of the string debug = True print("Running flagging version {}".format(version)) print("--------------------------------") if not os.path.exists(configpath): print('Specify a valid path to configuration information') print( '-- check magnetism_products.py -h for more options and requirements' ) sys.exit() print("1. Read and check validity of configuration data") config = GetConf(configpath) print("2. Activate logging scheme as selected in config") config = DefineLogger(config=config, category="DataProducts", job=os.path.basename(__file__), newname='mm-dp-flagging.log', debug=debug) name1 = "{}-flag".format(config.get('logname')) name2 = "{}-flag-lemitest".format(config.get('logname')) name3 = "{}-cleaning".format(config.get('logname')) name4 = "{}-archive".format(config.get('logname')) name5 = "{}-upload".format(config.get('logname')) statusmsg[name1] = 'flagging data sets successful' statusmsg[name2] = 'Lemitest not performed' statusmsg[name3] = 'Cleanup: cleaning database successful' statusmsg[name4] = 'Archive: not time for archiving' statusmsg[name5] = 'Upload: nothing to do' flagfilearchivepath = config.get('flagarchive', '') print(flagfilearchivepath) if not os.path.isdir(flagfilearchivepath): flagfilearchivepath = '' if not os.path.isdir(flagfilepath): flagfilepath = '' print("3. Connect databases and select first available") try: config = ConnectDatabases(config=config, debug=debug) db = config.get('primaryDB') connectdict = config.get('conncetedDB') except: statusmsg[name1] = 'database failed' if 'flag' in joblist: print("4. Dealing with flagging dictionary") try: #ok = True #if ok: for elem in flagdict: print(" -------------------------------------------") print(" Dealing with sensorgroup which starts with {}".format( elem)) print(" -------------------------------------------") # Get parameter timerange = flagdict[elem][0] keyspar = flagdict[elem][1] if keyspar in ['Default', 'default', 'All', 'all', '', None]: keys = None else: keys = keyspar.split(',') threshold = flagdict[elem][2] if threshold in [ 'Default', 'default', 'None', 'none', '', None ]: threshold = None windowpar = flagdict[elem][3] if windowpar in [ 'Default', 'default', 'None', 'none', '', None ]: window = None else: window = timedelta(seconds=windowpar) markall = flagdict[elem][4] lowlimit = flagdict[elem][5] if lowlimit in [ 'Default', 'default', 'None', 'none', '', None ]: lowlimit = None highlimit = flagdict[elem][6] if highlimit in [ 'Default', 'default', 'None', 'none', '', None ]: highlimit = None starttime = datetime.utcnow() - timedelta(seconds=timerange) print( " - Using the following parameter: keys={},threshold={},window={},limits={}" .format(keys, threshold, window, [lowlimit, highlimit])) # Checking available sensors sensorlist = dbselect(db, 'DataID', 'DATAINFO', 'SensorID LIKE "{}%"'.format(elem)) print(" -> Found {}".format(sensorlist)) print(" a) select 1 second or highest resolution data" ) # should be tested later again validsensors1 = [] determinesr = [] srlist = [] for sensor in sensorlist: res = dbselect(db, 'DataSamplingrate', 'DATAINFO', 'DataID="{}"'.format(sensor)) try: sr = float(res[0]) print(" - Sensor: {} -> Samplingrate: {}".format( sensor, sr)) if sr >= 1: validsensors1.append(sensor) srlist.append(sr) except: print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") print("Check sampling rate {} of {}".format( res, sensor)) print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") determinesr.append(sensor) print(" b) checking sampling rate of failed sensors") for sensor in determinesr: lastdata = dbgetlines(db, sensor, timerange) if lastdata.length()[0] > 0: sr = lastdata.samplingrate() print(" - Sensor: {} -> Samplingrate: {}".format( sensor, sr)) if sr >= 1: validsensors1.append(sensor) srlist.append(sr) print(" c) Check for recent data") validsensors = [] validsr = [] for idx, sensor in enumerate(validsensors1): last = dbselect(db, 'time', sensor, expert="ORDER BY time DESC LIMIT 1") if debug: print(" Last time", last) try: dbdate = last[0] except: print(" - No data found for {}".format(sensor)) try: if getstringdate(dbdate) > starttime: print(" - Valid data for {}".format(sensor)) validsensors.append(sensor) validsr.append(srlist[idx]) except: print(" Dateformat problem for {}".format(sensor)) print(" d) Flagging data") for idx, sensor in enumerate(validsensors): lines = int(timerange / validsr[idx]) lastdata = dbgetlines(db, sensor, lines) print(" - got {} datapoints".format( lastdata.length()[0])) if lastdata.length()[0] > 0: sensorid = "_".join(sensor.split('_')[:-1]) print(" - getting existing flags for {}".format( sensorid)) vflag = db2flaglist(db, sensorid, begin=datetime.strftime( starttime, "%Y-%m-%d %H:%M:%S")) print(" - found {} existing flags".format( len(vflag))) if len(vflag) > 0: try: print(" - removing existing flags") lastdata = lastdata.flag(vflag) lastdata = lastdata.remove_flagged() print(" ...success") except: print( " ------------------------------------------------" ) print( " -- Failed to apply flags TODO need to check that" ) flaglist = [] if threshold: print(" - determining new outliers") if debug: print("MARK all: ", markall) flagls = lastdata.flag_outlier(keys=keys, threshold=threshold, timerange=window, returnflaglist=True, markall=markall) # now check flaglist---- if more than 20 consecutive flags... then drop them flaglist = consecutive_check(flagls, remove=True) #if len(flagls) > len(flaglist)+1 and sensor.startswith("LEMI036_1"): #+1 to add some room # statusmsg[name2] = 'Step1: removed consecutive flags for {}: Found {}, Clean: {}'.format(sensor, len(flagls), len(flaglist)) print( " - new outlier flags: {}; after combination: {}" .format(len(flagls), len(flaglist))) if lowlimit: print(" - flagging data below lower limit") flaglow = lastdata.flag_range( keys=keys, below=lowlimit, text='below lower limit {}'.format(lowlimit), flagnum=3) if len(flaglist) == 0: flaglist = flaglow else: flaglist.extend(flaglow) if highlimit: print(" - flagging data above higher limit") flaghigh = lastdata.flag_range( keys=keys, above=highlimit, text='exceeding higher limit {}'.format( highlimit), flagnum=3) if len(flaglist) == 0: flaglist = flaghigh else: flaglist.extend(flaghigh) print(" -> RESULT: found {} new flags".format( len(flaglist))) if not debug and len(flaglist) > 0: for dbel in connectdict: dbt = connectdict[dbel] print( " -- Writing flags for sensors {} to DB {}" .format(sensor, dbel)) print(" -- New flags: {}".format( len(flaglist))) prevflaglist = db2flaglist(dbt, sensorid) if len(prevflaglist) > 0: lastdata.flagliststats(prevflaglist, intensive=True) else: print( " - no flags so far for this sensor") name3 = "{}-toDB-{}".format( config.get('logname'), dbel) statusmsg[ name3] = 'flags successfully written to DB' try: flaglist2db(dbt, flaglist) except: statusmsg[ name3] = 'flags could not be written to DB - disk full?' aftflaglist = db2flaglist(dbt, sensorid) lastdata.flagliststats(aftflaglist, intensive=True) except: print(" -> flagging failed") statusmsg[name1] = 'Step1: flagging failed' if 'upload' in joblist and flagfilepath: print("5. Upload flagging lists from files") filelist = [] print(" Searching for new flagging files") for fi in os.listdir(flagfilepath): if fi.endswith("flags.json") or fi.endswith("flags.pkl"): print(" -> found: {}".format(os.path.join(flagfilepath, fi))) filelist.append(os.path.join(flagfilepath, fi)) if len(filelist) > 0: for fi in filelist: fileflaglist = loadflags(fi) try: instnamel = os.path.basename(fi).split('_') instname = ["_".join(instnamel[:3])] # Get instnames from fileflaglist except: instname = [] try: flagdict = [{ "starttime": el[0], "endtime": el[1], "components": el[2].split(','), "flagid": el[3], "comment": el[4], "sensorid": el[5], "modificationdate": el[6] } for el in fileflaglist] instname2 = [el.get('sensorid') for el in flagdict] uniqueinstnames = list(set(instname2)) instname = uniqueinstnames if debug: print(" - Sensorname(s) extracted from flag file") except: if debug: print(" - Sensorname(s) extracted from file name") pass if len(fileflaglist) > 0: print(" - Loaded {} flags from file {} for {}".format( len(fileflaglist), fi, instname)) # get all flags from DB dbflaglist = [] for inst in instname: tmpdbflaglist = db2flaglist(db, inst) dbflaglist = combinelists(dbflaglist, tmpdbflaglist) print(" - {} flags in DB for {}".format( len(dbflaglist), instname)) # combine flaglist fileflaglist = combinelists(fileflaglist, dbflaglist) # clean flaglist - necessary here as dublicates might come from combinations flaglist = DataStream().flaglistclean(fileflaglist) print( " - {} flags remaining after cleaning. Delete {} replicates" .format(len(flaglist), len(fileflaglist) - len(flaglist))) print(" - Combining consecutives") flaglist = consecutive_check(flaglist) print(" - {} flags remaining after joining consecutives.". format(len(flaglist))) # copy file to ...json.uploaded date = datetime.strftime(datetime.utcnow(), "%Y%m%d%H%M") if not debug: copyfile(fi, "{}.uploaded{}".format(fi, date)) # delete existing flags in DB and fill with new 'clean' flaglist for dbel in connectdict: dbt = connectdict[dbel] print(" -- Writing flags to DB {}".format(dbel)) for inst in instname: flaglist2db(db, flaglist, mode='delete', sensorid=inst) # delete flagfile os.remove(fi) print(" -> Done") else: print( " -> debug: will not modify or upload any datasets: {}" .format(date)) statusmsg[ name5] = 'Upload: new flagging data sets uploaded' else: print( " -> Flaglist {} is empty. If pkl file check python version..." .format(fi)) if 'delete' in joblist and flagfilearchivepath: print("Not existing. Deleting content") # not yet available #delsensor = 'RCST7_20160114_0001' #delsensor = 'LEMI036_3_0001' #delcomment = 'aof - threshold 5.0 window 43200.0 sec' #delcomment = 'aof - threshold: 6, window: 600.0 sec' # Backup any data too bee deleted? parameter = 'comment' if delcomment in ['0', '1', '2', '3', '4']: parameter = 'flagnumber' print( " - found a valid flagnumber as value: removing flags with FlagID {}" .format(delcomment)) elif delcomment in KEYLIST: parameter = 'key' print( " - found a valid key as value: removing all flags for key {}". format(delcomment)) print(" - selected sensor {}".format(delsensor)) flaglist = db2flaglist(db, delsensor) print(" - got {} flags".format(len(flaglist))) toberemovedflaglist = DataStream().flaglistmod('select', flaglist, parameter=parameter, value=delcomment) print(" - will backup and then remove {} flags matching your criteria". format(len(toberemovedflaglist))) flagfile = os.path.join( flagfilearchivepath, 'flags_{}_backup_{}.json'.format( delsensor, datetime.strftime(datetime.utcnow(), "%Y%m%d%H%M"))) succ = saveflags(toberemovedflaglist, flagfile) if succ: print(" - backup saved to {}".format(flagfile)) remainingflaglist = DataStream().flaglistmod('delete', flaglist, parameter=parameter, value=delcomment) print(" - remaining {} flags".format(len(remainingflaglist))) if not debug: for dbel in connectdict: dbt = connectdict[dbel] print(" -- Writing flags to DB {}".format(dbel)) flaglist2db(dbt, remainingflaglist, mode='delete', sensorid=delsensor) print(" -> Done") else: print(" -> Debug selected - no changes made to DB") else: print( " Backup could not be saved - aborting - check directory permissions" ) if 'clean' in joblist: print("6. Cleaning flagging list") try: print(" Cleaning up all records") cumflag = [] stream = DataStream() flaglist = db2flaglist(db, 'all') if debug: print(" -> Found {} flags in database".format(len(flaglist))) print(" --------------------------------------") stream.flagliststats(flaglist, intensive=True) print(" --------------------------------------") currentyear = endtime.year yearlist = [i for i in range(2000, currentyear + 2)] for year in yearlist: startyear = year - 1 print(" Checking data from {} until {}".format( startyear, year)) beg = '{}-01-01'.format(startyear) end = '{}-01-01'.format(year) flaglist_tmp = db2flaglist(db, 'all', begin=beg, end=end) print(" -> Found {} flags in database between {} and {}". format(len(flaglist_tmp), startyear, year)) if len(flaglist_tmp) > 0: print(" - Cleaning up flaglist") clflaglist_tmp = stream.flaglistclean(flaglist_tmp, progress=True) print(" -> {} flags remaining".format( len(clflaglist_tmp))) if len(clflaglist_tmp) < consecutivethreshold: # TODO this method leads to a killed process sometimes... print(" - Combining consecutives") coflaglist_tmp = consecutive_check( clflaglist_tmp) #debug=debug) else: coflaglist_tmp = clflaglist_tmp print(" -> {} flags remaining".format( len(coflaglist_tmp))) if len(cumflag) == 0: cumflag = coflaglist_tmp else: cumflag.extend(coflaglist_tmp) if debug: print(" -> cleaned record contains {} flags".format( len(cumflag))) print(" --------------------------------------") stream.flagliststats(cumflag, intensive=True) print(" --------------------------------------") if not debug: for dbel in connectdict: dbt = connectdict[dbel] print(" -- Writing flags to DB {}".format(dbel)) flaglist2db(dbt, cumflag, mode='delete', sensorid='all') print(" -> cleaned flaglist uploaded to DB") statusmsg[name3] = 'Cleanup: cleaning database successful' except: print(" -> failure while cleaning up") statusmsg[name3] = 'Cleanup: failure' # schedule with crontab at February 1st 6:00 (analyze yearly) flagging -c /wic.cfg -j archive if 'archive' in joblist and flagfilearchivepath: print("7. Saving archive and deleting old db contents") print(" Archiving flags") print(" ---------------") print(" Every year in February - archive full year two years ago") print(" Delete all inputs older than two years from DB") ## e.g. Feb 2019 -> Keep only 2017 and 2018 in DB ## archive everything before 2017 ## delete everything before 2017 ## -> archive containsnow : flags_2016_final.pkl, # and 2015,etc ## -> flags_archive.pkl current backup (only monthly) ## -> DB contains 2017 to present ... 3 years max stream = DataStream() flaglist = db2flaglist(db, 'all') if debug: print(" -> Found {} flags in database".format(len(flaglist))) print(" --------------------------------------") stream.flagliststats(flaglist) print(" --------------------------------------") # Backup and export all old flags minyear = 2015 succ = False currentyear = endtime.year if currentyear - 3 > minyear: yearlist = [i for i in range(minyear, currentyear - 2)] else: yearlist = [2015] flaglist_tmp = [] for year in yearlist: startyear = 2000 if year > 2015: startyear = year - 1 print(" Archiving flaglist until {}".format(year)) flagfile = os.path.join(flagfilearchivepath, 'flags_{}-{}.json'.format(startyear, year)) beg = '{}-01-01'.format(startyear) end = '{}-01-01'.format(year) flaglist_tmp = db2flaglist(db, 'all', begin=beg, end=end) if len(flaglist_tmp) > 0: print(" -> Found {} flags in database between {} and {}". format(len(flaglist_tmp), startyear, year)) if os.path.isfile(flagfile): fileflaglist = loadflags(flagfile) print(" -> Found {} flags in file".format( len(fileflaglist))) flaglist_tmp.extend(fileflaglist) # Cleaning has been done already print(" - Saving flag archive to {}".format(flagfilepath)) succ = saveflags(flaglist_tmp, flagfile, overwrite=True) print(" -> Done") if succ: # drop all flags from flaglist print(" Droping all flags until year {}".format(year)) newflaglist = stream.flaglistmod('delete', flaglist, starttime='2000-01-01', endtime=end) print(" -> remaining amount of flags: {}".format( len(newflaglist))) # Cleaning has been done already print( " Uploading new list to database and deleting all other inputs" ) print(" --------------------------------------") stream.flagliststats(newflaglist) print(" --------------------------------------") if not debug: for dbel in connectdict: dbt = connectdict[dbel] print(" -- Writing flags to DB {}".format(dbel)) flaglist2db(dbt, newflaglist, mode='delete') print(" -> Done") else: print(" -> Debug selected - no changes made to DB") statusmsg[name4] = 'Archiving flags: done until {}'.format(year) else: print(" -> Problem with saving files - aborting") statusmsg[name4] = 'Archiving flags: file saving problem' print("------------------------------------------") print(" flagging finished") print("------------------------------------------") print("SUCCESS") if not debug: martaslog = ml(logfile=config.get('logfile'), receiver=config.get('notification')) martaslog.telegram['config'] = config.get('notificationconfig') martaslog.msg(statusmsg) else: print("Debug selected - statusmsg looks like:") print(statusmsg)
def main(argv): version = '1.0.0' configpath = '' statusmsg = {} debug=False endtime = datetime.utcnow() joblist = ['NEIC','AT'] stb = DataStream() sta = DataStream() errorcntAT = 0 errorcntNE = 0 uploadcheck = 1 path = '/tmp/neic_quakes.d' try: opts, args = getopt.getopt(argv,"hc:e:j:p:s:o:D",["config=","endtime=","joblist=","debug="]) except getopt.GetoptError: print ('quakes_import.py -c <config>') sys.exit(2) for opt, arg in opts: if opt == '-h': print ('-------------------------------------') print ('Description:') print ('-- quakes_import.py will determine the primary instruments --') print ('-----------------------------------------------------------------') print ('detailed description ..') print ('...') print ('...') print ('-------------------------------------') print ('Usage:') print ('python quakes_import.py -c <config>') print ('-------------------------------------') print ('Options:') print ('-c (required) : configuration data path') print ('-e : endtime, default is now') print ('-j : joblist: NEIC, AT') print ('-p : path for neic data') print ('-------------------------------------') print ('Application:') print ('python quakes_import.py -c /etc/marcos/analysis.cfg -p /home/cobs/ANALYSIS/Seismo/neic_quakes.d') sys.exit() elif opt in ("-c", "--config"): # delete any / at the end of the string configpath = os.path.abspath(arg) elif opt in ("-e", "--endtime"): # get an endtime endtime = arg.split(',') elif opt in ("-j", "--joblist"): # get an endtime joblist = arg.split(',') elif opt in ("-p", "--path"): path = arg elif opt in ("-D", "--debug"): # delete any / at the end of the string debug = True print ("Running flagging version {}".format(version)) print ("--------------------------------") if not os.path.exists(configpath): print ('Specify a valid path to configuration information') print ('-- check magnetism_products.py -h for more options and requirements') sys.exit() print ("1. Read and check validity of configuration data") config = GetConf(configpath) print ("2. Activate logging scheme as selected in config") config = DefineLogger(config=config, category = "DataProducts", job=os.path.basename(__file__), newname='mm-di-quakes.log', debug=debug) namea = "{}-quakes-AT".format(config.get('logname')) nameb = "{}-quakes-NEIC".format(config.get('logname')) currentvaluepath = config.get('currentvaluepath') print ("3. Connect databases and select first available") try: config = ConnectDatabases(config=config, debug=debug) db = config.get('primaryDB') connectdict = config.get('conncetedDB') except: statusmsg[name1] = 'database failed' print (" -> connected databases: {}".format(connectdict)) proxy = '' prox = config.get('proxy','') proxport = config.get('proxyport') if prox: proxy = "--proxy http://{}:{} ".format(prox,proxport) (startlong, startlat) = dbcoordinates(db, 'A2') if 'AT' in joblist: try: print ("Getting Austrian data") print ("---------------------") statusmsg[namea] = 'Austrian data added' if not stb.length()[0] > 0: # only load it once print (" - getting Austrian data from geoweb") stb = read('http://geoweb.zamg.ac.at/static/event/lastweek.csv') stb.header['DataFormat'] = 'NEICCSV' stb.header['DataSource'] = 'Austrian Seismological Service' stb.header['DataReferences'] = 'http://geoweb.zamg.ac.at/static/event/lastmonth.csv' if debug: print (" - Found :", stb.length()) dct = stb._get_key_names() poslon = KEYLIST.index(dct.get('longitude')) poslat = KEYLIST.index(dct.get('latitude')) lonar = stb.ndarray[poslon] latar = stb.ndarray[poslat] # calculate distance between points from pyproj import Geod g = Geod(ellps='WGS84') ar = [] for idx,el in enumerate(lonar): (az12, az21, dist) = g.inv(startlong, startlat, el, latar[idx]) ar.append(dist/1000.) pos = KEYLIST.index('var5') stb.header['col-var5'] = 'distance from COBS' stb.header['unit-col-var5'] = 'km' stb.ndarray[pos] = np.asarray(ar) # Add to DB like BLV data, Kp data # Please Note: you must manually insert a DATAID in DATAINFO to upload header data # insert into DATAINFO (DataID, SensorID) VALUES ('QUAKES','QUAKES'); stb.header['StationID'] = 'SGO' stb = stb.extract('f',2,'>=') if debug: print (" - Found :", stb.length()) if not debug: dbupdateDataInfo(db, 'QUAKES', stb.header) stb = fix_datastream_for_db(stb) if not debug: for dbel in connectdict: dbt = connectdict[dbel] print (" -- Writing AT Quakes to DB {}".format(dbel)) writeDB(dbt,stb,tablename='QUAKES',StationID='SGO') print (" -> Done") else: print (" - Debug selected: ") print (" last line of AT {}".format(stb.length())) print (" -> Austrian data has been added to all databases") print ("----------------------------") print (" Now writing last AT update into current value") #statusmsg[namea] = 'Austrian data added' errorcntAT = 0 # update upload time in current data file fulldict = getcurrentdata(currentvaluepath) valdict = fulldict.get('logging',{}) uploadtime = datetime.strftime(datetime.utcnow(),"%Y-%m-%d %H:%M") valdict['seismoATdata'] = [uploadtime,''] fulldict[u'logging'] = valdict if not debug: print (" Writing update time to current data") writecurrentdata(currentvaluepath, fulldict) print (" -> Updating time has been written to current data") print ("----------------------------") print (" DONE") print ("----------------------------") except: errorcntAT += 1 if errorcntAT > 1: message = True #statusmsg[namea] = 'Austrian data failed' fulldict = getcurrentdata(currentvaluepath) valdict = fulldict.get('logging',{}) try: lastupload = datetime.strptime(valdict.get('seismoATdata',['',''])[0],"%Y-%m-%d %H:%M") if not lastupload < datetime.utcnow()-timedelta(hours=uploadcheck): message = False except: message = True if message: print ('Austrian data failed') statusmsg[namea] = 'Austrian data failed' if 'NEIC' in joblist: #try: print ("Downloading NEIC data") print ("---------------------") statusmsg[nameb] = 'NEIC data added' if not sta.length()[0] > 0: # only load it once print (" - getting NEIC data from usgs") os.system('curl https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/4.5_week.csv {}-s > {}'.format(proxy,path)) sta = read(path) sta.header['DataFormat'] = 'NEICCSV' #sta.header['DataSource'] = 'US National Earthquake information center' #sta.header['DataReferences'] = 'http://earthquake.usgs.gov' dct = sta._get_key_names() poslon = KEYLIST.index(dct.get('longitude')) poslat = KEYLIST.index(dct.get('latitude')) lonar = sta.ndarray[poslon] latar = sta.ndarray[poslat] # calculate distance between points from pyproj import Geod g = Geod(ellps='WGS84') ar = [] for idx,el in enumerate(lonar): (az12, az21, dist) = g.inv(startlong, startlat, el, latar[idx]) ar.append(dist/1000.) pos = KEYLIST.index('var5') sta.header['col-var5'] = 'distance from COBS' sta.header['unit-col-var5'] = 'km' sta.ndarray[pos] = np.asarray(ar) # Add to DB like BLV data, Kp data # Please Note: you must manually insert a DATAID in DATAINFO to upload header data # insert into DATAINFO (DataID, SensorID) VALUES ('QUAKES','QUAKES'); sta.header['StationID'] = 'SGO' sta = sta.extract('f',5,'>=') print (" got {} quakes above magnitude 5".format(sta.length()[0])) if not debug: dbupdateDataInfo(db, 'QUAKES', sta.header) sta = fix_datastream_for_db(sta) if not debug: for dbel in connectdict: dbt = connectdict[dbel] print (" -- Writing NEIC Quakes to DB {}".format(dbel)) writeDB(dbt,sta,tablename='QUAKES',StationID='SGO') print (" -> done") else: print (" - Debug selected: ") print (" last line of NEIC {}".format(stb.length())) print ("NEIC data has been added") print ("----------------------------") print (" Now writing last NEIC update into current value") #statusmsg[nameb] = 'NEIC data added' errorcntNE = 0 # update upload time in current data file fulldict = getcurrentdata(currentvaluepath) valdict = fulldict.get('logging',{}) uploadtime = datetime.strftime(datetime.utcnow(),"%Y-%m-%d %H:%M") valdict['seismoNEICdata'] = [uploadtime,''] fulldict[u'logging'] = valdict if not debug: writecurrentdata(currentvaluepath, fulldict) """ except: print (" error encountered") errorcntNE+=1 if errorcntNE > 1: message = True fulldict = getcurrentdata(currentvaluepath) valdict = fulldict.get('logging',{}) try: lastupload = datetime.strptime(valdict.get('seismoNEICdata',['',''])[0],"%Y-%m-%d %H:%M") if not lastupload < datetime.utcnow()-timedelta(hours=uploadcheck): message = False except: message = True if message: statusmsg[nameb] = 'NEIC data failed' """ print ("------------------------------------------") print (" quakes_import finished") print ("------------------------------------------") print ("SUCCESS") if not debug: martaslog = ml(logfile=config.get('logfile'),receiver=config.get('notification')) martaslog.telegram['config'] = config.get('notificationconfig') martaslog.msg(statusmsg) pass else: print ("Debug selected - statusmsg looks like:") print (statusmsg)
import magpy.opt.cred as mpcred import json import getopt import pwd import socket import sys # for sys.version_info() scriptpath = os.path.dirname(os.path.realpath(__file__)) anacoredir = os.path.abspath(os.path.join(scriptpath, '..', 'core')) sys.path.insert(0, anacoredir) from analysismethods import DefineLogger, ConnectDatabases, getstringdate from martas import martaslog as ml from acquisitionsupport import GetConf2 as GetConf """ Methods in several scripts: ConnectDatabases() DefineLogger() getstringdate() """ def PrimaryVario(db, variolist, endtime=datetime.utcnow(), logname='', statusmsg={}, debug=False): """ Identify currently active variometer and f-instrument, which are recording now and have at least one day of data """ varioinst = '' ## Step 1: checking available data for variometers (need to cover one day and should not be older than one hour ## the first instrument fitting these conditions is selected for inst in variolist:
def main(argv): try: version = __version__ except: version = "1.0.0" configpath = '' statusmsg = {} joblist = ['vario', 'scalar'] debug = False endtime = None try: opts, args = getopt.getopt(argv, "hc:j:D", [ "config=", "joblist=", "endtime=", "debug=", ]) except getopt.GetoptError: print('magnetism_checkadj.py -c <config>') sys.exit(2) for opt, arg in opts: if opt == '-h': print('-------------------------------------') print('Description:') print('-- magnetism_checkadj.py will analyse magnetic data --') print( '-----------------------------------------------------------------' ) print('detailed description ..') print('...') print('...') print('-------------------------------------') print('Usage:') print('python magnetism_checkadj.py -c <config>') print('-------------------------------------') print('Options:') print('-c (required) : configuration data path') print('-j : vario, scalar') print('-e : endtime') print('-------------------------------------') print('Application:') print('python magnetism_checkadj.py -c /etc/marcos/analysis.cfg') sys.exit() elif opt in ("-c", "--config"): # delete any / at the end of the string configpath = os.path.abspath(arg) elif opt in ("-j", "--joblist"): # get a list of jobs (vario, scalar) joblist = arg.split(',') elif opt in ("-e", "--endtime"): endtime = arg elif opt in ("-D", "--debug"): # delete any / at the end of the string debug = True if debug: print("Running magnetism_checkadj version {} - debug mode".format( version)) print("---------------------------------------") if not os.path.exists(configpath): print('Specify a valid path to configuration information') print( '-- check magnetism_products.py -h for more options and requirements' ) sys.exit() if endtime: try: endtime = DataStream()._testtime(endtime) except: print("Endtime could not be interpreted - Aborting") sys.exit(1) else: endtime = datetime.utcnow() print("1. Read configuration data") config = GetConf(configpath) config = ConnectDatabases(config=config, debug=debug) print("2. Activate logging scheme as selected in config") config = DefineLogger(config=config, category="DataProducts", job=os.path.basename(__file__), newname='mm-dp-magdatacheck.log', debug=debug) if debug: print(" -> Config contents:") print(config) print("3. Get basic information of current data") config, statusmsg = GetPrimaryInstruments(config=config, statusmsg=statusmsg, fallback=False, debug=debug) if 'vario' in joblist: print("4. Variometer analysis") namecheck1 = "{}-check-adjusted".format(config.get('logname')) msg = CompareAdjustedVario(config=config, endtime=endtime, debug=debug) statusmsg[namecheck1] = msg if 'scalar' in joblist: print("5. Scalar analysis") namecheck2 = "{}-check-f".format(config.get('logname')) msg = CompareFieldStrength(config=config, endtime=endtime, debug=debug) statusmsg[namecheck2] = msg if not debug: martaslog = ml(logfile=config.get('logfile'), receiver=config.get('notification')) martaslog.telegram['config'] = config.get('notificationconfig') martaslog.msg(statusmsg) else: print("Debug selected - statusmsg looks like:") print(statusmsg)
def main(argv): """ METHODS: read_kpnow_data(source) read_swpam_data(source, limit=5) _execute_sql swtableinit() """ version = "1.0.0" kpsource = '/srv/archive/external/gfz/kp/' swsource = '/srv/archive/external/esa-nasa/ace/raw/' kpname = 'gfzkp*' swename = '*_swepam_1m.txt' magname = '*_mag_1m.txt' configpath = '' # is only necessary for monitoring sqllist = [] debug = False init = False # create table if TRUE statusmsg = {} usage = 'sw_extractor.py -c <config> -k <kpsource> -s <swesource> ' try: opts, args = getopt.getopt(argv,"hc:k:s:o:p:b:e:ID",["config=","kpsource=","swesource=","init=","debug=",]) except getopt.GetoptError: print(usage) sys.exit(2) for opt, arg in opts: if opt == '-h': print('-------------------------------------') print('Description:') print('sw-extractor.py extracts CME predictions from Scoreboard ') print('-------------------------------------') print('Usage:') print(usage) print('-------------------------------------') print('Options:') print('-c : configuration data ') print('-k : Kp source - default is ') print('-s : SWE source - default is ') print('-------------------------------------') print('Examples:') print('---------') print('python3 sw_extractor.py -c /home/cobs/CONF/wic.cfg -k /srv/archive/external/gfz/kp/gfzkp* -s /srv/archive/external/esa-nasa/ace/raw/ -D') print('---------') sys.exit() elif opt in ("-c", "--config"): configpath = os.path.abspath(arg) elif opt in ("-k", "--kpsource"): kpsource = os.path.abspath(arg) elif opt in ("-s", "--swesource"): swsource = os.path.abspath(arg) elif opt in ("-n", "--kpname"): kpname = arg elif opt in ("-w", "--swename"): swname = arg elif opt in ("-m", "--magname"): magname = arg elif opt in ("-I", "--init"): init = True elif opt in ("-D", "--debug"): debug = True if debug: print ("Running sw-extractor version:", version) # 1. conf and logger: # ########################### if debug: print ("Read and check validity of configuration data") print (" and activate logging scheme as selected in config") config = GetConf(configpath) config = DefineLogger(config=config, category = "Info", job=os.path.basename(__file__), newname='mm-info-sw.log', debug=debug) if debug: print (" -> Done") # 2. database: # ########################### try: config = ConnectDatabases(config=config, debug=debug) db = config.get('primaryDB') connectdict = config.get('conncetedDB') except: statusmsg[name1] = 'database failed' # 3. Read Kp data: # ########################### try: sqllist = read_kpnow_data(os.path.join(kpsource,kpname),debug=debug) statusmsg['Kp access'] = 'success' except: statusmsg['Kp access'] = 'failed' # 4. Read ACE swepam data: # ########################### try: newsql = read_swepam_data(os.path.join(swsource,swename),debug=debug) sqllist.extend(newsql) statusmsg['ACE swepam access'] = 'success' except: statusmsg['ACE swepam access'] = 'failed' # 5. Read ACE mag data: # ########################### try: newsql = read_mag_data(os.path.join(swsource,magname),debug=debug) sqllist.extend(newsql) statusmsg['ACE mag access'] = 'success' except: statusmsg['ACE mag access'] = 'failed' # 6. Read GIC data: # ########################### try: newsql = read_gicnow_data(db,source='GICAUT',maxsensor=9, minutes=5, debug=debug) sqllist.extend(newsql) statusmsg['GIC data access'] = 'success' except: statusmsg['GIC data access'] = 'failed' # 7. Read GOES data: # ########################### try: if debug: print ("Running GOES") goespath = '/srv/archive/external/esa-nasa/goes' newsql = read_xrs_data(os.path.join(goespath,'XRS_GOES16*'), debug=debug) sqllist.extend(newsql) statusmsg['XRS data access'] = 'success' except: statusmsg['XRS data access'] = 'failed' # 8. Read PREDSTORM data: # ########################### try: if debug: print ("Running PREDSTORM") predpath = '/srv/archive/external/helio4cast/predstorm' psql = read_predstorm_data(os.path.join(predpath,'PREDSTORM*'), debug=debug) sqllist.extend(psql) statusmsg['PREDSTORM data access'] = 'success' except: # no predstorm data between 23:00 and 2:00 MET # just put success message if hitting except in this time range statusmsg['PREDSTORM data access'] = 'failed' if time_between(datetime.utcnow().time(),dttime(21,0),dttime(0,0)): statusmsg['PREDSTORM data access'] = 'success' sqllist = [el for el in sqllist if el] if debug: print ("Debug selected - sql call looks like:") print (sqllist) else: for dbel in connectdict: db = connectdict[dbel] print (" -- Writing data to DB {}".format(dbel)) if init: swtableinit(db) if len(sqllist) > 0: _execute_sql(db,sqllist, debug=debug) # Logging section # ########################### if not debug and config: martaslog = ml(logfile=config.get('logfile'),receiver=config.get('notification')) martaslog.telegram['config'] = config.get('notificationconfig') martaslog.msg(statusmsg) else: print ("Debug selected - statusmsg looks like:") print (statusmsg)
def main(argv): try: version = __version__ except: version = "1.0.0" configpath = '' statusmsg = {} joblist = ['default', 'service'] debug = False endtime = None testplot = False try: opts, args = getopt.getopt(argv, "hc:j:e:DP", [ "config=", "joblist=", "endtime=", "debug=", "plot=", ]) except getopt.GetoptError: print('gamma_products.py -c <config>') sys.exit(2) for opt, arg in opts: if opt == '-h': print('-------------------------------------') print('Description:') print('-- gamma_products.py will analyse magnetic data --') print( '-----------------------------------------------------------------' ) print('detailed description ..') print('...') print('...') print('-------------------------------------') print('Usage:') print('python gamma_products.py -c <config>') print('-------------------------------------') print('Options:') print('-c (required) : configuration data path') print('-j : default, service') print('-e : endtime') print('-------------------------------------') print('Application:') print('python gamma_products.py -c /etc/marcos/analysis.cfg') sys.exit() elif opt in ("-c", "--config"): # delete any / at the end of the string configpath = os.path.abspath(arg) elif opt in ("-j", "--joblist"): # get a list of jobs (vario, scalar) joblist = arg.split(',') elif opt in ("-e", "--endtime"): endtime = arg elif opt in ("-D", "--debug"): # delete any / at the end of the string debug = True elif opt in ("-P", "--plot"): # delete any / at the end of the string testplot = True if debug: print("Running gamma_products version {} - debug mode".format(version)) print("---------------------------------------") if not os.path.exists(configpath): print('Specify a valid path to configuration information') print( '-- check magnetism_products.py -h for more options and requirements' ) sys.exit() if endtime: try: endtime = DataStream()._testtime(endtime) except: print("Endtime could not be interpreted - Aborting") sys.exit(1) else: endtime = datetime.utcnow() print("1. Read configuration data") config = GetConf(configpath) config = ConnectDatabases(config=config, debug=debug) print("2. Activate logging scheme as selected in config") config = DefineLogger(config=config, category="DataProducts", job=os.path.basename(__file__), newname='mm-dp-scaradon.log', debug=debug) config['testplot'] = testplot starttime = datetime.strftime(endtime - timedelta(days=7), "%Y-%m-%d") if 'default' in joblist: print("3. Create standard data table") statusmsg = CreateOldsProductsTables(config=config, statusmsg=statusmsg, start=starttime, end=endtime, debug=debug) if 'service' in joblist: print("4. Create Webservice table") statusmsg = CreateWebserviceTable(config=config, statusmsg=statusmsg, start=starttime, end=endtime, debug=debug) if not debug: martaslog = ml(logfile=config.get('logfile'), receiver=config.get('notification')) martaslog.telegram['config'] = config.get('notificationconfig') martaslog.msg(statusmsg) else: print("Debug selected - statusmsg looks like:") print(statusmsg)
def main(argv): version = __version__ configpath = '' statusmsg = {} path = '' dayrange = 1 debug = False endtime = datetime.utcnow() try: opts, args = getopt.getopt(argv, "hc:r:e:l:D", [ "config=", "range=", "endtime=", "debug=", ]) except getopt.GetoptError: print('job.py -c <config>') sys.exit(2) for opt, arg in opts: if opt == '-h': print('-------------------------------------') print('Description:') print('-- general_graph.py will plot sensor data --') print( '-----------------------------------------------------------------' ) print('detailed description ..') print('...') print('...') print('-------------------------------------') print('Usage:') print('python general_graph.py -c <config>') print('-------------------------------------') print('Options:') print('-c (required) : configuration data path') print('-r : range in days') print('-e : endtime') print('-------------------------------------') print('Application:') print('python general_graph.py -c /etc/marcos/analysis.cfg') print('python general_graph.py -c /etc/marcos/analysis.cfg') print('# debug run on my machine') print( 'python3 general_graph.py -c ../conf/wic.cfg -s debug -k x,y,z -f none -D' ) sys.exit() elif opt in ("-c", "--config"): # delete any / at the end of the string configpath = os.path.abspath(arg) elif opt in ("-r", "--range"): # range in days dayrange = int(arg) elif opt in ("-e", "--endtime"): # endtime of the plot endtime = arg elif opt in ("-D", "--debug"): # delete any / at the end of the string debug = True if debug: print("Running ... graph creator version {}".format(version)) if not os.path.exists(configpath): print('Specify a valid path to configuration information') print('-- check general_graph.py -h for more options and requirements') sys.exit(0) if endtime: try: endtime = DataStream()._testtime(endtime) except: print("Endtime could not be interpreted - Aborting") sys.exit(1) else: endtime = datetime.utcnow() #>> EDIT >>>>>>>>>>>>>>>>>>>>>>>> newloggername = 'mm-pp-myplot' category = "MyPlot" #<<<<<<<<<<<<<<<<<<<<<<<< EDIT << print("1. Read and check validity of configuration data") config = GetConf(configpath) print("2. Activate logging scheme as selected in config") config = DefineLogger(config=config, category=category, job=os.path.basename(__file__), newname=newloggername, debug=debug) monitorname = "{}-plot".format(config.get('logname')) print("3. Connect to databases") config = ConnectDatabases(config=config, debug=debug) #try: # print ("4. Read and Plot method") success = CreateDiagram(config=config, endtime=endtime, dayrange=dayrange, debug=debug) # statusmsg[namecheck1] = "success" # if not success: # statusmsg #except: # statusmsg[namecheck1] = "failure" if not debug: martaslog = ml(logfile=config.get('logfile'), receiver=config.get('notification')) martaslog.telegram['config'] = config.get('notificationconfig') martaslog.msg(statusmsg) else: print("Debug selected - statusmsg looks like:") print(statusmsg)
def main(argv): version = '1.0.0' configpath = '' statusmsg = {} debug = False dipath = '/srv/archive/WIC/DI' analyzepath = os.path.join(dipath, 'analyze') datapath = os.path.join(dipath, 'data') priminst = '/home/cobs/ANALYSIS/Logs/primaryinst.pkl' plotdir = '/home/cobs/ANALYSIS/Info/plots' endtime = datetime.utcnow() starttime = endtime - timedelta(days=380) pier = "A2" caption = '' channelconfig = '/home/cobs/ANALYSIS/Info/conf/tg_base.cfg' failedmsg = '' try: opts, args = getopt.getopt(argv, "hc:t:D", ["config=", "telegram=", "debug="]) except getopt.GetoptError: print('tg_base.py -c <config>') sys.exit(2) for opt, arg in opts: if opt == '-h': print('-------------------------------------') print('Description:') print('-- tg_base.py will obtain baseline plots --') print( '-----------------------------------------------------------------' ) print('detailed description ..') print('...') print('...') print('-------------------------------------') print('Usage:') print('python tg_base.py -c <config>') print('-------------------------------------') print('Options:') print('-c (required) : configuration data path') print('-t (required) : telegram channel configuration') print('-------------------------------------') print('Application:') print( 'python tg_base.py -c /etc/marcos/analysis.cfg -t /etc/marcos/telegram.cfg' ) sys.exit() elif opt in ("-c", "--config"): # delete any / at the end of the string configpath = os.path.abspath(arg) elif opt in ("-t", "--channel"): # delete any / at the end of the string channelconfig = os.path.abspath(arg) elif opt in ("-D", "--debug"): # delete any / at the end of the string debug = True print("Running tg_base version {}".format(version)) print("--------------------------------") if not os.path.exists(configpath): print('Specify a valid path to configuration information') print( '-- check magnetism_products.py -h for more options and requirements' ) sys.exit() print("1. Read and check validity of configuration data") config = GetConf(configpath) print("2. Activate logging scheme as selected in config") config = DefineLogger(config=config, category="Info", job=os.path.basename(__file__), newname='mm-info-base.log', debug=debug) name1 = "{}-tgbase".format(config.get('logname')) statusmsg[name1] = 'Baseline notification successful' print("3. Connect databases and select first available") try: config = ConnectDatabases(config=config, debug=debug) db = config.get('primaryDB') connectdict = config.get('conncetedDB') except: statusmsg[name1] = 'database failed' # SOME DEFINITIONS: datapath = config.get('dipath') dipath = os.path.join(config.get('dipath'), '..') analyzepath = os.path.join(dipath, 'analyze') pier = config.get('primarypier') if config.get('temporarygraphs'): plotdir = config.get('temporarygraphs') caption = '' # 1. get primary instruments: # ########################### #lst = pload(open(priminst,'rb')) config, statusmsg = GetPrimaryInstruments(config=config, statusmsg=statusmsg, debug=debug) variosens = config.get('primaryVario') scalarsens = config.get('primaryScalar') print("PRIMARY INSTRUMENTS: vario={}, scalar={}".format( variosens, scalarsens)) # 2. define BLV filename # ########################### blvname = "BLVcomp_{}_{}_{}.txt".format(variosens, scalarsens, pier) blvdata = os.path.join(datapath, blvname) print("BASEVALUE SOURCE: {}".format(blvdata)) # 3. Read BLV fiel and create BLV plot for the last year # ########################### plttitle = "{}: {} and {}".format(pier, variosens, scalarsens) caption = CreateBLVPlot(db, blvname, blvdata, starttime, endtime, plotdir, plttitle, debug=debug) # 4. read file list of *.txt files remaining in DI/analyse # ########################### failedmsg = GetFailed(analyzepath, endtime, debug=debug) # 5. send all info to telegramchannel # ########################### if not debug: with open(os.path.join(plotdir, 'basegraph.png'), "rb") as f: telegram_send.send(images=[f], captions=[caption], conf=channelconfig, parse_mode="markdown") if not failedmsg == '': telegram_send.send(messages=[failedmsg], conf=channelconfig, parse_mode="markdown") else: print("Debug selected") print("tg_base successfully finished") # 6. Logging section # ########################### if not debug: martaslog = ml(logfile=config.get('logfile'), receiver=config.get('notification')) martaslog.telegram['config'] = config.get('notificationconfig') martaslog.msg(statusmsg) else: print("Debug selected - statusmsg looks like:") print(statusmsg)
def main(argv): """ METHODS: extract_config() -> read analysis config get_readlist() -> get calls to read data chunks get_data() -> get_chunk_config() -> obtain details of chunk get_chunk_feature() -> get statistical features for each chunk - get_emd_features() - obtain_basic_emd_characteristics() - get_features() - get_wavelet_features() """ version = "1.0.0" swsource = 'https://kauai.ccmc.gsfc.nasa.gov/CMEscoreboard/' path = '' creddb = 'cobsdb' output = [] stime = '' etime = '' configpath = '' # is only necessary for monitoring cmedict = {} hours_threshold = 12 debug = False init = False # create table if TRUE statusmsg = {} meankpcrit = 4.5 maxkpcrit = 6.0 receivers = {'deutsch' : {'userid1': {'name':'roman leon', 'email':'*****@*****.**', 'language':'deutsch'}}} languagedict = {'english' : {'msgheader': "Coronal mass ejection - CME", 'msgnew':'New CME started at ', 'msgupdate':'Update on CME from ', 'msgarrival':'Estimated arrival: ', 'msgpred':'Expected geomagnetic activity (Kp): ', 'timezone':'UTC', 'msgref':'Based on experimental data from [CMEscoreboard](https://kauai.ccmc.gsfc.nasa.gov/CMEscoreboard/)', 'msguncert':'arrival time estimates usually have an uncertainty of +/- 7 hrs', 'channeltype':'telegram', 'channelconfig':'/etc/martas/telegram.cfg', }, 'deutsch' : {'msgheader': "Sonneneruption - CME", 'msgnew':'Neuer CME (koronaler Massenauswurf) am ', 'msgupdate':'Update zu CME vom ', 'msgarrival':'Geschätzte Ankunftszeit: ', 'msgpred':'Erwartete geomagnetische Aktivität (Kp): ', 'timezone':'UTC', 'msgref':'Basierend auf experimentellen Daten des [CMEscoreboard](https://kauai.ccmc.gsfc.nasa.gov/CMEscoreboard/)', 'msguncert':'geschätzte Ankunftszeiten sind meistens mit Fehlern von +/- 7 hrs behaftet', 'channeltype':'telegram', 'channelconfig':'/etc/martas/telegram_cobsnoti.cfg', } } #'channelconfig':'/etc/martas/tg_weltraum.cfg', #'channelconfig':'/etc/martas/tg_space.cfg', usage = 'cme-extractor.py -c <config> -s <source> -o <output> -p <path> -b <begin> -e <end>' try: opts, args = getopt.getopt(argv,"hc:s:o:p:b:e:ID",["config=","source=","output=","path=","begin=","end=","init=","debug=",]) except getopt.GetoptError: print(usage) sys.exit(2) for opt, arg in opts: if opt == '-h': print('-------------------------------------') print('Description:') print('cme-extractor.py extracts CME predictions from Scoreboard ') print('-------------------------------------') print('Usage:') print(usage) print('-------------------------------------') print('Options:') print('-c : configuration data ') print('-s : source - default is ') print('-o : output (list like db,file)') print('-p : path') print('-b : starttime') print('-e : endtime') print('-------------------------------------') print('Examples:') print('---------') print('python3 cme_extractor.py -c /home/cobs/CONF/wic.cfg -o file,db,telegram,email -p /srv/archive/external/esa-nasa/cme/ -D') print('---------') sys.exit() elif opt in ("-c", "--config"): configpath = os.path.abspath(arg) elif opt in ("-s", "--source"): swsource = arg elif opt in ("-o", "--output"): output = arg.split(',') elif opt in ("-p", "--path"): path = os.path.abspath(arg) elif opt in ("-b", "--starttime"): stime = arg elif opt in ("-e", "--endtime"): etime = arg elif opt in ("-I", "--init"): init = True elif opt in ("-D", "--debug"): debug = True if debug: print ("Running cme-extractor version:", version) print ("Selected output:", output) # 1. conf and logger: # ########################### if debug: print ("Read and check validity of configuration data") print (" and activate logging scheme as selected in config") config = GetConf(configpath) config = DefineLogger(config=config, category = "Info", job=os.path.basename(__file__), newname='mm-info-cme.log', debug=debug) if debug: print (" -> Done") # 2. database: # ########################### try: config = ConnectDatabases(config=config, debug=debug) db = config.get('primaryDB') connectdict = config.get('conncetedDB') except: statusmsg[name1] = 'database failed' try: if swsource.startswith('http'): cmedict = get_cme_dictionary_scoreboard(url=swsource) elif os.path.isfile(swsource): cmedict = read_memory(swsource) else: sys.exit() statusmsg['CMEaccess'] = 'success' except: statusmsg['CMEaccess'] = 'failed' if debug: print (cmedict) if not cmedict: statusmsg['CMEaccess'] = 'failed' # read memory and extract new and updated data if path: if os.path.isdir(path): path = os.path.join(path,"cme_{}.json".format(datetime.strftime(datetime.utcnow(),"%Y"))) # open existing json, extend dictionary, eventually update contents data = read_memory(path,debug=False) full, new, up = get_new_inputs(data,cmedict) if debug: print ('Saveing to path:', path) print ('new:', new) print ('update:', up) if 'file' in output and (new or up): print (" Dealing with job: file") success = write_memory(path, cmedict) if success: statusmsg['CME2file'] = 'success' print (" -> everything fine") else: statusmsg['CME2file'] = 'failed' print (" -> failed") if 'db' in output and creddb: print (" Dealing with job: db") # Only add data with arrival time +12h > now to database # delete data with arrivaltime+12h < now from db success = False statusmsg['CME2db'] = 'failed' try: if debug: print ("Accessing database ...") config = ConnectDatabases(config=config, debug=debug) db = config.get('primaryDB') connectdict = config.get('conncetedDB') if debug: print (" ... done") success = True except: pass try: for dbel in connectdict: db = connectdict[dbel] print (" -- Writing data to DB {}".format(dbel)) if init: swtableinit(db) if success: update_database(db,full, new, up, swsource,debug=debug) statusmsg['CME2db'] = 'success' print (" -> everything fine") except: print (" -> failed") if 'telegram' in output or 'email' in output: print (" Dealing with jobs: telegram and email") # Access memory of already send data, send update/new statusmsg['CME2telegram'] = 'success' statusmsg['CME2mail'] = 'success' total = new + up for el in total: # Construct markdown message for each language provided valdict = full.get(el) moveon = True # Test some criteria before sending message (e.g. not too old, k large enough) print (" Checking notification criteria ...") arr = dparser.parse(valdict.get('arrival'),fuzzy=True) if arr < datetime.utcnow(): print (" -> arrival time in the past") moveon = False kprange = [float(vl) for vl in valdict.get('KPrange').split('-')] if mean(kprange) < meankpcrit: print (" -> expected average activity too small (< {})".format(meankpcrit)) moveon = False if max(kprange) < maxkpcrit and el in up: print (" -> expected maximum activity too small (< {}) - skipping update".format(maxkpcrit)) moveon = False if moveon: for lang in languagedict: langdic = languagedict[lang] msghead = "*{}*".format(langdic.get('msgheader')) if el in new: msgbody = "\n\n{} {}\n".format(langdic.get('msgnew'), valdict.get('start')) else: msgbody = "\n\n{} {}\n".format(langdic.get('msgupdate'), valdict.get('start')) msgbody += "\n{} *{}* {}\n".format(langdic.get('msgarrival'), valdict.get('arrival') ,langdic.get('timezone')) msgbody += "{} {}\n".format(langdic.get('msgpred'), valdict.get('KPrange')) msgbody += '{}\n'.format(langdic.get('msgref')) msgbody += "" msg = msghead+msgbody if debug: print (msg) if not debug and 'email' in output and lang == 'deutsch': #if 'email' in output and lang == 'deutsch': #TODO e-mail is not yet working print ("Now starting e-mail") # read e-mail receiver list from dictionary #try: mailcfg = config.get('emailconfig','/etc/martas/mail.cfg') maildict = read_meta_data(mailcfg) maildict['Subject'] = msghead # email is a comma separated string reclist = receivers.get(lang,{}) for dic in reclist: userdic = reclist[dic] email = userdic.get('email') name = userdic.get('name') preferedlang = userdic.get('language') maildict['Text'] = msgbody maildict['To'] = email #print (" receivers are: {}".format(maildict['To'])) #### Stop here with debug mode for basic tests without memory and mails if preferedlang == lang: print (" ... sending mail in language {} now: {}".format(lang,maildict)) sm(maildict) print (" ... done") statusmsg['CME2mail'] = 'success' print (" -> email fine") #except: #statusmsg['CME2mail'] = 'failed' if not debug and 'telegram' in output: print ("Now starting telegram") try: telegram_send.send(messages=[msg],conf=langdic.get('channelconfig'),parse_mode="markdown") print (" -> telegram fine") except: statusmsg['CME2telegram'] = 'failed' print (" -> telegram failed") if debug and 'telegram' in output and lang == 'deutsch': telegram_send.send(messages=[msg],conf=config.get('notificationconfig'),parse_mode="markdown") print (" -> debug telegram fine") # Logging section # ########################### if not debug and config: martaslog = ml(logfile=config.get('logfile'),receiver=config.get('notification')) martaslog.telegram['config'] = config.get('notificationconfig') martaslog.msg(statusmsg) else: print ("Debug selected - statusmsg looks like:") print (statusmsg)
def main(argv): try: version = __version__ except: version = "1.0.1" configpath = '' statusmsg = {} debug=False force=False starttime = None endtime = None joblist = ["adjusted", "quasidefinitive", "addon"] newloggername = 'mm-dp-magnetism.log' try: opts, args = getopt.getopt(argv,"hc:j:s:e:l:FD",["config=","joblist=","starttime=","endtime=","loggername=","force=","debug=",]) except getopt.GetoptError: print ('magnetism_products.py -c <config>') sys.exit(2) for opt, arg in opts: if opt == '-h': print ('-------------------------------------') print ('Description:') print ('-- magnetism_products.py will analyse magnetic data --') print ('-----------------------------------------------------------------') print ('detailed description ..') print ('...') print ('...') print ('-------------------------------------') print ('Usage:') print ('python magnetism_products.py -c <config>') print ('-------------------------------------') print ('Options:') print ('-c (required) : configuration data path') print ('-j : adjusted, quasidefinitive,addon') print (' : adjusted -> calculated adjusted data with constant') print (' : baseline approximation') print (' : quasidefinitive -> calculated data with spline') print (' : baseline and only if observer flags are present') print (' : addon -> requires adjusted, create diagtram and k values') print ('-l : loggername') print ('-s : starttime') print ('-e : endtime') print ('-F : force redo of quasidefinitve (not updating current.data)') print ('-------------------------------------') print ('Application:') print ('python magnetism_products.py -c /etc/marcos/analysis.cfg') print ('python magnetism_products.py -c /etc/marcos/analysisGAM.cfg -j adjusted -l mm-dp-magnetism-GAM') print ('python magnetism_products.py -c /etc/marcos/analysisGAM.cfg -j quasidefinitive -s starttime -e endtime -F -l mm-dp-qd-GAM') sys.exit() elif opt in ("-c", "--config"): # delete any / at the end of the string configpath = os.path.abspath(arg) elif opt in ("-j", "--joblist"): # get a list of jobs e.g. "adjusted, quasidefinitive, addon" joblist = arg.split(',') elif opt in ("-s", "--starttime"): # define an endtime for the current analysis - default is now starttime = arg elif opt in ("-e", "--endtime"): # define an endtime for the current analysis - default is now endtime = arg elif opt in ("-l", "--loggername"): # define an endtime for the current analysis - default is now newloggername = arg elif opt in ("-F", "--force"): # delete any / at the end of the string force = True elif opt in ("-D", "--debug"): # delete any / at the end of the string debug = True print ("Running magpy_products version {}".format(version)) if not os.path.exists(configpath): print ('Specify a valid path to configuration information') print ('-- check magnetism_products.py -h for more options and requirements') sys.exit() if "addon" in joblist and not "adjusted" in joblist: print ("joblist input 'addon' requires 'adjusted' as well - therefore skipping 'addon' option" ) if starttime: try: starttime = DataStream()._testtime(starttime) except: print ("Starttime could not be interpreted - using None") if endtime: try: endtime = DataStream()._testtime(endtime) except: print ("Endtime could not be interpreted - Aborting") sys.exit(1) else: endtime = datetime.utcnow() print ("1. Read and check validity of configuration data") config = GetConf(configpath) success = ValidityCheckConfig(config) #if not success: # sys.exit(1) # ############################################################# print ("2. Activate logging scheme as selected in config") config = DefineLogger(config=config, category = "DataProducts", job=os.path.basename(__file__), newname=newloggername, debug=debug) if debug: print (" -> Config contents:") print (config) print ("3. Check all paths and eventually remount directories") success,statusmsg = ValidityCheckDirectories(config=config, statusmsg=statusmsg, debug=debug) print ("4. Loading current.data and getting primary instruments") config, statusmsg = GetPrimaryInstruments(config=config, statusmsg=statusmsg, debug=debug) print ("5. Connect to databases") config = ConnectDatabases(config=config, debug=debug) if "adjusted" in joblist: print ("6. Obtain adjusted data") mindata,statusmsg = AdjustedData(config=config, statusmsg=statusmsg, endtime=endtime, debug=debug) if mindata.length()[0]>0 and "addon" in joblist: print ("8. Diagrams") suc,statusmsg = CreateDiagram(mindata, config=config,statusmsg=statusmsg, endtime=endtime) print ("9. K Values") suc,statusmsg = KValues(mindata, config=config,statusmsg=statusmsg) if "quasidefinitive" in joblist: print ("10. Obtain quasidefinitive data") statusmsg = QuasidefinitiveData(config=config, statusmsg=statusmsg, starttime=starttime, endtime=endtime, force=force, debug=debug) if not debug: martaslog = ml(logfile=config.get('logfile'),receiver=config.get('notification')) martaslog.telegram['config'] = config.get('notificationconfig') martaslog.msg(statusmsg) else: print ("Debug selected - statusmsg looks like:") print (statusmsg)
def main(argv): version = '1.0.0' configpath = '' statusmsg = {} debug = False channelconfig = '' try: opts, args = getopt.getopt(argv, "hc:t:D", ["config=", "telegram=", "debug="]) except getopt.GetoptError: print('tg_base.py -c <config>') sys.exit(2) for opt, arg in opts: if opt == '-h': print('-------------------------------------') print('Description:') print('-- tg_base.py will obtain baseline plots --') print( '-----------------------------------------------------------------' ) print('detailed description ..') print('...') print('...') print('-------------------------------------') print('Usage:') print('python tg_base.py -c <config>') print('-------------------------------------') print('Options:') print('-c (required) : configuration data path') print('-t (required) : telegram channel configuration') print('-------------------------------------') print('Application:') print( 'python tg_base.py -c /etc/marcos/wic.cfg -t /etc/marcos/telegram.cfg' ) sys.exit() elif opt in ("-c", "--config"): # delete any / at the end of the string configpath = os.path.abspath(arg) elif opt in ("-t", "--channel"): # delete any / at the end of the string channelconfig = os.path.abspath(arg) elif opt in ("-D", "--debug"): # delete any / at the end of the string debug = True print("Running tg_quake version {}".format(version)) print("--------------------------------") if not os.path.exists(configpath): print('Specify a valid path to configuration information') print( '-- check magnetism_products.py -h for more options and requirements' ) sys.exit() # 1. conf and logger: # ########################### print("Read and check validity of configuration data") config = GetConf(configpath) print(" -> Done") print("Activate logging scheme as selected in config") config = DefineLogger(config=config, category="Info", job=os.path.basename(__file__), newname='mm-info-quake.log', debug=debug) print(" -> Done") # 2. database: # ########################### name1 = "{}-tgquake".format(config.get('logname')) statusmsg[name1] = 'successful' continueeval = True try: config = ConnectDatabases(config=config, debug=debug) db = config.get('primaryDB') connectdict = config.get('conncetedDB') except: statusmsg[name1] = 'database failed' # SOME DEFINITIONS: temporarypath = config.get('temporarydata') memorypath = os.path.join(temporarypath, 'lastquake.npy') currentvaluepath = config.get('currentvaluepath') if not channelconfig: channelconfig = config.get('notificationconfig') if not channelconfig: print("No message channel defined - aborting") sys.exit(1) # 3. get quakes: # ########################### msg = 'problem with basic list generation' try: lastquakes = get_quakes(db, debug=debug) msg = 'problem with selecting relavant quakes' relevantquakes = select_relevant_quakes(lastquakes, criteria={}, debug=debug) msg = 'problem with new quakes' relevantquakes = new_quakes(relevantquakes, memorypath=memorypath, debug=debug) except: statusmsg[name1] = msg relevantquakes = [] # 4. sending notification: # ########################### try: if len(relevantquakes) > 0: send_quake_message(relevantquakes, tgconfig=channelconfig, memorypath=memorypath, debug=debug) except: statusmsg[name1] = 'problem sending notification' continueeval = False # 5. writing current data # ########################### try: if len(relevantquakes ) > 0 and currentvaluepath and continueeval and not debug: write_current_data(relevantquakes, currentvaluepath, debug=debug) except: statusmsg[name1] = 'problem writing current data' print("tg_quake successfully finished") # 6. Logging section # ########################### if not debug: martaslog = ml(logfile=config.get('logfile'), receiver=config.get('notification')) martaslog.telegram['config'] = config.get('notificationconfig') martaslog.msg(statusmsg) else: print("Debug selected - statusmsg looks like:") print(statusmsg)
def main(argv): version = '1.0.0' configpath = '' statusmsg = {} debug = False channelconfig = '' try: opts, args = getopt.getopt(argv, "hc:t:D", ["config=", "telegram=", "debug="]) except getopt.GetoptError: print('tg_kval.py -c <config>') sys.exit(2) for opt, arg in opts: if opt == '-h': print('-------------------------------------') print('Description:') print('-- tg_kval.py will obtain baseline plots --') print( '-----------------------------------------------------------------' ) print('detailed description ..') print('...') print('...') print('-------------------------------------') print('Usage:') print('python tg_base.py -c <config>') print('-------------------------------------') print('Options:') print('-c (required) : configuration data path') print('-l (required) : language dictionary') print('-t (optional) : telegram channel configuration') print('-------------------------------------') print('Application:') print( 'python tg_kval.py -c /etc/marcos/analysis.cfg -t /etc/marcos/telegram.cfg' ) sys.exit() elif opt in ("-c", "--config"): # delete any / at the end of the string configpath = os.path.abspath(arg) elif opt in ("-t", "--channel"): # delete any / at the end of the string channelconfig = os.path.abspath(arg) elif opt in ("-D", "--debug"): # delete any / at the end of the string debug = True print("Running tg_kval version {}".format(version)) print("--------------------------------") if not os.path.exists(configpath): print('Specify a valid path to configuration information') print( '-- check magnetism_products.py -h for more options and requirements' ) sys.exit() # 1. conf and logger: # ########################### print("Read and check validity of configuration data") config = GetConf(configpath) print(" -> Done") print("Activate logging scheme as selected in config") config = DefineLogger(config=config, category="Info", job=os.path.basename(__file__), newname='mm-info-kavl.log', debug=debug) print(" -> Done") # 2. database: # ########################### name1 = "{}-kval".format(config.get('logname')) statusmsg[name1] = 'successful' try: config = ConnectDatabases(config=config, debug=debug) db = config.get('primaryDB') connectdict = config.get('conncetedDB') except: statusmsg[name1] = 'database failed' # SOME DEFINITIONS: currentvaluepath = config.get('currentvaluepath') if not channelconfig: channelconfig = config.get('notificationconfig') if not channelconfig: print("No message channel defined - aborting") sys.exit(1) # 3. get K values: # ########################### try: data = get_kvals(db, debug=debug) except: statusmsg[name1] = 'problem with list generation' data = DataStream() # 4. update Spaceweather database: # ########################### try: for dbel in connectdict: db = connectdict[dbel] print(" -- Writing data to DB {}".format(dbel)) k2sw_database(db, data, debug=False) statusmsg['K2SWDB'] = 'fine' except: statusmsg['K2SWDB'] = 'problem with uploading' # 5. sending notification: # ########################### if data.length()[0] > 0: try: send_kval_message(data, currentvaluepath=currentvaluepath, debug=debug) except: statusmsg[name1] = 'problem sending notification' print("tg_kval successfully finished") # 6. Logging section # ########################### if not debug: martaslog = ml(logfile=config.get('logfile'), receiver=config.get('notification')) martaslog.telegram['config'] = config.get('notificationconfig') martaslog.msg(statusmsg) else: print("Debug selected - statusmsg looks like:") print(statusmsg)
def main(argv): version = __version__ configpath = '' statusmsg = {} outpath='/tmp' sensordefpath='' sensordefs = {} dayrange = 3 plotstyle = 'magpy' # one in magpy, xxx starttime = None endtime = None newloggername = 'mm-pp-tilt.log' flaglist = [] plotname = 'debug' debug=False opacity = 0.7 fullday = False show = False confinex = False gridcolor = '#316931' dropflagged = False sensorid='LM_TILT01_0001' keylist=None try: opts, args = getopt.getopt(argv,"hc:i:o:y:s:e:r:l:D",["config=","input=","output=","style=","starttime=","endtime=","range=","loggername=","debug=",]) except getopt.GetoptError: print ('try general_graph.py -h for instructions') sys.exit(2) for opt, arg in opts: if opt == '-h': print ('-------------------------------------') print ('Description:') print ('-- general_graph.py will plot sensor data --') print ('-----------------------------------------------------------------') print ('detailed description ..') print ('...') print ('...') print ('-------------------------------------') print ('Usage:') print ('python general_graph.py -c <config>') print ('-------------------------------------') print ('Options:') print ('-c (required) : configuration data path') print ('-i : input json file for sensor information') print ('-o : output directory (or file) to save the graph') print ('-y : plot style') print ('-l : loggername e.g. mm-pp-tilt.log') print ('-s : starttime') print ('-e : endtime') print ('-r : range in days') print ('-------------------------------------') print ('Application:') print ('python3 general_graph.py -c ../conf/wic.cfg -i ../conf/sensordef_plot.json -e 2020-12-17') print ('# debug run on my machine') print ('python3 general_graph.py -c ../conf/wic.cfg -e 2020-12-17 -D') sys.exit() elif opt in ("-c", "--config"): # delete any / at the end of the string configpath = os.path.abspath(arg) elif opt in ("-i", "--input"): # delete any / at the end of the string sensordefpath = os.path.abspath(arg) elif opt in ("-o", "--output"): # delete any / at the end of the string outpath = os.path.abspath(arg) elif opt in ("-y", "--style"): # define a plotstyle plotstyle = arg elif opt in ("-s", "--starttime"): # starttime of the plot starttime = arg elif opt in ("-e", "--endtime"): # endtime of the plot endtime = arg elif opt in ("-r", "--range"): # range in days dayrange = int(arg) elif opt in ("-l", "--loggername"): # loggername newloggername = arg elif opt in ("-D", "--debug"): # delete any / at the end of the string debug = True if debug: print ("Running graph creator in analysis version {}".format(version)) if not os.path.exists(configpath): print ('Specify a valid path to configuration information') print ('-- check general_graph.py -h for more options and requirements') sys.exit() if not os.path.exists(sensordefpath): print ('Sensordefinitions not found...') if debug: print (' ... but debug selected - using dummy values') sensordefs = debugsensor # creating a dummy sensordefs file in tmp print (' ... and now creating an example in /tmp/sensordefinitions_default.json') WriteMemory('/tmp/sensordefinitions_default.json', sensordefs) else: print ('-- check general_graph.py -h for more options and requirements') sys.exit() if endtime: try: endtime = DataStream()._testtime(endtime) except: print ("Endtime could not be interpreted - Aborting") sys.exit(1) else: endtime = datetime.utcnow() if not starttime: starttime = endtime-timedelta(days=dayrange) else: try: starttime = DataStream()._testtime(starttime) dayrange = int((endtime-starttime).days) except: print ("Starttime could not be interpreted - Aborting") sys.exit(1) # general test environment: if debug and sensorid == 'travis': print (" basic code test successful") sys.exit(0) print ("1. Read and check validity of configuration data") config = GetConf(configpath) print ("2. Activate logging scheme as selected in config") config = DefineLogger(config=config, category = "TestPlotter", job=os.path.basename(__file__), newname=newloggername, debug=debug) print ("3. Connect to databases") config = ConnectDatabases(config=config, debug=debug) print ("4. Read sensordefinitions") if not sensordefs: print (sensordefpath) sensordefs = ReadMemory(sensordefpath) plotname = os.path.basename(sensordefpath).replace('.json','').replace('_plot','') print ("Plotname : ", plotname) statname = "plot-{}".format(plotname) pass else: statname = "plot-{}".format('debug') print ("4.1 Extracting some basic definitions from sensor configuartion") senspar = sensordefs.get('parameter',{}) if senspar.get('fullday','False') in ['True','true','TRUE',True]: fullday = True try: opacity = float(senspar.get('opacity',0.7)) except: pass if senspar.get('show','False') in ['True','true','TRUE',True]: show = True if senspar.get('confinex','False') in ['True','true','TRUE',True]: confinex = True gridcolor = senspar.get('gridcolor','#316931') bartrange = float(senspar.get('bartrange',0.06)) print (" Fullday: {}, Opacity: {}, Show: {}, Confinex: {}, Gridcolor: {}".format(fullday,opacity, show,confinex,gridcolor)) print ("5. Cycle through sensordefinitions") symbollist = [] specialdict = [] streamlist = [] keylist = [] filllist = [] paddinglist=[] annotatelist =[] colorlist=[] flaglist = [] for cnt,dataid in enumerate(sensordefs): if not dataid=='parameter': processname = "{}-{}".format(statname,dataid.replace("_","-")) statusmsg[processname] = "failure" useflags = False outlier = False dropflagged = False sensdict = sensordefs[dataid] revision = sensdict.get('revision','0001') print ("5.{}.1 Check SensorID - or whether DataID provided for {}".format(cnt+1,dataid)) sensorid, revision = CheckSensorID(dataid, revision, debug=debug) keys = sensdict.get('keys',[]) columns = sensdict.get('columns',[]) units = sensdict.get('units',[]) path = sensdict.get('source','') plotstyle = sensdict.get('plotstyle','line') flagtreatment = sensdict.get('flags','') if 'outlier' in flagtreatment: outlier = True dropflagged = True if 'flag' in flagtreatment or 'drop' in flagtreatment: useflags = True if 'drop' in flagtreatment: dropflagged = True filenamebegins = sensdict.get('filenamebegins','') filenameends = sensdict.get('filenameends','') mergepath = sensdict.get('basesource','') mergebegins = sensdict.get('basebegins','') mergeends = sensdict.get('baseends','') if keys: print ("5.{}.2 Read datastream for {}".format(cnt+1,dataid)) try: stream, fl = ReadDatastream(config=config, starttime=starttime, endtime=endtime, sensorid=sensorid, keylist=keys, revision=revision, flags=useflags, outlier=outlier, dropflagged=dropflagged, datapath=path, filenamebegins=filenamebegins, filenameends=filenameends,mergepath=mergepath, mergebegins=mergebegins, mergeends=mergeends, columns=columns, units=units, debug=debug) if stream and stream.length()[0]>1: print ("5.{}.3 Check out flagging and annotation".format(cnt+1)) if 'flag' in flagtreatment: print (" -> eventuallyadding existing standard flags from DB") flaglist = fl if 'quake' in flagtreatment: quakekey = sensdict.get('quakekey',keys[0]) print (" -> eventually adding QUAKES to column {}".format(quakekey)) fl = Quakes2Flags(config=config, endtime=endtime, timerange=dayrange+1, sensorid=sensorid, keylist=quakekey, debug=debug) flaglist = combinelists(flaglist,fl) if 'coil' in flagtreatment: print (" -> eventually adding COIL data to column xxx") pass if len(flaglist) > 0: print (" => total amount of {} flags added".format(len(flaglist))) stream = stream.flag(flaglist) print ("5.{}.4 Creating plot configuration lists".format(cnt+1)) streamlist.append(stream) keylist.append(keys) padding = sensdict.get('padding',[]) if not padding: padding = [0.0 for el in keys] paddinglist.append(padding) annotate = sensdict.get('annotate',[]) if not annotate: annotate = [False for el in keys] annotatelist.append(annotate) color = sensdict.get('color',[]) if not color: color = ['k' for el in keys] colorlist.extend(color) if plotstyle == 'line': symbol = ['-' for el in keys] elif plotstyle == 'bar': symbol = ['z' for el in keys] else: symbol = ['-' for el in keys] symbollist.extend(symbol) fill = sensdict.get('fill',[]) filllist.extend(fill) speciald = sensdict.get('specialdict',{}) print (speciald) #for el in speciald: # vals = speciald[el] # if isinstance(vals,list): # vals = [int(ele) for ele in vals] # speciald[el] = vals # print (speciald[el]) #print (specialdict) #specialdict.extend(speciald) print (" ==> section 5.{} done".format(cnt+1)) #print (specialdict) statusmsg[processname] = "success" except: print (" -- severe error in data treatment") pass else: print (" -- no keys defined - skipping this sensor") pass if len(streamlist) > 0: #mp.plotStreams(streamlist,keylist, fill=filllist, colorlist=colorlist, padding=paddinglist, annotate=annotatelist, gridcolor='#316931', confinex=True, opacity=0.7) print ("6. Creating plot") if os.path.isdir(outpath): # creating file name from sensorsdef input file fullplotpath = os.path.join(outpath,"{}_{}.png".format(plotname,datetime.strftime(endtime,"%Y-%m-%d"))) print (" -> Saving graph to {}".format(fullplotpath)) elif os.path.isfile(outpath): fullplotpath = outpath else: fullplotpath = '' CreateDiagram(streamlist,keylist, filllist=filllist, colorlist=colorlist, paddinglist=paddinglist, annotatelist=annotatelist, symbollist=symbollist, specialdict=specialdict, gridcolor=gridcolor, confinex=confinex, opacity=opacity, fullday=fullday, bartrange=bartrange, show=show, fullplotpath=fullplotpath, debug=debug) if not debug: martaslog = ml(logfile=config.get('logfile'),receiver=config.get('notification')) martaslog.telegram['config'] = config.get('notificationconfig') martaslog.msg(statusmsg) else: print ("Debug selected - statusmsg looks like:") print (statusmsg)
def main(argv): version = '1.0.0' configpath = '' statusmsg = {} debug=False sensor = '' outpath = '/tmp' duration=2 format_type='PYASCII' endtime = None try: opts, args = getopt.getopt(argv,"hc:s:o:e:d:f:D",["config=","sensor=","outputpath=","endtime=","duration=","format=","debug=",]) except getopt.GetoptError: print ('getprimary.py -c <config>') sys.exit(2) for opt, arg in opts: if opt == '-h': print ('-------------------------------------') print ('Description:') print ('-- getprimary.py will determine the primary instruments --') print ('-----------------------------------------------------------------') print ('detailed description ..') print ('...') print ('...') print ('-------------------------------------') print ('Usage:') print ('python getprimary.py -c <config>') print ('-------------------------------------') print ('Options:') print ('-c (required) : configuration data path') print ('-------------------------------------') print ('Application:') print ('python getprimary.py -c /etc/marcos/analysis.cfg') sys.exit() elif opt in ("-c", "--config"): # delete any / at the end of the string configpath = os.path.abspath(arg) elif opt in ("-s", "--sensor"): sensor = arg elif opt in ("-o", "--outputpath"): # delete any / at the end of the string outpath = os.path.abspath(arg) elif opt in ("-e", "--endtime"): # get an endtime endtime = arg elif opt in ("-d", "--duration"): duration = int(arg) elif opt in ("-f", "--format"): format_type = arg elif opt in ("-D", "--debug"): # delete any / at the end of the string debug = True print ("Running convert data version {}".format(version)) print ("--------------------------------") if not os.path.exists(configpath): print ('Specify a valid path to configuration information') print ('-- check magnetism_products.py -h for more options and requirements') sys.exit() if not sensor: print ('Specify a sensor name') sys.exit(0) if endtime: endtime = DataStream()._testtime(endtime) else: endtime = datetime.utcnow() print ("1. Read and check validity of configuration data") config = GetConf(configpath) print ("2. Activate logging scheme as selected in config") config = DefineLogger(config=config, category="DataProducts", job=os.path.basename(__file__), newname='mm-dp-convert{}.log'.format(sensor)) name1 = "{}-{}".format(config.get('logname'),sensor) statusmsg[name1] = 'successful' print ("3. Connect databases and select first available") try: config = ConnectDatabases(config=config, debug=debug) db = config.get('primaryDB') if debug: print (" -- success") except: if debug: print (" -- database failed") statusmsg[name1] = 'database failed' print ("4. Running conversion") try: starttime = datetime.strptime(datetime.strftime(endtime-timedelta(days=duration),"%Y-%m-%d"),"%Y-%m-%d") convert(db, sensor,outpath,starttime=starttime,endtime=endtime,format_type=format_type,debug=debug) except: if debug: print (" -- conversion failed") statusmsg[name1] = 'conversion failed' if not debug: martaslog = ml(logfile=config.get('logfile'),receiver=config.get('notification')) martaslog.telegram['config'] = config.get('notificationconfig') martaslog.msg(statusmsg) else: print ("Debug selected - statusmsg looks like:") print (statusmsg)