def tgplot(sensor, starttime, endtime, keys=None): """ DESCRIPTION plotting subroutine """ try: data = read(os.path.join(mqttpath,sensor,'*'),starttime=starttime, endtime=endtime) matplotlib.use('Agg') mp.plot(data, confinex=True, outfile=os.path.join(tmppath,'tmp.png')) return True except: return False
def CreateBLVPlot(db, blvname, blvdata, starttime, endtime, plotdir, plttitle, debug=False): print(" Loading absolute data: {}".format(blvdata)) absresult = read(blvdata, starttime=starttime, endtime=endtime) absresult = absresult.remove_flagged() print(" -> {} data points".format(absresult.length()[0])) try: blvflagname = blvname.replace("comp", "").replace(".txt", "") flags = db2flaglist(db, blvflagname) print(" Obtained {} flags".format(len(flags))) if len(flags) > 0: absresult = absresult.flag(flags) absresult = absresult.remove_flagged() except: print(" flagging failed") try: absresult = absresult._drop_nans('dx') absresult = absresult._drop_nans('dy') absresult = absresult._drop_nans('dz') print(" -> {} valid data points".format(absresult.length()[0])) func = absresult.fit(['dx', 'dy', 'dz'], fitfunc='spline', knotstep=0.3) print(" Saving to {}".format(plotdir)) if not debug: mp.plot(absresult, ['dx', 'dy', 'dz'], symbollist=['o', 'o', 'o'], padding=[2.5, 0.005, 2.5], function=func, plottitle=plttitle, outfile=os.path.join(plotdir, 'basegraph.png')) caption = "{}: Basevalues and adopted baseline".format( datetime.strftime(endtime, "%Y-%m-%d")) except: print(" -> error encountered when fitting data") caption = "Not enough data points for creating new baseline graph" if debug: print("Caption:", caption) return caption
def CreateWebserviceTable(config={}, statusmsg={}, start=datetime.utcnow() - timedelta(days=7), end=datetime.utcnow(), debug=False): # 1. read data rawdatapath = config.get('gammarawdata') meteopath = config.get('meteoproducts') result = DataStream() rcsg0path = config.get('rcsg0rawdata') name = "{}-servicetables".format(config.get('logname')) connectdict = config.get('conncetedDB') statusmsg[name] = 'gamma webservice table successfully created' try: if debug: print(" -----------------------------------") print(" Creating WebService database table") print(" -> Reading SCA Gamma data...") gammasca = read(os.path.join(rawdatapath, 'COBSEXP_2_*'), starttime=start, endtime=end) if gammasca.length()[0] > 0: gammasca.header['col-t1'] = 'T (tunnel)' gammasca.header['unit-col-t1'] = 'deg C' if debug: print(gammasca._get_key_headers()) print(" -> Done") except: statusmsg[name] = 'gamma table failed - critical' gammasca = DataStream() try: print(" -> Reading meteo data ...") meteo = read(os.path.join(meteopath, 'meteo-1min_*'), starttime=start, endtime=end) if debug: print(meteo._get_key_headers()) if meteo.length()[0] > 0: if debug: print(meteo.length()) meteo._move_column('y', 'var3') meteo._drop_column('y') #rain - keep meteo._drop_column('t1') meteo._drop_column('var4') meteo._move_column('f', 't2') #temp - keep -> add unit and description meteo._drop_column('f') meteo._drop_column('var2') # wind direction - remove meteo.header['col-t2'] = 'T (outside)' meteo.header['unit-col-t2'] = 'deg C' meteo.header['col-var3'] = 'rain' meteo.header['unit-col-var3'] = 'mm/h' # dropping potential string columns meteo._drop_column('str2') print(" -> Done") else: statusmsg[name] = 'no meteo data' print(" -> Done - no data") except: statusmsg[name] = 'meteo table failed' meteo = DataStream() # 2. join with other data from meteo if gammasca.length()[0] > 0 and meteo.length()[0] > 0: #meteo = meteo.filter() result = mergeStreams(gammasca, meteo) elif gammasca.length()[0] > 0: result = gammasca.copy() else: result = DataStream() # 3. add new meta information result.header['StationID'] = 'SGO' result.header['SensorID'] = 'GAMMASGO_adjusted_0001' result.header['DataID'] = 'GAMMASGO_adjusted_0001_0001' result.header['SensorElements'] = 'Counts,Temp,OutsideTemp,Voltage,rain' result.header['SensorKeys'] = 'x,t1,t2,var1,var3' result.header['SensorGroup'] = 'services' result.header['SensorName'] = 'GAMMASGO' result.header['SensorType'] = 'Radiometry' if debug: print(" Results", result.length()) if debug and config.get('testplot', False): mp.plot(result) # 4. export to DB as GAMMASGO_adjusted_0001_0001 in minute resolution if not debug: if result.length()[0] > 0: if len(connectdict) > 0: for dbel in connectdict: dbw = connectdict[dbel] # check if table exists... if not use: name3 = "{}-toDB-{}".format(config.get('logname'), dbel) statusmsg[name3] = 'gamma table successfully written to DB' try: writeDB(dbw, result) except: statusmsg[ name3] = 'gamma table could not be written to DB - disk full?' # else use #writeDB(dbw,datastream, tablename=...) print( " -> GAMMASGO_adjusted written to DB {}".format(dbel)) return statusmsg
def process_data(station, sensortype, sensors, basepath, fileext, date, plotvariables, **kwargs): ''' DEFINITION: STANDARD: Reads data, writes prelim data, plots & writes final data OPTIONAL: Baseline & declination correct, multiply stream, upload, produce plot of last 7 days. NOTE: Files must be saved under proper data format: --> /BASEPATH/TO/DATA/IAGA-ST-CODE/INSTRUMENT... (e.g: /srv/archive/magnetism/tse/lemi...) .../raw .../plots .../filtered Must also have file with header information under BASEPATH/CODE/CODE_headers.txt. PARAMETERS: Variables: - station: (str) IAGA-code for station - sensortype: (str) 'combined' or 'normal' combined = only for magnetic variometer + magnetometer normal = all other data - sensors: (str/list) Sensor name, 'ENV05_1_0001' Note: for 'combined' this is a list. ['LEMI025_22_0001', 'POS1_N432_0001'] - basepath: (str) Path to where data is stored. - fileext: (str/list) File extension of data file, e.g. 'bin', 'cdf', 'txt' Note: for 'combined' this is a list. ['bin', 'cdf'] - date: (str) Date of data in format %Y-%m-%d / YYYY-MM-DD. - plotvariables: (list) List of magpy keys to plot. Kwargs: - logger: (logging.Logger object) Logger for logging purposes. - decl: (float) Will rotate data by this value, if given - mult_factors: (dict) Will multiply corresponding stream keys with these factors - baseline: (dict) Will baseline correct corresponding keys with these factors - upload: (bool) If True, will upload files - sevendayplot: (bool) If True, will produce plot of last 7 days - prelim: (bool) If True, will save prelim file RETURNS: - True / False EXAMPLE: >>> APPLICATION: ''' decl = kwargs.get('decl') mult_factors = kwargs.get('mult_factors') baseline = kwargs.get('baseline') upload = kwargs.get('upload') prelim = kwargs.get('prelim') sevendayplot = kwargs.get('sevendayplot') logger = kwargs.get('logger') if not logger: logging.basicConfig(level=logging.INFO) logger = logging.getLogger(' %s ' % station.upper()) header_data = {} headersfile = os.path.join(basepath, station, '%s_headers.txt' % station) headers = open(headersfile, 'r') for line in headers: hdata = line.split() header_data[hdata[0].strip(':')] = hdata[1] #-------------------------------------------------------------------- # 1. READ DATA, REMOVE OUTLIERS, FILTER if sensortype == 'combined': vario_sensor = sensors[0] magn_sensor = sensors[1] v_datafile = '%s_%s.%s' % (vario_sensor, date, fileext[0]) m_datafile = '%s_%s.%s' % (magn_sensor, date, fileext[1]) v_datapath = os.path.join(basepath, station, vario_sensor, 'raw', v_datafile) m_datapath = os.path.join(basepath, station, magn_sensor, 'raw', m_datafile) logger.info("Reading files %s and %s..." % (v_datafile, m_datafile)) v_stream = read(v_datapath) m_stream = read(m_datapath) #v_stream.remove_outlier() #v_stream.remove_flagged() #m_stream.remove_outlier() #m_stream.remove_flagged() v_stream = v_stream.filter() m_stream = m_stream.filter() stream = mergeStreams(v_stream, m_stream) stream.header['col-f'] = 'F' stream.header['unit-col-f'] = 'nT' sensor = vario_sensor title = '%s-%s' % (vario_sensor, magn_sensor) elif sensortype == 'normal': sensor = sensors datafile = '%s_%s.%s' % (sensor, date, fileext) logger.info("Reading file %s..." % datafile) datapath = os.path.join(basepath, station, sensor, 'raw', datafile) stream = read(datapath) #stream.remove_outlier() #stream.remove_flagged() stream = stream.filter() title = sensor else: logger.error( "Wrong sensortype (%s). Options are 'combined' and 'normal'." % sensortype) for data_header in IAGA_headers: stream.header[data_header] = header_data[data_header] if sensor[:3].lower() == 'lem': stream.header['DataType'] = 'Magnetic' stream.header['DataComponents'] = 'x, y, z, F [nT]' stream.header['DataDigitalSampling'] = '0.1s, 5s' dx = 1000. * stream.header['DataCompensationX'] dy = 1000. * stream.header['DataCompensationY'] dz = 1000. * stream.header['DataCompensationZ'] stream.header['DataSensorOrientation'] = "%s, %s, %s" % (dx, dy, dz) elif sensor[:3].lower() == 'env': stream.header['DataType'] = 'Environmental' stream.header[ 'DataComponents'] = 'T (ambient) [C], RH [%], T (dewpoint) [C]' stream.header['DataDigitalSampling'] = '1s' stream._move_column(plotvariables[0], 'x') stream._move_column(plotvariables[1], 'y') stream._move_column(plotvariables[2], 'z') plotvariables = ['x', 'y', 'z'] elif sensor[:3].lower() == 'cs1': stream.header['DataType'] = 'Magnetic' stream.header['DataComponents'] = 'F [nT]' stream.header['DataDigitalSampling'] = '1s' filenamebegins = '%s_0002' % (title) #filenamebegins = '%s_%s_' % (station,title) if prelim: prelim_path = os.path.join(basepath, station, sensor, 'prelim') stream.write(prelim_path, filenamebegins=filenamebegins + '_', format_type='IAGA') logger.info("Preliminary data written to %s." % prelim_path) #-------------------------------------------------------------------- # 2. (OPTIONAL) ROTATE, MULTIPLY, BASELINE CORRECT # Steps for PRELIMINARY --> FINAL if decl: stream.rotation(alpha=decl) if mult_factors: stream.multiply(mult_factors) if baseline: stream.offset(baseline) #-------------------------------------------------------------------- # 3. PLOT sensorpadding = {'env': 0.5, 'pos': 10, 'lem': 5, 'cs1': 10} plotname = '%s_%s.png' % (filenamebegins, date) outfile = os.path.join(basepath, station, sensor, 'plots', plotname) mp.plot(stream, plotvariables, plottitle='%s %s (%s)' % (station.upper(), title, date), bgcolor='white', confinex=True, fullday=True, outfile=outfile, padding=sensorpadding[sensor[:3].lower()]) logger.info("Data plotted to %s." % outfile) #-------------------------------------------------------------------- # 4. SAVE & WRITE STREAM TO MINUTE FILE #filenamebegins = '%s_%s_' % (station,title) finalpath = os.path.join(basepath, station, sensor, filenamebegins) stream.write(finalpath, filenamebegins=filenamebegins + '_', format_type='IAGA') logger.info("Final data written to %s." % finalpath) #-------------------------------------------------------------------- # 5. UPLOAD (plot + filtered data) cred = 'cobshomepage' myproxy = mpcred.lc(cred, 'address') login = mpcred.lc(cred, 'user') passwd = mpcred.lc(cred, 'passwd') #passwd = 'ku7tag8!haus' # TODO CHANGE THIS BACK port = mpcred.lc(cred, 'port') ftppath = 'cmsjoomla/images/stories/currentdata/' upload = False if upload: try: filtered_file = '%s_%s.txt' % (filenamebegins, date) filtered_path = os.path.join(basepath, station, sensor, filenamebegins, filtered_file) logger.info("Uploading %s..." % filtered_path) ftpdatatransfer( localfile=filtered_path, ftppath=ftppath, # TODO myproxy=myproxy, port=port, login=login, passwd=passwd, raiseerror=True, logfile=os.path.join(basepath, station, '%s-transfer.log' % station)) except: logger.error("Uploading failed.") try: plot_file = '%s_%s.png' % (filenamebegins, date) plot_path = os.path.join(basepath, station, title, 'plots', plotfile) logger.info("Uploading %s..." % plot_path) ftpdatatransfer( localfile=plotpath, ftppath=ftppath, # TODO myproxy=myproxy, port=port, login=login, passwd=passwd, raiseerror=True, logfile=os.path.join(basepath, station, '%s-transfer.log' % station)) except: logger.error("Uploading failed.") #-------------------------------------------------------------------- # 6. CREATE 7-DAY PLOT (x, y, z, F) & UPLOAD if sevendayplot: today = datetime.utcnow() date = datetime.strftime(today, "%Y-%m-%d") datapath = os.path.join(basepath, station, sensor, filenamebegins, '*') startdate = datetime.strptime(date, '%Y-%m-%d') - timedelta(days=7) start = datetime.strftime(startdate, "%Y-%m-%d") + ' 00:00:00' end = date + ' 00:00:00' last7days = read(path_or_url=datapath, starttime=start, endtime=end) plotname = 'TSE_last7days.png' plotpath = os.path.join(basepath, station, '7dayplots', plotname) diff = eval(last7days.header['DataSensorOrientation']) last7days = last7days.offset(offsets={ 'x': -float(diff[0]), 'y': -float(diff[1]), 'z': -float(diff[2]) }) last7days = last7days.calc_f() fig = mp.plot(last7days, ['x', 'y', 'z', 'f'], plottitle='%s Magnetic Data (%s - %s)' % (station, start[:10], end[:10]), bgcolor='white', noshow=True, padding=5) axes = gcf().get_axes() day = datetime.strptime(start, '%Y-%m-%d %H:%M:%S') while day <= datetime.strptime(end, '%Y-%m-%d %H:%M:%S'): if day.weekday() in [5, 6]: # Saturday or Sunday t_start = day t_end = day + timedelta(days=1) for ax in axes: ax.axvspan(t_start, t_end, facecolor='green', alpha=0.3, linewidth=0) day += timedelta(days=1) ax.get_xaxis().set_major_formatter( matplotlib.dates.DateFormatter('%d.%b\n%H:%M')) plt.savefig(plotpath, savedpi=80) ftppath = 'zamg/images/graphs/magnetism/' oldftppath = 'cmsjoomla/images/stories/currentdata/tse' scptransfer(plotpath, '94.136.40.103:' + ftppath, passwd)
deltaF=-0.258, diid='A2_WIC.txt', pier='A2', expD=4.0, expI=64.0, starttime='2015-06-01', endtime='2016-02-01', db=db, dbadd=True) print absresult.length() absresult.write('/srv/archive/WIC/DI/data', coverage='all', filenamebegins="BLV_FGE_S0252_0001_GSM90_14245_0002_A2_oc_min", format_type="PYSTR", mode='replace') data1 = read( '/srv/archive/WIC/DI/data/BLV_FGE_S0252_0001_GSM90_14245_0002_A2_oc_min.txt' ) mp.plot(data1, ['dx', 'dy', 'dz'], symbollist=['o', 'o', 'o']) data2 = read( '/srv/archive/WIC/DI/data/BLV_FGE_S0252_0001_GSM90_14245_0002_A2.txt') sub = subtractStreams(data1, data2) mp.plot(sub, ['dx', 'dy', 'dz'], symbollist=['o', 'o', 'o']) #writeDB(db,absresult,'BLV_FGE_S0252_0001_GSM90_14245_0002_A2')
def main(argv): creddb = '' # c dipath = '' # a variolist = '' # v variodataidlist = '' # j scalarlist = '' # s scalardataidlist = '' # k pierlist = '' # p abstypelist = '' # y azimuthlist = '' # z archive = '' # w (e.g. /srv/archive) identifier = 'BLV' # f stationid = 'wic' # t fallbackvariopath = '' # o fallbackscalarpath = '' # l begin = '1900-01-01' # b end = datetime.strftime(datetime.utcnow(), "%Y-%m-%d") # e expD = 3 # d expI = 64 # i compensation = False # m rotation = False # q dbadd = False # n addBLVdb = False # n flagging = False # g createarchive = False # r webdir = '/var/www/joomla/images/didaten/' # TODO add option webuser = '******' webgroup = 'www-data' defaultuser = '******' defaultgroup = 'cobs' debug = False flaglist = [] keepremote = False getremote = False remotecred = '' remotepath = '' variopath = '' # scalarpath = '' # try: opts, args = getopt.getopt( argv, "hc:a:v:j:s:k:o:mql:b:e:t:z:d:i:p:y:w:f:ngrx:u:D", [ "cred=", "dipath=", "variolist=", "variodataidlist=", "scalarlist=", "scalardataidlist=", "variopath=", "compensation=", "rotation=", "scalarpath=", "begin=", "end=", "stationid=", "pierlist=", "abstypelist=", "azimuthlist=", "expD=", "expI=", "write=", "identifier=", "add2DB=", "flag=", "createarchive=", "webdir=", "keepremote", "debug=" ]) except getopt.GetoptError: print( 'di.py -c <creddb> -a <dipath> -v <variolist> -j <variodataidlist> -s <scalarlist> -o <variopath> -m <compensation> -q <rotation> -l <scalarpath> -b <startdate> -e <enddate> -t <stationid> -p <pierlist> -z <azimuthlist> -y <abstypelist> -d <expectedD> -i <expectedI> -w <writepath> -f<identifier> -n <add2DB> -g <flag> -r <createarchive> -x <webdir> -u <user> --keepremote' ) sys.exit(2) for opt, arg in opts: if opt == '-h': print('-------------------------------------') print('Description:') print( 'dianalysis.py reads DI measurements and calculates DI values.' ) print('Provide variometer and scalar data for correction.') print('Returns di values, f and collimation angles.') print( 'A number of additional option allow for archiving, validity tests,' ) print( 'and output redirection. If variometer data is provided, base values' ) print('are calculated. ') print('') print('-------------------------------------') print('Usage:') print( 'di.py -c <creddb> -a <dipath> -v <variolist> -j <variodataidlist> -s <scalarlist> -o <variopath> -l <scalarpath> -m <compensation> -q <rotation> -b <startdate> -e <enddate> -t <stationid> -p <pierlist> -z <azimuthlist> -y <abstypelist> -d <expectedD> -i <expectedI> -w <writepath> -n <add2DB> -g <flag> -r <createarchive> -x <webdir> -u <user> --keepremote' ) print('-------------------------------------') print('Options:') print( '-c : provide the shortcut to the data bank credentials' ) print( '-a (required) : path to DI data - can be either a real path to a local directory' ) print( ' or a credential shortcut for a remote connection ' ) print( '-w (required) : archivepath for writing results and eventually accessing data.' ) print(' e.g. /srv/archive. or /tmp') print( ' below this folder the following structure will be implemented:' ) print(' -/srv/archive/"StationID"/DI/analyse : ') print( ' folder with raw data to be analyzed' ) print(' -/srv/archive/"StationID"/DI/data : ') print( ' folder with calculated di results' ) print( ' Name will be set to "BLV_" + variometerID + pier.' ) print( ' This plain text file can be opend and' ) print( ' analyzed with the MagPy stream package.' ) print(' -/srv/archive/"StationID"/DI/raw : ') print( ' archivefolder with successfully analyzed raw data' ) print( '-f : idetifier for BLV data (in database and filename) - default is BLV' ) print( '-v : variolist - comma separated list of variometer ids' ) print( '-j : variodataidlist - specify the dataids to be used fro each vario' ) print(' Default: 0002 for each variometer sensor') print('-o : path to variometer data') print( '-m (no input) : apply compensation field values to variometer data' ) print('-q (no input) : apply rotation to variometer data') print('-s : scalarpath - path to scalar data') print( '-k : scalardataidlist - specify the dataids to be used fro each scalar' ) print(' Default: 0002 for each scalar sensor') print( '-b : startdate - begin of analysis (not yet active)' ) print( '-e : enddate - default is today (not yet active)') print( '-t (required) : ID of the station i.e. the Observatory code (required if' ) print(' not in meta data of DI measurements)') print( '-z : list of astronomic azimuths of the mark from the measurement pier' ) print( ' : Azimuthlist needs either to be empty or to have the same order ' ) print(' and length of the pierlist.') print( ' use "False" if the specific value should be taken from the' ) print(' originalfile/database') print(' e.g. -p D3,H6 -z False,156.678') print( '-y : comma separated list of absolute data types for each pier' ) print(' : "di" for standard theodolite or "autodif" ') print(' : e.g. -p D3,H6 -m di,autodif ') print('-d : expected declination') print('-i : expected inclination') print( '-p (required) : name/number of the pier, comma separated list' ) print('-n (no input) : add di and basevalues to data base') print( '-g (no input) : read flaglist from DB if db is opened and add flags' ) print( '-r : move successfully analyzed files to raw archive' ) print('-x : directory to copy non-analyzed files to.') print( ' : can be a www directory at which PHP-scripts are used to edit data.' ) print('-u : define user for which jobs are performed') print(' : e.g. cobs:cobsgroup') print( '--keepremote : Don t delete remote files after dowloading them' ) print('-------------------------------------') print('Examples:') print('1. Running on MARCOS servers:') print( 'python di.py -c wic -a cobshome,cobenzlabs -v "FGE_S0252_0001"' ) print(' -s "POS1_N432_0001" -j 0002 -b 2014-01-01') print( ' -w /media/DAE2-4808/archive -d 3 -i 64 -t wic -p H1,A7,A2,A16' ) print( ' -y di,di,di,autodif -z False,179.8978,180.1391,267.3982' ) print('2. Running it with manually provided data links:') print( 'python di.py -c wic -a /media/DAE2-4808/archive/WIC/DI/analyze' ) print( ' -v "FGE_S0252_0001" -s "POS1_N432_0001" -j 0002 -b 2014-02-01' ) print( ' -e 2014-05-01 -w /media/DAE2-4808/archive -d 3 -i 64 -t wic' ) print(' -p H1,A7,A2,A16 -y di,di,di,autodif -r') print(' -z False,179.8978,180.1391,267.3982 -u user:group') print('python di.py -c cobs -a cobshomepage,cobenzlabs ') print( ' -v DIDD_3121331_0002,LEMI025_1_0002 -s DIDD_3121331_0002' ) print(' -j 0001,0001 -b 2014-10-01 -e 2014-10-07') print(' -w /srv/archive -d 3 -i 64 -t wik -p D -n -r') sys.exit() elif opt in ("-c", "--creddb"): creddb = arg elif opt in ("-a", "--dipath"): dipath = arg elif opt in ("-w", "--archive"): archive = arg elif opt in ("-f", "--identifier"): identifier = arg elif opt in ("-v", "--variolist"): variolist = arg.split(',') elif opt in ("-j", "--variodataidlist"): variodataidlist = arg.split(',') elif opt in ("-s", "--scalarlist"): scalarlist = arg.split(',') elif opt in ("-k", "--scalardataidlist"): scalardataidlist = arg.split(',') elif opt in ("-o", "--variopath"): fallbackvariopath = arg elif opt in ("-m", "--compensation"): compensation = True elif opt in ("-q", "--rotation"): rotation = True elif opt in ("-l", "--scalarpath"): fallbackscalarpath = arg elif opt in ("-b", "--begin"): begin = arg elif opt in ("-e", "--end"): end = arg elif opt in ("-t", "--stationid"): stationid = arg elif opt in ("-p", "--pierlist"): pierlist = arg.split(',') elif opt in ("-z", "--azimuthlist"): azimuthlist = arg.split(',') elif opt in ("-y", "--abstypelist"): abstypelist = arg.split(',') elif opt in ("-x", "--webdir"): webdir = arg elif opt in ("-u", "--user"): user = arg.split(':') if len(user) > 1: defaultuser = user[0] defaultgroup = user[1] elif opt in ("-d", "--expectedD"): try: expD = float(arg) except: print("expected declination needs to be a float") sys.exit() elif opt in ("-i", "--expectedI"): try: expI = float(arg) except: print("expected inclination needs to be a float") sys.exit() elif opt in ("-n", "--add2db"): dbadd = True addBLVdb = True elif opt in ("-g", "--flag"): flagging = True elif opt in ("--keepremote"): keepremote = True elif opt in ("-r", "--createarchive"): createarchive = True elif opt in ("-D", "--debug"): debug = True print("-------------------------------------") print("Starting di analysis ... MARTAS version {}".format(__version__)) print("-------------------------------------") if dipath == '': print(' Specify the path to the DI data: -a /path/to/my/data !') print(' -- check dianalysis.py -h for more options and requirements') sys.exit() if archive == '': print( ' Specify an Archive path for writing results: -w /path/to/my/archive !' ) print(' -- check dianalysis.py -h for more options and requirements') sys.exit() if variolist == '': variolist = [] if scalarlist == '': scalarlist = [] if azimuthlist == '': azimuthlist = [] if abstypelist == '' or len(abstypelist) == 0: abstypelist = ['di' for elem in pierlist] if variodataidlist == '': variodataidlist = [] if scalardataidlist == '': scalardataidlist = [] if len(variodataidlist) == 0: variodataidlist = ['0002' for elem in variolist] else: if not len(variolist) == len(variodataidlist): print( ' You need to specify a specific DataID for each variometer: e.g. -j 0002,0001' ) print( ' -- check dianalysis.py -h for more options and requirements') sys.exit() if len(scalardataidlist) == 0: scalardataidlist = ['0002' for elem in scalarlist] else: if not len(scalarlist) == len(scalardataidlist): print( ' You need to specify a specific DataID for each variometer: e.g. -j 0002,0001' ) print( ' -- check dianalysis.py -h for more options and requirements') sys.exit() if not len(abstypelist) == 0: if not len(abstypelist) == len(pierlist): print( ' Abstypelist needs to have the same order and length of the pierlist' ) print( ' -- check dianalysis.py -h for more options and requirements') sys.exit() try: test = datetime.strptime(begin, "%Y-%m-%d") print(test) except: print(' Date format for begin seems to be wrong: -b 2013-11-22') print(' -- check dianalysis.py -h for more options and requirements') sys.exit() try: datetime.strptime(end, "%Y-%m-%d") except: print(' Date format for end seems to be wrong: -e 2013-11-22') print(' -- check dianalysis.py -h for more options and requirements') sys.exit() if pierlist == []: print( ' Specify a list of the measurement piers containing at list one element: -p [Pier2]' ) print(' -- check dianalysis.py -h for more options and requirements') sys.exit() if not len(azimuthlist) == 0: if not len(azimuthlist) == len(pierlist): print( ' Azimuthlist needs to have the same order and length of the pierlist' ) print( ' -- check dianalysis.py -h for more options and requirements') sys.exit() if stationid == '': print(' Specify a station name e.g. your observatory code') print(' -- check dianalysis.py -h for more options and requirements') sys.exit() else: stationid = stationid.upper() if not creddb == '': print(" Accessing data bank ...") try: db = mysql.connect(host=mpcred.lc(creddb, 'host'), user=mpcred.lc(creddb, 'user'), passwd=mpcred.lc(creddb, 'passwd'), db=mpcred.lc(creddb, 'db')) print(" ... success") except: print(" ... failure - check your credentials") sys.exit() else: db = False if not fallbackvariopath == '': if not fallbackvariopath.endswith('*'): fallbackvariopath = os.path.join(fallbackvariopath, '*') variopath = fallbackvariopath if not fallbackscalarpath == '': if not fallbackscalarpath.endswith('*'): fallbackscalarpath = os.path.join(fallbackscalarpath, '*') scalarpath = fallbackscalarpath if variolist == []: if fallbackvariopath == '': print( ' !! You have not provided any variometer information at all') if scalarlist == []: if fallbackscalarpath == '': print(' You have not provided any independent scalar information') print( ' -> we asume this data is provided along with the DI files') # ----------------------------------------------------- # a) Basic information # ----------------------------------------------------- if debug: print(" -------------------------------------") print(" Archive", archive) print(" Variolist", variolist) print(" Abstypelist", abstypelist) print(" Dipath", dipath) # ----------------------------------------------------- # b) Getting new raw data from the input server # ----------------------------------------------------- print(" -------------------------------------") print(" Identifying DI data source") if not os.path.exists(dipath): print(" Checking given DI path for credential information ...") try: credlist = mpcred.sc() credshort = [elem[0] for elem in credlist] print(" ... found credentials") except: print( " dipath {} not existing - credentials not accessible - aborting" .format(dipath)) sys.exit() try: dic = dipath.split(',') print(dic) print(len(dic)) if len(dic) == 2: remotecred = dic[0] remotepath = dic[1] print( " Using credentials {} to get DI data from the remote path {}" .format(dic[0], dic[1])) elif len(dic) == 1: remotecred = dic[0] remotepath = '' print(" Using credentials {} to get DI data".format(dic[0])) else: print( " -> could not interprete dipath in terms of credential information" ) sys.exit() if remotecred in credshort: getremote = True else: print( " -> dipath %s not existing - credentials not existing - aborting" % dipath) except: print( " -> dipath %s not existing - credentials not existing - aborting" % dipath) sys.exit() if getremote == False: sys.exit() else: print(" Found directory at specified dipath location") # Getting data from the webdir (eventually edited and corrected) if createarchive and not webdir == '': print(" -------------------------------------") print(" Createarchive and webdir selected ...") dipath = os.path.join(archive, stationid, 'DI', 'analyze') for pier in pierlist: diid = pier + '_' + stationid + '.txt' for infile in iglob(os.path.join(webdir, '*' + diid)): # Testing whether file exists: if os.path.exists( os.path.join(dipath, os.path.split(infile)[1])): print(" Deleting:", os.path.join(dipath, os.path.split(infile)[1])) os.remove(os.path.join(dipath, os.path.split(infile)[1])) print(" Retrieving from webdir: ", infile) shutil.copy(infile, dipath) # Setting permission to defaultuser even if started the job uid = pwd.getpwnam(defaultuser)[2] gid = grp.getgrnam(defaultgroup)[2] os.chown(os.path.join(dipath, os.path.split(infile)[1]), uid, gid) # Deleting file from web dir try: os.remove(os.path.join(webdir, os.path.split(infile)[1])) except: print(" !! No permissions to modify webdirectory") pass # copy all files from web directory to the analysis folder if getremote: delete = True if keepremote: delete = False print(" -------------------------------------") print( " Getting remote data - deleting downloaded data from remote source set to {}" .format(delete)) dipath = os.path.join(archive, stationid, 'DI', 'analyze') for pier in pierlist: if not os.path.exists(dipath): os.makedirs(dipath) diid = pier + '_' + stationid + '.txt' try: port = mpcred.lc(remotecred, 'port') except: port = 21 ftpget(mpcred.lc(remotecred, 'address'), mpcred.lc(remotecred, 'user'), mpcred.lc(remotecred, 'passwd'), remotepath, os.path.join(archive, stationid, 'DI', 'analyze'), diid, port=port, delete=delete) if debug: print(" -------------------------------------") print( " DI data defined and collected - now starting the analysis for variometer: {}" .format(variolist)) print(" ") # ----------------------------------------------------- # c) analyze all files in the local analysis directory and put successfully analyzed data to raw # ----------------------------------------------------- for pier in pierlist: print("######################################################") print("Starting analysis for pier ", pier) print("######################################################") abspath = dipath diid = pier + '_' + stationid + '.txt' for vario in variolist: dataid = variodataidlist[variolist.index(vario)] if os.path.exists( os.path.join(archive, stationid, vario, vario + '_' + dataid)): variopath = os.path.join(archive, stationid, vario, vario + '_' + dataid, vario + '*') else: variopath = vario if not os.path.exists(variopath): print( " -> No variometerdata found in the specified paths/IDs - using dummy path" ) variopath = '/tmp/*' print(" -> Using Variometerdata at:", variopath) for scalar in scalarlist: # Define paths for variometer and scalar data scalarid = scalardataidlist[scalarlist.index(scalar)] if os.path.exists( os.path.join(archive, stationid, scalar, scalar + '_' + scalarid)): scalarpath = os.path.join(archive, stationid, scalar, scalar + '_' + scalarid, scalar + '*') else: scalarpath = scalar if not os.path.exists(scalarpath): print( " -> No scalar data found in the specified paths/IDs - using dummy path" ) scalarpath = '/tmp/*' print(" -> Using Scalar data at:", scalarpath) # ALPHA and delta needs to be provided with the database print(" -------------------------------------") print( " Extracting delta and rotation parameters ... should not be necessary as this is be done by absoluteAnalysis provided a database is connected" ) deltaF = 0.0 alpha = 0.0 beta = 0.0 """ if db: print(" ") alpha = dbgetfloat(db, 'DATAINFO', vario, 'DataSensorAzimuth') if not isNumber(alpha): alpha = 0.0 beta = dbgetfloat(db, 'DATAINFO', vario, 'DataSensorTilt') if not isNumber(beta): beta = 0.0 deltaF = dbgetfloat(db, 'DATAINFO', scalar, 'DataDeltaF') if not isNumber(deltaF): deltaF = 0.0 else: # eventually add an input option # load a scalar file from path and get delta F from header try: scal = read(scalarpath,starttime=begin,endtime=begin) try: scal = applyDeltas(db,scal) deltaF = 0.0 except: deltaF = scal.header['DataDeltaF'] except: deltaF = 0.0 try: var = read(variopath,starttime=begin,endtime=begin) try: var = applyDeltas(db,var) except: pass # TODO this is wrong -> but clarify whether a correction is necessary at all alpha = var.header['DataSensorAzimuth'] beta = var.header['DataSensorTilt'] except: alpha = 0.0 beta = 0.0 print("using alpha, beta, deltaF:", alpha, beta, deltaF) """ print(" -------------------------------------") print( " Extracting azimuth data ... should be contained in DI files, can be provided as option, is contained in PIERS table of DB" ) # Azimuths are usually contained in the DI files ## Eventually overriding azimuths in DI files if len(azimuthlist) > 0: azimuth = azimuthlist[pierlist.index(pier)] if azimuth == 'False' or azimuth == 'false': azimuth = False else: azimuth = False if azimuth: print( " -> Overriding (eventual) DI files data with an azimuth of {} deg" .format(azimuth)) else: print(" -> Using azimuth from DI file") if len(abstypelist) > 0: abstype = abstypelist[pierlist.index(pier)] if abstype == 'False' or abstype == 'false': abstype = False else: abstype = False if abstype: print(" -> Selected type of absolute measurements is {}". format(abstype)) else: print(" -> Absolute measurement type taken from DI file") # TODO ... Get azimuth data from PIERS table if db: print( " Checking azimuth in PIERS table of the database ...") val = dbselect(db, 'AzimuthDictionary', 'PIERS', 'PierID like "{}"'.format(pier))[0] print("Found ", val) print(" -------------------------------------") movetoarchive = False if createarchive and variolist.index( vario) == len(variolist) - 1 and scalarlist.index( scalar) == len(scalarlist) - 1: print( " Running analysis - and moving successfully analyzed files to raw directory" ) movetoarchive = os.path.join(archive, stationid, 'DI', 'raw') else: print( " Running analysis - and keeping files in analyze directory" ) absstream = absoluteAnalysis(abspath, variopath, scalarpath, expD=expD, expI=expI, diid=diid, stationid=stationid, abstype=abstype, azimuth=azimuth, pier=pier, alpha=alpha, deltaF=deltaF, starttime=begin, endtime=end, db=db, dbadd=dbadd, compensation=compensation, magrotation=rotation, movetoarchive=movetoarchive, deltaD=0.0000000001, deltaI=0.0000000001) print(" -> Done") # ----------------------------------------------------- # d) write data to a file and sort it, write it again # (workaround to get sorting correctly) # ----------------------------------------------------- print(" -------------------------------------") if absstream and absstream.length()[0] > 0: print(" Writing {} data line(s) ...".format( absstream.length()[0])) absstream.write(os.path.join(archive, stationid, 'DI', 'data'), coverage='all', mode='replace', filenamebegins=identifier + '_' + vario + '_' + scalar + '_' + pier) try: # Reload all data, delete old file and write again to get correct ordering newabsstream = read( os.path.join( archive, stationid, 'DI', 'data', identifier + '_' + vario + '_' + scalar + '_' + pier + '*')) os.remove( os.path.join( archive, stationid, 'DI', 'data', identifier + '_' + vario + '_' + scalar + '_' + pier + '.txt')) # delete file from hd newabsstream.write(os.path.join( archive, stationid, 'DI', 'data'), coverage='all', mode='replace', filenamebegins=identifier + '_' + vario + '_' + scalar + '_' + pier) except: print(" Stream apparently not existing...") print(" -> Done") if addBLVdb: # SensorID necessary.... print(" Adding data to the data bank ... ") #newabsstream.header["SensorID"] = vario writeDB(db, absstream, tablename=identifier + '_' + vario + '_' + scalar + '_' + pier) #stream2db(db,newabsstream,mode='force',tablename=identifier+'_'+vario+'_'+scalar+'_'+pier) print(" -> Done") # ----------------------------------------------------- # f) get flags and apply them to data # ----------------------------------------------------- if db and flagging and addBLVdb: newabsstream = readDB( db, identifier + '_' + vario + '_' + scalar + '_' + pier) flaglist = db2flaglist( db, identifier + '_' + vario + '_' + scalar + '_' + pier) elif addBLVdb: newabsstream = readDB( db, identifier + '_' + vario + '_' + scalar + '_' + pier) flaglist = [] if len(flaglist) > 0: flabsstream = newabsstream.flag(flaglist) #for i in range(len(flaglist)): # flabsstream = newabsstream.flag_stream(flaglist[i][2],flaglist[i][3],flaglist[i][4],flaglist[i][0],flaglist[i][1]) flabsstream.write(os.path.join(archive, stationid, 'DI', 'data'), coverage='all', filenamebegins=identifier + '_' + vario + '_' + scalar + '_' + pier) pltabsstream = flabsstream.remove_flagged() # ----------------------------------------------------- # h) fit baseline and plot # ----------------------------------------------------- try: #pltabsstream = read(os.path.join(archive,stationid,'DI','data',identifier+'_'+vario+'_'+scalar+'_'+pier+'*')) pltabsstream.trim(starttime=datetime.utcnow() - timedelta(days=380)) # fit baseline using the parameters defined in db (if parameters not available then skip fitting) #absstream = absstream.fit(['dx','dy','dz'],poly,4) savename = identifier + '_' + vario + '_' + scalar + '_' + pier + '.png' #absstream = absstream.extract('f',98999,'<') mp.plot(pltabsstream, ['dx', 'dy', 'dz'], symbollist=['o', 'o', 'o'], plottitle=vario + '_' + scalar + '_' + pier, outfile=os.path.join(archive, stationid, 'DI', 'graphs', savename)) #absstream.plot(['dx','dy','dz'],symbollist=['o','o','o'],plottitle=vario+'_'+scalar+'_'+pier,outfile=os.path.join(archive,stationid,'DI','graphs',savename)) except: pass # ----------------------------------------------------- # j) move files from analyze folder to web folder # ----------------------------------------------------- # move only if createarchive is selected if createarchive: print(" -------------------------------------") print( " Dont mind the error message - works only if su at cron is running this job" ) filelst = [] for infile in iglob( os.path.join(archive, stationid, 'DI', 'analyze', '*.txt')): print("Processing ", infile) filelst.append(infile) destination = '/var/www/joomla/images/didaten/' infilename = os.path.split(infile) print(infilename) try: shutil.copy(infile, destination) #perform changes to privs if not webuser == '': uid = pwd.getpwnam(webuser)[2] gid = grp.getgrnam(webgroup)[2] os.chown(os.path.join(destination, infilename[1]), uid, gid) except: print("Webdir not accessible - finishing") pass print("----------------------------------------------------------------") print("di.py app finished") print("----------------------------------------------------------------") print("SUCCESS")