示例#1
0
def handler(obj):
    try:
        fname = obj.filename.split("/")[-1]
        item = action_table.get(fname)
        if item == None:
            return apache.HTTP_NOT_FOUND

        (action, multipar) = item

        args = {}
        if obj.args != None:
            for arg in obj.args.split('&'):
                pv = arg.split('=', 1)
                if pv[0] in multipar:
                    if pv[0] not in args:
                        args[pv[0]] = []
                        
                    if len(pv) == 1:
                        args[pv[0]].append("")
                    else:
                        args[pv[0]].append(urllib.unquote(pv[1]))
                else:
                    if len(pv) == 1:
                        args[pv[0]] = ""
                    else:
                        args[pv[0]] = urllib.unquote(pv[1])

        action(obj, args)

    except ArclinkAuthFailed, e:
        obj.err_headers_out['WWW-Authenticate'] = 'Basic realm="%s"' % (e.dcname,)
        logs.debug("unauthorized")
        return apache.HTTP_UNAUTHORIZED
示例#2
0
def submit_email(to, subj, text):
		"""
		Sends an email with given subject and text to a specified address using the specified SMTP host.

		@arguments: to,   a string storing the email address of the recipient
					subj, a string defining the email subject
					text, a string containing the email message text
		"""
		# I am not able to set sender to [email protected] with smtplib.
		# It is forced to [email protected], causing the message to be rejected
		# by the GFZ mail server. -Andres

		#msg = MIMEText("")
		#msg['Subject'] = subj
		#msg['From'] = EMAIL_ADDR
		#msg['To'] = to
		#msg.set_payload(text)

		#server = smtplib.SMTP(SMTP_SERVER)
		#server.sendmail(EMAIL_ADDR, to, msg.as_string())
		#server.quit()

		cmd = "%s -I'From: %s' -I'To: %s' -I'Subject: %s' -a'Message-ID:' -A'X-Loop: %s' | %s -f'%s' -- '%s'" % \
			(FORMAIL_BIN, EMAIL_FROM, to, subj, EMAIL_ADDR, SENDMAIL_BIN, EMAIL_ADDR, to)

		logs.debug("executing cmd: %s" % cmd)

		fd = os.popen(cmd, "w")
		try:
			fd.write(text)
		finally:
			fd.close()
示例#3
0
	def __parse_request(self, line):
		"""
		Gets a request line in Breq_fast format to match it against the corresponding pattern.
		If successful the request list will be completed; the fail string otherwise.

		@arguments: line, a request line in Breq_fast format
		"""
		loc = "*"
		self.request = "%s\n%s" % (self.request, line)
		m = re.search("\s+".join(self.__reqlist),line)
		if m:
			d = m.groupdict()

			logs.debug("request_line: %s" % line)
				
			# catch two digit year inputs
			if d["beg_2year"]:
				if int(d["beg_2year"]) > 50:
					d["beg_4year"] = "19%s" % d["beg_2year"]
				else:
					d["beg_4year"] = "20%s" % d["beg_2year"]
			if d["end_2year"]:
				if int(d["end_2year"]) > 50:
					d["end_4year"] = "19%s" % d["end_2year"]
				else:
					d["end_4year"] = "20%s" % d["end_2year"]

			# some users have problems with time...
			if int(d["beg_hour"]) > 23:
				d["beg_hour"] = "23"
			if int(d["end_hour"]) > 23:
				d["end_hour"] = "23"
			if int(d["beg_min"]) > 59:
				d["beg_min"] = "59"
			if int(d["end_min"]) > 59:
				d["end_min"] = "59"
			if int(d["beg_sec"]) > 59:
				d["beg_sec"] = "59"
			if int(d["end_sec"]) > 59:
				d["end_sec"] = "59"
			
			try:
				beg_time = datetime.datetime(int(d["beg_4year"]),int(d["beg_month"]),int(d["beg_day"]),
											int(d["beg_hour"]),int(d["beg_min"]),int(d["beg_sec"]))
				end_time = datetime.datetime(int(d["end_4year"]),int(d["end_month"]),int(d["end_day"]),
											int(d["end_hour"]),int(d["end_min"]),int(d["end_sec"]))
			except ValueError, e:
				self.failstr = "%s%s [error: wrong begin or end time: %s]\n" % (self.failstr, line, e)
				return
			
			# expand network and station
			for (network, station) in self.__expand_net_station(d["network"], d["station"], beg_time, end_time):

				cha_list = re.findall("([\w?\*]+)\s*",d["cha_list"])
				if len(cha_list) == int(d['cha_num'])+1:
					loc = cha_list.pop()
				for cha in cha_list:
					cha = re.sub("[?]+","*",cha)
					self.reqlist.append((str(network),str(station),cha,loc,beg_time,end_time, {}, set()))
					logs.debug("reqlist.append: %s %s %s %s" % (str(network),str(station),cha,loc))
示例#4
0
def parse_breqfast_from_handler(req, fh):
    parser = BreqParser()
    parser.parse_email_from_handler(fh)
    req.content = parser.reqlist
    logs.debug("")
    if parser.failstr:
        logs.error(parser.failstr)
    else:
        logs.info("parsed %d lines from breqfast message" % len(req.content))
示例#5
0
文件: sds.py 项目: aemanov/seiscomp3
    def iterdata(self, time1, time2, net, sta, cha, loc):
        lasttime = None
        lastrecord = None
        recstream = []

        if self.__is_iso(time1, time2, net, sta, os.path.exists):
            recstream.append(("isoarchive", self.isodir))
        if self.__is_sds(time1, time2, net, sta, cha, loc, self.archdir, os.path.exists, os.listdir):
            recstream.append(("sdsarchive", self.archdir))
        if self.__is_sds(time1, time2, net, sta, cha, loc, self.nrtdir, os.path.exists, os.listdir):
            recstream.append(("sdsarchive", self.nrtdir))
        
        if not recstream and self.exists_db(time1, time2, net, sta, cha, loc):
            raise TemporaryUnavailabilityException

        for (service, source) in recstream:
            if lastrecord:
                try:
                    etime = lastrecord.endTime()
                except Core.ValueException:
                    logs.warning("SDS: record.endTime() raises Core.ValueException! Resulting SEED file maybe incorrect!")
                    etime = lastrecord.startTime()
                timetuple = time.strptime(etime.toString("%Y-%m-%d %H:%M:%S"), "%Y-%m-%d %H:%M:%S")                
                lasttime = datetime.datetime(*timetuple[:6])+datetime.timedelta(seconds=1) # avoids dublettes
                if lasttime >= time2:
                    break
                time1 = lasttime
                lastrecord = None
                
            self._recstream = IO.RecordStream.Create(service)
            if not self._recstream:
                logs.error("Could not fetch recordstream service '%s'" % service)
                raise StopIteration
            
            if not self._recstream.setSource(source):
                logs.error("Could not set recordstream source '%s'" % source)
                self._recstream = None
                raise StopIteration

            logs.debug("%s %s: addStream for %s-%s" % (service, source, str(time1), str(time2)))
            self._recstream.addStream(net,sta,loc,cha,Core.Time.FromString(str(time1),"%Y-%m-%d %H:%M:%S"),
                                      Core.Time.FromString(str(time2),"%Y-%m-%d %H:%M:%S"))

            try:
                recinput = IO.RecordInput(self._recstream, Core.Array.DOUBLE, Core.Record.SAVE_RAW)
                record = recinput.next()
                
                while record:
                    yield record.raw().str()
                    lastrecord = record
                    record = recinput.next()                

            except Core.GeneralException, e:
                logs.error(e.what())
            except Exception, e:
                logs.error("SDS: Unexpected exception occured: %s" % e)
示例#6
0
    def run(self):
        try:
            seiscompRoot = self.commandline().unrecognizedOptions()[0]
            sys.stderr.write("root directory: %s\n" % seiscompRoot)

            try:
                DCID = self.configGetString("datacenterID")

            except:
                logs.error("datacenterID not found in global.cfg")
                return False

            networkRestricted = {}
            incompleteResponse = {}

            global instdb
            instdb = Instruments(DCID)

            self.__load_file(loadGains, os.path.join(seiscompRoot, "config", "gain.dlsv"))

            # for backwards compatibility
            self.__load_file(loadGains, os.path.join(seiscompRoot, "config", "gain.tab.out"))
            self.__load_file(loadGains, os.path.join(seiscompRoot, "config", "gain.tab"))

            try:
                self.__load_file(instdb.load_db, os.path.join(seiscompRoot, "resp", "inst.db"))
                self.__load_file(instdb.load_sensor_attr, os.path.join(seiscompRoot, "resp", "sensor_attr.csv"))
                self.__load_file(instdb.load_datalogger_attr, os.path.join(seiscompRoot, "resp", "datalogger_attr.csv"))

            except (IOError, NettabError), e:
                logs.error("fatal error: " + str(e))
                return False

            sc3Inv = seiscomp3.DataModel.Inventory()
            inventory = InventoryWrapper(sc3Inv, DCID)

            existingNetworks = set()
            existingStations = set()

            for f in glob.glob(os.path.join(seiscompRoot, "key", "network_*")):
                try:
                    logs.debug("processing " + f)
                    netCode = f.split("/network_")[-1]
                    try:
                        kf = Keyfile(f)
                    except IOError, e:
                        logs.error(str(e))
                        continue

                    existingNetworks.add(netCode)
                    networkRestricted[netCode] = False

                    inventory.updateNetwork(netCode, kf)

                except ValueError, e:
                    logs.error("%s: %s" % (f, str(e)))
示例#7
0
    def __init__(self, appName):
        # initialize SC3 environment
        env = seiscomp3.System.Environment_Instance()

        # set up logging
        self.__syslog = seiscomp3.Logging.SyslogOutput()
        self.__syslog.open(appName, syslog_facility)

        for (v, c) in ((1, "error"), (2, "warning"), (2, "notice"),
                       (3, "info"), (4, "debug")):
            if verbosity >= v:
                self.__syslog.subscribe(seiscomp3.Logging.getGlobalChannel(c))

        logs.debug = seiscomp3.Logging.debug
        logs.info = seiscomp3.Logging.info
        logs.notice = seiscomp3.Logging.notice
        logs.warning = seiscomp3.Logging.warning
        logs.error = seiscomp3.Logging.error

        logs.notice("Starting webinterface")

        # load SC3 config files from all standard locations (SEISCOMP_ROOT must be set)
        self.__cfg = seiscomp3.Config.Config()
        env.initConfig(self.__cfg, appName, env.CS_FIRST, env.CS_LAST, True)

        self.__action_table = {}
        self.__modules = {}

        # Common config variables
        self.server_folder = self.getConfigString('SERVER_FOLDER', None)

        if not self.server_folder:
            err = "%s: Cannot find server root, configuration not loaded" % (
                appName)
            raise Exception(err)

        if not os.path.exists(self.server_folder):
            err = "%s: Server root directory not found" % (appName)
            raise Exception(err)

        # Add inventory cache here, to be accessible to all modules
        inventory = os.path.join(self.server_folder, 'data',
                                 'Arclink-inventory.xml')
        self.ic = InventoryCache(inventory)

        # Load all modules in given directory.
        # Modules must contain a class WI_Module, whose __init__() takes
        # WebInterface object (our self) as an argument and calls our
        # addAction().

        #for f in glob.glob(os.path.join(env.shareDir(), "plugins", "webinterface", "*.py")):
        for f in glob.glob(
                os.path.join(self.server_folder, "wsgi", "modules", "*.py")):
            self.__load_module(f)

        logs.debug(str(self))
示例#8
0
def parse_native_from_handler(req, fd):
    rqline = fd.readline()
    while rqline:
        rqline = rqline.strip()
        if not rqline:
            rqline = fd.readline()
            logs.debug("skipping empty request line")
            continue
            
        rqsplit = rqline.split()
        if len(rqsplit) < 3:
            logs.error("invalid request line: '%s'" % (rqline,))
            rqline = fd.readline()
            continue

        try:
            start_time = datetime.datetime(*map(int, rqsplit[0].split(",")))
            end_time = datetime.datetime(*map(int, rqsplit[1].split(",")))
        except ValueError, e:
            logs.error("syntax error (%s): '%s'" % (str(e), rqline))
            rqline = fd.readline()
            continue

        network = rqsplit[2]
        station = "."
        channel = "."
        location = "."

        i = 3
        if len(rqsplit) > 3 and rqsplit[3] != ".":
            station = rqsplit[3]
            i += 1
            if len(rqsplit) > 4 and rqsplit[4] != ".":
                channel = rqsplit[4]
                i += 1
                if len(rqsplit) > 5 and rqsplit[5] != ".":
                    location = rqsplit[5]
                    i += 1
                    
        while len(rqsplit) > i and rqsplit[i] == ".":
            i += 1
        
        constraints = {}
        for arg in rqsplit[i:]:
            pv = arg.split('=', 1)
            if len(pv) != 2:
                raise ArclinkError, "invalid request syntax"
            
            constraints[pv[0]] = pv[1]

        req.add(network, station, channel, location, start_time, end_time,
            constraints)
        
        rqline = fd.readline()
示例#9
0
def parse_native_from_handler(req, fd):
    rqline = fd.readline()
    while rqline:
        rqline = rqline.strip()
        if not rqline:
            rqline = fd.readline()
            logs.debug("skipping empty request line")
            continue
            
        rqsplit = rqline.split()
        if len(rqsplit) < 3:
            logs.error("invalid request line: '%s'" % (rqline,))
            rqline = fd.readline()
            continue

        try:
            start_time = datetime.datetime(*map(int, rqsplit[0].split(",")))
            end_time = datetime.datetime(*map(int, rqsplit[1].split(",")))
        except ValueError, e:
            logs.error("syntax error (%s): '%s'" % (str(e), rqline))
            rqline = fd.readline()
            continue

        network = rqsplit[2]
        station = "."
        channel = "."
        location = "."

        i = 3
        if len(rqsplit) > 3 and rqsplit[3] != ".":
            station = rqsplit[3]
            i += 1
            if len(rqsplit) > 4 and rqsplit[4] != ".":
                channel = rqsplit[4]
                i += 1
                if len(rqsplit) > 5 and rqsplit[5] != ".":
                    location = rqsplit[5]
                    i += 1
                    
        while len(rqsplit) > i and rqsplit[i] == ".":
            i += 1
        
        constraints = {}
        for arg in rqsplit[i:]:
            pv = arg.split('=', 1)
            if len(pv) != 2:
                raise ArclinkError, "invalid request syntax"
            
            constraints[pv[0]] = pv[1]

        req.add(network, station, channel, location, start_time, end_time,
            constraints)
        
        rqline = fd.readline()
示例#10
0
    def __init__(self, appName):
        # initialize SC3 environment
        env = seiscomp3.System.Environment_Instance()

        # set up logging
        self.__syslog = seiscomp3.Logging.SyslogOutput()
        self.__syslog.open(appName, syslog_facility)

        for (v, c) in ((1, "error"), (2, "warning"), (2, "notice"), (3, "info"), (4, "debug")):
            if verbosity >= v:
                self.__syslog.subscribe(seiscomp3.Logging.getGlobalChannel(c))

        logs.debug = seiscomp3.Logging.debug
        logs.info = seiscomp3.Logging.info
        logs.notice = seiscomp3.Logging.notice
        logs.warning = seiscomp3.Logging.warning
        logs.error = seiscomp3.Logging.error

        logs.notice("Starting webinterface")

        # load SC3 config files from all standard locations (SEISCOMP_ROOT must be set)
        self.__cfg = seiscomp3.Config.Config()
        env.initConfig(self.__cfg, appName, env.CS_FIRST, env.CS_LAST, True)

        self.__action_table = {}
        self.__modules = {}

        # Common config variables
        self.server_folder = self.getConfigString('SERVER_FOLDER', None)

        if not self.server_folder:
            err="%s: Cannot find server root, configuration not loaded" % (appName)
            raise Exception(err)

        if not os.path.exists(self.server_folder):
            err="%s: Server root directory not found" % (appName)
            raise Exception(err)

        # Add inventory cache here, to be accessible to all modules
        inventory = os.path.join(self.server_folder, 'data', 'Arclink-inventory.xml')
        self.ic = InventoryCache(inventory)

        # Load all modules in given directory.
        # Modules must contain a class WI_Module, whose __init__() takes
        # WebInterface object (our self) as an argument and calls our
        # addAction().

        #for f in glob.glob(os.path.join(env.shareDir(), "plugins", "webinterface", "*.py")):
        for f in glob.glob(os.path.join(self.server_folder, "wsgi", "modules", "*.py")):
            self.__load_module(f)

        logs.debug(str(self))
示例#11
0
def application(environ, start_response):
    """Main WSGI handler that processes client requests and calls
    the proper functions.

    Begun by Javier Quinteros <*****@*****.**>,
    GEOFON team, June 2013

    """

    # Read the URI and save the first word in fname
    #fname = environ['PATH_INFO'].split("/")[-1]
    #fname = environ['PATH_INFO'].lstrip('/').split("/")[0]
    # print "environ['PATH_INFO'].lstrip('/')", environ['PATH_INFO'].lstrip('/')

    fname = environ['PATH_INFO']

    if not len(fname):
        fname = 'default'

    logs.debug('fname: %s' % (fname))

    item = wi.getAction(fname)
    logs.debug('item: %s' % (str(item)))

    # Among others, this will filter wrong function names,
    # but also the favicon.ico request, for instance.
    if item is None:
        status = '404 Not Found'
        return send_html_response(status, 'Error! ' + status, start_response)

    (action, multipar) = item
    parameters = {}

    try:
        form = cgi.FieldStorage(fp=environ['wsgi.input'], environ=environ)

    except ValueError, e:
        if str(e) == "Maximum content length exceeded":
            # Add some user-friendliness (this message triggers an alert box on the client)
            return send_plain_response("400 Bad Request",
                                       "maximum request size exceeded",
                                       start_response)

        return send_plain_response("400 Bad Request", str(e), start_response)
示例#12
0
    def __expand_net_station(self, network, station, beg_time, end_time):
        """
		Mathias, 31.05.2007
		
		Expands a possibly wildcarded station code field by looking up in the inventory.
		Allowed wildcards: '?' and '*'
		
		@arguments: string network:       lookup for stations in this network (e.g. GE or G* or ?? or * or *E )
					string station:       one station entry, (e.g. SNAA or SN?? or ???? or * or SN* or ? )
					beg_time, end_time:   begin & end time: limit search for net/stations to selected time window
		
		@return: list of expanded (network, station) codes tuples
		"""
        net_station_list = []

        if re.search(r'[?\*]+', station):
            logs.debug("*** expanding %s for network: %s" % (station, network))

            # FIXME: begin=None, end=None
            db = self.mgr.get_inventory(network=re.sub("[?]+", "*", network),
                                        station=re.sub("[?]+", "*", station),
                                        begin=None,
                                        end=None)

            netlist = []
            netlist = db.network.keys()
            logs.debug("- netlist: %s" % netlist)

            for net_code in netlist:
                for net in db.network[net_code].itervalues():
                    for sta_code in net.station:
                        #logs.debug("- net/stacode: %s %s" % (net_code, sta_code))
                        s = re.sub("^[?]$", "*", station)  # single ? --> *
                        s = re.sub("[?]", ".", s)  # SN?? --> SN..
                        s = re.sub("[\*]", ".*",
                                   "^" + s + "$")  # S*AA --> S.*AA
                        #logs.debug("- station filter: %s" % s)
                        if re.match(s, sta_code):
                            net_station_list.append((net_code, sta_code))

        else:
            net_station_list.append((network, station))

        return net_station_list
示例#13
0
    def __expand_net_station(self, network, station, beg_time, end_time):
        """
		Mathias, 31.05.2007
		
		Expands a possibly wildcarded station code field by looking up in the inventory.
		Allowed wildcards: '?' and '*'
		
		@arguments: string network:       lookup for stations in this network (e.g. GE or G* or ?? or * or *E )
					string station:       one station entry, (e.g. SNAA or SN?? or ???? or * or SN* or ? )
					beg_time, end_time:   begin & end time: limit search for net/stations to selected time window
		
		@return: list of expanded (network, station) codes tuples
		"""
        net_station_list = []

        if re.search(r"[?\*]+", station):
            logs.debug("*** expanding %s for network: %s" % (station, network))

            # FIXME: begin=None, end=None
            db = self.mgr.get_inventory(
                network=re.sub("[?]+", "*", network), station=re.sub("[?]+", "*", station), begin=None, end=None
            )

            netlist = []
            netlist = db.network.keys()
            logs.debug("- netlist: %s" % netlist)

            for net_code in netlist:
                for net in db.network[net_code].itervalues():
                    for sta_code in net.station:
                        # logs.debug("- net/stacode: %s %s" % (net_code, sta_code))
                        s = re.sub("^[?]$", "*", station)  # single ? --> *
                        s = re.sub("[?]", ".", s)  # SN?? --> SN..
                        s = re.sub("[\*]", ".*", "^" + s + "$")  # S*AA --> S.*AA
                        # logs.debug("- station filter: %s" % s)
                        if re.match(s, sta_code):
                            net_station_list.append((net_code, sta_code))

        else:
            net_station_list.append((network, station))

        return net_station_list
示例#14
0
def application(environ, start_response):
    """Main WSGI handler that processes client requests and calls
    the proper functions.

    Begun by Javier Quinteros <*****@*****.**>,
    GEOFON team, June 2013

    """

    # Read the URI and save the first word in fname
    #fname = environ['PATH_INFO'].split("/")[-1]
    #fname = environ['PATH_INFO'].lstrip('/').split("/")[0]
    # print "environ['PATH_INFO'].lstrip('/')", environ['PATH_INFO'].lstrip('/')

    fname = environ['PATH_INFO']

    if not len(fname):
       fname = 'default'

    logs.debug('fname: %s' % (fname))

    item = wi.getAction(fname)
    logs.debug('item: %s' % (str(item)))

    # Among others, this will filter wrong function names,
    # but also the favicon.ico request, for instance.
    if item is None:
       status = '404 Not Found'
       return send_html_response(status, 'Error! ' + status, start_response)

    (action, multipar) = item
    parameters = {}

    try:
        form = cgi.FieldStorage(fp=environ['wsgi.input'], environ=environ)

    except ValueError, e:
        if str(e) == "Maximum content length exceeded":
            # Add some user-friendliness (this message triggers an alert box on the client)
            return send_plain_response("400 Bad Request", "maximum request size exceeded", start_response)

        return send_plain_response("400 Bad Request", str(e), start_response)
示例#15
0
    def send_notifiers(self, group):
        Nsize = DataModel.Notifier.Size()

        if Nsize > 0:
            logs.warning("trying to apply %d changes..." % Nsize)
        else:
            logs.notice("no changes to apply")
            return 0

        Nmsg = DataModel.Notifier.GetMessage(True)
            
        it = Nmsg.iter()
        msg = DataModel.NotifierMessage()

        maxmsg = 100
        sent = 0
        mcount = 0

        try:
            try:
                while it.get():
                    msg.attach(DataModel.Notifier_Cast(it.get()))
                    mcount += 1
                    if msg and mcount == maxmsg:
                        sent += mcount
                        logs.debug("sending message (%5.1f %%)" % (sent / float(Nsize) * 100.0))
                        self.send(group, msg)
                        msg.clear()
                        mcount = 0
                        self.sync("fill-db")
                    it.next()
            except:
                pass
        finally:
            if msg.size():
                logs.debug("sending message (%5.1f %%)" % 100.0)
                self.send(group, msg)
                msg.clear()
                self.sync("fill-db")

        return mcount
示例#16
0
def purge(obj, args):
    myaddr = args.get("arclink")
    req_id = args.get("req_id")

    try:
        (host, port) = myaddr.split(':')
        port = int(port)
    except (AttributeError, ValueError):
        raise ArclinkError, "invalid ArcLink address"

    if req_id is None or len(req_id) == 0:
        raise ArclinkError, "missing request ID "

    (sesskey, user, passwd) = init_session(obj, args)

    arcl.open_connection(host, port, user, passwd, timeout=ARCLINK_TIMEOUT)
    try:
        logs.debug("connected to %s at %s" %
                   (arcl.software, arcl.organization))

        arcl_status = arcl.purge(req_id)
        logs.debug("request deleted")

    finally:
        logs.debug("closing connection")
        arcl.close_connection()

    url = "http://%s%s/status?sesskey=%s&arclink=%s" % \
        (obj.hostname, obj.uri.rsplit("/", 1)[0], sesskey, myaddr)

    obj.headers_out['Location'] = url
    raise apache.SERVER_RETURN, apache.HTTP_MOVED_PERMANENTLY
示例#17
0
def purge(obj, args):
    myaddr = args.get("arclink")
    req_id = args.get("req_id")
    
    try:
        (host, port) = myaddr.split(':')
        port = int(port)
    except (AttributeError, ValueError):
        raise ArclinkError, "invalid ArcLink address"

    if req_id == None or len(req_id) == 0:
        raise ArclinkError, "missing request ID "

    (sesskey, user, passwd) = init_session(obj, args)
    
    arcl.open_connection(host, port, user, passwd, timeout=ARCLINK_TIMEOUT)
    try:
        logs.debug("connected to %s at %s" % (arcl.software, arcl.organization))

        arcl_status = arcl.purge(req_id)
        logs.debug("request deleted")

    finally:
        logs.debug("closing connection")
        arcl.close_connection()

    url = "http://%s%s/status?sesskey=%s&arclink=%s" % \
        (obj.hostname, obj.uri.rsplit("/", 1)[0], sesskey, myaddr)
        
    obj.headers_out['Location'] = url
    raise apache.SERVER_RETURN, apache.HTTP_MOVED_PERMANENTLY
示例#18
0
def download(obj, args):
    myaddr = args.get("arclink")
    req_id = args.get("req_id")
    vol_id = args.get("vol_id")

    try:
        (host, port) = myaddr.split(':')
        port = int(port)
    except (AttributeError, ValueError):
        raise ArclinkError, "invalid ArcLink address"

    if req_id is None or len(req_id) == 0:
        raise ArclinkError, "missing request ID "

    (sesskey, user, passwd) = init_session(obj, args)

    arcl.open_connection(host, port, user, passwd, timeout=ARCLINK_TIMEOUT)
    try:
        logs.debug("connected to %s at %s" %
                   (arcl.software, arcl.organization))

        arcl_status = arcl.get_status(req_id)
        logs.debug("got request status")

        req_type = arcl_status.request[0].type
        fext = ''
        if "compression=bzip2" in arcl_status.request[0].args.split():
            obj.content_type = "application/x-bzip2"
            fext = '.bz2'
        elif req_type == "WAVEFORM" or req_type == "RESPONSE":
            obj.content_type = "application/x-seed"
        else:
            obj.content_type = "application/xml"
            fext = '.xml'
        obj.headers_out[
            'Content-Disposition'] = 'attachment; filename=ArclinkRequest_%s%s' % (
                str(req_id), fext)

        arcl.download_data(obj, req_id, vol_id)
        logs.debug("download finished")

    finally:
        logs.debug("closing connection")
        arcl.close_connection()
示例#19
0
def download(obj, args):
    myaddr = args.get("arclink")
    req_id = args.get("req_id")
    vol_id = args.get("vol_id")
    
    try:
        (host, port) = myaddr.split(':')
        port = int(port)
    except (AttributeError, ValueError):
        raise ArclinkError, "invalid ArcLink address"

    if req_id == None or len(req_id) == 0:
        raise ArclinkError, "missing request ID "

    (sesskey, user, passwd) = init_session(obj, args)
    
    arcl.open_connection(host, port, user, passwd, timeout=ARCLINK_TIMEOUT)
    try:
        logs.debug("connected to %s at %s" % (arcl.software, arcl.organization))

        arcl_status = arcl.get_status(req_id)
        logs.debug("got request status")

        req_type = arcl_status.request[0].type
        fext = ''
        if "compression=bzip2" in arcl_status.request[0].args.split():
            obj.content_type = "application/x-bzip2"
            fext = '.bz2'
        elif req_type == "WAVEFORM" or req_type == "RESPONSE":
            obj.content_type = "application/x-seed"
        else:
            obj.content_type = "application/xml"
            fext = '.xml'
        obj.headers_out['Content-Disposition'] = 'attachment; filename=ArclinkRequest_%s%s' % (str(req_id), fext)

        
        arcl.download_data(obj, req_id, vol_id)
        logs.debug("download finished")

    finally:
        logs.debug("closing connection")
        arcl.close_connection()
示例#20
0
def status(obj, args):
    myaddr = args.get("arclink")
    req_id = args.get("req_id", "ALL")

    try:
        (host, port) = myaddr.split(':')
        port = int(port)
    except (AttributeError, ValueError):
        raise ArclinkError, "invalid ArcLink address"

    (sesskey, user, passwd) = init_session(obj, args)

    arcl.open_connection(host, port, user, passwd, timeout=ARCLINK_TIMEOUT)
    try:
        arcl.send_command("USER_IP %s" % obj.remote_host)
    except:
        pass

    try:
        obj.content_type = "text/html; charset=UTF-8"
        obj.write(status_head)

        logs.debug("connected to %s at %s" %
                   (arcl.software, arcl.organization))

        arcl_status = arcl.get_status(req_id)
        logs.debug("got request status")

        for req in arcl_status.request:
            if req.error:
                req_status = "ERROR"
            elif req.ready:
                req_status = "READY"
            else:
                req_status = "PROCESSING"

            obj.write('Request ID: %s, Type: %s, Args: %s<br>\n' % \
                (req.id, req.type, req.args))
            obj.write('Status: %s, Size: %d, Info: %s<br>\n' % \
                (req_status, req.size, req.message))

            if req.user != "":
                obj.write('User: %s, Institution: %s<br>\n' %
                          (req.user, req.institution))

            if req.ready and not req.error and req.size > 0:
                obj.write('<a href="download?sesskey=%s&arclink=%s&req_id=%s">Download!</a>&nbsp;' % \
                    (sesskey, myaddr, req.id))

            obj.write('<a href="purge?sesskey=%s&arclink=%s&req_id=%s">Delete!</a><br>' % \
                (sesskey, myaddr, req.id))

            for vol in req.volume:
                obj.write(4 * '&nbsp;' + 'Volume ID: %s, Status: %s, Size: %d, Info: %s<br>\n' % \
                    (vol.id, arclink_status_string(vol.status), vol.size, vol.message))

                if vol.status == STATUS_OK or vol.status == STATUS_WARN:
                    obj.write(4 * '&nbsp;' + '<a href="download?sesskey=%s&arclink=%s&req_id=%s&vol_id=%s">Download!</a><br>' % \
                        (sesskey, myaddr, req.id, vol.id))

                for rqln in vol.line:
                    obj.write('<br>\n')
                    obj.write(8 * '&nbsp;' +
                              '<font face="courier">%s</font><br>\n' %
                              (rqln.content, ))
                    obj.write(8 * '&nbsp;' + 'Status: %s, Size: %d, Info: %s<br>\n' % \
                        (arclink_status_string(rqln.status), rqln.size, rqln.message))

            obj.write('<br>\n')

    finally:
        logs.debug("closing connection")
        arcl.close_connection()

    obj.write(status_tail)
示例#21
0
    def run(self):
        try:
            seiscompRoot = self.commandline().unrecognizedOptions()[0]
            sys.stderr.write("root directory: %s\n" % seiscompRoot)

            try:
                DCID = self.configGetString("datacenterID")

            except:
                logs.error("datacenterID not found in global.cfg")
                return False

            networkRestricted = {}
            incompleteResponse = {}

            global instdb
            instdb = Instruments(DCID)

            self.__load_file(loadGains,
                             os.path.join(seiscompRoot, "config", "gain.dlsv"))

            # for backwards compatibility
            self.__load_file(
                loadGains, os.path.join(seiscompRoot, "config",
                                        "gain.tab.out"))
            self.__load_file(loadGains,
                             os.path.join(seiscompRoot, "config", "gain.tab"))

            try:
                self.__load_file(instdb.load_db,
                                 os.path.join(seiscompRoot, "resp", "inst.db"))
                self.__load_file(
                    instdb.load_sensor_attr,
                    os.path.join(seiscompRoot, "resp", "sensor_attr.csv"))
                self.__load_file(
                    instdb.load_datalogger_attr,
                    os.path.join(seiscompRoot, "resp", "datalogger_attr.csv"))

            except (IOError, NettabError) as e:
                logs.error("fatal error: " + str(e))
                return False

            sc3Inv = seiscomp3.DataModel.Inventory()
            inventory = InventoryWrapper(sc3Inv, DCID)

            existingNetworks = set()
            existingStations = set()

            for f in glob.glob(os.path.join(seiscompRoot, "key", "network_*")):
                try:
                    logs.debug("processing " + f)
                    netCode = f.split("/network_")[-1]
                    try:
                        kf = Keyfile(f)
                    except IOError as e:
                        logs.error(str(e))
                        continue

                    existingNetworks.add(netCode)
                    networkRestricted[netCode] = False

                    inventory.updateNetwork(netCode, kf)

                except ValueError as e:
                    logs.error("%s: %s" % (f, str(e)))

            for f in glob.glob(os.path.join(seiscompRoot, "key", "station_*")):
                try:
                    logs.debug("processing " + f)
                    (netCode, staCode) = f.split("/station_")[-1].split('_', 1)
                    try:
                        kf = Keyfile(f)
                    except IOError as e:
                        logs.error(str(e))
                        continue

                    existingStations.add((netCode, staCode))

                    if netCode not in existingNetworks:
                        logs.warning(
                            "network %s does not exist, ignoring station %s" %
                            (netCode, staCode))
                        continue

                    if not hasattr(kf, "latitude") or not kf.latitude:
                        logs.warning("missing latitude for %s %s" %
                                     (netCode, staCode))
                        continue

                    if not hasattr(kf, "longitude") or not kf.longitude:
                        logs.warning("missing longitude for %s %s" %
                                     (netCode, staCode))
                        continue

                    if not hasattr(kf, "elevation") or not kf.elevation:
                        logs.warning("missing elevation for %s %s" %
                                     (netCode, staCode))
                        continue

                    if not hasattr(kf, "depth1") or not kf.depth1:
                        logs.warning(
                            "missing depth of primary sensor for %s %s" %
                            (netCode, staCode))
                        continue

                    if decimal.Decimal(kf.latitude) == decimal.Decimal("0.0") and \
                            decimal.Decimal(kf.longitude) == decimal.Decimal("0.0"):
                        logs.warning("missing coordinates for %s %s" %
                                     (netCode, staCode))
                        continue

                    if not hasattr(kf, "orientation1") or not kf.orientation1:
                        logs.warning(
                            "missing orientation of primary sensor for %s %s, using default"
                            % (netCode, staCode))
                        kf.orientation1 = "Z 0.0 -90.0; N 0.0 0.0; E 90.0 0.0"

                    if not hasattr(kf, "orientation2"):
                        kf.orientation2 = ""

                    if not hasattr(kf, "unit1") or not kf.unit1:
                        logs.warning(
                            "missing unit of primary sensor for %s %s, using M/S"
                            % (netCode, staCode))
                        kf.unit1 = "M/S"

                    if not hasattr(kf, "unit2"):
                        logs.warning(
                            "missing unit of secondary sensor for %s %s, using M/S**2"
                            % (netCode, staCode))
                        kf.unit2 = "M/S**2"

                    if not hasattr(kf, "type"):
                        kf.type = ""

                    restricted = False

                    # TODO: Make restricted part of the key file

                    if not inventory.updateStation(netCode, staCode,
                                                   restricted, kf):
                        try:
                            incNet = incompleteResponse[netCode]

                        except KeyError:
                            incNet = set()
                            incompleteResponse[netCode] = incNet

                        incNet.add(staCode)

                except ValueError as e:
                    logs.error("%s: %s" % (f, str(e)))

            for (netCode, restricted) in networkRestricted.items():
                inventory.setNetworkRestricted(netCode, restricted)

            for (netCode, network) in inventory.networks.items():
                if netCode not in existingNetworks:
                    logs.notice("deleting network %s from inventory" %
                                (netCode, ))
                    inventory.obj.remove(network.obj)

            for ((netCode, staCode), station) in inventory.stations.items():
                if netCode in existingNetworks and (
                        netCode, staCode) not in existingStations:
                    logs.notice("deleting station %s_%s from inventory" %
                                (netCode, staCode))
                    inventory.networks[netCode].obj.remove(station.obj)

            if incompleteResponse:
                logs.info(
                    "The following stations are missing full response data")
                logs.info("Use dlsv2inv if needed")

                # for netCode in sorted(incompleteResponse.keys()):
                #    logs.info("%s: %s" % (netCode, " ".join(sorted(list(incompleteResponse[netCode])))))
                tmpDict = sortDictionary(incompleteResponse)
                for netCode in list(tmpDict.keys()):
                    tmpSortedList = list(tmpDict[netCode])
                    tmpSortedList.sort()
                    logs.info("%s: %s" % (netCode, " ".join(tmpSortedList)))

            ar = seiscomp3.IO.XMLArchive()
            if not self.output:
                sys.stderr.write("Writing output to stdout\n")
                if not ar.create("-"):
                    sys.stderr.write("Cannot open open stdout\n")
                    return False
            else:
                sys.stderr.write("Writing output to %s\n" % self.output)
                if not ar.create(self.output):
                    sys.stderr.write("Cannot open open %s\n" % self.output)
                    return False

            ar.setFormattedOutput(self.commandline().hasOption("formatted"))
            ar.writeObject(sc3Inv)

        except Exception:
            logs.print_exc()

        return True
示例#22
0
            # Add some user-friendliness (this message triggers an alert box on the client)
            return send_plain_response("400 Bad Request",
                                       "maximum request size exceeded",
                                       start_response)

        return send_plain_response("400 Bad Request", str(e), start_response)

    if form:
        for k in form.keys():
            if k in multipar:
                parameters[k] = form.getlist(k)

            else:
                parameters[k] = form.getfirst(k)

    logs.debug('parameters: %s' % (parameters))

    body = []

    # body.extend(["%s: %s" % (key, value)
    #     for key, value in environ.iteritems()])

    # status = '200 OK'
    # return send_plain_response(status, body, start_response)

    logs.debug('Calling %s' % action)

    try:
        res_string = action(environ, parameters)

    except PlsRedirect as redir:
示例#23
0
    def iterdata(self, time1, time2, net, sta, cha, loc):
        lasttime = None
        lastrecord = None
        recstream = []

        if self.__is_iso(time1, time2, net, sta, os.path.exists):
            recstream.append(("isoarchive", self.isodir))
        if self.__is_sds(time1, time2, net, sta, cha, loc, self.archdir,
                         os.path.exists, os.listdir):
            recstream.append(("sdsarchive", self.archdir))
        if self.__is_sds(time1, time2, net, sta, cha, loc, self.nrtdir,
                         os.path.exists, os.listdir):
            recstream.append(("sdsarchive", self.nrtdir))

        if not recstream and self.exists_db(time1, time2, net, sta, cha, loc):
            raise TemporaryUnavailabilityException

        for (service, source) in recstream:
            if lastrecord:
                try:
                    etime = lastrecord.endTime()
                except Core.ValueException:
                    logs.warning(
                        "SDS: record.endTime() raises Core.ValueException! Resulting SEED file maybe incorrect!"
                    )
                    etime = lastrecord.startTime()
                timetuple = time.strptime(etime.toString("%Y-%m-%d %H:%M:%S"),
                                          "%Y-%m-%d %H:%M:%S")
                lasttime = datetime.datetime(
                    *timetuple[:6]) + datetime.timedelta(
                        seconds=1)  # avoids dublettes
                if lasttime >= time2:
                    break
                time1 = lasttime
                lastrecord = None

            self._recstream = IO.RecordStream.Create(service)
            if not self._recstream:
                logs.error("Could not fetch recordstream service '%s'" %
                           service)
                raise StopIteration

            if not self._recstream.setSource(source):
                logs.error("Could not set recordstream source '%s'" % source)
                self._recstream = None
                raise StopIteration

            logs.debug("%s %s: addStream for %s-%s" %
                       (service, source, str(time1), str(time2)))
            self._recstream.addStream(
                net, sta, loc, cha,
                Core.Time.FromString(str(time1), "%Y-%m-%d %H:%M:%S"),
                Core.Time.FromString(str(time2), "%Y-%m-%d %H:%M:%S"))

            try:
                recinput = IO.RecordInput(self._recstream, Core.Array.DOUBLE,
                                          Core.Record.SAVE_RAW)
                record = recinput.next()

                while record:
                    yield record.raw().str()
                    lastrecord = record
                    record = recinput.next()

            except Core.GeneralException, e:
                logs.error(e.what())
            except Exception, e:
                logs.error("SDS: Unexpected exception occured: %s" % e)
示例#24
0
                        kf = Keyfile(f)
                    except IOError, e:
                        logs.error(str(e))
                        continue

                    existingNetworks.add(netCode)
                    networkRestricted[netCode] = False

                    inventory.updateNetwork(netCode, kf)

                except ValueError, e:
                    logs.error("%s: %s" % (f, str(e)))

            for f in glob.glob(os.path.join(seiscompRoot, "key", "station_*")):
                try:
                    logs.debug("processing " + f)
                    (netCode, staCode) = f.split("/station_")[-1].split('_', 1)
                    try:
                        kf = Keyfile(f)
                    except IOError, e:
                        logs.error(str(e))
                        continue

                    existingStations.add((netCode, staCode))

                    if netCode not in existingNetworks:
                        logs.warning("network %s does not exist, ignoring station %s" % (netCode, staCode))
                        continue

                    if not hasattr(kf, "latitude") or not kf.latitude:
                        logs.warning("missing latitude for %s %s" % (netCode, staCode))
示例#25
0
def status(obj, args):
    myaddr = args.get("arclink")
    req_id = args.get("req_id", "ALL")
    
    try:
        (host, port) = myaddr.split(':')
        port = int(port)
    except (AttributeError, ValueError):
        raise ArclinkError, "invalid ArcLink address"

    (sesskey, user, passwd) = init_session(obj, args)
    
    arcl.open_connection(host, port, user, passwd, timeout=ARCLINK_TIMEOUT)
    try:
        arcl.send_command("USER_IP %s" % obj.remote_host)
    except:
        pass

    try:
        obj.content_type = "text/html; charset=UTF-8"
        obj.write(status_head)
    
        logs.debug("connected to %s at %s" % (arcl.software, arcl.organization))

        arcl_status = arcl.get_status(req_id)
        logs.debug("got request status")

        for req in arcl_status.request:
            if req.error:
                req_status = "ERROR"
            elif req.ready:
                req_status = "READY"
            else:
                req_status = "PROCESSING"

            obj.write('Request ID: %s, Type: %s, Args: %s<br>\n' % \
                (req.id, req.type, req.args))
            obj.write('Status: %s, Size: %d, Info: %s<br>\n' % \
                (req_status, req.size, req.message))
            
            if req.user != "":
                obj.write('User: %s, Institution: %s<br>\n' % (req.user, req.institution))

            if req.ready and not req.error and req.size > 0:
                obj.write('<a href="download?sesskey=%s&arclink=%s&req_id=%s">Download!</a>&nbsp;' % \
                    (sesskey, myaddr, req.id))

            obj.write('<a href="purge?sesskey=%s&arclink=%s&req_id=%s">Delete!</a><br>' % \
                (sesskey, myaddr, req.id))

            for vol in req.volume:
                obj.write(4 * '&nbsp;' + 'Volume ID: %s, Status: %s, Size: %d, Info: %s<br>\n' % \
                    (vol.id, arclink_status_string(vol.status), vol.size, vol.message))

                if vol.status == STATUS_OK or vol.status == STATUS_WARN:
                    obj.write(4 * '&nbsp;' + '<a href="download?sesskey=%s&arclink=%s&req_id=%s&vol_id=%s">Download!</a><br>' % \
                        (sesskey, myaddr, req.id, vol.id))
                
                for rqln in vol.line:
                    obj.write('<br>\n')
                    obj.write(8 * '&nbsp;' + '<font face="courier">%s</font><br>\n' % (rqln.content,))
                    obj.write(8 * '&nbsp;' + 'Status: %s, Size: %d, Info: %s<br>\n' % \
                        (arclink_status_string(rqln.status), rqln.size, rqln.message))
     
            obj.write('<br>\n')
            
    finally:
        logs.debug("closing connection")
        arcl.close_connection()

    obj.write(status_tail)
示例#26
0
def start():
    """
	Checks request spool directory for files => iterating and processing
	"""

    while True:
        names = set()
        checklist = [
            f for f in os.listdir(os.path.join(SPOOL_DIR, "check"))
            if os.path.isfile(os.path.join(SPOOL_DIR, "check", f))
            and not f.endswith("_checking")
        ]

        if not checklist:
            break

        for fname in checklist:
            fname = os.path.join(SPOOL_DIR, "check", fname)
            basename = os.path.basename(fname)
            m = re.match("^.+/(?P<req_name>.+)[_](?P<breq_id>\w+[_]\d+)$",
                         fname)
            if m:
                (req_name, breq_id) = (m.group("req_name"), m.group("breq_id"))
                if req_name in names:
                    continue

                names.add(req_name)

                sys.stderr.write("working on: %s %s\n" % (req_name, breq_id))
            else:
                os.rename(fname, fname.replace("_checking", "_fail"))
                logs.error("Parsing of Breq_fast name and ID in %s failed" %
                           fname)
                continue

            ### redirect the logging output to a logfile ###
            set_logger(
                os.path.join(BREQ_DIR, req_name, breq_id, "breq_mail.log"))

            ### mark the processed file with suffix _checking ###
            logs.debug("checking file %s" % fname)
            os.rename(fname, "_".join((fname, "checking")))
            fname = "_".join((fname, "checking"))
            logs.debug("rename file in %s" % fname)

            ### parse the original breq_fast email ###
            email = os.path.join(BREQ_DIR, req_name, breq_id, "breq_mail.org")
            parser = BreqParser()
            parser.parse_email(email)
            logs.debug("parsing email %s" % email)

            ### create the response email message after checking this email ###
            emailmsg = check_request(email, basename, parser)
            emailmsg = "%s\n\nThis request has the request ID: %s_%s\n\n%s\n" % (
                emailmsg, req_name, breq_id, _emailextro)
            emailmsg = "%s\n\nbreq_fast request header:\n%s" % (emailmsg,
                                                                parser.head)
            emailmsg = "%s\nbreq_fast request lines:%s\n" % (emailmsg,
                                                             parser.request)
            emailaddr = EMAIL_ADDR
            try:
                emailaddr = parser.tokendict["email"]
            except KeyError:
                pass

            errorstate = False
            if os.path.exists(
                    os.path.join(SPOOL_DIR, "make", basename + "_running")):
                ### email was sent before crash, don't send it again
                logs.debug("email notification was already sent")
                os.unlink(
                    os.path.join(SPOOL_DIR, "make", basename + "_running"))
                errorstate = True

            else:
                submit_email(
                    emailaddr,
                    "breq_fast request %s_%s checked" % (req_name, breq_id),
                    emailmsg)
                logs.debug("email submitted with message: %s" % emailmsg)

            ### mark the processed file with suffix _done and move it to the check/done-dir in SPOOL_DIR ###
            shutil.move(
                fname,
                os.path.join(SPOOL_DIR, "check", "done", basename + "_done"))
            logs.debug("move file %s to check/done dir" % fname)
            fname = os.path.join(SPOOL_DIR, "make", basename)
            logs.debug("now look for file %s" % fname)

            if (os.path.exists(fname)):
                ### mark the processed file with suffix _running ###
                os.rename(fname, "_".join((fname, "running")))
                fname = "_".join((fname, "running"))
                logs.debug("rename file in %s" % fname)

                try:
                    ### submit the request to arclink server ###
                    emailmsg = submit_request(parser, req_name, breq_id)

                    ### submit the email containing the processing status of the Breq_fast request
                    submit_email(
                        emailaddr, "breq_fast request %s_%s processed" %
                        (req_name, breq_id), emailmsg)
                    logs.debug("email submitted with message: %s" % emailmsg)

                except (ArclinkError, socket.error), e:
                    logs.error("quit processing: " + str(e))

                    if not errorstate:
                        #submit_email("admin", "breqfast failure", str(e))
                        pass

                    shutil.move(
                        os.path.join(SPOOL_DIR, "check", "done",
                                     basename + "_done"),
                        os.path.join(SPOOL_DIR, "check", basename))
                    break

                if errorstate:
                    #submit_email("admin", "breqfast OK", "")
                    pass

                ### mark the processed file with suffix _done and move it to the make/done-dir in SPOOL_DIR ###
                shutil.move(
                    fname,
                    os.path.join(SPOOL_DIR, "make", "done",
                                 basename + "_done"))
                logs.debug("move file %s in make/done dir" % fname)
示例#27
0
def start():
    """
	Checks request spool directory for files => iterating and processing
	"""

    for fname in os.listdir(os.path.join(SPOOL_DIR, "check")):
        fname = os.path.join(SPOOL_DIR, "check", fname)
        if os.path.isfile(fname) and not fname.endswith("_checking"):
            basename = os.path.basename(fname)
            m = re.match("^.+/(?P<req_name>.+)[_](?P<breq_id>\w+[_]\d+)$",
                         fname)
            if m:
                (req_name, breq_id) = (m.group("req_name"), m.group("breq_id"))
                sys.stderr.write("working on: %s %s\n" % (req_name, breq_id))
            else:
                os.rename(fname, fname.replace("_checking", "_fail"))
                logs.error("Parsing of Breq_fast name and ID in %s failed" %
                           fname)
                continue

            ### redirect the logging output to a logfile ###
            set_logger(
                os.path.join(BREQ_DIR, req_name, breq_id, "breq_mail.log"))

            ### mark the processed file with suffix _checking ###
            logs.debug("checking file %s" % fname)
            os.rename(fname, "_".join((fname, "checking")))
            fname = "_".join((fname, "checking"))
            logs.debug("rename file in %s" % fname)

            ### parse the original breq_fast email ###
            email = os.path.join(BREQ_DIR, req_name, breq_id, "breq_mail.org")
            parser = BreqParser()
            parser.parse_email(email)
            logs.debug("parsing email %s" % email)

            ### create the response email message after checking this email ###
            emailmsg = check_request(email, basename, parser)
            emailmsg = "%s\n\nThis request has the request ID: %s_%s\n\n%s\n" % (
                emailmsg, req_name, breq_id, _emailextro)
            emailmsg = "%s\n\nbreq_fast request header:\n%s" % (emailmsg,
                                                                parser.head)
            emailmsg = "%s\nbreq_fast request lines:%s\n" % (emailmsg,
                                                             parser.request)
            emailaddr = EMAIL_ADDR
            try:
                emailaddr = parser.tokendict["email"]
            except KeyError:
                pass
            submit_email(
                emailaddr,
                "breq_fast request %s_%s checked" % (req_name, breq_id),
                emailmsg)
            logs.debug("email submitted with message: %s" % emailmsg)

            ### mark the processed file with suffix _done and move it to the check/done-dir in SPOOL_DIR ###
            shutil.move(
                fname,
                os.path.join(SPOOL_DIR, "check", "done", basename + "_done"))
            logs.debug("move file %s to check/done dir" % fname)
            fname = os.path.join(SPOOL_DIR, "make", basename)
            logs.debug("now look for file %s" % fname)

            if (os.path.exists(fname)):
                ### mark the processed file with suffix _running ###
                os.rename(fname, "_".join((fname, "running")))
                fname = "_".join((fname, "running"))
                logs.debug("rename file in %s" % fname)

                ### submit the request to arclink server ###
                emailmsg = submit_request(parser, req_name, breq_id)

                ### submit the email containing the processing status of the Breq_fast request
                submit_email(
                    emailaddr,
                    "breq_fast request %s_%s processed" % (req_name, breq_id),
                    emailmsg)
                logs.debug("email submitted with message: %s" % emailmsg)
                ### mark the processed file with suffix _done and move it to the make/done-dir in SPOOL_DIR ###
                shutil.move(
                    fname,
                    os.path.join(SPOOL_DIR, "make", "done",
                                 basename + "_done"))
                logs.debug("move file %s in make/done dir" % fname)
示例#28
0
                        kf = Keyfile(f)
                    except IOError, e:
                        logs.error(str(e))
                        continue

                    existingNetworks.add(netCode)
                    networkRestricted[netCode] = False

                    inventory.updateNetwork(netCode, kf)

                except ValueError, e:
                    logs.error("%s: %s" % (f, str(e)))

            for f in glob.glob(os.path.join(seiscompRoot, "key", "station_*")):
                try:
                    logs.debug("processing " + f)
                    (netCode, staCode) = f.split("/station_")[-1].split('_', 1)
                    try:
                        kf = Keyfile(f)
                    except IOError, e:
                        logs.error(str(e))
                        continue

                    existingStations.add((netCode, staCode))

                    if netCode not in existingNetworks:
                        logs.warning(
                            "network %s does not exist, ignoring station %s" % (netCode, staCode))
                        continue

                    if not hasattr(kf, "latitude") or not kf.latitude:
示例#29
0
    def __parse_request(self, line):
        """
		Gets a request line in Breq_fast format to match it against the corresponding pattern.
		If successful the request list will be completed; the fail string otherwise.

		@arguments: line, a request line in Breq_fast format
		"""
        loc = "*"
        self.request = "%s\n%s" % (self.request, line)
        m = re.search("\s+".join(self.__reqlist), line)
        if m:
            d = m.groupdict()

            logs.debug("request_line: %s" % line)

            # catch two digit year inputs
            if d["beg_2year"]:
                if int(d["beg_2year"]) > 50:
                    d["beg_4year"] = "19%s" % d["beg_2year"]
                else:
                    d["beg_4year"] = "20%s" % d["beg_2year"]
            if d["end_2year"]:
                if int(d["end_2year"]) > 50:
                    d["end_4year"] = "19%s" % d["end_2year"]
                else:
                    d["end_4year"] = "20%s" % d["end_2year"]

            # some users have problems with time...
            if int(d["beg_hour"]) > 23:
                d["beg_hour"] = "23"
            if int(d["end_hour"]) > 23:
                d["end_hour"] = "23"
            if int(d["beg_min"]) > 59:
                d["beg_min"] = "59"
            if int(d["end_min"]) > 59:
                d["end_min"] = "59"
            if int(d["beg_sec"]) > 59:
                d["beg_sec"] = "59"
            if int(d["end_sec"]) > 59:
                d["end_sec"] = "59"

            try:
                beg_time = datetime.datetime(int(d["beg_4year"]),
                                             int(d["beg_month"]),
                                             int(d["beg_day"]),
                                             int(d["beg_hour"]),
                                             int(d["beg_min"]),
                                             int(d["beg_sec"]))
                end_time = datetime.datetime(int(d["end_4year"]),
                                             int(d["end_month"]),
                                             int(d["end_day"]),
                                             int(d["end_hour"]),
                                             int(d["end_min"]),
                                             int(d["end_sec"]))
            except:
                self.failstr = "%s%s  [error: wrong begin or end time]\n" % (
                    self.failstr, line)
                return

            # expand network and station
            for (network,
                 station) in self.__expand_net_station(d["network"],
                                                       d["station"], beg_time,
                                                       end_time):

                cha_list = re.findall("([\w?\*]+)\s*", d["cha_list"])
                if len(cha_list) == int(d['cha_num']) + 1:
                    loc = cha_list.pop()
                for cha in cha_list:
                    cha = re.sub("[?]+", "*", cha)
                    self.reqlist.append((str(network), str(station), cha, loc,
                                         beg_time, end_time, {}, set()))
                    logs.debug("reqlist.append: %s %s %s %s" %
                               (str(network), str(station), cha, loc))
        else:
            self.failstr = "%s%s\n" % (self.failstr, line)
示例#30
0
    except ValueError, e:
        if str(e) == "Maximum content length exceeded":
            # Add some user-friendliness (this message triggers an alert box on the client)
            return send_plain_response("400 Bad Request", "maximum request size exceeded", start_response)

        return send_plain_response("400 Bad Request", str(e), start_response)

    if form:
        for k in form.keys():
            if k in multipar:
                parameters[k] = form.getlist(k)

            else:
                parameters[k] = form.getfirst(k)

    logs.debug('parameters: %s' % (parameters))

    body = []

    # body.extend(["%s: %s" % (key, value)
    #     for key, value in environ.iteritems()])

    # status = '200 OK'
    # return send_plain_response(status, body, start_response)

    logs.debug('Calling %s' % action)

    try:
        res_string = action(environ, parameters)

    except PlsRedirect as redir:
示例#31
0
def process_options():
    parser = OptionParser(
        usage=
        "usage: %prog [-h|--help] [OPTIONS] -u USER -o OUTPUTFILE REQUEST",
        version="%prog v" + VERSION,
        add_help_option=False)

    parser.set_defaults(address="webdc.eu:18001",
                        request_format="native",
                        data_format="mseed",
                        no_resp_dict=False,
                        rebuild_volume=False,
                        proxymode=False,
                        timeout=300,
                        retries=5,
                        SSLpasswordFile="dcidpasswords.txt")

    parser.add_option("-h",
                      "--help",
                      action="store_true",
                      dest="showhelp",
                      default=False)
    parser.add_option("-l",
                      "--longhelp",
                      action="store_true",
                      dest="showlonghelp",
                      default=False)

    parser.add_option(
        "-w",
        "--password-file",
        type="string",
        dest="SSLpasswordFile",
        help=
        "file containing passwords used for decryption of encrypted data (default %default)"
    )

    parser.add_option(
        "-a",
        "--address",
        type="string",
        dest="address",
        help="address of primary ArcLink node (default %default)")

    foptions = ("native", "breqfast")
    parser.add_option(
        "-f",
        "--request-format",
        type="choice",
        dest="request_format",
        choices=foptions,
        help="request format: breqfast, native (default %default)")

    koptions = ("mseed", "mseed4k", "fseed", "dseed", "inv", "inventory")
    parser.add_option(
        "-k",
        "--data-format",
        type="choice",
        dest="data_format",
        choices=koptions,
        help=
        "data format: mseed, mseed4k, fseed, dseed, inv[entory] (default %default)"
    )

    parser.add_option(
        "-n",
        "--no-resp-dict",
        action="store_true",
        dest="no_resp_dict",
        help="avoid using response dictionary (default %default)")

    parser.add_option("-g",
                      "--rebuild-volume",
                      action="store_true",
                      dest="rebuild_volume",
                      help="rebuild SEED volume (default %default)")

    parser.add_option("-p",
                      "--proxy",
                      action="store_true",
                      dest="proxymode",
                      help="proxy mode, no routing (default %default)")

    parser.add_option("-t",
                      "--timeout",
                      type="int",
                      dest="timeout",
                      help="timeout in seconds (default %default)")

    parser.add_option("-x",
                      "--retries",
                      type="int",
                      dest="retries",
                      help="download retries (default %default)")

    parser.add_option("-v",
                      action="callback",
                      callback=add_verbosity,
                      help="increase verbosity level")

    parser.add_option("-q",
                      action="callback",
                      callback=add_quietness,
                      help="decrease verbosity level")

    parser.add_option("-u",
                      "--user",
                      type="string",
                      dest="user",
                      help="user's e-mail address")

    parser.add_option("-o",
                      "--output-file",
                      type="string",
                      dest="output_file",
                      help="file where downloaded data is written")

    (options, args) = parser.parse_args()

    if options.showhelp or options.showlonghelp:
        parser.print_help()
        if options.showlonghelp:
            print """
About ArcLink Protocol
====================== 

The ArcLink  is a protocol  used to request distributed  archive seismological
data. Today  it gives you access  to several European  data archives (European
Integrated Data Archive - EIDA)  that are supporting the protocol developed by
GEOFON   ([email protected])  at   the   GeoForschungZentrum,  Potsdam,
Germany.

You can find more information about it at the SeisComp3 and GEOFON web pages:

 * http://www.seiscomp3.org/
 * http://geofon.gfz-potsdam.de/ 

ArcLink Password File (for decryption)
======================================

In this file (default: dcidpasswords.txt) you can store your private passwords
given by different  data centers. Each data center  that you request encrypted
data will send you a different password.

The format of the  file is really simple: just the data  center ID followed by
the password that you received. One  data center ID and password per line. Any
empty lines or lines starting with # are ignored.

Example:

gfz password1
odc password2
ipgp password3

The data  center ID and password  can be found on  the automatically generated
e-mail that  you received from each  data center. (You will  only receive this
email if  you have been  authorized to download  encrypted data, and  you have
tried to download it.)

Input File Format
=================

ArcLink Fetch  program supports  two different input  formats for  the request
file.  It  supports  the traditional  BREQ  FAST  format,  and its own  native
format. Both formats  contains the same information and  they differ slightly.

Native Format:
--------------

The native format has the following format: 

YYYY,MM,DD,HH,MM,SS YYYY,MM,DD,HH,MM,SS Network Station Channel [Location]

the Channel, Station and Location, can contains wildcards (*) and the Location
field is optional. For matching all locations please use the '*' symbol.

Example:

2010,02,18,12,00,00 2010,02,18,12,10,00 GE WLF BH*
2010,02,18,12,00,00 2010,02,18,12,10,00 GE VSU BH* 00

BREQ FAST Format:
-----------------

The  BREQ FAST  format is  a  standard format  used on  seismology to  request
data. Each header line start with '.' and the request lines have the following
format:

Station Network {Time Start} {Time End} {Number of Channels} N x Channels Location

Time Specification should have the following format: YYYY MM DD HH MM SS.TTTT

Please read more about the BREQ FAST format at:

http://www.iris.edu/manuals/breq_fast.htm

"""
            sys.exit()

    errors = []
    warnings = []
    if options.user == None:
        errors.append("Username required")

    if options.output_file == None:
        errors.append("Output file required")

    if options.data_format.upper() != "FSEED" and options.rebuild_volume:
        errors.append("-g is only applicable to FSEED format")

    if len(args) == 0:
        errors.append("No request file supplied")
    else:
        for reqfile in args:
            if not os.path.exists(reqfile):
                errors.append("Request file '%s' not found." % reqfile)

    SSLpasswordDict = {}
    if os.path.exists(options.SSLpasswordFile):
        fd = open(options.SSLpasswordFile)
        line = fd.readline()
        while line:
            line = line.strip()
            if line and line[0] != "#":
                try:
                    (dcid, password) = line.split()
                    SSLpasswordDict[dcid] = password
                except ValueError:
                    logs.error(options.SSLpasswordFile + " invalid line: " +
                               line)
                    fd.close()
                    sys.exit()
            line = fd.readline()
    else:
        if options.SSLpasswordFile != parser.defaults['SSLpasswordFile']:
            errors.append("Supplied password file (%s) not found" %
                          options.SSLpasswordFile)
        else:
            warnings.append("Default password file (%s) not found" %
                            options.SSLpasswordFile)

    if len(errors) > 0:
        logs.error("\n** ArcLink Fetch %s **\n" % VERSION)
        parser.print_usage()
        logs.error("Errors detected on the command line:")
        for item in errors:
            logs.error("\t%s" % item)
        print ""

    if len(warnings) > 0:
        logs.debug("Warnings detected on the command line:")
        for item in warnings:
            logs.debug("\t%s" % item)
        print ""

    if len(errors) > 0:
        sys.exit()

    return (SSLpasswordDict, options.address, options.request_format,
            options.data_format, not options.no_resp_dict,
            options.rebuild_volume, options.proxymode, options.user,
            options.timeout, options.retries, options.output_file, args[0])
示例#32
0
def start():
    """
	Checks request spool directory for files => iterating and processing
	"""

    while True:
        names = set()
        checklist = [
            f
            for f in os.listdir(os.path.join(SPOOL_DIR, "check"))
            if os.path.isfile(os.path.join(SPOOL_DIR, "check", f)) and not f.endswith("_checking")
        ]

        if not checklist:
            break

        for fname in checklist:
            fname = os.path.join(SPOOL_DIR, "check", fname)
            basename = os.path.basename(fname)
            m = re.match("^.+/(?P<req_name>.+)[_](?P<breq_id>\w+[_]\d+)$", fname)
            if m:
                (req_name, breq_id) = (m.group("req_name"), m.group("breq_id"))
                if req_name in names:
                    continue

                names.add(req_name)

                sys.stderr.write("working on: %s %s\n" % (req_name, breq_id))
            else:
                os.rename(fname, fname.replace("_checking", "_fail"))
                logs.error("Parsing of Breq_fast name and ID in %s failed" % fname)
                continue

                ### redirect the logging output to a logfile ###
            set_logger(os.path.join(BREQ_DIR, req_name, breq_id, "breq_mail.log"))

            ### mark the processed file with suffix _checking ###
            logs.debug("checking file %s" % fname)
            os.rename(fname, "_".join((fname, "checking")))
            fname = "_".join((fname, "checking"))
            logs.debug("rename file in %s" % fname)

            ### parse the original breq_fast email ###
            email = os.path.join(BREQ_DIR, req_name, breq_id, "breq_mail.org")
            parser = BreqParser()
            parser.parse_email(email)
            logs.debug("parsing email %s" % email)

            ### create the response email message after checking this email ###
            emailmsg = check_request(email, basename, parser)
            emailmsg = "%s\n\nThis request has the request ID: %s_%s\n\n%s\n" % (
                emailmsg,
                req_name,
                breq_id,
                _emailextro,
            )
            emailmsg = "%s\n\nbreq_fast request header:\n%s" % (emailmsg, parser.head)
            emailmsg = "%s\nbreq_fast request lines:%s\n" % (emailmsg, parser.request)
            emailaddr = EMAIL_ADDR
            try:
                emailaddr = parser.tokendict["email"]
            except KeyError:
                pass

            errorstate = False
            if os.path.exists(os.path.join(SPOOL_DIR, "make", basename + "_running")):
                ### email was sent before crash, don't send it again
                logs.debug("email notification was already sent")
                os.unlink(os.path.join(SPOOL_DIR, "make", basename + "_running"))
                errorstate = True

            else:
                submit_email(emailaddr, "breq_fast request %s_%s checked" % (req_name, breq_id), emailmsg)
                logs.debug("email submitted with message: %s" % emailmsg)

                ### mark the processed file with suffix _done and move it to the check/done-dir in SPOOL_DIR ###
            shutil.move(fname, os.path.join(SPOOL_DIR, "check", "done", basename + "_done"))
            logs.debug("move file %s to check/done dir" % fname)
            fname = os.path.join(SPOOL_DIR, "make", basename)
            logs.debug("now look for file %s" % fname)

            if os.path.exists(fname):
                ### mark the processed file with suffix _running ###
                os.rename(fname, "_".join((fname, "running")))
                fname = "_".join((fname, "running"))
                logs.debug("rename file in %s" % fname)

                try:
                    ### submit the request to arclink server ###
                    emailmsg = submit_request(parser, req_name, breq_id)

                    ### submit the email containing the processing status of the Breq_fast request
                    submit_email(emailaddr, "breq_fast request %s_%s processed" % (req_name, breq_id), emailmsg)
                    logs.debug("email submitted with message: %s" % emailmsg)

                except (ArclinkError, socket.error), e:
                    logs.error("quit processing: " + str(e))

                    if not errorstate:
                        # submit_email("admin", "breqfast failure", str(e))
                        pass

                    shutil.move(
                        os.path.join(SPOOL_DIR, "check", "done", basename + "_done"),
                        os.path.join(SPOOL_DIR, "check", basename),
                    )
                    break

                if errorstate:
                    # submit_email("admin", "breqfast OK", "")
                    pass

                    ### mark the processed file with suffix _done and move it to the make/done-dir in SPOOL_DIR ###
                shutil.move(fname, os.path.join(SPOOL_DIR, "make", "done", basename + "_done"))
                logs.debug("move file %s in make/done dir" % fname)