def preprocess_link(self, referrer, url): # Modify and filter URLs before crawling if not url: return None fields = urlsplit(urljoin( referrer, url))._asdict() # convert to absolute URLs and split fields['path'] = re.sub(r'/$', '', fields['path']) # remove trailing "/" fields['fragment'] = '' # remove targets within a page fields = SplitResult(**fields) if fields.scheme == 'http': httpurl = newurl = fields.geturl() httpsurl = httpurl.replace('http:', 'https:', 1) elif fields.scheme == 'https': httpsurl = newurl = fields.geturl() httpurl = httpsurl.replace('https:', 'http:', 1) else: # Filter the URL without 'http' or 'https' return None if httpurl not in self.url_set and httpsurl not in self.url_set: # Filter URL that already exists in set return newurl else: return None
def build_url(url): url_result = {UrlParser.QUERY: "", UrlParser.FRAGMENT: ""} if not url or UrlParser.SCHEME not in url or not url[UrlParser.SCHEME]: raise Exception("UrlParser:build_url", "Url dictionary is empty or missing key values") url_result[UrlParser.SCHEME] = url[UrlParser.SCHEME] if UrlParser.NETLOC in url and url[UrlParser.NETLOC]: if ( UrlParser.USERNAME in url and url[UrlParser.USERNAME] and url[UrlParser.USERNAME] in url[UrlParser.NETLOC] ): url_result[UrlParser.NETLOC] = url[UrlParser.NETLOC] if UrlParser.NETLOC not in url_result: url_result[UrlParser.NETLOC] = url[UrlParser.HOSTNAME] if UrlParser.PORT in url and url[UrlParser.PORT]: url_result[UrlParser.NETLOC] += str(url[UrlParser.PORT]) if UrlParser.USERNAME in url and url[UrlParser.USERNAME]: credentials = "{}@".format(url[UrlParser.USERNAME]) if UrlParser.PASSWORD in url and url[UrlParser.PASSWORD]: credentials = "{}:{}@".format(url[UrlParser.USERNAME], url[UrlParser.PASSWORD]) url_result[UrlParser.NETLOC] = credentials + url_result[UrlParser.NETLOC] url_result[UrlParser.PATH] = url[UrlParser.FILENAME] if UrlParser.PATH in url and url[UrlParser.PATH]: url_result[UrlParser.PATH] = url[UrlParser.PATH] + "/" + url_result[UrlParser.PATH] url_result[UrlParser.PATH] = re.sub("//+", "/", url_result[UrlParser.PATH]) if UrlParser.QUERY in url and url[UrlParser.QUERY]: url_result[UrlParser.QUERY] = url[UrlParser.QUERY] result = SplitResult(**url_result) return result.geturl()
def build_url(url): url_result = {UrlParser.QUERY: '', UrlParser.FRAGMENT: ''} if not url or UrlParser.SCHEME not in url or not url[UrlParser.SCHEME]: raise Exception('UrlParser:build_url', 'Url dictionary is empty or missing key values') url_result[UrlParser.SCHEME] = url[UrlParser.SCHEME] if UrlParser.NETLOC in url and url[UrlParser.NETLOC]: if UrlParser.USERNAME in url \ and url[UrlParser.USERNAME] \ and url[UrlParser.USERNAME] in url[UrlParser.NETLOC]: url_result[UrlParser.NETLOC] = url[UrlParser.NETLOC] if UrlParser.NETLOC not in url_result: url_result[UrlParser.NETLOC] = url[UrlParser.HOSTNAME] if UrlParser.PORT in url and url[UrlParser.PORT]: url_result[UrlParser.NETLOC] += str(url[UrlParser.PORT]) if UrlParser.USERNAME in url and url[UrlParser.USERNAME]: credentials = '{}@'.format(url[UrlParser.USERNAME]) if UrlParser.PASSWORD in url and url[UrlParser.PASSWORD]: credentials = '{}:{}@'.format(url[UrlParser.USERNAME], url[UrlParser.PASSWORD]) url_result[UrlParser.NETLOC] = credentials + url_result[UrlParser.NETLOC] url_result[UrlParser.PATH] = url[UrlParser.FILENAME] if UrlParser.PATH in url and url[UrlParser.PATH]: url_result[UrlParser.PATH] = url[UrlParser.PATH] + '/' + url_result[UrlParser.PATH] url_result[UrlParser.PATH] = re.sub('//+', '/', url_result[UrlParser.PATH]) if UrlParser.QUERY in url and url[UrlParser.QUERY]: url_result[UrlParser.QUERY] = url[UrlParser.QUERY] result = SplitResult(**url_result) return result.geturl()
def _parseurl(self, url): ret = urlsplit(url) self.username = ret.username self.password = ret.password if ret.port <> None: n = SplitResult(ret.scheme, ret.hostname + ":" + ret.port.__str__(), ret.path, ret.query, ret.fragment) else: n = SplitResult(ret.scheme, ret.hostname, ret.path, ret.query, ret.fragment) self.url = n.geturl()
def clean_link(url): o = urlsplit(url) if not o.scheme.lower() in ALLOWED_URL_SCHEMES: return None o = SplitResult(o.scheme, o.netloc, o.path, o.query, '') while o.query and __utm_matcher.search(o.query): query = __utm_matcher.sub('', o.query) o = SplitResult(o.scheme, o.netloc, o.path, query, '') return o.geturl()
def urlsplit(url, scheme="", allow_fragments=True): """Parse a URL into 5 components: <scheme>://<netloc>/<path>?<query>#<fragment> Return a 5-tuple: (scheme, netloc, path, query, fragment). Note that we don't break the components up in smaller bits (e.g. netloc is a single string) and we don't expand % escapes.""" allow_fragments = bool(allow_fragments) key = url, scheme, allow_fragments, type(url), type(scheme) cached = _parse_cache.get(key, None) if cached: return cached if len(_parse_cache) >= MAX_CACHE_SIZE: # avoid runaway growth clear_cache() netloc = query = fragment = "" i = url.find(":") if i > 0: if url[:i] == "http": # optimize the common case scheme = url[:i].lower() url = url[i + 1 :] if url[:2] == "//": netloc, url = _splitnetloc(url, 2) if ("[" in netloc and "]" not in netloc) or ( "]" in netloc and "[" not in netloc ): raise ValueError("Invalid IPv6 URL") if allow_fragments and "#" in url: url, fragment = url.split("#", 1) if "?" in url: url, query = url.split("?", 1) v = SplitResult(scheme, netloc, url, query, fragment) _parse_cache[key] = v return v for c in url[:i]: if c not in scheme_chars: break else: # make sure "url" is not actually a port number (in which case # "scheme" is really part of the path) rest = url[i + 1 :] if not rest or any(c not in "0123456789" for c in rest): # not a port number scheme, url = url[:i].lower(), rest if url[:2] == "//": netloc, url = _splitnetloc(url, 2) if ("[" in netloc and "]" not in netloc) or ( "]" in netloc and "[" not in netloc ): raise ValueError("Invalid IPv6 URL") if allow_fragments and "#" in url: url, fragment = url.split("#", 1) if "?" in url: url, query = url.split("?", 1) v = SplitResult(scheme, netloc, url, query, fragment) _parse_cache[key] = v return v
def with_port(url_str): try: port = settings.PORT except AttributeError: port = None if port == 80: port = None url_split = urlsplit(url_str) if port: if not url_split.port and url_split.netloc: scheme, netloc, url, query, fragment = url_split netloc += ":%s" % port url_split = SplitResult(scheme, netloc, url, query, fragment) return url_split.geturl()
def with_port(url_str): try: port = settings.PORT except AttributeError: port = None if port == 80: port = None url_split = urlsplit(url_str) if port: if not url_split.port and url_split.netloc: scheme, netloc, url, query, fragment = url_split netloc += ":%s" % port url_split = SplitResult(scheme, netloc, url, query, fragment) return url_split.geturl()
def assertApiUrlEqual(self, *args, **kwargs): """ Allows equality comparison of two or more URLs agnostic of API version. This is done by prepending '/api/vx' (where x is equal to the `version` keyword argument or API_CURRENT_VERSION) to each string passed as a positional argument if that URL doesn't already start with that string. Example usage: url = '/api/v1/apps/app/bastacorp/' self.assertApiUrlEqual(url, '/apps/app/bastacorp1/') # settings.API_CURRENT_VERSION = 2 url = '/api/v1/apps/app/bastacorp/' self.assertApiUrlEqual(url, '/apps/app/bastacorp/', version=1) """ PATH = 2 version = kwargs.get('version', settings.API_CURRENT_VERSION) urls = list(args) prefix = '/api/v%d' % version for idx, url in enumerate(urls): urls[idx] = list(urlsplit(url)) if not urls[idx][PATH].startswith(prefix): urls[idx][PATH] = prefix + urls[idx][PATH] urls[idx] = SplitResult(*urls[idx]) eq_(*urls)
def _urlsplit(url, scheme="", allow_fragments=True): """ Parse a URL into 5 components: <scheme>://<netloc>/<path>?<query>#<fragment> Return a 5-tuple: (scheme, netloc, path, query, fragment). Note that we don't break the components up in smaller bits (e.g. netloc is a single string) and we don't expand % escapes. """ if _coerce_args: url, scheme, _coerce_result = _coerce_args(url, scheme) allow_fragments = bool(allow_fragments) netloc = query = fragment = "" i = url.find(":") if i > 0: for c in url[:i]: if c not in scheme_chars: break else: start = i + 1 scheme, url = url[:i].lower(), url[start:] if url[:2] == "//": netloc, url = _splitnetloc(url, 2) if ("[" in netloc and "]" not in netloc) or ( "]" in netloc and "[" not in netloc ): raise ValueError("Invalid IPv6 URL") if allow_fragments and "#" in url: url, fragment = url.split("#", 1) if "?" in url: url, query = url.split("?", 1) v = SplitResult(scheme, netloc, url, query, fragment) return _coerce_result(v) if _coerce_args else v
def _request(scheme=None, netloc=None, path=None, query=None, fragment=None): split = SplitResult(scheme=scheme, netloc=netloc, path=path, query=query, fragment=None) return urlunsplit(split)
def _urlsplit(url, scheme='', allow_fragments=True): """Parse a URL into 5 components: <scheme>://<netloc>/<path>?<query>#<fragment> Return a 5-tuple: (scheme, netloc, path, query, fragment). Note that we don't break the components up in smaller bits (e.g. netloc is a single string) and we don't expand % escapes.""" if _coerce_args: url, scheme, _coerce_result = _coerce_args(url, scheme) allow_fragments = bool(allow_fragments) netloc = query = fragment = '' i = url.find(':') if i > 0: for c in url[:i]: if c not in scheme_chars: break else: scheme, url = url[:i].lower(), url[i + 1:] if url[:2] == '//': netloc, url = _splitnetloc(url, 2) if (('[' in netloc and ']' not in netloc) or (']' in netloc and '[' not in netloc)): raise ValueError("Invalid IPv6 URL") if allow_fragments and '#' in url: url, fragment = url.split('#', 1) if '?' in url: url, query = url.split('?', 1) v = SplitResult(scheme, netloc, url, query, fragment) return _coerce_result(v) if _coerce_args else v
def wrapped_url(api_url): if api_url: parsed = urlsplit(api_url) path = parsed.path.replace('.json', '').replace('api/','').replace('pages/','').strip('/') path = reverse("web_page_wrapper", args=[path]) return SplitResult('', '', path, parsed.query, parsed.fragment).geturl() else: return api_url
def build(self): query = "&".join({"{}={}".format(k, v) for k, v in self.params.items()}) return urlunsplit(SplitResult( scheme=self.scheme, netloc=self.netloc, path=self.path, query=query, fragment=None))
def _populate_urlsplit_tuple(self, scheme='', netloc='', path='', query='', fragment=''): return SplitResult(scheme, netloc, path, query, fragment)
def find_next_indexes(soup): ''' next page for an album index or an album page ''' indexes = soup.findAll('a', 'pix-navi-page') urls = [] if indexes: max_p = max([int(tag.string) for tag in indexes if tag.string.isdigit()]) result = urlsplit(httplib.html_unescape(indexes[0]['href'])) #i don't want patch urllib.unquote. bug description: http://bugs.python.org/issue1712522 #quick fix is convert to ascii. query_dict = parse_qs(result.query.encode('ascii')) for p in range(1, max_p + 1): query_dict['p'] = p result = SplitResult(result.scheme, result.netloc, result.path, urlencode(query_dict, doseq=True), result.fragment) urls.append(result.geturl()) return urls
def __init__(self, url): """ """ # urlsplit will parse what it can from the provided string. raw = urlsplit(url) if not raw.path: raise ValueError("Invalid argument for MIB source: %s" % url) scheme = raw.scheme if not scheme: scheme = "file" if not raw.netloc else "http" path = raw.path if scheme == "file" and not path.startswith("/"): path = os.path.abspath("./" + path) cooked = SplitResult(scheme, raw.netloc, path, raw.query, raw.fragment) self._url = cooked.geturl() self._scheme = scheme self._path = cooked.path self._filename = os.path.split(cooked.path)[-1]
def __init__(self, url): """ """ # urlsplit will parse what it can from the provided string. raw = urlsplit(url) if not raw.path: raise ValueError("Invalid argument for MIB source: %s" % url) scheme = raw.scheme if not scheme: scheme = "file" if not raw.netloc else "http" path = raw.path if scheme == "file" and not path.startswith("/"): path = os.path.abspath("./" + path) cooked = SplitResult(scheme, raw.netloc, path, raw.query, raw.fragment) self._url = cooked.geturl() self._scheme = scheme self._path = cooked.path self._filename = os.path.split(cooked.path)[-1]
def hide_thunder(url): ## tweak image display settings.. url_parts = urlsplit(url) # parse query from url url_query_args = parse_qs(url_parts.query) # parse our query args pcmd_list = list(url_query_args["pcmd"][0]) # split into list for editing pcmd_list[9] = "0" # disable Thunder display url_query_args["pcmd"] = "".join(pcmd_list) # save it back fixed_url = urlunsplit( SplitResult(scheme="", netloc="", path=url_parts.path, query=urlencode(url_query_args, True), fragment="")) # re-join url parts return fixed_url
def get_scrape_url(tracker_url, info_hash): if 'announce' in tracker_url: v = urlsplit(tracker_url) sr = SplitResult(v.scheme, v.netloc, v.path.replace('announce', 'scrape'), v.query, v.fragment) result = urlunsplit(sr) else: log.debug( '`announce` not contained in tracker url, guessing scrape address.' ) result = tracker_url + '/scrape' result += '&' if '?' in result else '?' result += 'info_hash=%s' % quote(info_hash.decode('hex')) return result
def _login(self): login_url = urljoin(self.base_url, '/manage_main') # would be the propper way to login, but is not supported by geckodriver/ chromedriver yet # self.driver.switch_to.alert.authenticate(self.login, self.password) # Disabled because it works only in firefox # if self.driver == 'Firefox': # self.driver.get(login_url) # self.driver.switch_to.alert.send_keys(self.login + Keys.TAB + self.password) # self.driver.switch_to.alert.accept() # else: components = urlsplit(login_url) credentials = '%s:%s@' % (self.login, self.password) components_with_auth = SplitResult(components[0], credentials + components[1], *components[2:]) self.driver.get(urlunsplit(components_with_auth))
def assertApiUrlEqual(self, *args, **kwargs): """ Allows equality comparison of two or more URLs agnostic of API version. This is done by prepending '/api/vx' (where x is equal to the `version` keyword argument or API_CURRENT_VERSION) to each string passed as a positional argument if that URL doesn't already start with that string. Also accepts 'netloc' and 'scheme' optional keyword arguments to compare absolute URLs. Example usage: url = '/api/v1/apps/app/bastacorp/' self.assertApiUrlEqual(url, '/apps/app/bastacorp1/') # settings.API_CURRENT_VERSION = 2 url = '/api/v1/apps/app/bastacorp/' self.assertApiUrlEqual(url, '/apps/app/bastacorp/', version=1) """ # Constants for the positions of the URL components in the tuple # returned by urlsplit. Only here for readability purposes. SCHEME = 0 NETLOC = 1 PATH = 2 version = kwargs.get('version', settings.API_CURRENT_VERSION) scheme = kwargs.get('scheme', None) netloc = kwargs.get('netloc', None) urls = list(args) prefix = '/api/v%d' % version for idx, url in enumerate(urls): urls[idx] = list(urlsplit(url)) if not urls[idx][PATH].startswith(prefix): urls[idx][PATH] = prefix + urls[idx][PATH] if scheme and not urls[idx][SCHEME]: urls[idx][SCHEME] = scheme if netloc and not urls[idx][NETLOC]: urls[idx][NETLOC] = netloc urls[idx] = SplitResult(*urls[idx]) eq_(*urls)
def pdf_echo_loopback(self): ''' Adjunct endpoint used with above PDF test echo page that proxies the generated PDF back to the test page ''' # set default PDF server URI pdfServerUri = '%s://%s/services/pdfserver/renderpdf' % (splunk.getDefault('protocol'), cherrypy.config.get('mgmtHostPort')) # get alternate PDF server URI; values seem to be varied so we normalize alertSettings = en.getEntity('configs/conf-alert_actions', 'email', namespace='search') if alertSettings.get('reportServerURL') and alertSettings['reportServerURL'].strip(): pdfServerUri = alertSettings['reportServerURL'].strip() url = urlsplit(pdfServerUri) if len(url.path)<2: url = url._asdict() url['path'] = '/services/pdfserver/renderpdf' pdfServerUri = urlunsplit(SplitResult(**url)) # determine the external address that is most likely accessible urlparts = urlparse.urlparse(pdfServerUri) ai = socket.getaddrinfo(urlparts.hostname, int(urlparts.port or 80), socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE)[0] af, socktype, proto, canonname, hostport = ai appserverHost = alertSettings.get('hostname') and alertSettings['hostname'].strip() if appserverHost: logger.info('using configured appserver hostname "%s"' % appserverHost) else: s = socket.socket(af, socktype, proto) s.connect(hostport) sockname = s.getsockname() logger.info('most promising interface looks like %s' % sockname[0]) appserverHost = sockname[0] appserverProtocol = 'https' if splunk.util.normalizeBoolean(cherrypy.config.get('enableSplunkWebSSL', False)) else 'http' # create a fake sso-bypass session utilizing the user's current sessionKey active_session = cherrypy.serving.session session_args = ('timeout', 'clean_freq', 'storage_path', 'servers') args = dict([ (arg_name, getattr(active_session, arg_name)) for arg_name in session_args if hasattr(active_session, arg_name)]) fake_session = cherrypy.serving.session.__class__(**args) fake_session['sessionKey'] = cherrypy.session['sessionKey'] fake_session['SSO_DISABLE'] = 1 fake_session.save() fake_session.release_lock() # set GET args args = { 'target': '%s://%s:%s%s/debug/pdf_echo' % ( appserverProtocol, appserverHost if af == socket.AF_INET else '[%s]' % appserverHost, cherrypy.config['httpport'], cherrypy.request.script_name ), 'mode': 'default', 'session': fake_session.id } # fetch the SSL certificate, if any cert = cherrypy.request.app.root.report.get_cert() if cert: args['cert'] = cert logger.info('Testing PDF server=%s on URI=%s' % (pdfServerUri, args['target'])) # make a request to the registered PDF server for the echo page timeout = 20 h = httplib2.Http(timeout=timeout, disable_ssl_certificate_validation=True) start = time.time() try: serverResponse, serverContent = h.request(pdfServerUri, method='POST', body=urllib.urlencode(args)) except: if time.time() - start > (timeout-1): cherrypy.response.headers['content-type'] = 'text/plain' return "Timed out while waiting for a response" raise cherrypy.response.headers['content-type'] = 'application/pdf' return serverContent
def requestPDF(self, **kw): """ Expects a valid splunk session key to be passed in along with the url to be rendered to PDF Complete parameter list: session_key (required) request_path (required) paperSize - 'a4', 'letter', etc or dimensions in mm '200x400' - default 'letter' orientation - 'portrait' or 'landscape' - default 'portrait' title - Title of report - default 'Splunk Report' override_disposition owner """ request_path = kw.get('request_path') if not request_path: raise SimpleError(400, "Invalid request_path supplied") print_session_key = kw.get('session_key') if not print_session_key: if cherrypy.config.get('debug_report_server'): logger.warn('Using debug user for report server') print_session_key = splunk.auth.getSessionKey( 'admin', 'changeme', hostPath=self.splunkd_urlhost) else: raise SimpleError(400, "Invalid session key supplied") settings = en.getEntity(ALERT_ACTIONS_ENTITY, 'email', namespace='system', sessionKey=print_session_key, owner='nobody') enabled = splunk.util.normalizeBoolean( settings.get('reportServerEnabled')) if not enabled: raise SimpleError(400, 'PDF server is not enabled') report_server_url = settings.get('reportServerURL') if isinstance(report_server_url, basestring): report_server_url = report_server_url.strip() url = urlsplit(report_server_url) if url.netloc and len(url.path) < 2: # user has specified the protocol://host:port only url = url._asdict() url['path'] = DEFAULT_SERVICES_URL report_server_url = urlunsplit(SplitResult(**url)) elif report_server_url is None: report_server_url = DEFAULT_SERVICES_URL else: raise SimpleError(500, "reportServerURL is invalid") if not report_server_url: report_server_url = DEFAULT_SERVICES_URL papersize = kw.get('papersize') if not papersize: papersize = settings.get('reportPaperSize', 'letter') orientation = kw.get('orientation') if not orientation: orientation = settings.get('reportPaperOrientation', 'portrait') title = kw.get('title') if not title: title = settings.get('reportTitle', _('Splunk Report')) owner = kw.get('owner', 'nobody') print_session = self.build_session(owner, print_session_key) try: data = { 'session': print_session.id, 'target': request_path, 'papersize': papersize, 'orientation': orientation, 'title': title, 'footer_right': _('Generated by Splunk at %(time)s') % dict(time='&D'), 'mode': 'splunk' } # see if splunkweb is running in SSL mode; if so pass the certificate to the pdf server cert = self.get_cert() if cert: data['cert'] = cert try: logger.info("Appserver dispatching report request to '%s'" % report_server_url) server_response, server_content = splunk.rest.simpleRequest( report_server_url, postargs=data, rawResult=True) except Exception, e: logger.error( "Appserver failed to dispatch report request to %s: %s" % (report_server_url, e)) raise SimpleError( 500, "Appserver failed to dispatch report request to %s: %s" % (report_server_url, e)) if server_response.status == 404: logger.error( "Appserver got a 404 response while contacting the PDF server at %s - Check that the PDF Server app is installed and that reportServerURL is correct" % report_server_url) raise SimpleError( 500, "Appserver got a 404 response while contacting the PDF server at %s - Check that the PDF Server app is installed and that reportServerURL is correct" % report_server_url) elif server_response.status != 200: if server_content and server_content[0] == '>': logger.error( "Appserver received error from PDF server at %s: %s" % (report_server_url, server_content[1:])) raise SimpleError( server_response.status, "PDF server at %s returned error: %s" % (report_server_url, server_content[1:])) logger.error( "Appserver failed to dispatch report request to %s: %s - %s" % (report_server_url, server_response.status, server_response.reason)) raise SimpleError( 500, "Appserver failed to dispatch report request to %s: %s %s" % (report_server_url, server_response.status, server_response.reason)) # relay the response through to the requester cherrypy.response.headers['content-type'] = server_response[ 'content-type'] cherrypy.response.headers['content-length'] = server_response[ 'content-length'] if kw.get('override_disposition'): cherrypy.response.headers['content-disposition'] = kw[ 'override_disposition'] elif 'content-disposition' in server_response: cherrypy.response.headers[ 'content-disposition'] = server_response[ 'content-disposition'] cherrypy.response.body = server_content return cherrypy.response.body
from circonus.collectd.df import get_df_graph_data from circonus.collectd.graph import get_collectd_graph_data from circonus.collectd.memory import get_memory_graph_data from circonus.collectd.interface import get_interface_graph_data from circonus.tag import get_tags_with, get_telemetry_tag, is_taggable from requests import codes as status_codes from requests.exceptions import HTTPError import requests API_PROTOCOL = "https" API_LOCATION = "api.circonus.com" API_VERSION = 2 API_BASE_SPLIT = SplitResult(scheme=API_PROTOCOL, netloc=API_LOCATION, path="/v%d" % API_VERSION, query="", fragment="") API_BASE_URL = urlunsplit(API_BASE_SPLIT) log = logging.getLogger(__name__) def get_api_url(resource_type_or_cid): """Get a valid fully qualified Circonus API URL for the given resource type or ``cid``. :param str resource_type_or_cid: The resource type or ``cid`` representing a specific resource. :return: The API URL. :rtype: :py:class:`str` """
def handle_distrib(self, message): """React to a file dispatch message. """ pathname1, pathname2 = message.split(" ") dummy, filename = os.path.split(pathname1) # TODO: Should not make any assumptions on filename formats, should # load a description of it from a config file instead. if pathname1.endswith(".hmf"): risestr, satellite = filename[:-4].split("_", 1) risetime = datetime.strptime(risestr, "%Y%m%d%H%M%S") pname = pass_name(risetime, satellite) swath = self._received_passes.get(pname, { "satellite": satellite, "start_time": risetime }) swath["type"] = "binary" if satellite == "FENGYUN_1D": swath["format"] = "CHRPT" else: swath["format"] = "HRPT" swath["instrument"] = ("avhrr/3", "mhs", "amsu") swath["level"] = "0" elif filename.startswith("P042") or filename.startswith("P154"): pds = {} pds["format"] = filename[0] pds["apid1"] = filename[1:8] pds["apid2"] = filename[8:15] pds["apid3"] = filename[15:22] pds["time"] = datetime.strptime(filename[22:33], "%y%j%H%M%S") pds["nid"] = filename[33] pds["ufn"] = filename[34:36] pds["extension"] = filename[36:40] if pds["apid1"][:3] == "042": satellite = "TERRA" elif pds["apid1"][:3] == "154": satellite = "AQUA" else: raise ValueError("Unrecognized satellite ID: " + pds["apid1"][:3]) risetime = pds["time"] pname = pass_name(risetime, satellite) swath = self._received_passes.get(pname, { "satellite": satellite, "start_time": risetime }) instruments = { "0064": "modis", "0141": "ceres+y", "0157": "ceres-y", "0261": "amsu-a1", "0262": "amsu-a1", "0290": "amsu-a2", "0342": "hsb", "0402": "amsr-e", "0404": "airs", "0405": "airs", "0406": "airs", "0407": "airs", "0414": "airs", "0415": "airs", "0419": "airs", "0957": "gbad", } swath["instrument"] = instruments.get(pds["apid1"][3:], pds["apid1"][3:]) swath["format"] = "PDS" swath["type"] = "binary" swath["level"] = "0" swath["number"] = int(pds["ufn"]) # NPP RDRs elif filename.startswith("R") and filename.endswith(".h5"): # Occassionaly RT-STPS produce files with a nonstandard file # naming, lacking the 'RNSCA' field. We will try to deal with this # below (Adam - 2013-06-04): mda = {} idx_start = 0 mda["format"] = filename[0] if filename.startswith("RATMS-RNSCA"): mda["instrument"] = "atms" elif filename.startswith("RCRIS-RNSCA"): mda["instrument"] = "cris" elif filename.startswith("RNSCA-RVIRS"): mda["instrument"] = "viirs" else: if filename.startswith("RATMS_npp"): mda["instrument"] = "atms" elif filename.startswith("RCRIS_npp"): mda["instrument"] = "cris" else: logger.warning("Seems to be a NPP/JPSS RDR " + "file but name is not standard!") logger.warning("filename = " + filename) return None idx_start = -6 mda["start_time"] = datetime.strptime( filename[idx_start + 16:idx_start + 33], "d%Y%m%d_t%H%M%S") end_time = datetime.strptime( filename[idx_start + 16:idx_start + 25] + " " + filename[idx_start + 35:idx_start + 42], "d%Y%m%d e%H%M%S") mda["orbit"] = filename[idx_start + 45:idx_start + 50] # FIXME: swath start and end time is granule dependent. # Get the end time as well! - Adam 2013-06-03: satellite = "NPP" start_time = mda["start_time"] pname = pass_name(start_time, satellite) swath = self._received_passes.get(pname, { "satellite": satellite, "start_time": start_time }) swath['end_time'] = end_time swath["instrument"] = mda["instrument"] swath["format"] = "RDR" swath["type"] = "HDF5" swath["level"] = "0" # metop elif filename[4:12] == "_HRP_00_": instruments = { "AVHR": "avhrr", "ASCA": "ascat", "AMSA": "amsu-a", "ASCA": "ascat", "ATOV": "atovs", "AVHR": "avhrr/3", "GOME": "gome", "GRAS": "gras", "HIRS": "hirs/4", "IASI": "iasi", "MHSx": "mhs", "SEMx": "sem", "ADCS": "adcs", "SBUV": "sbuv", "HKTM": "vcdu34" } satellites = {"M02": "METOP-A", "M01": "METOP-B"} satellite = satellites[filename[12:15]] risetime = datetime.strptime(filename[16:31], "%Y%m%d%H%M%SZ") #falltime = datetime.strptime(filename[16:47], "%Y%m%d%H%M%SZ") pname = pass_name(risetime, satellite) swath = self._received_passes.get(pname, { "satellite": satellite, "start_time": risetime }) swath["instrument"] = instruments[filename[:4]] swath["format"] = "EPS" swath["type"] = "binary" swath["level"] = "0" else: return None if pathname2.endswith(filename): uri = pathname2 else: uri = os.path.join(pathname2, filename) url = urlsplit(uri) if url.scheme in ["", "file"]: scheme = "ssh" netloc = self._emitter uri = urlunsplit( SplitResult(scheme, netloc, url.path, url.query, url.fragment)) elif url.scheme == "ftp": scheme = "ssh" netloc = url.hostname uri = urlunsplit( SplitResult(scheme, netloc, url.path, url.query, url.fragment)) swath["filename"] = os.path.split(url.path)[1] swath["uri"] = uri return swath
def main(argv=None): """Load data into a Cheshire3 database based on parameters in argv.""" global argparser, session, server, db if argv is None: args = argparser.parse_args() else: args = argparser.parse_args(argv) if irods is None: raise MissingDependencyException('icheshire3-load script', 'irods (PyRods)' ) session = Session() server = SimpleServer(session, args.serverconfig) if args.database is None: try: dbid = identify_database(session, os.getcwd()) except EnvironmentError as e: server.log_critical(session, e.message) return 1 server.log_debug( session, "database identifier not specified, discovered: {0}".format(dbid)) else: dbid = args.database try: db = server.get_object(session, dbid) except ObjectDoesNotExistException: msg = """Cheshire3 database {0} does not exist. Please provide a different database identifier using the --database option. """.format(dbid) server.log_critical(session, msg) return 2 else: # Allow for multiple data arguments docFac = db.get_object(session, 'defaultDocumentFactory') for dataArg in args.data: if dataArg.startswith('irods://'): parsed = urlsplit(dataArg) else: # Examine current environment status, myEnv = irods.getRodsEnv() try: host = myEnv.getRodsHost() except AttributeError: host = myEnv.rodsHost # Port try: myEnv.getRodsPort() except AttributeError: port = myEnv.rodsPort # User try: username = myEnv.getRodsUserName() except AttributeError: username = myEnv.rodsUserName netloc = '{0}@{1}:{2}'.format(username, host, port) try: cqm = myEnv.getRodsCwd() except AttributeError: cwd = myEnv.rodsCwd path = '/'.join([cwd, dataArg]) parsed = SplitResult('irods', netloc, path, None, None) dataArg = urlunsplit(parsed) server.log_debug(session, dataArg) if args.format is None or not args.format.startswith('i'): fmt = 'irods' else: fmt = args.format server.log_debug(session, fmt) try: docFac.load(session, dataArg, args.cache, fmt, args.tagname, args.codec) except MissingDependencyException as e: server.log_critical(session, e.reason) missingDependencies = e.dependencies raise MissingDependencyException('cheshire3-load script', missingDependencies) wf = db.get_object(session, 'buildIndexWorkflow') wf.process(session, docFac)