def __init__(self, partName, namespace=None): Observable.__init__(self) if not correctNameRe.match(partName): raise Fields2XmlException('Invalid name: "%s"' % partName) self._identifier = None self._partName = partName self._namespace = namespace
def __init__(self, type, interval=1.0, **kwargs): Observable.__init__(self, **kwargs) self._interval = interval self._lastReportTime = 0 self._type = type self._addCount = 0 self._deleteCount = 0
def __init__(self, reactor, host=None, port=None, period=None, verbose=None, prio=None, name=None, err=None, autoStart=True, schedule=None, retryAfterErrorTime=30, compress=True): Observable.__init__(self, name=name) self._reactor = reactor self._host = host self._port = port self._schedule = schedule if schedule is None: if period is None: period = 1 else: warn("Please use schedule instead of period.", DeprecationWarning) # since 2013-02-20 self._schedule = Schedule(period=period) elif not period is None: raise ValueError("Using both schedule and period is invalid") self._prio = prio self._err = err or sys.stderr self._paused = not autoStart self._currentTimer = None self._currentProcess = None self._sok = None self._errorState = None self._retryAfterErrorTime = retryAfterErrorTime self._compress = compress if autoStart and (not self._host or not self._port): raise ValueError("Unless autoStart is set to False host and port need to be specified.") if verbose in [True, False]: warn('Verbose flag is deprecated', DeprecationWarning)
def __init__(self, schemaPath): Observable.__init__(self) try: self._schema = XMLSchema(parse(open(schemaPath))) except XMLSchemaParseError, e: print e.error_log.last_error raise
def __init__(self, reactor, conversion=None, **kwargs): Observable.__init__(self, **kwargs) self._conversion = Conversion() if conversion is None else conversion self._dynamicHtml = DynamicHtml([dynamicPath], reactor=reactor, notFoundPage='notFound', additionalGlobals={ 'VERSION': version, 'allCoreInfo': self._allCoreInfo, 'parseCql': parseString, 'cqlToExpression': cqlToExpression, 'cql2string': cql2string, 'dumps': dumps, } ) self._internalTree = be((Observable(), (PathFilter('/', excluding=['/static', '/version']), (self._dynamicHtml,) ), (PathFilter('/static'), (PathRename(lambda path: path[len('/static'):]), (FileServer(staticPath),) ) ), (PathFilter('/version'), (StringServer("Meresco Lucene version %s" % version, ContentTypePlainText),) ), ))
def __init__(self, host=None, port=None, core=None): Observable.__init__(self) self._host = host self._port = port self._core = core if core is not None: self.observable_name = lambda: core
def __init__(self, name, settings, host=None, port=None, readonly=False, **kwargs): Observable.__init__(self, name=name) self._host, self._port = host, port self.settings = settings self._fieldRegistry = settings.fieldRegistry self._name = name self._readonly = readonly
def __init__(self, host=None, port=None, core=None): Observable.__init__(self) self._host = host self._port = port self._core = core if core is not None: self.observable_name = lambda: core
def __init__(self, reactor, conversion=None, **kwargs): Observable.__init__(self, **kwargs) self._conversion = Conversion() if conversion is None else conversion self._dynamicHtml = DynamicHtml( [dynamicPath], reactor=reactor, notFoundPage='notFound', additionalGlobals={ 'VERSION': version, 'allCoreInfo': self._allCoreInfo, 'parseCql': parseString, 'cqlToExpression': cqlToExpression, 'cql2string': cql2string, 'dumps': dumps, }) self._internalTree = be(( Observable(), (PathFilter('/', excluding=['/static', '/version']), (self._dynamicHtml, )), (PathFilter('/static'), (PathRename(lambda path: path[len('/static'):]), (FileServer(staticPath), ))), (PathFilter('/version'), (StringServer("Meresco Lucene version %s" % version, ContentTypePlainText), )), ))
def __init__(self, reactor, oaiPath, **kwargs): Observable.__init__(self, **kwargs) self._outside = Transparent() oaiJsonInfo = OaiJsonInfo() self._dynamicHtml = DynamicHtml( [dynamicPath], reactor=reactor, notFoundPage='notFound', additionalGlobals={ 'VERSION': VERSION, 'oaiPath': oaiPath, 'strftime': strftime, 'localtime': localtime, }) self._internalTree = be(( Observable(), (PathFilter('/', excluding=['/static', '/version', '/json']), ( self._dynamicHtml, ( oaiJsonInfo, (self._outside, ), ), )), (PathFilter('/json'), ( oaiJsonInfo, (self._outside, ), )), (PathFilter('/static'), (PathRename(lambda path: path[len('/static'):]), (FileServer(staticPath), ))), (PathFilter('/version'), (StringServer("Meresco Oai version %s" % VERSION, ContentTypePlainText), )), ))
def __init__(self, searchApiBaseUrl='https://api.histograph.io/search', **kwargs): Observable.__init__(self, **kwargs) parsedUrl = parseAbsoluteUrl(searchApiBaseUrl) self._host = parsedUrl.host self._port = parsedUrl.port self._path = parsedUrl.path self._ssl = (parsedUrl.scheme == 'https')
def __init__(self, defaultRecordSchema, pageSize=10, maximumRecordNumber=1000, defaultMaxTermsFacet=10, useOriginalPath=False, getItemsFromObserver=False, version=VERSION, enableCollectLog=False, **kwargs): Observable.__init__(self, **kwargs) self._version = version self._defaultRecordSchema = defaultRecordSchema self._pageSize = pageSize self._maximumRecordNumber = maximumRecordNumber self._defaultMaxTermsFacet = defaultMaxTermsFacet self._argumentLimits = dict( page_size=pageSize, maximum_record_number=maximumRecordNumber, default_facet_terms_count=defaultMaxTermsFacet, default_record_schema=defaultRecordSchema, ) self._determinePath = lambda **kwargs: kwargs['path'] if useOriginalPath: self._determinePath = lambda **kwargs: kwargs.get( 'originalPath', kwargs['path']) self._getItems = self._observerGetItems if getItemsFromObserver else self._defaultGetItems self._collectLogForScope = collectLogForScope if enableCollectLog else lambda **kwargs: None
def __init__(self, path, metadataPrefix, workingDirectory, set=None, xWait=True, partition=None, err=None, verb=None, autoCommit=True, incrementalHarvestSchedule=_UNSPECIFIED, restartAfterFinish=False, userAgentAddition=None, name=None): Observable.__init__(self, name=name) self._userAgent = _USER_AGENT + ('' if userAgentAddition is None else ' (%s)' % userAgentAddition) self._path = path self._metadataPrefix = metadataPrefix isdir(workingDirectory) or makedirs(workingDirectory) self._stateFilePath = join(workingDirectory, "harvester.state") self._set = set self._xWait = xWait self._partition = partition self._err = err or stderr self._verb = verb or 'ListRecords' self._autoCommit = autoCommit if restartAfterFinish and incrementalHarvestSchedule: raise ValueError("In case restartAfterFinish==True, incrementalHarvestSchedule must not be set") self._restartAfterFinish = restartAfterFinish if incrementalHarvestSchedule is _UNSPECIFIED and not restartAfterFinish: incrementalHarvestSchedule = Schedule(timeOfDay='00:00') self._incrementalHarvestSchedule = incrementalHarvestSchedule self._resumptionToken = None self._from = None self._errorState = None self._earliestNextRequestTime = 0 self._readState() self._identifierFilePath = join(workingDirectory, "harvester.identifier") if isfile(self._identifierFilePath): self._identifier = open(self._identifierFilePath).read().strip() else: self._identifier = str(uuid4()) open(self._identifierFilePath, 'w').write(self._identifier)
def __init__(self, partName, namespace=None): Observable.__init__(self) if not correctNameRe.match(partName): raise Fields2XmlException('Invalid name: "%s"' % partName) self._identifier = None self._partName = partName self._namespace = namespace
def __init__(self, schemaPath): Observable.__init__(self) try: self._schema = XMLSchema(parse(open(schemaPath))) except XMLSchemaParseError, e: print e.error_log.last_error raise
def __init__(self, fromKwarg, toKwarg=None, name=None): if not fromKwarg: raise ValueError( "'fromKwarg' should contain a keyword argument name.") Observable.__init__(self, name=name) self._fromKwarg = fromKwarg self._toKwarg = toKwarg if toKwarg else self._fromKwarg
def __init__(self, count, field, allowOverrideField=False, maximumCount=None, suggestMode=None): Observable.__init__(self) self._count = count self._field = field self._suggestMode = suggestMode self._allowOverrideField = allowOverrideField self._maximumCount = maximumCount
def __init__(self, reactor, message=None, initialSchedule=None, schedule=None, errorSchedule=Schedule(period=15), autoStart=True, prio=None, name=None): Observable.__init__(self, name=name) self._reactor = reactor self._message = message or 'handle' self._initialSchedule = initialSchedule self._schedule = schedule self._errorSchedule = errorSchedule # Pause as soon as possible, please. self._pause = not autoStart self._prio = prio self._busy = False self._currentTimer = None self._errorState = None if not (self._pause or self._schedule): raise ValueError( 'When autoStart is enabled, a schedule is required.')
def __init__(self, host=None, port=None, path=None, name=None, conversion=None): Observable.__init__(self, name=name) self._host = host self._port = port self._path = '' if path is None else path self._path += '/__lucene_remote__' self._conversion = Conversion() if conversion is None else conversion
def __init__(self, allowed=None, disallowed=None, **kwargs): Observable.__init__(self, **kwargs) if allowed and not disallowed: self._allowedMessage = lambda message: message in allowed elif disallowed and not allowed: self._allowedMessage = lambda message: message not in disallowed else: raise ValueError('Use disallowed or allowed')
def __init__(self, included=None, excluded=None): Observable.__init__(self) if included and excluded or not (included or excluded): raise ValueError("Use included OR excluded") if included: self._allowed = lambda name: name in included else: self._allowed = lambda name: not name in excluded
def __init__(self, included=None, excluded=None): Observable.__init__(self) if included and excluded or not (included or excluded): raise ValueError("Use included OR excluded") if included: self._allowed = lambda name: name in included else: self._allowed = lambda name: not name in excluded
def __init__(self, verbose=False, truncate_chars=300): Observable.__init__(self) self._verbose = verbose self._truncate_chars = truncate_chars self._fieldslist = [] self._wcp_collection = None self._nids_aut_enriched = set() self._record_pids = set()
def __init__(self, allowed=None, disallowed=None, **kwargs): Observable.__init__(self, **kwargs) if allowed and not disallowed: self._allowedMessage = lambda message: message in allowed elif disallowed and not allowed: self._allowedMessage = lambda message: message not in disallowed else: raise ValueError('Use disallowed or allowed')
def __init__(self, serviceIdentifier, periodicDownloadName, sourceServiceType, sourceServiceIdentifier=None, pollIntervalConfigSelector=None, name=None, **kwargs): Observable.__init__(self, name) self._serviceIdentifier = serviceIdentifier self._periodicDownloadName = periodicDownloadName self._pollIntervalConfigSelector = pollIntervalConfigSelector or oaiDownloadConfigSelector self._sourceServiceType = sourceServiceType self._sourceServiceIdentifier = sourceServiceIdentifier self._extraKwargs = kwargs
def __init__(self, attributeXpaths=None, namespaceMap=None, sendAsList=False, namespaces=None): Observable.__init__(self) self._attributeXpaths = attributeXpaths or [] self._sendAsList = sendAsList if namespaceMap: warn("Please use 'namespaces=...'", DeprecationWarning) namespaces=namespaceMap self.xpath = _namespaces.copyUpdate(namespaces or {}).xpath
def __init__(self, should=None, could=None, namespaces=None, namespaceMap=None): Observable.__init__(self) if namespaceMap: warn("Please use 'namespaces=...'", DeprecationWarning) namespaces=namespaceMap self.xpath = _namespaces.copyUpdate(namespaces or {}).xpath self._should = _init(should) self._could = _init(could)
def __init__(self, log=None, scopeNames=None, argumentsSelection=None, numberOfRecordsSelection=None, **kwargs): Observable.__init__(self, **kwargs) self._log = self.call if log is None else log self._scopeNames = () if scopeNames is None else scopeNames self._argumentSelectionScope = 'sru' if argumentsSelection is None else argumentsSelection['scope'] self._argumentSelectionKey = 'arguments' if argumentsSelection is None else argumentsSelection['key'] self._numberOfRecordsSelectionScope = 'sru' if numberOfRecordsSelection is None else numberOfRecordsSelection['scope'] self._numberOfRecordsSelectionKey = 'numberOfRecords' if numberOfRecordsSelection is None else numberOfRecordsSelection['key']
def __init__(self, name=None, stderr=stderr, sendRecordData=True, logErrors=True, enableCollectLog=False): Observable.__init__(self, name=name) self._stderr = stderr self._logErrors = logErrors self._sendRecordData = sendRecordData self._collectLogForScope = lambda **kwargs: None if enableCollectLog: self._collectLogForScope = collectLogForScope
def __init__(self, home="/", name=None): Observable.__init__(self, name=name) self._home = home self._actions = { 'create': self.handleCreate, 'update': self.handleUpdate, 'delete': self.handleDelete, }
def __init__(self, handlers=None, **kwargs): Observable.__init__(self, **kwargs) self._handlers = [] for handler in handlers or []: self._handlers.append( dict(path=handler[0], handleRequest=handler[1])) self.paths = ['/get'] + [h['path'] for h in self._handlers]
def __init__(self, title, description, link, antiUnaryClause='', **sruArgs): Observable.__init__(self) self._title = title self._description = description self._link = link self._antiUnaryClause = antiUnaryClause self._sortKeys = sruArgs.get('sortKeys', None) self._maximumRecords = sruArgs.get('maximumRecords', 10)
def __init__(self, multiLevelFieldsDict): Observable.__init__(self) self._multiLevelFields = multiLevelFieldsDict for key, multilevelFields in multiLevelFieldsDict.items(): for multilevelField in multilevelFields: assert len( multilevelField ) == 3, 'multilevelFields for %s should be a list of tuples with: levelField, maximumCount, sorted' % key
def __init__(self, schemaPath): Observable.__init__(self) try: with open(schemaPath) as fp: self._schema = XMLSchema(parse(fp)) except XMLSchemaParseError as e: print(e.error_log.last_error) raise
def __init__(self, title, description, link, antiUnaryClause='', **sruArgs): Observable.__init__(self) self._title = title self._description = description self._link = link self._antiUnaryClause = antiUnaryClause self._sortKeys = sruArgs.get('sortKeys', None) self._maximumRecords = sruArgs.get('maximumRecords', 10)
def __init__(self, allowed=None, disallowed=None, **kwargs): Observable.__init__(self, **kwargs) # self._getLongAsLxml = False if allowed and not disallowed: self._allowedCollection = lambda message: message in allowed elif disallowed and not allowed: self._allowedCollection = lambda message: message not in disallowed else: raise ValueError('Use disallowed or allowed')
def __init__(self, action, loginPath, home="/", name=None): Observable.__init__(self, name=name) self._action = action self._loginPath = loginPath self._home = home self._actions = { 'changepassword': self.handleChangePassword, 'remove': self.handleRemove, }
def __init__(self, allowed=None, disallowed=None, **kwargs): Observable.__init__(self, **kwargs) # self._getLongAsLxml = False if allowed and not disallowed: self._allowedCollection = lambda message: message in allowed elif disallowed and not allowed: self._allowedCollection = lambda message: message not in disallowed else: raise ValueError('Use disallowed or allowed')
def __init__(self, xpathList, fromKwarg, toKwarg=None, namespaces=None, namespaceMap=None): Observable.__init__(self) self._xpaths = xpathList self._fromKwarg = fromKwarg self._toKwarg = toKwarg if toKwarg else self._fromKwarg if namespaceMap: warn("Please use 'namespaces=...'", DeprecationWarning) namespaces = namespaceMap self.xpath = oftenUsedNamespaces.copyUpdate(namespaces or {}).xpath
def __init__(self, scopeName=None, name=None, **kwargs): if scopeName is None and name is None: raise TypeError("No name set for LogCollectorScope.") elif scopeName is None: scopeName = name if name is None: name = scopeName Observable.__init__(self, name=name, **kwargs) self._scopeName = scopeName
def __init__(self, serviceIdentifier, flag, name=None): Observable.__init__(self, name=name) self._serviceIdentifier = serviceIdentifier self._flag = flag self._flagSet = flag.default self._seed = randomWord(20) self._cookiekey = 'flagcheck%s=' % randomWord(10) self._debugTimeout = 60 self._debugIds = TimedDictionary(self._debugTimeout)
def __init__(self): Observable.__init__(self) config = ConfigParser.ConfigParser() config.read( join(dirname(abspath(__file__)), '..', 'conf', 'application.ini')) self._pidgraph_api = config.get('IndexServer', 'pidGraphUrl') self._pidgraph_enabled = config.getboolean('IndexServer', 'pidGraphIsEnabled') print "PidGraph API:", self._pidgraph_api, ". IsEnabled:", self._pidgraph_enabled
def __init__(self, scopeName=None, name=None, **kwargs): if scopeName is None and name is None: raise TypeError("No name set for LogCollectorScope.") elif scopeName is None: scopeName = name if name is None: name = scopeName Observable.__init__(self, name=name, **kwargs) self._scopeName = scopeName
def __init__(self, secret, softwareVersion=None, name=None): Observable.__init__(self, name=name) self._secret = secret self._softwareVersion = softwareVersion self._prefix = '/service/' self._actions = { 'update': (self.handleUpdate, 'POST'), 'list': (self.handleList, 'GET'), }
def __init__(self, extraRecordDataNewStyle=True, drilldownSortBy=DRILLDOWN_SORTBY_COUNT, extraXParameters=None, includeQueryTimes=False, drilldownMaximumMaximumResults=None, enableCollectLog=False): Observable.__init__(self) self._drilldownSortBy = drilldownSortBy self._extraRecordDataNewStyle = extraRecordDataNewStyle self._extraXParameters = set(extraXParameters or []) self._extraXParameters.add("x-recordSchema") self._includeQueryTimes = includeQueryTimes self._drilldownMaximumMaximumResults = drilldownMaximumMaximumResults self._drilldownMaximumTerms = DEFAULT_MAXIMUM_TERMS if self._drilldownMaximumMaximumResults is None else min(DEFAULT_MAXIMUM_TERMS, self._drilldownMaximumMaximumResults) self._collectLogForScope = collectLogForScope if enableCollectLog else lambda **kwargs: None
def __init__(self, extraRecordDataNewStyle=True, drilldownSortBy=DRILLDOWN_SORTBY_COUNT, extraXParameters=None, includeQueryTimes=False, drilldownMaximumMaximumResults=None, enableCollectLog=False): Observable.__init__(self) self._drilldownSortBy = drilldownSortBy self._extraRecordDataNewStyle = extraRecordDataNewStyle self._extraXParameters = set(extraXParameters or []) self._extraXParameters.add("x-recordSchema") self._includeQueryTimes = includeQueryTimes self._drilldownMaximumMaximumResults = drilldownMaximumMaximumResults self._drilldownMaximumTerms = DEFAULT_MAXIMUM_TERMS if self._drilldownMaximumMaximumResults is None else min(DEFAULT_MAXIMUM_TERMS, self._drilldownMaximumMaximumResults) self._collectLogForScope = collectLogForScope if enableCollectLog else lambda **kwargs: None
def __init__(self, transactionName, partname="solr", singularValueFields=None, isSingularValueField=None): Observable.__init__(self) self._transactionName = transactionName self._partname = partname if singularValueFields and isSingularValueField: raise ValueError("Use either 'singularValueFields' or 'isSingularValueField'") self._isSingularValueField = isSingularValueField if singularValueFields: singularValueFields = set(singularValueFields) self._isSingularValueField = lambda name: name in singularValueFields
def __init__(self, statePath, log=None, name=None, deleteAll=False, harvestInterval=24*60*60, errorInterval=10): Observable.__init__(self, name=name) self._statePath = statePath if not isdir(statePath): makedirs(statePath) self._state = State.load(filePath=join(self._statePath, 'state')) self._events = _Events(self._statePath) self._logWrite = (lambda aString: None) if log is None else log.write self._deleteAll = deleteAll self._harvestInterval = harvestInterval self._errorInterval = errorInterval
def __init__(self, resultsFrom, matches=None, dedupFieldName=None, dedupSortFieldName=None, dedupByDefault=True, drilldownFieldnamesTranslate=lambda s: s): Observable.__init__(self) self._resultsFrom = resultsFrom self._matches = matches or [] self._cores = set(cSpec['core'] for match in self._matches for cSpec in match) self._dedupFieldName = dedupFieldName self._dedupSortFieldName = dedupSortFieldName self._dedupByDefault = dedupByDefault self._drilldownFieldnamesTranslate = drilldownFieldnamesTranslate self._clusteringEnabled = True self._extraFilterQueries = ExtractFilterQueries(self._cores)
def __init__(self, path, keys, snapshotInterval=3600): Observable.__init__(self) self._path = path self._snapshotFilename = join(self._path, snapshotFilename) self._txlogFileName = join(self._path, 'txlog') self._txlogFile = None self._keys = keys self._snapshotInterval = snapshotInterval self._lastSnapshot = (1970, 1, 1, 0, 0, 0) self._data = Aggregator(Top100sFactory()) self._readState()
def __init__(self, fieldRegistry, untokenizedFieldnames, indexFieldFactory, rewriteIdentifier=None, **kwargs): Observable.__init__(self, **kwargs) self._fieldRegistry = fieldRegistry self._untokenizedFieldnames = untokenizedFieldnames self._indexFieldFactory = indexFieldFactory self._rewriteIdentifier = rewriteIdentifier or (lambda i: i)
def __init__(self, path, keys, snapshotInterval=3600): Observable.__init__(self) self._path = path self._snapshotFilename = join(self._path, snapshotFilename) self._txlogFileName = join(self._path, 'txlog') self._txlogFile = None self._keys = keys self._snapshotInterval = snapshotInterval self._lastSnapshot = (1970, 1, 1, 0, 0, 0) self._data = Aggregator(Top100sFactory()) self._readState()
def __init__(self, host=None, port=None, description='Meresco SRU', modifiedDate=None, database=None, defaultRecordSchema="dc", defaultRecordPacking="xml", maximumMaximumRecords=None, wsdl=None, oldAndWrongStyleSortKeys=False): Observable.__init__(self) self._host = host self._port = port self._description = description self._modifiedDate = modifiedDate self._database = database self._defaultRecordSchema = defaultRecordSchema self._defaultRecordPacking = defaultRecordPacking self._maximumMaximumRecords = maximumMaximumRecords self._wsdl = wsdl self._oldAndWrongStyleSortKeys = oldAndWrongStyleSortKeys
def __init__(self, count, field, allowOverrideField=False, maximumCount=None, suggestMode=None): Observable.__init__(self) self._count = count self._field = field self._suggestMode = suggestMode self._allowOverrideField = allowOverrideField self._maximumCount = maximumCount
def __init__(self, logfileDir, enabled=True): Observable.__init__(self) self._enabled = enabled self._logfileDir = logfileDir #self._logger = logging.getLogger('Logger') #self._logger.setLevel(logging.WARNING) #self._formatter = logging.Formatter("%(asctime)s %(message)s", "%Y-%m-%dT%H:%M:%SZ") if not isdir(self._logfileDir): makedirs(self._logfileDir) print "RSS Logger directory:", self._logfileDir
def __init__(self, host=None, port=None, path=None, name=None, conversion=None): Observable.__init__(self, name=name) self._host = host self._port = port self._path = '' if path is None else path self._path += '/__lucene_remote__' self._conversion = Conversion() if conversion is None else conversion
def __init__(self, should=None, could=None, namespaces=None, namespaceMap=None): Observable.__init__(self) if namespaceMap: warn("Please use 'namespaces=...'", DeprecationWarning) namespaces = namespaceMap self.xpath = _namespaces.copyUpdate(namespaces or {}).xpath self._should = _init(should) self._could = _init(could)