Пример #1
0
            if field == "embargoedUntilDate":
                ans = u"%sZ" % unicode(o).split(".")[0]
                document[field].append(unicode(ans).encode("utf-8"))
            else:
                document[field].append(unicode(o).encode("utf-8"))
        else:
            document["text"].append(unicode(o).encode("utf-8"))
    document = dict(document)
    return document


if __name__ == "__main__":
    item_id = "mond_ay2"
    silo_name = "DataFinder"
    # r = Redis()
    c = Config()
    redis_section = "redis"
    worker_section = "worker_solr"
    worker_number = "1"
    worker_section = "worker_solr"

    rq = Redis()

    b = BroadcastToRedis("localhost", "silochanges")
    b.creation(silo_name, item_id)

    #    line = rq.spop("{\"_timestamp\": \"2013-05-16T18:03:24.125059\", \"type\": \"c\", \"id\": \"datafinder_1\", \"silo\": \"DataFinder\"}")

    #    msg = json.loads(line)

    user_name = "admin"
from ConfigParser import ConfigParser

OPTIONAL_SUPERVISOR_PROCESS_OPTIONS = ['user']

if __name__ == "__main__":
  if not os.path.exists("workers_enabled"):
    os.mkdir("workers_enabled")
  else:
    # clear out the enabled directory
    for conf in os.listdir("workers_enabled"):
      if os.path.isfile(os.path.join("workers_enabled", conf)):
        os.remove(os.path.join("workers_enabled", conf))

  if not os.path.exists("workers_available"):
    os.mkdir("workers_available")
  c = Config()
  base_superv_conf = "supervisord.conf.base"
  if len(sys.argv) == 2:
    # use a different base supervisor file
    base_superv_conf = sys.argv[1]
  supervisord_config = Config(base_superv_conf)

  if 'supervisor' in c.sections():
    supervisord_config.add_section('inet_http_server')
    params = {'username':'******',
              'password':'******',
              'port':'127.0.0.1:9001'}
    for key in params:
      if c.has_option('supervisor', key):
        supervisord_config.set('inet_http_server', key, c.get('supervisor', key))
      else:
Пример #3
0
#!/usr/bin/env python

from redisqueue import RedisQueue

from LogConfigParser import Config

import sys

from time import sleep

if __name__ == "__main__":
  c = Config()
  redis_section = "redis"
  worker_number = sys.argv[1]
  worker_section = sys.argv[2]
  if len(sys.argv) == 4:
    if "redis_%s" % sys.argv[3] in c.sections():
      redis_section = "redis_%s" % sys.argv[3]

  rq = RedisQueue(c.get(worker_section, "listento"), "logger_%s" % worker_number,
                  db=c.get(redis_section, "db"), 
                  host=c.get(redis_section, "host"), 
                  port=c.get(redis_section, "port")
                  )

  with open(c.get(worker_section, "logfile"), "a+") as logfile:
    while(True):
      line = rq.pop()
      if line:
        try:
          if line.endswith("\n"):
Пример #4
0
  while True:
    if this_modified > last_modified:
      last_modified = this_modified
      """ File was modified, so read new lines, look for error keywords """
      while True:
        line = fh.readline()
        if not line: break	
        print line
        r.submit_line(line)

    watcher = os.stat(log_filename)
    this_modified = watcher.st_mtime
    time.sleep(1)

if __name__=='__main__':
  c = Config()
  redis_section = "redis"
  worker_number = sys.argv[1]
  worker_section = sys.argv[2]
  if len(sys.argv) == 4:
    if "redis_%s" % sys.argv[3] in c.sections():
      redis_section = "redis_%s" % sys.argv[3]
      
  # from config:
  redis_config = dict(c.items(redis_section))
  
  if 'port' in redis_config:
    redis_config['port'] = int(redis_config['port'])
  servicename = c.get(worker_section, "servicename")
  queuename = c.get(worker_section, "pushto")
  logfilename = c.get(worker_section, "logfile")
Пример #5
0
                document[field].append(unicode(o).encode("utf-8"))
        else:
            document["text"].append(unicode(o).encode("utf-8"))
    document = dict(document)
    if "publisher" in document:
        # TODO: SOLR not accepting multiple values for publisher though multivalued fields. Fix this
        del document["publisher"]
    if debug:
        f = codecs.open("/var/log/databank/solr_doc.log", "w", "utf-8")
        f.write(simplejson.dumps(document))
        f.close()
    return document


if __name__ == "__main__":
    c = Config()
    redis_section = "redis"
    worker_section = "worker_solr"
    worker_number = sys.argv[1]
    hours_before_commit = 1
    if len(sys.argv) == 3:
        if "redis_%s" % sys.argv[2] in c.sections():
            redis_section = "redis_%s" % sys.argv[2]

    rq = RedisQueue(
        c.get(worker_section, "listento"),
        "solr_%s" % worker_number,
        db=c.get(redis_section, "db"),
        host=c.get(redis_section, "host"),
        port=c.get(redis_section, "port"),
        errorqueue=c.get(worker_section, "errorq"),
Пример #6
0
            #        fn = unicode(o).split('/datasets/')
            #        if len(fn) == 2 and fn[1]:
            #            document['filename'].append(unicode(fn[1]).encode("utf-8"))
            if field == "embargoedUntilDate":
                ans = u"%sZ"%unicode(o).split('.')[0]
                document[field].append(unicode(ans).encode("utf-8"))
            else:
                document[field].append(unicode(o).encode("utf-8"))
        else:
            document['text'].append(unicode(o).encode("utf-8"))
    document = dict(document)
    return document


if __name__ == "__main__":
    c = Config()
    redis_section = "redis"
    worker_section = "worker_solr"
    worker_number = sys.argv[1]
    hours_before_commit = 1 
    if len(sys.argv) == 3:
        if "redis_%s" % sys.argv[2] in c.sections():
            redis_section = "redis_%s" % sys.argv[2]

    rq = RedisQueue(c.get(worker_section, "listento"), "solr_%s" % worker_number,
                  db=c.get(redis_section, "db"), 
                  host=c.get(redis_section, "host"), 
                  port=c.get(redis_section, "port"),
                  errorqueue=c.get(worker_section, "errorq")
                 )
    #DB_ROOT = c.get(worker_section, "dbroot")
Пример #7
0
#!/usr/bin/env python

from redisqueue import RedisQueue

from LogConfigParser import Config

import sys

from time import sleep

if __name__ == "__main__":
  c = Config()
  redis_section = "redis"
  worker_section = "worker_broker"
  worker_number = sys.argv[1]
  if len(sys.argv) == 3:
    if "redis_%s" % sys.argv[2] in c.sections():
      redis_section = "redis_%s" % sys.argv[2]

  rq = RedisQueue(c.get(worker_section, "listento"), "broker_%s" % worker_number,
                  db=c.get(redis_section, "db"), 
                  host=c.get(redis_section, "host"), 
                  port=c.get(redis_section, "port")
                  )
  if c.has_option(worker_section, "fanout_status_queue"):
    # keep a queue of messages to deliver for a given push'd item
    # better resumeability at the cost of more redis operations
    topushq = RedisQueue(c.get(worker_section, "fanout_status_queue"), "fanout_broker_%s" % worker_number,
                  db=c.get(redis_section, "db"), 
                  host=c.get(redis_section, "host"), 
                  port=c.get(redis_section, "port")
Пример #8
0
                if '/datasets/' in o:
                    fn = unicode(o).split('/datasets/')
                    if len(fn) == 2 and fn[1]:
                        document['filename'].append(unicode(fn[1]).encode("utf-8"))
            if field == "embargoedUntilDate":
                ans = u"%sZ"%unicode(o).split('.')[0]
                document[field].append(unicode(ans).encode("utf-8"))
            else:
                document[field].append(unicode(o).encode("utf-8"))
        else:
            document['text'].append(unicode(o).encode("utf-8"))
    document = dict(document)
    return document

if __name__ == "__main__":
    c = Config()
    redis_section = "redis"
    worker_section = "worker_solr"
    worker_number = sys.argv[1]
    hours_before_commit = 1
    if len(sys.argv) == 3:
        if "redis_%s" % sys.argv[2] in c.sections():
            redis_section = "redis_%s" % sys.argv[2]

    rq = RedisQueue(c.get(worker_section, "listento"), "solr_%s" % worker_number,
                  db=c.get(redis_section, "db"), 
                  host=c.get(redis_section, "host"), 
                  port=c.get(redis_section, "port"),
                  errorqueue=c.get(worker_section, "errorq")
                 )
    DB_ROOT = c.get(worker_section, "dbroot")
# process version (for provenance)
PROCESS = "iucr_redis_tally"
PROCESS_VERSION = "0.1"

nss = {"prism": "http://prismstandard.org/namespaces/1.2/basic/",
       "dc": "http://purl.org/dc/elements/1.1/",
       "dcterms": "http://purl.org/dc/terms/",
       "ov":"http://open.vocab.org/terms/",
       "foaf":"http://xmlns.com/foaf/0.1/",
           }

from uuid import uuid4
import hashlib

if __name__ == "__main__":
  c = Config()
  redis_section = "redis"
  store_section = "ofs"
  worker_section = "worker_%s" % PROCESS
  worker_number = sys.argv[1]
  if len(sys.argv) == 3:
    if "redis_%s" % sys.argv[2] in c.sections():
      redis_section = "redis_%s" % sys.argv[2]

  rq = RedisQueue(c.get(worker_section, "listento"), "%s_%s" % (PROCESS, worker_number),
                  db=c.get(redis_section, "db"), 
                  host=c.get(redis_section, "host"), 
                  port=c.get(redis_section, "port")
                  )
  r = Redis(db=c.get(redis_section, "db"),
                  host=c.get(redis_section, "host"),
Пример #10
0
import logging

logger = logging.getLogger("pirus2_dispatcher")
logger.setLevel(logging.INFO)
# create console handler and set level to debug
ch = logging.StreamHandler()
# create formatter
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
# add formatter to ch
ch.setFormatter(formatter)
# add ch to logger
logger.addHandler(ch)

if __name__ == "__main__":
  c = Config()
  redis_section = "redis"
  worker_number = sys.argv[1]
  worker_section = "worker_pirus2"
  if len(sys.argv) == 3:
    if "redis_%s" % sys.argv[2] in c.sections():
      redis_section = "redis_%s" % sys.argv[2]

  rq = RedisQueue(c.get(worker_section, "listento"), "pirus_%s" % worker_number,
                  db=c.get(redis_section, "db"), 
                  host=c.get(redis_section, "host"), 
                  port=c.get(redis_section, "port")
                  )

  try:
    plugin_name = c.get(worker_section, "repository_plugin")