if field == "embargoedUntilDate":
ans = u"%sZ" % unicode(o).split(".")[0]
document[field].append(unicode(ans).encode("utf-8"))
else:
document[field].append(unicode(o).encode("utf-8"))
else:
document["text"].append(unicode(o).encode("utf-8"))
document = dict(document)
return document
if __name__ == "__main__":
item_id = "mond_ay2"
silo_name = "DataFinder"
# r = Redis()
c = Config()
redis_section = "redis"
worker_section = "worker_solr"
worker_number = "1"
worker_section = "worker_solr"
rq = Redis()
b = BroadcastToRedis("localhost", "silochanges")
b.creation(silo_name, item_id)
# line = rq.spop("{\"_timestamp\": \"2013-05-16T18:03:24.125059\", \"type\": \"c\", \"id\": \"datafinder_1\", \"silo\": \"DataFinder\"}")
# msg = json.loads(line)
user_name = "admin"
from ConfigParser import ConfigParser
OPTIONAL_SUPERVISOR_PROCESS_OPTIONS = ['user']
if __name__ == "__main__":
if not os.path.exists("workers_enabled"):
os.mkdir("workers_enabled")
else:
# clear out the enabled directory
for conf in os.listdir("workers_enabled"):
if os.path.isfile(os.path.join("workers_enabled", conf)):
os.remove(os.path.join("workers_enabled", conf))
if not os.path.exists("workers_available"):
os.mkdir("workers_available")
c = Config()
base_superv_conf = "supervisord.conf.base"
if len(sys.argv) == 2:
# use a different base supervisor file
base_superv_conf = sys.argv[1]
supervisord_config = Config(base_superv_conf)
if 'supervisor' in c.sections():
supervisord_config.add_section('inet_http_server')
params = {'username':'******',
'password':'******',
'port':'127.0.0.1:9001'}
for key in params:
if c.has_option('supervisor', key):
supervisord_config.set('inet_http_server', key, c.get('supervisor', key))
else:
#!/usr/bin/env python
from redisqueue import RedisQueue
from LogConfigParser import Config
import sys
from time import sleep
if __name__ == "__main__":
c = Config()
redis_section = "redis"
worker_number = sys.argv[1]
worker_section = sys.argv[2]
if len(sys.argv) == 4:
if "redis_%s" % sys.argv[3] in c.sections():
redis_section = "redis_%s" % sys.argv[3]
rq = RedisQueue(c.get(worker_section, "listento"), "logger_%s" % worker_number,
db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port")
)
with open(c.get(worker_section, "logfile"), "a+") as logfile:
while(True):
line = rq.pop()
if line:
try:
if line.endswith("\n"):
while True:
if this_modified > last_modified:
last_modified = this_modified
""" File was modified, so read new lines, look for error keywords """
while True:
line = fh.readline()
if not line: break
print line
r.submit_line(line)
watcher = os.stat(log_filename)
this_modified = watcher.st_mtime
time.sleep(1)
if __name__=='__main__':
c = Config()
redis_section = "redis"
worker_number = sys.argv[1]
worker_section = sys.argv[2]
if len(sys.argv) == 4:
if "redis_%s" % sys.argv[3] in c.sections():
redis_section = "redis_%s" % sys.argv[3]
# from config:
redis_config = dict(c.items(redis_section))
if 'port' in redis_config:
redis_config['port'] = int(redis_config['port'])
servicename = c.get(worker_section, "servicename")
queuename = c.get(worker_section, "pushto")
logfilename = c.get(worker_section, "logfile")
document[field].append(unicode(o).encode("utf-8"))
else:
document["text"].append(unicode(o).encode("utf-8"))
document = dict(document)
if "publisher" in document:
# TODO: SOLR not accepting multiple values for publisher though multivalued fields. Fix this
del document["publisher"]
if debug:
f = codecs.open("/var/log/databank/solr_doc.log", "w", "utf-8")
f.write(simplejson.dumps(document))
f.close()
return document
if __name__ == "__main__":
c = Config()
redis_section = "redis"
worker_section = "worker_solr"
worker_number = sys.argv[1]
hours_before_commit = 1
if len(sys.argv) == 3:
if "redis_%s" % sys.argv[2] in c.sections():
redis_section = "redis_%s" % sys.argv[2]
rq = RedisQueue(
c.get(worker_section, "listento"),
"solr_%s" % worker_number,
db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port"),
errorqueue=c.get(worker_section, "errorq"),
# fn = unicode(o).split('/datasets/')
# if len(fn) == 2 and fn[1]:
# document['filename'].append(unicode(fn[1]).encode("utf-8"))
if field == "embargoedUntilDate":
ans = u"%sZ"%unicode(o).split('.')[0]
document[field].append(unicode(ans).encode("utf-8"))
else:
document[field].append(unicode(o).encode("utf-8"))
else:
document['text'].append(unicode(o).encode("utf-8"))
document = dict(document)
return document
if __name__ == "__main__":
c = Config()
redis_section = "redis"
worker_section = "worker_solr"
worker_number = sys.argv[1]
hours_before_commit = 1
if len(sys.argv) == 3:
if "redis_%s" % sys.argv[2] in c.sections():
redis_section = "redis_%s" % sys.argv[2]
rq = RedisQueue(c.get(worker_section, "listento"), "solr_%s" % worker_number,
db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port"),
errorqueue=c.get(worker_section, "errorq")
)
#DB_ROOT = c.get(worker_section, "dbroot")
#!/usr/bin/env python
from redisqueue import RedisQueue
from LogConfigParser import Config
import sys
from time import sleep
if __name__ == "__main__":
c = Config()
redis_section = "redis"
worker_section = "worker_broker"
worker_number = sys.argv[1]
if len(sys.argv) == 3:
if "redis_%s" % sys.argv[2] in c.sections():
redis_section = "redis_%s" % sys.argv[2]
rq = RedisQueue(c.get(worker_section, "listento"), "broker_%s" % worker_number,
db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port")
)
if c.has_option(worker_section, "fanout_status_queue"):
# keep a queue of messages to deliver for a given push'd item
# better resumeability at the cost of more redis operations
topushq = RedisQueue(c.get(worker_section, "fanout_status_queue"), "fanout_broker_%s" % worker_number,
db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port")
if '/datasets/' in o:
fn = unicode(o).split('/datasets/')
if len(fn) == 2 and fn[1]:
document['filename'].append(unicode(fn[1]).encode("utf-8"))
if field == "embargoedUntilDate":
ans = u"%sZ"%unicode(o).split('.')[0]
document[field].append(unicode(ans).encode("utf-8"))
else:
document[field].append(unicode(o).encode("utf-8"))
else:
document['text'].append(unicode(o).encode("utf-8"))
document = dict(document)
return document
if __name__ == "__main__":
c = Config()
redis_section = "redis"
worker_section = "worker_solr"
worker_number = sys.argv[1]
hours_before_commit = 1
if len(sys.argv) == 3:
if "redis_%s" % sys.argv[2] in c.sections():
redis_section = "redis_%s" % sys.argv[2]
rq = RedisQueue(c.get(worker_section, "listento"), "solr_%s" % worker_number,
db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port"),
errorqueue=c.get(worker_section, "errorq")
)
DB_ROOT = c.get(worker_section, "dbroot")
# process version (for provenance)
PROCESS = "iucr_redis_tally"
PROCESS_VERSION = "0.1"
nss = {"prism": "http://prismstandard.org/namespaces/1.2/basic/",
"dc": "http://purl.org/dc/elements/1.1/",
"dcterms": "http://purl.org/dc/terms/",
"ov":"http://open.vocab.org/terms/",
"foaf":"http://xmlns.com/foaf/0.1/",
}
from uuid import uuid4
import hashlib
if __name__ == "__main__":
c = Config()
redis_section = "redis"
store_section = "ofs"
worker_section = "worker_%s" % PROCESS
worker_number = sys.argv[1]
if len(sys.argv) == 3:
if "redis_%s" % sys.argv[2] in c.sections():
redis_section = "redis_%s" % sys.argv[2]
rq = RedisQueue(c.get(worker_section, "listento"), "%s_%s" % (PROCESS, worker_number),
db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port")
)
r = Redis(db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
import logging
logger = logging.getLogger("pirus2_dispatcher")
logger.setLevel(logging.INFO)
# create console handler and set level to debug
ch = logging.StreamHandler()
# create formatter
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
# add formatter to ch
ch.setFormatter(formatter)
# add ch to logger
logger.addHandler(ch)
if __name__ == "__main__":
c = Config()
redis_section = "redis"
worker_number = sys.argv[1]
worker_section = "worker_pirus2"
if len(sys.argv) == 3:
if "redis_%s" % sys.argv[2] in c.sections():
redis_section = "redis_%s" % sys.argv[2]
rq = RedisQueue(c.get(worker_section, "listento"), "pirus_%s" % worker_number,
db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port")
)
try:
plugin_name = c.get(worker_section, "repository_plugin")