def __init__(self,
inflate=None,
_id=None,
extra_omits=None,
extra_fields=None,
river=None):
from vars import invalidate, scripts_home
from conf import public_user
from InformaCamUtils.elasticsearch import Elasticsearch
self.es = Elasticsearch(river=river)
self.emit_omits = copy.deepcopy(emit_omits)
if extra_omits is not None:
self.emit_omits.extend(extra_omits)
self.locked_fields = copy.deepcopy(locked_fields)
if extra_fields is not None:
self.locked_fields.extend(extra_fields)
if _id is None:
if inflate is None:
return
try:
self._id = inflate['_id']
del inflate['_id']
self.date_admitted = time.time() * 1000 #in milliseconds!
self.inflate(inflate)
if not self.es.index(self):
self.invalid = {
"error_code": invalidate['codes']['unindexible'],
"reason": invalidate['reasons']['unindexible']
}
from conf import log_root
f = open("%sreindex.txt" % log_root, "a+")
f.write("%s\n" % self._id)
f.close()
print "SHIT:\n %s" % self.invalid
except KeyError as e:
print e
return
else:
asset = self.es.get(_id, river)
if asset is not None:
self._id = _id
self.inflate(asset)
else:
self.invalid = {
"error_code": invalidate['codes']['asset_non_existent'],
"reason": invalidate['reasons']['asset_non_existent']
}
def reindex():
from vars import mime_type_map
from InformaCamUtils.elasticsearch import Elasticsearch
elasticsearch = Elasticsearch()
elasticsearch.createIndex(reindex=True)
for mode in ["sources", "submissions"]:
for root_, dirs_, files_ in os.walk(os.path.join(assets_root, mode)):
for dir_ in dirs_:
for root, dirs, files in os.walk(
os.path.join(assets_root, mode, dir_)):
data = {
'_id': dir_,
'asset_path': os.path.join(assets_root, mode, dir_)
}
for file in files:
if mode == "submissions":
if re.match(r'^(low_|med_|thumb_|high_)', file):
continue
else:
comps = file.split(".")
if len(comps) == 2 and re.match(
r'(mkv|jpg)', comps[1]):
data['file_name'] = file
for mt, ext in mime_type_map.iteritems():
if ext == comps[1]:
data['mime_type'] = mt
break
break
elif mode == "sources":
if re.match(r'^(credentials|publicKey|baseImage_)',
file):
continue
else:
data['file_name'] = file
break
print data
try:
if mode == "submissions":
submission = Submission(inflate=data, reindex=True)
print submission.emit()
elif mode == "sources":
source = Source(inflate=data, reindex=True)
except exceptions.ConnectionError as e:
print e
return
except AttributeError as e:
continue
def get(self):
res = Res()
q = False
clauses = []
op = None
print urllib2.unquote(self.request.query)
for k,v in parseRequest(urllib2.unquote(self.request.query)).iteritems():
if k == "operator":
op = v
else:
clauses.append({
"field" : k,
k : v
})
el = Elasticsearch(river="j3m")
#el = Elasticsearch(river="j3m,submissions")
if len(clauses) == 1:
q = el.query({"clauses" : clauses})
elif len(clauses) == 0:
q = el.query({
"clauses": [
{
"field" : "get_all",
"get_all" : False
}
]
})
else:
if op is None: op = "and"
q = el.query({
"operator" : op,
"clauses" : clauses
})
if q is not False:
res.data = q
res.result = 200
self.finish(res.emit())
def initElasticsearch(): elasticsearch = Elasticsearch() elasticsearch.createIndex(reindex=True)
