def __init__(self, inflate=None, _id=None, extra_omits=None, extra_fields=None, river=None): from vars import invalidate, scripts_home from conf import public_user from InformaCamUtils.elasticsearch import Elasticsearch self.es = Elasticsearch(river=river) self.emit_omits = copy.deepcopy(emit_omits) if extra_omits is not None: self.emit_omits.extend(extra_omits) self.locked_fields = copy.deepcopy(locked_fields) if extra_fields is not None: self.locked_fields.extend(extra_fields) if _id is None: if inflate is None: return try: self._id = inflate['_id'] del inflate['_id'] self.date_admitted = time.time() * 1000 #in milliseconds! self.inflate(inflate) if not self.es.index(self): self.invalid = { "error_code": invalidate['codes']['unindexible'], "reason": invalidate['reasons']['unindexible'] } from conf import log_root f = open("%sreindex.txt" % log_root, "a+") f.write("%s\n" % self._id) f.close() print "SHIT:\n %s" % self.invalid except KeyError as e: print e return else: asset = self.es.get(_id, river) if asset is not None: self._id = _id self.inflate(asset) else: self.invalid = { "error_code": invalidate['codes']['asset_non_existent'], "reason": invalidate['reasons']['asset_non_existent'] }
def reindex(): from vars import mime_type_map from InformaCamUtils.elasticsearch import Elasticsearch elasticsearch = Elasticsearch() elasticsearch.createIndex(reindex=True) for mode in ["sources", "submissions"]: for root_, dirs_, files_ in os.walk(os.path.join(assets_root, mode)): for dir_ in dirs_: for root, dirs, files in os.walk( os.path.join(assets_root, mode, dir_)): data = { '_id': dir_, 'asset_path': os.path.join(assets_root, mode, dir_) } for file in files: if mode == "submissions": if re.match(r'^(low_|med_|thumb_|high_)', file): continue else: comps = file.split(".") if len(comps) == 2 and re.match( r'(mkv|jpg)', comps[1]): data['file_name'] = file for mt, ext in mime_type_map.iteritems(): if ext == comps[1]: data['mime_type'] = mt break break elif mode == "sources": if re.match(r'^(credentials|publicKey|baseImage_)', file): continue else: data['file_name'] = file break print data try: if mode == "submissions": submission = Submission(inflate=data, reindex=True) print submission.emit() elif mode == "sources": source = Source(inflate=data, reindex=True) except exceptions.ConnectionError as e: print e return except AttributeError as e: continue
def get(self): res = Res() q = False clauses = [] op = None print urllib2.unquote(self.request.query) for k,v in parseRequest(urllib2.unquote(self.request.query)).iteritems(): if k == "operator": op = v else: clauses.append({ "field" : k, k : v }) el = Elasticsearch(river="j3m") #el = Elasticsearch(river="j3m,submissions") if len(clauses) == 1: q = el.query({"clauses" : clauses}) elif len(clauses) == 0: q = el.query({ "clauses": [ { "field" : "get_all", "get_all" : False } ] }) else: if op is None: op = "and" q = el.query({ "operator" : op, "clauses" : clauses }) if q is not False: res.data = q res.result = 200 self.finish(res.emit())
def initElasticsearch(): elasticsearch = Elasticsearch() elasticsearch.createIndex(reindex=True)