def __init__(self, inflate=None, _id=None, extra_omits=None, extra_fields=None, river=None): from vars import invalidate, scripts_home from conf import public_user from InformaCamUtils.elasticsearch import Elasticsearch self.es = Elasticsearch(river=river) self.emit_omits = copy.deepcopy(emit_omits) if extra_omits is not None: self.emit_omits.extend(extra_omits) self.locked_fields = copy.deepcopy(locked_fields) if extra_fields is not None: self.locked_fields.extend(extra_fields) if _id is None: if inflate is None: return try: self._id = inflate['_id'] del inflate['_id'] self.date_admitted = time.time() * 1000 #in milliseconds! self.inflate(inflate) if not self.es.index(self): self.invalid = { "error_code": invalidate['codes']['unindexible'], "reason": invalidate['reasons']['unindexible'] } from conf import log_root f = open("%sreindex.txt" % log_root, "a+") f.write("%s\n" % self._id) f.close() print "SHIT:\n %s" % self.invalid except KeyError as e: print e return else: asset = self.es.get(_id, river) if asset is not None: self._id = _id self.inflate(asset) else: self.invalid = { "error_code": invalidate['codes']['asset_non_existent'], "reason": invalidate['reasons']['asset_non_existent'] }
def reindex(): from vars import mime_type_map from InformaCamUtils.elasticsearch import Elasticsearch elasticsearch = Elasticsearch() elasticsearch.createIndex(reindex=True) for mode in ["sources", "submissions"]: for root_, dirs_, files_ in os.walk(os.path.join(assets_root, mode)): for dir_ in dirs_: for root, dirs, files in os.walk( os.path.join(assets_root, mode, dir_)): data = { '_id': dir_, 'asset_path': os.path.join(assets_root, mode, dir_) } for file in files: if mode == "submissions": if re.match(r'^(low_|med_|thumb_|high_)', file): continue else: comps = file.split(".") if len(comps) == 2 and re.match( r'(mkv|jpg)', comps[1]): data['file_name'] = file for mt, ext in mime_type_map.iteritems(): if ext == comps[1]: data['mime_type'] = mt break break elif mode == "sources": if re.match(r'^(credentials|publicKey|baseImage_)', file): continue else: data['file_name'] = file break print data try: if mode == "submissions": submission = Submission(inflate=data, reindex=True) print submission.emit() elif mode == "sources": source = Source(inflate=data, reindex=True) except exceptions.ConnectionError as e: print e return except AttributeError as e: continue
def reindex(): from vars import mime_type_map from InformaCamUtils.elasticsearch import Elasticsearch elasticsearch = Elasticsearch() elasticsearch.createIndex(reindex=True) for mode in ["sources", "submissions"]: for root_, dirs_, files_ in os.walk(os.path.join(assets_root, mode)): for dir_ in dirs_: for root, dirs, files in os.walk( os.path.join(assets_root, mode, dir_)): data = { '_id' : dir_, 'asset_path' : os.path.join(assets_root, mode, dir_) } for file in files: if mode == "submissions": if re.match(r'^(low_|med_|thumb_|high_)', file): continue else: comps = file.split(".") if len(comps) == 2 and re.match(r'(mkv|jpg)', comps[1]): data['file_name'] = file for mt, ext in mime_type_map.iteritems(): if ext == comps[1]: data['mime_type'] = mt break break elif mode == "sources": if re.match(r'^(credentials|publicKey|baseImage_)', file): continue else: data['file_name'] = file break print data try: if mode == "submissions": submission = Submission(inflate=data, reindex=True) print submission.emit() elif mode == "sources": source = Source(inflate=data, reindex=True) except exceptions.ConnectionError as e: print e sys.exit() except AttributeError as e: continue
def get(self): res = Res() q = False clauses = [] op = None print urllib2.unquote(self.request.query) for k,v in parseRequest(urllib2.unquote(self.request.query)).iteritems(): if k == "operator": op = v else: clauses.append({ "field" : k, k : v }) el = Elasticsearch(river="j3m") #el = Elasticsearch(river="j3m,submissions") if len(clauses) == 1: q = el.query({"clauses" : clauses}) elif len(clauses) == 0: q = el.query({ "clauses": [ { "field" : "get_all", "get_all" : False } ] }) else: if op is None: op = "and" q = el.query({ "operator" : op, "clauses" : clauses }) if q is not False: res.data = q res.result = 200 self.finish(res.emit())
def __init__(self, inflate=None, _id=None, extra_omits=None, extra_fields=None, river=None): from vars import invalidate, scripts_home from conf import public_user from InformaCamUtils.elasticsearch import Elasticsearch self.es = Elasticsearch(river=river) self.emit_omits = copy.deepcopy(emit_omits) if extra_omits is not None: self.emit_omits.extend(extra_omits) self.locked_fields = copy.deepcopy(locked_fields) if extra_fields is not None: self.locked_fields.extend(extra_fields) if _id is None: if inflate is None: return try: self._id = inflate['_id'] del inflate['_id'] self.date_admitted = time.time() * 1000 #in milliseconds! self.inflate(inflate) if not self.es.index(self): self.invalid = { "error_code" : invalidate['codes']['unindexible'], "reason" : invalidate['reasons']['unindexible'] } from conf import log_root f = open("%sreindex.txt" % log_root, "a+") f.write("%s\n" % self._id) f.close() print "SHIT:\n %s" % self.invalid except KeyError as e: print e return else: asset = self.es.get(_id, river) if asset is not None: self._id = _id self.inflate(asset) else: self.invalid = { "error_code": invalidate['codes']['asset_non_existent'], "reason" : invalidate['reasons']['asset_non_existent'] }
class Asset(): def __init__(self, inflate=None, _id=None, extra_omits=None, extra_fields=None, river=None): from vars import invalidate, scripts_home from conf import public_user from InformaCamUtils.elasticsearch import Elasticsearch self.es = Elasticsearch(river=river) self.emit_omits = copy.deepcopy(emit_omits) if extra_omits is not None: self.emit_omits.extend(extra_omits) self.locked_fields = copy.deepcopy(locked_fields) if extra_fields is not None: self.locked_fields.extend(extra_fields) if _id is None: if inflate is None: return try: self._id = inflate['_id'] del inflate['_id'] self.date_admitted = time.time() * 1000 #in milliseconds! self.inflate(inflate) if not self.es.index(self): self.invalid = { "error_code": invalidate['codes']['unindexible'], "reason": invalidate['reasons']['unindexible'] } from conf import log_root f = open("%sreindex.txt" % log_root, "a+") f.write("%s\n" % self._id) f.close() print "SHIT:\n %s" % self.invalid except KeyError as e: print e return else: asset = self.es.get(_id, river) if asset is not None: self._id = _id self.inflate(asset) else: self.invalid = { "error_code": invalidate['codes']['asset_non_existent'], "reason": invalidate['reasons']['asset_non_existent'] } def makeDir(self, path): from InformaCamUtils.funcs import ShellReader, AsTrueValue from conf import public_user ShellReader(["mkdir", path]) self.asset_path = path self.save() ShellReader([ "chown", "-R", "%(usr)s:%(usr)s" % { 'usr': public_user }, self.asset_path ]) os.umask(022) def addFile(self, file_name, content): try: content = base64.b64decode(content) except TypeError: content += "=" * ((4 - len(content) % 4) % 4) content = base64.b64decode(content) file = open(os.path.join(self.asset_path, file_name), 'wb+') file.write(content) file.close() return True def importAssets(self, file_name): pass def save(self): return self.es.update(self) def invalidate(self, error_code, reason): self.invalid = {"error_code": error_code, "reason": reason} self.save() def delete(self, deleteAssets=False): self.es.delete(self) if deleteAssets and hasattr(self, 'asset_path'): from conf import assets_root pardir = os.path.abspath(os.path.join(self.asset_path, os.pardir)) if pardir in [ os.path.join(assets_root, "submissions"), os.path.join(assets_root, "sources"), os.path.join(assets_root, "data") ]: print "DELETING %s!" % self._id import shutil shutil.rmtree(self.asset_path) def emit(self, exclude=None): emit = {} for key, value in self.__dict__.iteritems(): if not key in self.emit_omits: if type(value) is unicode: emit[key] = str(value) else: emit[key] = value if exclude is not None: for e in exclude: try: del emit[e] except KeyError as e: pass return emit def inflate(self, inflate): for key, value in inflate.iteritems(): setattr(self, key, value)
def initElasticsearch(): elasticsearch = Elasticsearch() elasticsearch.createIndex(reindex=True)
class Asset(): def __init__(self, inflate=None, _id=None, extra_omits=None, extra_fields=None, river=None): from vars import invalidate, scripts_home from conf import public_user from InformaCamUtils.elasticsearch import Elasticsearch self.es = Elasticsearch(river=river) self.emit_omits = copy.deepcopy(emit_omits) if extra_omits is not None: self.emit_omits.extend(extra_omits) self.locked_fields = copy.deepcopy(locked_fields) if extra_fields is not None: self.locked_fields.extend(extra_fields) if _id is None: if inflate is None: return try: self._id = inflate['_id'] del inflate['_id'] self.date_admitted = time.time() * 1000 #in milliseconds! self.inflate(inflate) if not self.es.index(self): self.invalid = { "error_code" : invalidate['codes']['unindexible'], "reason" : invalidate['reasons']['unindexible'] } from conf import log_root f = open("%sreindex.txt" % log_root, "a+") f.write("%s\n" % self._id) f.close() print "SHIT:\n %s" % self.invalid except KeyError as e: print e return else: asset = self.es.get(_id, river) if asset is not None: self._id = _id self.inflate(asset) else: self.invalid = { "error_code": invalidate['codes']['asset_non_existent'], "reason" : invalidate['reasons']['asset_non_existent'] } def makeDir(self, path): from InformaCamUtils.funcs import ShellReader, AsTrueValue from conf import public_user ShellReader(["mkdir", path]) self.asset_path = path self.save() ShellReader([ "chown", "-R", "%(usr)s:%(usr)s" % {'usr' : public_user}, self.asset_path ]) os.umask(022) def addFile(self, file_name, content): try: content = base64.b64decode(content) except TypeError: content += "=" * ((4 - len(content) % 4) % 4) content = base64.b64decode(content) file = open(os.path.join(self.asset_path, file_name), 'wb+') file.write(content) file.close() return True def importAssets(self, file_name): pass def save(self): return self.es.update(self) def invalidate(self, error_code, reason): self.invalid = { "error_code" : error_code, "reason" : reason } self.save() def delete(self, deleteAssets=False): self.es.delete(self) if deleteAssets and hasattr(self, 'asset_path'): from conf import assets_root pardir = os.path.abspath(os.path.join(self.asset_path, os.pardir)) if pardir in [ os.path.join(assets_root, "submissions"), os.path.join(assets_root, "sources"), os.path.join(assets_root, "data")]: print "DELETING %s!" % self._id import shutil shutil.rmtree(self.asset_path) def emit(self, exclude=None): emit = {} for key, value in self.__dict__.iteritems(): if not key in self.emit_omits: if type(value) is unicode: emit[key] = str(value) else: emit[key] = value if exclude is not None: for e in exclude: try: del emit[e] except KeyError as e: pass return emit def inflate(self, inflate): for key, value in inflate.iteritems(): setattr(self, key, value)