def get(self, _id): res = Res() if passesParameterFilter(_id): source = ICSource(_id=_id) print len(_id) if not hasattr(source, "invalid"): res.data = source.emit() res.result = 200 else: res.reason = source.invalid self.finish(res.emit())
def post(self, _id): res = Res() if passesParameterFilter(_id): source = ICSource(_id=_id) for k,v in parseRequest(self.request.body).iteritems(): if k not in source.locked_fields: setattr(source, k, v) if source.save(): res.data = source.emit() res.result = 200 self.finish(res.emit())
def get(self, source_id): """Returns the specified Source. required: _id """ res = ServerResponse() if passesParameterFilter(source_id): source = ICSource(_id = source_id) if not hasattr(source, 'invalid'): res.data = source.emit() res.result = 200 else: res.reason = source.invalid self.write(res.emit())
def post(self): """Add a new Source to the database. Only the server may utilize this method. required parameters in post: _id package_name package_content """ res = ServerResponse() params = parseRequest(self.request.body) if params is not None: source = ICSource(inflate={'_id' : params['_id']}) if source.addFile(params['package_name'], params['package_content']): if source.importAssets(params['package_name']): res.result = 200 res.data = source.emit() else: source.invalidate( invalidate['codes']['source_invalid_public_credentials'], invalidate['reasons']['source_invalid_public_credentials'] ) res.reason = source.invalid self.write(res.emit())
def reindex(): from vars import mime_type_map from InformaCamUtils.elasticsearch import Elasticsearch elasticsearch = Elasticsearch() elasticsearch.createIndex(reindex=True) for mode in ["sources", "submissions"]: for root_, dirs_, files_ in os.walk(os.path.join(assets_root, mode)): for dir_ in dirs_: for root, dirs, files in os.walk( os.path.join(assets_root, mode, dir_)): data = { '_id': dir_, 'asset_path': os.path.join(assets_root, mode, dir_) } for file in files: if mode == "submissions": if re.match(r'^(low_|med_|thumb_|high_)', file): continue else: comps = file.split(".") if len(comps) == 2 and re.match( r'(mkv|jpg)', comps[1]): data['file_name'] = file for mt, ext in mime_type_map.iteritems(): if ext == comps[1]: data['mime_type'] = mt break break elif mode == "sources": if re.match(r'^(credentials|publicKey|baseImage_)', file): continue else: data['file_name'] = file break print data try: if mode == "submissions": submission = Submission(inflate=data, reindex=True) print submission.emit() elif mode == "sources": source = Source(inflate=data, reindex=True) except exceptions.ConnectionError as e: print e return except AttributeError as e: continue
def watch(only_sources=False, only_submissions=False, only_imports=False): """For each subscribed repository, this class sends new media to our Data API. """ clients = [] mode = None if only_submissions: mode = "submissions" elif only_sources: mode = "sources" print "running watch... (mode=%s)" % mode for sync_type in sync: if sync_type == "google_drive": if not only_imports: from InformaCamData.drive_client import DriveClient clients.append(DriveClient(mode=mode)) elif sync_type == "globaleaks": if not only_imports: from InformaCamData.globaleaks_client import GlobaleaksClient clients.append(GlobaleaksClient(mode=mode)) elif sync_type == "import": from InformaCamData.import_client import ImportClient clients.append(ImportClient(mode=mode)) for client in clients: if not client.usable: print "client not usable" continue for asset in client.listAssets(omit_absorbed=True): mime_type = client.getAssetMimeType(asset) if not mime_type in client.mime_types.itervalues(): continue if mime_type == client.mime_types['zip']: if only_submissions: continue data = { '_id': client.getFileNameHash(asset), 'file_name': client.getFileName(asset), 'package_content': b64encode(client.pullFile(asset)).rstrip("="), 'sync_source': sync_type } print "%s is a source" % data['file_name'] try: source = Source(inflate=data) except exceptions.ConnectionError as e: print e return if hasattr(source, "invalid"): print source.invalid continue else: if only_sources: continue data = { '_id': client.getFileNameHash(asset), 'file_name': client.getFileName(asset), 'mime_type': mime_type, 'package_content': b64encode(client.pullFile(asset)).rstrip("="), 'sync_source': sync_type } print "%s is a submission" % data['file_name'] file_name_segments = getBespokeFileExtension(data['file_name']) if file_name_segments is None: print "could not map file extension for %s" % data[ 'file_name'] file_name_segments = [''] if file_name_segments[-1] != client.mime_type_map[mime_type]: data['file_name'] = "%s.%s" % ( data['file_name'], client.mime_type_map[mime_type]) try: submission = Submission(inflate=data) except exceptions.ConnectionError as e: print e return if hasattr(submission, "invalid"): print submission.invalid continue client.absorb(asset) client.lockFile(asset) client.updateLog()