def get(self, _id): res = Res() if passesParameterFilter(_id): source = ICSource(_id=_id) print len(_id) if not hasattr(source, "invalid"): res.data = source.emit() res.result = 200 else: res.reason = source.invalid self.finish(res.emit())
def get(self, _id): res = Res() if passesParameterFilter(_id): source = ICSource(_id=_id) print len(_id) if not hasattr(source, "invalid"): res.data = source.emit() res.result = 200 else: res.reason = source.invalid self.finish(res.emit())
def post(self, _id): res = Res() if passesParameterFilter(_id): source = ICSource(_id=_id) for k,v in parseRequest(self.request.body).iteritems(): if k not in source.locked_fields: setattr(source, k, v) if source.save(): res.data = source.emit() res.result = 200 self.finish(res.emit())
def post(self, _id): res = Res() if passesParameterFilter(_id): source = ICSource(_id=_id) for k,v in parseRequest(self.request.body).iteritems(): if k not in source.locked_fields: setattr(source, k, v) if source.save(): res.data = source.emit() res.result = 200 self.finish(res.emit())
def get(self, source_id): """Returns the specified Source. required: _id """ res = ServerResponse() if passesParameterFilter(source_id): source = ICSource(_id = source_id) if not hasattr(source, 'invalid'): res.data = source.emit() res.result = 200 else: res.reason = source.invalid self.write(res.emit())
def post(self):
"""Add a new Source to the database.
Only the server may utilize this method.
required parameters in post:
_id
package_name
package_content
"""
res = ServerResponse()
params = parseRequest(self.request.body)
if params is not None:
source = ICSource(inflate={'_id' : params['_id']})
if source.addFile(params['package_name'], params['package_content']):
if source.importAssets(params['package_name']):
res.result = 200
res.data = source.emit()
else:
source.invalidate(
invalidate['codes']['source_invalid_public_credentials'],
invalidate['reasons']['source_invalid_public_credentials']
)
res.reason = source.invalid
self.write(res.emit())
def reindex():
from vars import mime_type_map
from InformaCamUtils.elasticsearch import Elasticsearch
elasticsearch = Elasticsearch()
elasticsearch.createIndex(reindex=True)
for mode in ["sources", "submissions"]:
for root_, dirs_, files_ in os.walk(os.path.join(assets_root, mode)):
for dir_ in dirs_:
for root, dirs, files in os.walk(
os.path.join(assets_root, mode, dir_)):
data = {
'_id': dir_,
'asset_path': os.path.join(assets_root, mode, dir_)
}
for file in files:
if mode == "submissions":
if re.match(r'^(low_|med_|thumb_|high_)', file):
continue
else:
comps = file.split(".")
if len(comps) == 2 and re.match(
r'(mkv|jpg)', comps[1]):
data['file_name'] = file
for mt, ext in mime_type_map.iteritems():
if ext == comps[1]:
data['mime_type'] = mt
break
break
elif mode == "sources":
if re.match(r'^(credentials|publicKey|baseImage_)',
file):
continue
else:
data['file_name'] = file
break
print data
try:
if mode == "submissions":
submission = Submission(inflate=data, reindex=True)
print submission.emit()
elif mode == "sources":
source = Source(inflate=data, reindex=True)
except exceptions.ConnectionError as e:
print e
return
except AttributeError as e:
continue
def watch(only_sources=False, only_submissions=False, only_imports=False):
"""For each subscribed repository, this class sends new media to our Data API.
"""
clients = []
mode = None
if only_submissions:
mode = "submissions"
elif only_sources:
mode = "sources"
print "running watch... (mode=%s)" % mode
for sync_type in sync:
if sync_type == "google_drive":
if not only_imports:
from InformaCamData.drive_client import DriveClient
clients.append(DriveClient(mode=mode))
elif sync_type == "globaleaks":
if not only_imports:
from InformaCamData.globaleaks_client import GlobaleaksClient
clients.append(GlobaleaksClient(mode=mode))
elif sync_type == "import":
from InformaCamData.import_client import ImportClient
clients.append(ImportClient(mode=mode))
for client in clients:
if not client.usable:
print "client not usable"
continue
for asset in client.listAssets(omit_absorbed=True):
mime_type = client.getAssetMimeType(asset)
if not mime_type in client.mime_types.itervalues():
continue
if mime_type == client.mime_types['zip']:
if only_submissions:
continue
data = {
'_id':
client.getFileNameHash(asset),
'file_name':
client.getFileName(asset),
'package_content':
b64encode(client.pullFile(asset)).rstrip("="),
'sync_source':
sync_type
}
print "%s is a source" % data['file_name']
try:
source = Source(inflate=data)
except exceptions.ConnectionError as e:
print e
return
if hasattr(source, "invalid"):
print source.invalid
continue
else:
if only_sources:
continue
data = {
'_id':
client.getFileNameHash(asset),
'file_name':
client.getFileName(asset),
'mime_type':
mime_type,
'package_content':
b64encode(client.pullFile(asset)).rstrip("="),
'sync_source':
sync_type
}
print "%s is a submission" % data['file_name']
file_name_segments = getBespokeFileExtension(data['file_name'])
if file_name_segments is None:
print "could not map file extension for %s" % data[
'file_name']
file_name_segments = ['']
if file_name_segments[-1] != client.mime_type_map[mime_type]:
data['file_name'] = "%s.%s" % (
data['file_name'], client.mime_type_map[mime_type])
try:
submission = Submission(inflate=data)
except exceptions.ConnectionError as e:
print e
return
if hasattr(submission, "invalid"):
print submission.invalid
continue
client.absorb(asset)
client.lockFile(asset)
client.updateLog()
