def get(self, submission_id):
"""Returns the specified Submission.
required:
_id
"""
res = ServerResponse()
if passesParameterFilter(submission_id):
submission = ICSubmission(_id = submission_id)
if not hasattr(submission, 'invalid'):
res.data = submission.emit()
res.result = 200
else:
res.reason = submission.invalid
if len(self.request.query) > 3:
with_query = submission.search(parseRequest(self.request.query))
res.data['refined_search'] = {}
if hasattr(with_query, 'annotations'):
res.data['refined_search']['annotations'] = with_query.annotations
self.write(res.emit())
def reindex():
from vars import mime_type_map
from InformaCamUtils.elasticsearch import Elasticsearch
elasticsearch = Elasticsearch()
elasticsearch.createIndex(reindex=True)
for mode in ["sources", "submissions"]:
for root_, dirs_, files_ in os.walk(os.path.join(assets_root, mode)):
for dir_ in dirs_:
for root, dirs, files in os.walk(
os.path.join(assets_root, mode, dir_)):
data = {
'_id': dir_,
'asset_path': os.path.join(assets_root, mode, dir_)
}
for file in files:
if mode == "submissions":
if re.match(r'^(low_|med_|thumb_|high_)', file):
continue
else:
comps = file.split(".")
if len(comps) == 2 and re.match(
r'(mkv|jpg)', comps[1]):
data['file_name'] = file
for mt, ext in mime_type_map.iteritems():
if ext == comps[1]:
data['mime_type'] = mt
break
break
elif mode == "sources":
if re.match(r'^(credentials|publicKey|baseImage_)',
file):
continue
else:
data['file_name'] = file
break
print data
try:
if mode == "submissions":
submission = Submission(inflate=data, reindex=True)
print submission.emit()
elif mode == "sources":
source = Source(inflate=data, reindex=True)
except exceptions.ConnectionError as e:
print e
return
except AttributeError as e:
continue
def get(self, _id): res = Res() if passesParameterFilter(_id): submission = ICSubmission(_id=_id) if not hasattr(submission, "invalid"): res.data = submission.emit() res.result = 200 else: res.reason = submission.invalid self.finish(res.emit())
def get(self, _id): res = Res() if passesParameterFilter(_id): submission = ICSubmission(_id=_id) if not hasattr(submission, "invalid"): res.data = submission.emit() res.result = 200 else: res.reason = submission.invalid self.finish(res.emit())
def reindex():
from vars import mime_type_map
from InformaCamUtils.elasticsearch import Elasticsearch
elasticsearch = Elasticsearch()
elasticsearch.createIndex(reindex=True)
for mode in ["sources", "submissions"]:
for root_, dirs_, files_ in os.walk(os.path.join(assets_root, mode)):
for dir_ in dirs_:
for root, dirs, files in os.walk(
os.path.join(assets_root, mode, dir_)):
data = {
'_id' : dir_,
'asset_path' : os.path.join(assets_root, mode, dir_)
}
for file in files:
if mode == "submissions":
if re.match(r'^(low_|med_|thumb_|high_)', file):
continue
else:
comps = file.split(".")
if len(comps) == 2 and re.match(r'(mkv|jpg)', comps[1]):
data['file_name'] = file
for mt, ext in mime_type_map.iteritems():
if ext == comps[1]:
data['mime_type'] = mt
break
break
elif mode == "sources":
if re.match(r'^(credentials|publicKey|baseImage_)', file):
continue
else:
data['file_name'] = file
break
print data
try:
if mode == "submissions":
submission = Submission(inflate=data, reindex=True)
print submission.emit()
elif mode == "sources":
source = Source(inflate=data, reindex=True)
except exceptions.ConnectionError as e:
print e
sys.exit()
except AttributeError as e:
continue
def post(self):
"""Add a new Submission to the database.
Only the server may utilize this method.
required parameters in post:
_id
package_name
package_content
"""
res = ServerResponse()
params = parseRequest(self.request.body)
if params is not None:
submission = ICSubmission(inflate={'_id' : params['_id']})
if submission.addFile(params['package_name'], params['package_content']):
if submission.importAssets(params['package_name']):
submission.setMimeType(params['mime_type'])
res.result = 200
res.data = submission.emit()
else:
submission.invalidate(
invalidate['codes']['asset_non_existent'],
invalidate['reasons']['asset_non_existent']
)
res.reason = submission.invalid
self.write(res.emit())
def post(self, _id): res = Res() if passesParameterFilter(_id): submission = ICSubmission(_id=_id) if not hasattr(submission, "invalid"): for k,v in parseRequest(self.request.body).iteritems(): if k not in submission.locked_fields: setattr(source, k, v) if submission.save(): res.data = submission.emit() res.result = 200 self.finish(res.emit())
def post(self, _id): res = Res() if passesParameterFilter(_id): submission = ICSubmission(_id=_id) if not hasattr(submission, "invalid"): for k,v in parseRequest(self.request.body).iteritems(): if k not in submission.locked_fields: setattr(source, k, v) if submission.save(): res.data = submission.emit() res.result = 200 self.finish(res.emit())
def __init__(self, submission):
from InformaCamModels.submission import Submission
from InformaCamModels.collection import Collection
from InformaCamModels.j3m import J3M
# unzip
self.asset_path = submission.asset_path.replace(
"/submissions/", "/collections/")
try:
os.makedirs(self.asset_path)
except OSError as e:
print e
print submission.emit()
unzip = ShellThreader([
"unzip",
"%s/%s" % (submission.asset_path, submission.file_name), "-d",
self.asset_path
])
unzip.start()
unzip.join()
print unzip.output
assets = []
media = []
sensor_captures = []
j3m_verified = False
j3m = None
rx = r'\s*inflating: (.*)'
for line in unzip.output:
#for line in unzip_output:
asset_match = re.findall(rx, line)
if len(asset_match) == 1:
assets.append(asset_match[0].strip())
r_j3m = r'.*/log.j3m'
r_caches = r'.*/informaCaches/.*'
for asset in assets:
j3m_match = re.match(r_j3m, asset)
if j3m_match is not None:
print "J3M: %s" % asset
j3m_verified = self.validateManifest()
j3m = J3M(path_to_j3m="%s.manifest" % asset)
cache_match = re.match(r_caches, asset)
if cache_match is not None:
print "ADD CACHE: %s" % asset
sensor_captures.append(asset)
else:
mime_type = getMimeTypeFromFile(asset)
print mime_type
if mime_type in [mime_types['image'], mime_types['video']]:
print "HERE IS A NEW SUB:%s" % asset
f = open(asset, 'rb')
data = {
'_id': hashString(asset),
'file_name': os.path.basename(asset),
'mime_type': mime_type,
'package_content': b64encode(f.read()).rstrip("="),
'sync_source': submission.sync_source
}
f.close()
if data['file_name'][
-4:] != ".%s" % mime_type_map[mime_type]:
data['file_name'] = "%s.%s" % (
data['file_name'], mime_type_map[mime_type])
try:
new_submission = Submission(inflate=data)
media.append(new_submission._id)
except exceptions.ConnectionError as e:
print e
return
inflate = {'_id': submission._id, 'asset_path': self.asset_path}
if len(media) > 0:
inflate['attached_media'] = media
if len(sensor_captures) > 0:
inflate['sensor_captures'] = sensor_captures
if j3m is not None:
inflate['j3m_id'] = j3m._id
inflate['dateCreated'] = j3m.genealogy['dateCreated']
collection = Collection(inflate=inflate)
print "NEW COLLECTION: %s" % collection._id
# delete self as submission
submission.delete()
def watch(only_sources=False, only_submissions=False, only_imports=False):
"""For each subscribed repository, this class sends new media to our Data API.
"""
clients = []
mode = None
if only_submissions:
mode = "submissions"
elif only_sources:
mode = "sources"
print "running watch... (mode=%s)" % mode
for sync_type in sync:
if sync_type == "google_drive":
if not only_imports:
from InformaCamData.drive_client import DriveClient
clients.append(DriveClient(mode=mode))
elif sync_type == "globaleaks":
if not only_imports:
from InformaCamData.globaleaks_client import GlobaleaksClient
clients.append(GlobaleaksClient(mode=mode))
elif sync_type == "import":
from InformaCamData.import_client import ImportClient
clients.append(ImportClient(mode=mode))
for client in clients:
if not client.usable:
print "client not usable"
continue
for asset in client.listAssets(omit_absorbed=True):
mime_type = client.getAssetMimeType(asset)
if not mime_type in client.mime_types.itervalues():
continue
if mime_type == client.mime_types['zip']:
if only_submissions:
continue
data = {
'_id':
client.getFileNameHash(asset),
'file_name':
client.getFileName(asset),
'package_content':
b64encode(client.pullFile(asset)).rstrip("="),
'sync_source':
sync_type
}
print "%s is a source" % data['file_name']
try:
source = Source(inflate=data)
except exceptions.ConnectionError as e:
print e
return
if hasattr(source, "invalid"):
print source.invalid
continue
else:
if only_sources:
continue
data = {
'_id':
client.getFileNameHash(asset),
'file_name':
client.getFileName(asset),
'mime_type':
mime_type,
'package_content':
b64encode(client.pullFile(asset)).rstrip("="),
'sync_source':
sync_type
}
print "%s is a submission" % data['file_name']
file_name_segments = getBespokeFileExtension(data['file_name'])
if file_name_segments is None:
print "could not map file extension for %s" % data[
'file_name']
file_name_segments = ['']
if file_name_segments[-1] != client.mime_type_map[mime_type]:
data['file_name'] = "%s.%s" % (
data['file_name'], client.mime_type_map[mime_type])
try:
submission = Submission(inflate=data)
except exceptions.ConnectionError as e:
print e
return
if hasattr(submission, "invalid"):
print submission.invalid
continue
client.absorb(asset)
client.lockFile(asset)
client.updateLog()