def delete(self, context_gus, *uriargs):
"""
Request: adminContextDesc
Response: None
Errors: InvalidInputFormat, ContextGusNotFound
"""
context_iface = Context()
receivertip_iface = ReceiverTip()
internaltip_iface = InternalTip()
whistlebtip_iface = WhistleblowerTip()
comment_iface = Comment()
receiver_iface = Receiver()
file_iface = File()
# This DELETE operation, its permanent, and remove all the reference
# a Context has within the system (in example: remove associated Tip,
# remove
try:
context_desc = yield context_iface.get_single(context_gus)
tips_related_blocks = yield receivertip_iface.get_tips_by_context(context_gus)
print "Tip that need to be deleted, associated with the context",\
context_gus, ":", len(tips_related_blocks)
for tip_block in tips_related_blocks:
internaltip_id = tip_block.get('internaltip')['internaltip_id']
yield whistlebtip_iface.delete_access_by_itip(internaltip_id)
yield receivertip_iface.massive_delete(internaltip_id)
yield comment_iface.delete_comment_by_itip(internaltip_id)
yield file_iface.delete_file_by_itip(internaltip_id)
# and finally, delete the InternalTip
yield internaltip_iface.tip_delete(internaltip_id)
# (Just consistency check)
receivers_associated = yield receiver_iface.get_receivers_by_context(context_gus)
print "receiver associated by context POV:", len(receivers_associated),\
"receiver associated by context DB-field:", len(context_desc['receivers'])
# Align all the receiver associated to the context, that the context cease to exist
yield receiver_iface.align_context_delete(context_desc['receivers'], context_gus)
# TODO delete stats associated with context ?
# TODO delete profile associated with the context
# Finally, delete the context
yield context_iface.delete_context(context_gus)
self.set_status(200)
except ContextGusNotFound, e:
self.set_status(e.http_status)
self.write({'error_message': e.error_message, 'error_code' : e.error_code})
def delete_context(self, context_gus):
"""
This DELETE operation, its permanent, and remove all the reference
a Context has within the system (Tip, File, submission...)
"""
store = self.getStore()
# Get context description, just to verify that context_gus is valid
context_iface = Context(store)
context_desc = context_iface.get_single(context_gus)
# Collect tip by context and iter on the list
receivertip_iface = ReceiverTip(store)
tips_related_blocks = receivertip_iface.get_tips_by_context(context_gus)
internaltip_iface = InternalTip(store)
whistlebtip_iface = WhistleblowerTip(store)
file_iface = File(store)
comment_iface = Comment(store)
# For every InternalTip, delete comment, wTip, rTip and Files
for tip_block in tips_related_blocks:
internaltip_id = tip_block.get('internaltip')['internaltip_id']
whistlebtip_iface.delete_access_by_itip(internaltip_id)
receivertip_iface.massive_delete(internaltip_id)
comment_iface.delete_comment_by_itip(internaltip_id)
file_iface.delete_file_by_itip(internaltip_id)
# and finally, delete the InternalTip
internaltip_iface.tip_delete(internaltip_id)
# (Just a consistency check - need to be removed)
receiver_iface = Receiver(store)
receivers_associated = receiver_iface.get_receivers_by_context(context_gus)
print "receiver associated by context POV:", len(receivers_associated),\
"receiver associated by context DB-field:", len(context_desc['receivers'])
# Align all the receiver associated to the context, that the context cease to exist
receiver_iface.align_context_delete(context_desc['receivers'], context_gus)
# Get the profile list related to context_gus and delete all of them
profile_iface = PluginProfiles(store)
profile_list = profile_iface.get_profiles_by_contexts([ context_gus ])
for prof in profile_list:
profile_iface.delete_profile(prof['profile_gus'])
# Get the submission list under the context, and delete all of them
submission_iface = Submission(store)
submission_list = submission_iface.get_all()
for single_sub in submission_list:
submission_iface.submission_delete(single_sub['submission_gus'], wb_request=False)
# Finally, delete the context
context_iface.delete_context(context_gus)
self.returnData(context_desc)
self.returnCode(200)
return self.prepareRetVals()
def add_file(self, submission_gus, file_name, content_type, file_size):
store = self.getStore('add_file')
try:
submission_r = store.find(Submission, Submission.submission_gus==submission_gus).one()
except NotOneError:
store.close()
raise SubmissionGusNotFound
if not submission_r:
store.close()
raise SubmissionGusNotFound
new_file = File()
new_file.file_gus = ret_file_gus = unicode(idops.random_file_gus())
new_file.name = file_name
new_file.content_type = content_type
new_file.size = file_size
submission_r.files.update({ new_file.file_gus : file_name })
log.debug("Added file %s in submission %s with name %s" % (ret_file_gus, submission_gus, file_name))
store.add(new_file)
store.commit()
store.close()
return ret_file_gus
def post(self, submission_gus, *args):
"""
Parameter: submission_gus
Request: Unknown
Response: Unknown
Errors: SubmissionGusNotFound, SubmissionConcluded
POST in fileHandlers need to be refactored-engineered
"""
submission_iface = Submission()
try:
submission_desc = yield submission_iface.get_single(submission_gus)
if submission_desc['finalize']:
raise SubmissionConcluded
results = []
# XXX will this ever be bigger than 1?
file_array, files = self.request.files.popitem()
for file in files:
start_time = time.time()
file_request = { 'filename' : file.get('filename'),
'content_type' : file.get('content_type'),
'file_size' : len(file['body']),
'submission_gus' : submission_gus,
'context_gus' : submission_desc['context_gus'],
'description' : ''
}
print "file_request", file_request, "\n"
file_iface = File()
file_desc = yield file_iface.new(file_request)
log.debug("Created file from %s with file_gus %s" % (file_request['filename'], file_desc['file_gus'] ))
result = self.process_file(file, submission_gus, file_desc['file_gus'])
result['elapsed_time'] = time.time() - start_time
results.append(result)
response = json.dumps(results, separators=(',',':'))
if 'application/json' in self.request.headers.get('Accept'):
self.set_header('Content-Type', 'application/json')
self.set_status(200)
self.write(response)
except InvalidInputFormat, e:
self.set_status(e.http_status)
self.write({'error_message': e.error_message, 'error_code' : e.error_message})
def get_files(self, submission_gus):
file_iface = File(self.getStore())
filelist = file_iface.get_all_by_submission(submission_gus)
self.returnData(filelist)
self.returnCode(200)
return self.prepareRetVals()
def delete(self, tip_token, *uriargs):
"""
Request: None
Response: None
Errors: ForbiddenOperation, TipGusNotFound
When an uber-receiver decide to "total delete" a Tip, is handled by this call.
"""
try:
if not is_receiver_token(tip_token):
raise ForbiddenOperation
receivertip_iface = ReceiverTip()
receivers_map = yield receivertip_iface.get_receivers_by_tip(tip_token)
if not receivers_map['actor']['can_delete_submission']:
raise ForbiddenOperation
# sibilings_tips has the keys: 'sibilings': [$] 'requested': $
sibilings_tips = yield receivertip_iface.get_sibiligs_by_tip(tip_token)
# delete all the related tip
for sibiltip in sibilings_tips['sibilings']:
yield receivertip_iface.personal_delete(sibiltip['tip_gus'])
# and the tip of the called
yield receivertip_iface.personal_delete(sibilings_tips['requested']['tip_gus'])
# extract the internaltip_id, we need for the next operations
itip_id = sibilings_tips['requested']['internaltip_id']
file_iface = File()
# remove all the files: XXX think if delivery method need to be inquired
files_list = yield file_iface.get_files_by_itip(itip_id)
print "TODO remove file_list", files_list
comment_iface = Comment()
# remove all the comments based on a specific itip_id
comments_list = yield comment_iface.delete_comment_by_itip(itip_id)
internaltip_iface = InternalTip()
# finally, delete the internaltip
internaltip_iface.tip_delete(sibilings_tips['requested']['internaltip_id'])
self.set_status(200)
except ForbiddenOperation, e:
self.set_status(e.http_status)
self.write({'error_message' : e.error_message, 'error_code' : e.error_code})
def get(self, file_gus):
filelocation = os.path.join(config.advanced.submissions_dir, file_gus)
file_iface = File()
requestedfileinfo = yield file_iface.get_single(file_gus)
print "Download of", requestedfileinfo
self.render(filelocation, missing=[], info={
"name": requestedfileinfo['name'],
"file": "%s, type=%s, size=%s" % \
(str(requestedfileinfo['name']), str(requestedfileinfo['content_type']), str(requestedfileinfo['size']))
}
)
def get(self, submission_gus, *args):
"""
Parameters: submission_gus
Request: None
Response: Unknown
Errors: Unknown
GET return list of files uploaded in this submission
Doubt: Is this API needed ? because in JQueryFileUploader do not exists
but in our GL-API-Style design could. At the moment the client
do not plan to use them.
"""
file_iface = File()
filelist = yield file_iface.get_all_by_submission(submission_gus)
self.write(json.dumps(filelist))
self.set_status(200)
def operation(self):
"""
Goal of this function is to check all the new files and validate
thru the configured SystemSettings
possible marker in the file are:
'not processed', 'ready', 'blocked', 'stored'
defined in File._marker
"""
plugin_type = u'fileprocess'
file_iface = File()
profile_iface = PluginProfiles()
not_processed_file = yield file_iface.get_file_by_marker(file_iface._marker[0])
print "FileProcess", not_processed_file
for single_file in not_processed_file:
profile_associated = yield profile_iface.get_profiles_by_contexts([ single_file['context_gus'] ] )
for p_cfg in profile_associated:
if p_cfg['plugin_type'] != plugin_type:
continue
print "processing", single_file['file_name'], "using the profile", p_cfg['profile_gus'], "configured for", p_cfg['plugin_name']
plugin = PluginManager.instance_plugin(p_cfg['plugin_name'])
try:
tempfpath = os.path.join(config.advanced.submissions_dir, single_file['file_gus'])
except AttributeError:
# XXX hi level danger Log - no directory present to perform file analysis
continue
return_code = plugin.do_fileprocess(tempfpath, p_cfg['admin_settings'])
# Todo Log/stats in both cases
if return_code:
yield file_iface.flip_mark(single_file['file_gus'], file_iface._marker[1]) # ready
else:
yield file_iface.flip_mark(single_file['file_gus'], file_iface._marker[2]) # blocked
def delete_tip(self, tip_gus):
store = self.getStore()
receivertip_iface = ReceiverTip(store)
receivers_map = receivertip_iface.get_receivers_by_tip(tip_gus)
if not receivers_map['actor']['can_delete_submission']:
raise ForbiddenOperation
# sibilings_tips has the keys: 'sibilings': [$] 'requested': $
sibilings_tips = receivertip_iface.get_sibiligs_by_tip(tip_gus)
# delete all the related tip
for sibiltip in sibilings_tips['sibilings']:
receivertip_iface.personal_delete(sibiltip['tip_gus'])
# and the tip of the called
receivertip_iface.personal_delete(sibilings_tips['requested']['tip_gus'])
# extract the internaltip_id, we need for the next operations
itip_id = sibilings_tips['requested']['internaltip_id']
# remove all the files: XXX think if delivery method need to be inquired
file_iface = File(store)
files_list = file_iface.get_files_by_itip(itip_id)
# remove all the comments based on a specific itip_id
comment_iface = Comment(store)
comments_list = comment_iface.delete_comment_by_itip(itip_id)
internaltip_iface = InternalTip(store)
# finally, delete the internaltip
internaltip_iface.tip_delete(sibilings_tips['requested']['internaltip_id'])
# XXX Notify Tip removal to the receivers ?
# XXX ask to the deleter a comment about the action, notifiy this comment ?
self.returnCode(200)
return self.prepareRetVals()
def new_files(self, submission_gus, request):
store = self.getStore()
submission_desc = Submission(store).get_single(submission_gus)
if submission_desc['finalize']:
raise SubmissionConcluded
result_list = []
file_array, files = request.files.popitem()
for single_file in files:
start_time = time.time()
file_request = { 'filename' : single_file.get('filename'),
'content_type' : single_file.get('content_type'),
'file_size' : len(single_file['body']),
'submission_gus' : submission_gus,
'context_gus' : submission_desc['context_gus'],
'description' : ''
}
file_iface = File(store)
file_desc = file_iface.new(file_request)
print "Created file from %s with file_gus %s" % (file_request['filename'], file_desc['file_gus'])
result = self._dump_file(single_file, submission_gus, file_desc['file_gus'])
result['elapsed_time'] = time.time() - start_time
result_list.append(result)
self.returnData(result_list)
self.returnCode(200)
return self.prepareRetVals()
def delivery(self):
"""
Goal of delivery is checks if some delivery is configured for a context/receiver combo,
and if is, just delivery the file in the requested way.
If not, store in the DB and permit downloading.
"""
plugin_type = u'delivery'
store = self.getStore()
file_iface = File(store)
receivertip_iface = ReceiverTip(store)
receivercfg_iface = ReceiverConfs(store)
profile_iface = PluginProfiles(store)
ready_files = file_iface.get_file_by_marker(file_iface._marker[1]) # ready
for single_file in ready_files:
# from every file, we need to find the ReceiverTip with the same InternalTip.id
# This permit to found effectively the receiver that need the file available
print "Delivery management for", single_file['file_name']
#rtip_list = receivertip_iface.get_tips_by_itip(single_file['internaltip_id'])
#for rtip in rtip_list:
# --------------------------------------------------------------------
# This code is not yet executed until GPG delivery plugin is not ready
# Ok, we had a valid an appropriate receiver configuration for the delivery task
#related_profile = profile_iface.get_single(receiver_conf['profile_gus'])
#settings_dict = { 'admin_settings' : related_profile['admin_settings'],
# 'receiver_settings' : receiver_conf['receiver_settings']}
#plugin = PluginManager.instance_plugin(related_profile['plugin_name'])
# TODO Update delivery information
#return_code = plugin.do_delivery(settings_dict, single_file)
tempfpath = os.path.join(config.advanced.submissions_dir, single_file['file_gus'])
file_iface.add_content_from_fs(single_file['file_gus'], tempfpath)
file_iface.flip_mark(single_file['file_gus'], file_iface._marker[3]) # stored
def fileprocess(self):
store = self.getStore()
file_iface = File(store)
not_processed_file = file_iface.get_file_by_marker(file_iface._marker[0])
associated_itip = {}
new_files = {}
for single_file in not_processed_file:
itid = single_file['internaltip_id']
# if InternalTip.id is 0, mean that Submission is not finalized!
# the file remain marked as 'not processed'.
if not itid:
continue
# collect for logs/info/flow
associated_itip.update({ itid : InternalTip(store).get_single(itid) })
# this file log do not contain hash nor path: it's fine anyway
new_files.update({ single_file['file_gus'] : single_file })
try:
tempfpath = os.path.join(config.advanced.submissions_dir, single_file['file_gus'])
# XXX Access check + stats + length integrity
except AttributeError:
# XXX high level danger Log
continue
validate_file = self.do_fileprocess_validation(store,single_file['context_gus'], tempfpath)
# compute hash, SHA256 in non blocking mode (from utils/random.py)
filehash = get_file_checksum(tempfpath)
print "Processed:", single_file['file_name'], filehash, "validator response:", validate_file
if validate_file:
file_iface.flip_mark(single_file['file_gus'], file_iface._marker[1], filehash) # ready
else:
file_iface.flip_mark(single_file['file_gus'], file_iface._marker[2], filehash) # blocked
return (associated_itip, new_files)
def fileprocess(self):
plugin_type = u'fileprocess'
store = self.getStore()
file_iface = File(store)
profile_iface = PluginProfiles(store)
not_processed_file = file_iface.get_file_by_marker(file_iface._marker[0])
for single_file in not_processed_file:
profile_associated = profile_iface.get_profiles_by_contexts([ single_file['context_gus'] ] )
for p_cfg in profile_associated:
if p_cfg['plugin_type'] != plugin_type:
continue
print "processing", single_file['file_name'], "using the profile", p_cfg['profile_gus'], "configured for", p_cfg['plugin_name']
plugin = PluginManager.instance_plugin(p_cfg['plugin_name'])
try:
tempfpath = os.path.join(config.advanced.submissions_dir, single_file['file_gus'])
except AttributeError:
# XXX hi level danger Log - no directory present to perform file analysis
continue
return_code = plugin.do_fileprocess(tempfpath, p_cfg['admin_settings'])
# Todo Log/stats in both cases
if return_code:
file_iface.flip_mark(single_file['file_gus'], file_iface._marker[1]) # ready
else:
file_iface.flip_mark(single_file['file_gus'], file_iface._marker[2]) # blocked
def get(self, what, *uriargs):
"""
Parameters: None
Response: Unknown
Errors: None
/dump/overview GET should return up to all the tables of GLBackend
"""
expected = [ 'itip', 'wtip', 'rtip', 'receivers', 'comment',
'profiles', 'rcfg', 'file', 'submission', 'contexts', 'plugins', 'all', 'count' ]
outputDict = {}
if what == 'receivers' or what == 'all' or what == 'count':
receiver_iface = Receiver()
receiver_list = yield receiver_iface.get_all()
if what != 'count':
outputDict.update({ 'receivers_elements' : len(receiver_list), 'receivers' : receiver_list})
else:
outputDict.update({ 'receivers_elements' : len(receiver_list)})
if what == 'itip' or what == 'all' or what == 'count':
itip_iface = InternalTip()
itip_list = yield itip_iface.get_all()
if what != 'count':
outputDict.update({ 'internaltips_elements' : len(itip_list), 'internaltips' : itip_list })
else:
outputDict.update({ 'internaltips_elements' : len(itip_list)})
if what == 'rtip' or what == 'all' or what == 'count':
rtip_iface = ReceiverTip()
rtip_list = yield rtip_iface.get_all()
if what != 'count':
outputDict.update({ 'rtip_elements' : len(rtip_list), 'receivers_tips' : rtip_list })
else:
outputDict.update({ 'rtip_elements' : len(rtip_list)})
if what == 'wtip' or what == 'all' or what == 'count':
wtip_iface = WhistleblowerTip()
wtip_list = yield wtip_iface.get_all()
if what != 'count':
outputDict.update({ 'wtip_elements' : len(wtip_list), 'whistleblower_tips' : wtip_list })
else:
outputDict.update({ 'wtip_elements' : len(wtip_list)})
if what == 'comment' or what == 'all' or what == 'count':
comment_iface = Comment()
comment_list = yield comment_iface.get_all()
if what != 'count':
outputDict.update({ 'comment_elements' : len(comment_list), 'comments' : comment_list })
else:
outputDict.update({ 'comment_elements' : len(comment_list)})
if what == 'profiles' or what == 'all' or what == 'count':
profile_iface = PluginProfiles()
profile_list = yield profile_iface.get_all()
if what != 'count':
outputDict.update({ 'profiles_elements' : len(profile_list), 'profiles' : profile_list })
else:
outputDict.update({ 'profiles_elements' : len(profile_list)})
if what == 'plugins' or what == 'all' or what == 'count':
plugin_list = yield PluginManager.get_all()
if what != 'count':
outputDict.update({ 'plugins_elements' : len(plugin_list), 'plugins' : plugin_list })
else:
outputDict.update({ 'plugins_elements' : len(plugin_list) })
if what == 'rcfg' or what == 'all' or what == 'count':
rconf_iface = ReceiverConfs()
rconf_list = yield rconf_iface.get_all()
if what != 'count':
outputDict.update({ 'rcfg_elements' : len(rconf_list), 'settings' : rconf_list })
else:
outputDict.update({ 'rcfg_elements' : len(rconf_list)})
if what == 'submission' or what == 'all' or what == 'count':
submission_iface = Submission()
submission_list = yield submission_iface.get_all()
if what != 'count':
outputDict.update({ 'submission_elements' : len(submission_list), 'submissions' : submission_list })
else:
outputDict.update({ 'submission_elements' : len(submission_list)})
if what == 'file' or what == 'all' or what == 'count':
file_iface = File()
file_list = yield file_iface.get_all()
if what != 'count':
outputDict.update({ 'file_elements' : len(file_list), 'files' : file_list })
else:
outputDict.update({ 'file_elements' : len(file_list)})
if what == 'contexts' or what == 'all' or what == 'count':
context_iface = Context()
context_list = yield context_iface.get_all()
if what != 'count':
outputDict.update({ 'contexts_elements' : len(context_list), 'contexts' : context_list })
else:
outputDict.update({ 'contexts_elements' : len(context_list)})
if not what in expected:
self.set_status(405)
else:
self.set_status(200)
self.write(outputDict)
self.finish()