def handle_package_upload(
real_src,
relative_src,
client_id,
configuration,
submit_mrslfiles,
dst,
):
"""A file package was uploaded (eg. .zip file). Extract the content and
submit mrsl files if submit_mrsl_files is True.
"""
logger = configuration.logger
msg = ''
status = True
logger.info("handle_package_upload %s %s %s" % \
(real_src, relative_src, dst))
client_dir = client_id_dir(client_id)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
# Unpack in same directory unless real_dst is given
if not dst:
real_dst = os.path.abspath(os.path.dirname(real_src))
elif os.path.isabs(dst):
real_dst = os.path.abspath(dst)
else:
real_dst = os.path.join(base_dir, dst)
real_dst += os.sep
mrslfiles_to_parse = []
real_src_lower = real_src.lower()
if real_src_lower.endswith('.zip'):
# Handle .zip file
msg += "Received '%s' for unpacking. " % relative_src
try:
zip_object = zipfile.ZipFile(real_src, 'r', allowZip64=True)
except Exception, exc:
logger.error("open zip failed: %s" % exc)
msg += 'Could not open zipfile: %s! ' % exc
return (False, msg)
logger.info("unpack entries of %s to %s" % \
(real_src, real_dst))
for zip_entry in zip_object.infolist():
entry_filename = force_utf8(zip_entry.filename)
msg += 'Extracting: %s . ' % entry_filename
# write zip_entry to disk
# IMPORTANT: we must abs-expand for valid_user_path_name check
# otherwise it will incorrectly fail on e.g. abc/
# dir entry in archive
local_zip_entry_name = os.path.join(real_dst, entry_filename)
valid_status, valid_err = valid_user_path_name(
entry_filename, os.path.abspath(local_zip_entry_name),
base_dir)
if not valid_status:
status = False
msg += "Filename validation error: %s! " % valid_err
continue
# create sub dir(s) if missing
zip_entry_dir = os.path.dirname(local_zip_entry_name)
if not os.path.isdir(zip_entry_dir):
msg += 'Creating dir %s . ' % entry_filename
try:
os.makedirs(zip_entry_dir, 0775)
except Exception, exc:
logger.error("create directory failed: %s" % exc)
msg += 'Error creating directory: %s! ' % exc
status = False
continue
if os.path.isdir(local_zip_entry_name):
logger.debug("nothing more to do for dir entry: %s" % \
local_zip_entry_name)
continue
try:
zip_data = zip_object.read(zip_entry.filename)
except Exception, exc:
logger.error("read data in %s failed: %s" % \
(zip_entry.filename, exc))
msg += 'Error reading %s :: %s! ' % (zip_entry.filename, exc)
status = False
continue
return (False, msg)
logger.info("unpack entries of %s to %s" % \
(real_src, real_dst))
for tar_entry in tar_object:
entry_filename = force_utf8(tar_entry.name)
msg += 'Extracting: %s . ' % entry_filename
# write tar_entry to disk
# IMPORTANT: we must abs-expand for valid_user_path_name check
# otherwise it will incorrectly fail on e.g. abc/
# dir entry in archive
local_tar_entry_name = os.path.join(real_dst, entry_filename)
valid_status, valid_err = valid_user_path_name(
entry_filename, os.path.abspath(local_tar_entry_name),
base_dir)
if not valid_status:
status = False
msg += "Filename validation error: %s! " % valid_err
continue
# Found empty dir - make sure dirname doesn't strip to parent
if tar_entry.isdir():
logger.debug("empty dir %s - include in parent creation" % \
local_tar_entry_name)
local_tar_entry_name += os.sep
# create sub dir(s) if missing
def handle_package_upload(
real_src,
relative_src,
client_id,
configuration,
submit_mrslfiles,
dst,
):
"""A file package was uploaded (eg. .zip file). Extract the content and
submit mrsl files if submit_mrsl_files is True.
"""
logger = configuration.logger
msg = ''
status = True
logger.info("handle_package_upload %s %s %s" % \
(real_src, relative_src, dst))
client_dir = client_id_dir(client_id)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
# Unpack in same directory unless real_dst is given
if not dst:
real_dst = os.path.abspath(os.path.dirname(real_src))
elif os.path.isabs(dst):
real_dst = os.path.abspath(dst)
else:
real_dst = os.path.join(base_dir, dst)
real_dst += os.sep
mrslfiles_to_parse = []
real_src_lower = real_src.lower()
if real_src_lower.endswith('.zip'):
# Handle .zip file
msg += "Received '%s' for unpacking. " % relative_src
try:
zip_object = zipfile.ZipFile(real_src, 'r')
except Exception, exc:
logger.error("open zip failed: %s" % exc)
msg += 'Could not open zipfile: %s! ' % exc
return (False, msg)
logger.info("unpack entries of %s to %s" % \
(real_src, real_dst))
for zip_entry in zip_object.infolist():
msg += 'Extracting: %s . ' % zip_entry.filename
# write zip_entry to disk
local_zip_entry_name = os.path.join(real_dst, zip_entry.filename)
valid_status, valid_err = valid_user_path_name(
zip_entry.filename, local_zip_entry_name, base_dir)
if not valid_status:
status = False
msg += "Filename validation error: %s! " % valid_err
continue
# create sub dir(s) if missing
zip_entry_dir = os.path.dirname(local_zip_entry_name)
if not os.path.isdir(zip_entry_dir):
msg += 'Creating dir %s . ' % zip_entry.filename
try:
os.makedirs(zip_entry_dir, 0775)
except Exception, exc:
logger.error("create directory failed: %s" % exc)
msg += 'Error creating directory: %s! ' % exc
status = False
continue
if os.path.isdir(local_zip_entry_name):
logger.info("nothing more to do for dir entry: %s" % \
local_zip_entry_name)
continue
# TODO: can we detect and ignore symlinks?
# Zip format is horribly designed/documented:
# http://www.pkware.com/documents/casestudies/APPNOTE.TXT
# I haven't managed to find a way to detect symlinks. Thus
# they are simply created as files containing the name they
# were supposed to link to: This is inconsistent but safe :-S
# write file - symbolic links are written as files! (good for
# security)
if not write_file(zip_object.read(zip_entry.filename),
local_zip_entry_name,
logger) and \
not os.path.exists(local_zip_entry_name):
msg += 'Error unpacking %s to disk! ' % zip_entry.filename
status = False
continue
# get the size as the OS sees it
try:
__ = os.path.getsize(local_zip_entry_name)
except Exception, exc:
logger.warning("unpack may have failed: %s" % exc)
msg += \
'File %s unpacked, but could not get file size %s! '\
% (zip_entry.filename, exc)
status = False
continue
tar_file_content = tarfile.TarFile.open(real_src)
except Exception, exc:
logger.error("open tar failed: %s" % exc)
msg += 'Could not open .tar file: %s! ' % exc
return (False, msg)
logger.info("unpack entries of %s to %s" % \
(real_src, real_dst))
for tar_entry in tar_object:
msg += 'Extracting: %s . ' % tar_entry.name
# write tar_entry to disk
local_tar_entry_name = os.path.join(real_dst, tar_entry.name)
valid_status, valid_err = valid_user_path_name(
tar_entry.name, local_tar_entry_name, base_dir)
if not valid_status:
status = False
msg += "Filename validation error: %s! " % valid_err
continue
# Found empty dir - make sure dirname doesn't strip to parent
if tar_entry.isdir():
logger.info("empty dir %s - include in parent creation" % \
local_tar_entry_name)
local_tar_entry_name += os.sep
# create sub dir(s) if missing
tar_entry_dir = os.path.dirname(local_tar_entry_name)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_title=True, op_header=False)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
# TODO: all non-file fields should be validated!!
# Input fields are mostly file stuff so do not validate it
validate_args = dict([(key, user_arguments_dict.get(key, val)) for \
(key, val) in defaults.items()])
(validate_status, accepted) = validate_input_and_cert(
validate_args,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
if not correct_handler('POST'):
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Only accepting POST requests to prevent unintended updates'})
return (output_objects, returnvalues.CLIENT_ERROR)
output_objects.append({'object_type': 'header', 'text'
: '%s submit job/file' % configuration.short_title})
submitstatuslist = []
fileuploadobjs = []
filenumber = 0
file_fields = int(accepted.get('file_fields', -1)[-1])
save_as_default = (accepted['save_as_default'][-1] != 'False')
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
mrsl = ''
while True:
(content, file_type) = handle_form_input(filenumber,
user_arguments_dict, configuration)
if not content:
if filenumber < file_fields:
# blank field but file_fields indicates more fields
filenumber += 1
continue
# no field count and no data for filenumber found
break
# always append mrsltextarea if available!
try:
mrsl = user_arguments_dict['mrsltextarea_%s' % filenumber][0]
content += mrsl
except:
pass
content += '\n'
mrslfiles_to_parse = []
submit_mrslfiles = False
submitmrsl_key = 'submitmrsl_%s' % filenumber
if user_arguments_dict.has_key(submitmrsl_key):
val = str(user_arguments_dict[submitmrsl_key][0]).upper()
if val == 'ON' or val == 'TRUE':
submit_mrslfiles = True
fileuploadobj = {'object_type': 'fileuploadobj',
'submitmrsl': submit_mrslfiles}
if file_type == 'plain':
# get filename
filename_key = 'FILENAME_%s' % filenumber
if not user_arguments_dict.has_key(filename_key):
output_objects.append(
{'object_type': 'error_text','text'
: ("The specified file_type is 'plain', but a filename" \
"value was not found. The missing control should be " \
"named %s") % filename_key})
return (output_objects, returnvalues.CLIENT_ERROR)
filename_val = convert_control_value_to_line(filename_key,
user_arguments_dict)
if not filename_val:
if filenumber < file_fields:
# blank field but file_fields indicates more fields
filenumber += 1
continue
output_objects.append(
{'object_type': 'error_text', 'text'
: 'No filename found - please make sure you provide a " \
"file to upload'})
return (output_objects, returnvalues.CLIENT_ERROR)
local_filename = base_dir + filename_val
valid_status, valid_err = valid_user_path_name(filename_val,
local_filename,
base_dir)
if not valid_status:
output_objects.append(
{'object_type': 'error_text', 'text': valid_err})
return (output_objects, returnvalues.CLIENT_ERROR)
# A new filename was created, write content to file
if not write_file(content, local_filename, logger):
output_objects.append({'object_type': 'error_text',
'text': 'Could not write: %s' % local_filename})
return (output_objects, returnvalues.SYSTEM_ERROR)
fileuploadobj['saved'] = True
# msg += "%s created!" % local_filename
fileuploadobj['name'] = os.sep\
+ convert_control_value_to_line(filename_key,
user_arguments_dict)
if local_filename.upper().endswith('.MRSL')\
and submit_mrslfiles:
mrslfiles_to_parse.append[local_filename]
elif file_type == 'fileupload':
# An input type=file was found
fileupload_key = 'fileupload_%s_0_0' % filenumber
# if not fileitem.filename:
if not user_arguments_dict.has_key(fileupload_key
+ 'filename'):
output_objects.append({'object_type': 'error_text',
'text': 'NO FILENAME error'})
return (output_objects, returnvalues.CLIENT_ERROR)
base_name = strip_dir(user_arguments_dict[fileupload_key
+ 'filename'])
if not base_name:
if filenumber < file_fields:
# blank field but file_fields indicates more fields
# output_objects.append({'object_type': 'text', 'text':
# 'skip item %d' % filenumber})
filenumber += 1
continue
output_objects.append(
{'object_type': 'error_text', 'text'
: 'No filename found - please make sure you provide a " \
"file to upload'})
return (output_objects, returnvalues.CLIENT_ERROR)
extract_packages = False
extract_key = 'extract_%s' % filenumber
if user_arguments_dict.has_key(extract_key):
val = str(user_arguments_dict[extract_key][0]).upper()
if val == 'ON' or val == 'TRUE':
extract_packages = True
remote_filename = ''
default_remotefilename_key = 'default_remotefilename_%s'\
% filenumber
if user_arguments_dict.has_key(default_remotefilename_key):
remote_filename = \
user_arguments_dict[default_remotefilename_key][0]
# remotefilename overwrites default_remotefilename if it exists
remotefilename_key = 'remotefilename_%s' % filenumber
if user_arguments_dict.has_key(remotefilename_key):
remote_filename = \
user_arguments_dict[remotefilename_key][0]
if not remote_filename:
remote_filename = base_name
# if remote_filename is a directory, use client's local filename
# for the last part of the filename
if remote_filename.strip().endswith(os.sep):
remote_filename += base_name
if not user_arguments_dict.has_key(fileupload_key):
output_objects.append({'object_type': 'error_text',
'text': 'File content not found!'})
return (output_objects, returnvalues.CLIENT_ERROR)
local_filename = os.path.abspath(base_dir + remote_filename)
valid_status, valid_err = valid_user_path_name(remote_filename,
local_filename,
base_dir)
if not valid_status:
output_objects.append(
{'object_type': 'error_text', 'text': valid_err})
return (output_objects, returnvalues.CLIENT_ERROR)
if not os.path.isdir(os.path.dirname(local_filename)):
try:
os.makedirs(os.path.dirname(local_filename), 0777)
except Exception:
fileuploadobj['message'] = \
{'object_type': 'error_text',
'text': 'Exception creating dirs %s'\
% os.path.dirname(local_filename)}
fileuploadobj['name'] = remote_filename
# reads uploaded file into memory
binary = user_arguments_dict.has_key('%s_is_encoded'
% fileupload_key)
if binary:
data = user_arguments_dict[fileupload_key][-1]
data = str(base64.decodestring(data))
else:
data = user_arguments_dict[fileupload_key][-1]
# write file in memory to disk
if not write_file(data, local_filename,
configuration.logger):
output_objects.append(
{'object_type': 'error_text',
'text': 'Error writing file in memory to disk'})
return (output_objects, returnvalues.SYSTEM_ERROR)
fileuploadobj['saved'] = True
# Tell the client about the current settings (extract and submit)
# extract_str = "Extract files from packages (.zip, .tar.gz, .tgz, .tar.bz2): "
# if extract_packages:
# extract_str += "ON"
# else:
# extract_str += "OFF"
# output_objects.append({"object_type":"text", "text":extract_str})
fileuploadobj['extract_packages'] = extract_packages
# submit_str = "Submit mRSL files to parser (including .mRSL files in packages!): "
# if submit_mrslfiles:
# submit_str += "ON"
# else:
# submit_str += "OFF"
# output_objects.append({"object_type":"text", "text":submit_str})
# handle file package
if extract_packages\
and (local_filename.upper().endswith('.ZIP')
or local_filename.upper().endswith('.TAR.GZ')
or local_filename.upper().endswith('.TGZ')
or local_filename.upper().endswith('.TAR.BZ2')):
(status, msg) = handle_package_upload(local_filename,
remote_filename, client_id, configuration,
submit_mrslfiles, os.path.dirname(local_filename))
if status:
if submit_mrslfiles:
if isinstance(msg, basestring):
output_objects.append(
{'object_type': 'error_text',
'text': 'Error in submit: %s' % msg})
else:
submitstatuslist = msg
else:
output_objects.append({'object_type': 'text',
'text': msg})
else:
if submit_mrslfiles:
if isinstance(msg, basestring):
output_objects.append(
{'object_type': 'error_text',
'text': 'Error in unpack: %s' % msg})
else:
submitstatuslist = msg
else:
output_objects.append({'object_type': 'error_text',
'text': 'Problems unpacking: %s' % msg})
else:
# output_objects.append({"object_type":"text", "text":msg})
# a "normal" (non-package) file was uploaded
try:
output_objects.append({'object_type': 'text', 'text'
: 'File saved: %s' % remote_filename})
except Exception, err:
output_objects.append({'object_type': 'error_text',
'text'
: 'File seems to be saved, but could not get file size %s'
% err})
return (output_objects, returnvalues.SYSTEM_ERROR)
fileuploadobj['size'] = os.path.getsize(local_filename)
fileuploadobj['name'] = remote_filename
# Check if the extension is .mRSL
if local_filename.upper().endswith('.MRSL')\
and submit_mrslfiles:
# A .mrsl file was uploaded!
# output_objects.append({"object_type":"text", "text":
# "File name on MiG server: %s"
# % (remote_filename)})
mrslfiles_to_parse.append(local_filename)
else:
# mrsl file created by html controls. create filename. Loop until
# a filename that do not exits is created
html_generated_mrsl_dir = base_dir + 'html_generated_mrsl'
if os.path.exists(html_generated_mrsl_dir)\
and not os.path.isdir(html_generated_mrsl_dir):
# oops, user might have created a file with the same name
output_objects.append(
{'object_type': 'error_text', 'text'
: 'Please make sure %s does not exist or is a directory!'
% 'html_generated_mrsl/'})
return (output_objects, returnvalues.CLIENT_ERROR)
if not os.path.isdir(html_generated_mrsl_dir):
os.mkdir(html_generated_mrsl_dir)
while True:
time_c = time.gmtime()
timestamp = '%s_%s_%s__%s_%s_%s' % (
time_c[1],
time_c[2],
time_c[0],
time_c[3],
time_c[4],
time_c[5],
)
local_filename = html_generated_mrsl_dir\
+ '/TextAreaAt_' + timestamp + '.mRSL'
if not os.path.isfile(local_filename):
break
# A new filename was created, write content to file
if not write_file(content, local_filename, logger):
output_objects.append(
{'object_type': 'error_text',
'text': 'Could not write: %s' % local_filename})
return (output_objects, returnvalues.SYSTEM_ERROR)
fileuploadobj['name'] = os.sep\
+ 'html_generated_mrsl/TextAreaAt_' + timestamp\
+ '.mRSL'
fileuploadobj['size'] = os.path.getsize(local_filename)
mrslfiles_to_parse.append(local_filename)
fileuploadobjs.append(fileuploadobj)
# Submit selected file(s)
for mrslfile in mrslfiles_to_parse:
# do not reveal full path of mrsl file to client
relative_filename = os.sep + mrslfile.replace(base_dir, '')
submitstatus = {'object_type': 'submitstatus',
'name': relative_filename}
(status, newmsg, job_id) = new_job(mrslfile, client_id,
configuration, False, True)
if not status:
# output_objects.append({"object_type":"error_text", "text":"%s"
# % newmsg})
submitstatus['status'] = False
submitstatus['message'] = newmsg
else:
# return (output_objects, returnvalues.CLIENT_ERROR)
submitstatus['status'] = True
submitstatus['job_id'] = job_id
# output_objects.append({"object_type":"text", "text":"%s"
# % newmsg})
submitstatuslist.append(submitstatus)
# prepare next loop
filenumber += 1
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_title=True, op_header=False)
client_dir = client_id_dir(client_id)
status = returnvalues.OK
defaults = signature()[1]
# TODO: do we need to cover more non-file fields?
# All non-file fields must be validated
validate_args = dict([(key, user_arguments_dict.get(key, val)) for \
(key, val) in defaults.items()])
# IMPORTANT: we must explicitly inlude CSRF token
validate_args[csrf_field] = user_arguments_dict.get(
csrf_field, ['AllowMe'])
(validate_status, accepted) = validate_input_and_cert(
validate_args,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
output_objects.append({
'object_type':
'header',
'text':
'%s file handling' % configuration.short_title
})
submitstatuslist = []
fileuploadobjs = []
filenumber = 0
file_fields = int(accepted.get('file_fields', -1)[-1])
save_as_default = (accepted['save_as_default'][-1] != 'False')
if not safe_handler(configuration, 'post', op_name, client_id,
get_csrf_limit(configuration), accepted):
output_objects.append({
'object_type':
'error_text',
'text':
'''Only accepting
CSRF-filtered POST requests to prevent unintended updates'''
})
return (output_objects, returnvalues.CLIENT_ERROR)
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(
os.path.join(configuration.user_home, client_dir)) + os.sep
mrsl = ''
while True:
(content, file_type) = handle_form_input(filenumber,
user_arguments_dict,
configuration)
if not content:
if filenumber < file_fields:
# blank field but file_fields indicates more fields
filenumber += 1
continue
# no field count and no data for filenumber found
break
# always append mrsltextarea if available!
try:
mrsl = user_arguments_dict['mrsltextarea_%s' % filenumber][0]
content += mrsl
except:
pass
content += '\n'
mrslfiles_to_parse = []
submit_mrslfiles = False
submitmrsl_key = 'submitmrsl_%s' % filenumber
if configuration.site_enable_jobs and \
user_arguments_dict.has_key(submitmrsl_key):
val = str(user_arguments_dict[submitmrsl_key][0]).upper()
if val == 'ON' or val == 'TRUE':
submit_mrslfiles = True
fileuploadobj = {
'object_type': 'fileuploadobj',
'submitmrsl': submit_mrslfiles
}
if file_type == 'plain':
# get filename
filename_key = 'FILENAME_%s' % filenumber
if not user_arguments_dict.has_key(filename_key):
output_objects.append(
{'object_type': 'error_text','text'
: ("The specified file_type is 'plain', but a filename" \
"value was not found. The missing control should be " \
"named %s") % filename_key})
return (output_objects, returnvalues.CLIENT_ERROR)
filename_val = convert_control_value_to_line(
filename_key, user_arguments_dict)
if not filename_val:
if filenumber < file_fields:
# blank field but file_fields indicates more fields
filenumber += 1
continue
output_objects.append({
'object_type':
'error_text',
'text':
'No filename found - please make sure you provide a " \
"file to upload'
})
return (output_objects, returnvalues.CLIENT_ERROR)
local_filename = base_dir + filename_val
valid_status, valid_err = valid_user_path_name(
filename_val, local_filename, base_dir)
if not valid_status:
output_objects.append({
'object_type': 'error_text',
'text': valid_err
})
return (output_objects, returnvalues.CLIENT_ERROR)
# A new filename was created, write content to file
if not write_file(content, local_filename, logger):
logger.error("%s failed to write plain file %s" % \
(op_name, local_filename))
output_objects.append({
'object_type':
'error_text',
'text':
'Could not write: %s' % local_filename
})
return (output_objects, returnvalues.SYSTEM_ERROR)
logger.info("%s wrote plain file %s" % (op_name, local_filename))
fileuploadobj['saved'] = True
# msg += "%s created!" % local_filename
fileuploadobj['name'] = os.sep\
+ convert_control_value_to_line(filename_key,
user_arguments_dict)
if local_filename.upper().endswith('.MRSL')\
and submit_mrslfiles:
mrslfiles_to_parse.append(local_filename)
elif file_type == 'fileupload':
# An input type=file was found
fileupload_key = 'fileupload_%s_0_0' % filenumber
# if not fileitem.filename:
if not user_arguments_dict.has_key(fileupload_key + 'filename'):
output_objects.append({
'object_type': 'error_text',
'text': 'NO FILENAME error'
})
return (output_objects, returnvalues.CLIENT_ERROR)
base_name = strip_dir(user_arguments_dict[fileupload_key +
'filename'])
if not base_name:
if filenumber < file_fields:
# blank field but file_fields indicates more fields
# output_objects.append({'object_type': 'text', 'text':
# 'skip item %d' % filenumber})
filenumber += 1
continue
output_objects.append({
'object_type':
'error_text',
'text':
'No filename found - please make sure you provide a " \
"file to upload'
})
return (output_objects, returnvalues.CLIENT_ERROR)
extract_packages = False
extract_key = 'extract_%s' % filenumber
if user_arguments_dict.has_key(extract_key):
val = str(user_arguments_dict[extract_key][0]).upper()
if val == 'ON' or val == 'TRUE':
extract_packages = True
remote_filename = ''
default_remotefilename_key = 'default_remotefilename_%s'\
% filenumber
if user_arguments_dict.has_key(default_remotefilename_key):
remote_filename = \
user_arguments_dict[default_remotefilename_key][0]
# remotefilename overwrites default_remotefilename if it exists
remotefilename_key = 'remotefilename_%s' % filenumber
if user_arguments_dict.has_key(remotefilename_key):
remote_filename = \
user_arguments_dict[remotefilename_key][0]
if not remote_filename:
remote_filename = base_name
# if remote_filename is a directory, use client's local filename
# for the last part of the filename
if remote_filename.strip().endswith(os.sep):
remote_filename += base_name
if not user_arguments_dict.has_key(fileupload_key):
output_objects.append({
'object_type': 'error_text',
'text': 'File content not found!'
})
return (output_objects, returnvalues.CLIENT_ERROR)
local_filename = os.path.abspath(base_dir + remote_filename)
valid_status, valid_err = valid_user_path_name(
remote_filename, local_filename, base_dir)
if not valid_status:
output_objects.append({
'object_type': 'error_text',
'text': valid_err
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not os.path.isdir(os.path.dirname(local_filename)):
try:
os.makedirs(os.path.dirname(local_filename), 0775)
except Exception:
fileuploadobj['message'] = \
{'object_type': 'error_text',
'text': 'Exception creating dirs %s'\
% os.path.dirname(local_filename)}
fileuploadobj['name'] = remote_filename
# reads uploaded file into memory
binary = user_arguments_dict.has_key('%s_is_encoded' %
fileupload_key)
if binary:
data = user_arguments_dict[fileupload_key][-1]
data = str(base64.decodestring(data))
else:
data = user_arguments_dict[fileupload_key][-1]
# write file in memory to disk
if not write_file(data, local_filename, configuration.logger):
logger.error("%s failed to write upload file %s" % \
(op_name, local_filename))
output_objects.append({
'object_type':
'error_text',
'text':
'Error writing file in memory to disk'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
logger.info("%s wrote upload file %s" % (op_name, local_filename))
fileuploadobj['saved'] = True
# Tell the client about the current settings (extract and submit)
# extract_str = "Extract files from packages (.zip, .tar.gz, .tgz, .tar.bz2): "
# if extract_packages:
# extract_str += "ON"
# else:
# extract_str += "OFF"
# output_objects.append({"object_type":"text", "text":extract_str})
fileuploadobj['extract_packages'] = extract_packages
# submit_str = "Submit mRSL files to parser (including .mRSL files in packages!): "
# if submit_mrslfiles:
# submit_str += "ON"
# else:
# submit_str += "OFF"
# output_objects.append({"object_type":"text", "text":submit_str})
# handle file package
if extract_packages\
and (local_filename.upper().endswith('.ZIP')
or local_filename.upper().endswith('.TAR.GZ')
or local_filename.upper().endswith('.TGZ')
or local_filename.upper().endswith('.TAR.BZ2')):
(upload_status,
msg) = handle_package_upload(local_filename, remote_filename,
client_id, configuration,
submit_mrslfiles,
os.path.dirname(local_filename))
if upload_status:
if submit_mrslfiles:
if isinstance(msg, basestring):
output_objects.append({
'object_type':
'error_text',
'text':
'Error in submit: %s' % msg
})
else:
submitstatuslist = msg
else:
output_objects.append({
'object_type': 'text',
'text': msg
})
else:
if submit_mrslfiles:
if isinstance(msg, basestring):
output_objects.append({
'object_type':
'error_text',
'text':
'Error in unpack: %s' % msg
})
else:
submitstatuslist = msg
else:
output_objects.append({
'object_type':
'error_text',
'text':
'Problems unpacking: %s' % msg
})
else:
# output_objects.append({"object_type":"text", "text":msg})
# a "normal" (non-package) file was uploaded
try:
output_objects.append({
'object_type':
'text',
'text':
'File saved: %s' % remote_filename
})
except Exception, err:
output_objects.append({
'object_type':
'error_text',
'text':
'File seems to be saved, but could not get file size %s'
% err
})
return (output_objects, returnvalues.SYSTEM_ERROR)
fileuploadobj['size'] = os.path.getsize(local_filename)
fileuploadobj['name'] = remote_filename
# Check if the extension is .mRSL
if local_filename.upper().endswith('.MRSL')\
and submit_mrslfiles:
# A .mrsl file was uploaded!
# output_objects.append({"object_type":"text", "text":
# "File name on MiG server: %s"
# % (remote_filename)})
mrslfiles_to_parse.append(local_filename)
else:
# mrsl file created by html controls. create filename. Loop until
# a filename that do not exits is created
html_generated_mrsl_dir = base_dir + 'html_generated_mrsl'
if os.path.exists(html_generated_mrsl_dir)\
and not os.path.isdir(html_generated_mrsl_dir):
# oops, user might have created a file with the same name
output_objects.append({
'object_type':
'error_text',
'text':
'Please make sure %s does not exist or is a directory!' %
'html_generated_mrsl/'
})
return (output_objects, returnvalues.CLIENT_ERROR)
if not os.path.isdir(html_generated_mrsl_dir):
os.mkdir(html_generated_mrsl_dir)
while True:
time_c = time.gmtime()
timestamp = '%s_%s_%s__%s_%s_%s' % (
time_c[1],
time_c[2],
time_c[0],
time_c[3],
time_c[4],
time_c[5],
)
local_filename = html_generated_mrsl_dir\
+ '/TextAreaAt_' + timestamp + '.mRSL'
if not os.path.isfile(local_filename):
break
# A new filename was created, write content to file
if not write_file(content, local_filename, logger):
output_objects.append({
'object_type':
'error_text',
'text':
'Could not write: %s' % local_filename
})
return (output_objects, returnvalues.SYSTEM_ERROR)
fileuploadobj['name'] = os.sep\
+ 'html_generated_mrsl/TextAreaAt_' + timestamp\
+ '.mRSL'
fileuploadobj['size'] = os.path.getsize(local_filename)
mrslfiles_to_parse.append(local_filename)
fileuploadobjs.append(fileuploadobj)
# Submit selected file(s)
for mrslfile in mrslfiles_to_parse:
# do not reveal full path of mrsl file to client
relative_filename = os.sep + mrslfile.replace(base_dir, '')
submitstatus = {
'object_type': 'submitstatus',
'name': relative_filename
}
(new_status, newmsg, job_id) = new_job(mrslfile, client_id,
configuration, False, True)
if not new_status:
# output_objects.append({"object_type":"error_text", "text":"%s"
# % newmsg})
submitstatus['status'] = False
submitstatus['message'] = newmsg
else:
# return (output_objects, returnvalues.CLIENT_ERROR)
submitstatus['status'] = True
submitstatus['job_id'] = job_id
# output_objects.append({"object_type":"text", "text":"%s"
# % newmsg})
submitstatuslist.append(submitstatus)
# prepare next loop
filenumber += 1
