def handle_package_upload( real_src, relative_src, client_id, configuration, submit_mrslfiles, dst, ): """A file package was uploaded (eg. .zip file). Extract the content and submit mrsl files if submit_mrsl_files is True. """ logger = configuration.logger msg = '' status = True logger.info("handle_package_upload %s %s %s" % \ (real_src, relative_src, dst)) client_dir = client_id_dir(client_id) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep # Unpack in same directory unless real_dst is given if not dst: real_dst = os.path.abspath(os.path.dirname(real_src)) elif os.path.isabs(dst): real_dst = os.path.abspath(dst) else: real_dst = os.path.join(base_dir, dst) real_dst += os.sep mrslfiles_to_parse = [] real_src_lower = real_src.lower() if real_src_lower.endswith('.zip'): # Handle .zip file msg += "Received '%s' for unpacking. " % relative_src try: zip_object = zipfile.ZipFile(real_src, 'r', allowZip64=True) except Exception, exc: logger.error("open zip failed: %s" % exc) msg += 'Could not open zipfile: %s! ' % exc return (False, msg) logger.info("unpack entries of %s to %s" % \ (real_src, real_dst)) for zip_entry in zip_object.infolist(): entry_filename = force_utf8(zip_entry.filename) msg += 'Extracting: %s . ' % entry_filename # write zip_entry to disk # IMPORTANT: we must abs-expand for valid_user_path_name check # otherwise it will incorrectly fail on e.g. abc/ # dir entry in archive local_zip_entry_name = os.path.join(real_dst, entry_filename) valid_status, valid_err = valid_user_path_name( entry_filename, os.path.abspath(local_zip_entry_name), base_dir) if not valid_status: status = False msg += "Filename validation error: %s! " % valid_err continue # create sub dir(s) if missing zip_entry_dir = os.path.dirname(local_zip_entry_name) if not os.path.isdir(zip_entry_dir): msg += 'Creating dir %s . ' % entry_filename try: os.makedirs(zip_entry_dir, 0775) except Exception, exc: logger.error("create directory failed: %s" % exc) msg += 'Error creating directory: %s! ' % exc status = False continue if os.path.isdir(local_zip_entry_name): logger.debug("nothing more to do for dir entry: %s" % \ local_zip_entry_name) continue try: zip_data = zip_object.read(zip_entry.filename) except Exception, exc: logger.error("read data in %s failed: %s" % \ (zip_entry.filename, exc)) msg += 'Error reading %s :: %s! ' % (zip_entry.filename, exc) status = False continue
return (False, msg) logger.info("unpack entries of %s to %s" % \ (real_src, real_dst)) for tar_entry in tar_object: entry_filename = force_utf8(tar_entry.name) msg += 'Extracting: %s . ' % entry_filename # write tar_entry to disk # IMPORTANT: we must abs-expand for valid_user_path_name check # otherwise it will incorrectly fail on e.g. abc/ # dir entry in archive local_tar_entry_name = os.path.join(real_dst, entry_filename) valid_status, valid_err = valid_user_path_name( entry_filename, os.path.abspath(local_tar_entry_name), base_dir) if not valid_status: status = False msg += "Filename validation error: %s! " % valid_err continue # Found empty dir - make sure dirname doesn't strip to parent if tar_entry.isdir(): logger.debug("empty dir %s - include in parent creation" % \ local_tar_entry_name) local_tar_entry_name += os.sep # create sub dir(s) if missing
def handle_package_upload( real_src, relative_src, client_id, configuration, submit_mrslfiles, dst, ): """A file package was uploaded (eg. .zip file). Extract the content and submit mrsl files if submit_mrsl_files is True. """ logger = configuration.logger msg = '' status = True logger.info("handle_package_upload %s %s %s" % \ (real_src, relative_src, dst)) client_dir = client_id_dir(client_id) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep # Unpack in same directory unless real_dst is given if not dst: real_dst = os.path.abspath(os.path.dirname(real_src)) elif os.path.isabs(dst): real_dst = os.path.abspath(dst) else: real_dst = os.path.join(base_dir, dst) real_dst += os.sep mrslfiles_to_parse = [] real_src_lower = real_src.lower() if real_src_lower.endswith('.zip'): # Handle .zip file msg += "Received '%s' for unpacking. " % relative_src try: zip_object = zipfile.ZipFile(real_src, 'r') except Exception, exc: logger.error("open zip failed: %s" % exc) msg += 'Could not open zipfile: %s! ' % exc return (False, msg) logger.info("unpack entries of %s to %s" % \ (real_src, real_dst)) for zip_entry in zip_object.infolist(): msg += 'Extracting: %s . ' % zip_entry.filename # write zip_entry to disk local_zip_entry_name = os.path.join(real_dst, zip_entry.filename) valid_status, valid_err = valid_user_path_name( zip_entry.filename, local_zip_entry_name, base_dir) if not valid_status: status = False msg += "Filename validation error: %s! " % valid_err continue # create sub dir(s) if missing zip_entry_dir = os.path.dirname(local_zip_entry_name) if not os.path.isdir(zip_entry_dir): msg += 'Creating dir %s . ' % zip_entry.filename try: os.makedirs(zip_entry_dir, 0775) except Exception, exc: logger.error("create directory failed: %s" % exc) msg += 'Error creating directory: %s! ' % exc status = False continue if os.path.isdir(local_zip_entry_name): logger.info("nothing more to do for dir entry: %s" % \ local_zip_entry_name) continue # TODO: can we detect and ignore symlinks? # Zip format is horribly designed/documented: # http://www.pkware.com/documents/casestudies/APPNOTE.TXT # I haven't managed to find a way to detect symlinks. Thus # they are simply created as files containing the name they # were supposed to link to: This is inconsistent but safe :-S # write file - symbolic links are written as files! (good for # security) if not write_file(zip_object.read(zip_entry.filename), local_zip_entry_name, logger) and \ not os.path.exists(local_zip_entry_name): msg += 'Error unpacking %s to disk! ' % zip_entry.filename status = False continue # get the size as the OS sees it try: __ = os.path.getsize(local_zip_entry_name) except Exception, exc: logger.warning("unpack may have failed: %s" % exc) msg += \ 'File %s unpacked, but could not get file size %s! '\ % (zip_entry.filename, exc) status = False continue
tar_file_content = tarfile.TarFile.open(real_src) except Exception, exc: logger.error("open tar failed: %s" % exc) msg += 'Could not open .tar file: %s! ' % exc return (False, msg) logger.info("unpack entries of %s to %s" % \ (real_src, real_dst)) for tar_entry in tar_object: msg += 'Extracting: %s . ' % tar_entry.name # write tar_entry to disk local_tar_entry_name = os.path.join(real_dst, tar_entry.name) valid_status, valid_err = valid_user_path_name( tar_entry.name, local_tar_entry_name, base_dir) if not valid_status: status = False msg += "Filename validation error: %s! " % valid_err continue # Found empty dir - make sure dirname doesn't strip to parent if tar_entry.isdir(): logger.info("empty dir %s - include in parent creation" % \ local_tar_entry_name) local_tar_entry_name += os.sep # create sub dir(s) if missing tar_entry_dir = os.path.dirname(local_tar_entry_name)
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_title=True, op_header=False) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] # TODO: all non-file fields should be validated!! # Input fields are mostly file stuff so do not validate it validate_args = dict([(key, user_arguments_dict.get(key, val)) for \ (key, val) in defaults.items()]) (validate_status, accepted) = validate_input_and_cert( validate_args, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) if not correct_handler('POST'): output_objects.append( {'object_type': 'error_text', 'text' : 'Only accepting POST requests to prevent unintended updates'}) return (output_objects, returnvalues.CLIENT_ERROR) output_objects.append({'object_type': 'header', 'text' : '%s submit job/file' % configuration.short_title}) submitstatuslist = [] fileuploadobjs = [] filenumber = 0 file_fields = int(accepted.get('file_fields', -1)[-1]) save_as_default = (accepted['save_as_default'][-1] != 'False') # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath(os.path.join(configuration.user_home, client_dir)) + os.sep mrsl = '' while True: (content, file_type) = handle_form_input(filenumber, user_arguments_dict, configuration) if not content: if filenumber < file_fields: # blank field but file_fields indicates more fields filenumber += 1 continue # no field count and no data for filenumber found break # always append mrsltextarea if available! try: mrsl = user_arguments_dict['mrsltextarea_%s' % filenumber][0] content += mrsl except: pass content += '\n' mrslfiles_to_parse = [] submit_mrslfiles = False submitmrsl_key = 'submitmrsl_%s' % filenumber if user_arguments_dict.has_key(submitmrsl_key): val = str(user_arguments_dict[submitmrsl_key][0]).upper() if val == 'ON' or val == 'TRUE': submit_mrslfiles = True fileuploadobj = {'object_type': 'fileuploadobj', 'submitmrsl': submit_mrslfiles} if file_type == 'plain': # get filename filename_key = 'FILENAME_%s' % filenumber if not user_arguments_dict.has_key(filename_key): output_objects.append( {'object_type': 'error_text','text' : ("The specified file_type is 'plain', but a filename" \ "value was not found. The missing control should be " \ "named %s") % filename_key}) return (output_objects, returnvalues.CLIENT_ERROR) filename_val = convert_control_value_to_line(filename_key, user_arguments_dict) if not filename_val: if filenumber < file_fields: # blank field but file_fields indicates more fields filenumber += 1 continue output_objects.append( {'object_type': 'error_text', 'text' : 'No filename found - please make sure you provide a " \ "file to upload'}) return (output_objects, returnvalues.CLIENT_ERROR) local_filename = base_dir + filename_val valid_status, valid_err = valid_user_path_name(filename_val, local_filename, base_dir) if not valid_status: output_objects.append( {'object_type': 'error_text', 'text': valid_err}) return (output_objects, returnvalues.CLIENT_ERROR) # A new filename was created, write content to file if not write_file(content, local_filename, logger): output_objects.append({'object_type': 'error_text', 'text': 'Could not write: %s' % local_filename}) return (output_objects, returnvalues.SYSTEM_ERROR) fileuploadobj['saved'] = True # msg += "%s created!" % local_filename fileuploadobj['name'] = os.sep\ + convert_control_value_to_line(filename_key, user_arguments_dict) if local_filename.upper().endswith('.MRSL')\ and submit_mrslfiles: mrslfiles_to_parse.append[local_filename] elif file_type == 'fileupload': # An input type=file was found fileupload_key = 'fileupload_%s_0_0' % filenumber # if not fileitem.filename: if not user_arguments_dict.has_key(fileupload_key + 'filename'): output_objects.append({'object_type': 'error_text', 'text': 'NO FILENAME error'}) return (output_objects, returnvalues.CLIENT_ERROR) base_name = strip_dir(user_arguments_dict[fileupload_key + 'filename']) if not base_name: if filenumber < file_fields: # blank field but file_fields indicates more fields # output_objects.append({'object_type': 'text', 'text': # 'skip item %d' % filenumber}) filenumber += 1 continue output_objects.append( {'object_type': 'error_text', 'text' : 'No filename found - please make sure you provide a " \ "file to upload'}) return (output_objects, returnvalues.CLIENT_ERROR) extract_packages = False extract_key = 'extract_%s' % filenumber if user_arguments_dict.has_key(extract_key): val = str(user_arguments_dict[extract_key][0]).upper() if val == 'ON' or val == 'TRUE': extract_packages = True remote_filename = '' default_remotefilename_key = 'default_remotefilename_%s'\ % filenumber if user_arguments_dict.has_key(default_remotefilename_key): remote_filename = \ user_arguments_dict[default_remotefilename_key][0] # remotefilename overwrites default_remotefilename if it exists remotefilename_key = 'remotefilename_%s' % filenumber if user_arguments_dict.has_key(remotefilename_key): remote_filename = \ user_arguments_dict[remotefilename_key][0] if not remote_filename: remote_filename = base_name # if remote_filename is a directory, use client's local filename # for the last part of the filename if remote_filename.strip().endswith(os.sep): remote_filename += base_name if not user_arguments_dict.has_key(fileupload_key): output_objects.append({'object_type': 'error_text', 'text': 'File content not found!'}) return (output_objects, returnvalues.CLIENT_ERROR) local_filename = os.path.abspath(base_dir + remote_filename) valid_status, valid_err = valid_user_path_name(remote_filename, local_filename, base_dir) if not valid_status: output_objects.append( {'object_type': 'error_text', 'text': valid_err}) return (output_objects, returnvalues.CLIENT_ERROR) if not os.path.isdir(os.path.dirname(local_filename)): try: os.makedirs(os.path.dirname(local_filename), 0777) except Exception: fileuploadobj['message'] = \ {'object_type': 'error_text', 'text': 'Exception creating dirs %s'\ % os.path.dirname(local_filename)} fileuploadobj['name'] = remote_filename # reads uploaded file into memory binary = user_arguments_dict.has_key('%s_is_encoded' % fileupload_key) if binary: data = user_arguments_dict[fileupload_key][-1] data = str(base64.decodestring(data)) else: data = user_arguments_dict[fileupload_key][-1] # write file in memory to disk if not write_file(data, local_filename, configuration.logger): output_objects.append( {'object_type': 'error_text', 'text': 'Error writing file in memory to disk'}) return (output_objects, returnvalues.SYSTEM_ERROR) fileuploadobj['saved'] = True # Tell the client about the current settings (extract and submit) # extract_str = "Extract files from packages (.zip, .tar.gz, .tgz, .tar.bz2): " # if extract_packages: # extract_str += "ON" # else: # extract_str += "OFF" # output_objects.append({"object_type":"text", "text":extract_str}) fileuploadobj['extract_packages'] = extract_packages # submit_str = "Submit mRSL files to parser (including .mRSL files in packages!): " # if submit_mrslfiles: # submit_str += "ON" # else: # submit_str += "OFF" # output_objects.append({"object_type":"text", "text":submit_str}) # handle file package if extract_packages\ and (local_filename.upper().endswith('.ZIP') or local_filename.upper().endswith('.TAR.GZ') or local_filename.upper().endswith('.TGZ') or local_filename.upper().endswith('.TAR.BZ2')): (status, msg) = handle_package_upload(local_filename, remote_filename, client_id, configuration, submit_mrslfiles, os.path.dirname(local_filename)) if status: if submit_mrslfiles: if isinstance(msg, basestring): output_objects.append( {'object_type': 'error_text', 'text': 'Error in submit: %s' % msg}) else: submitstatuslist = msg else: output_objects.append({'object_type': 'text', 'text': msg}) else: if submit_mrslfiles: if isinstance(msg, basestring): output_objects.append( {'object_type': 'error_text', 'text': 'Error in unpack: %s' % msg}) else: submitstatuslist = msg else: output_objects.append({'object_type': 'error_text', 'text': 'Problems unpacking: %s' % msg}) else: # output_objects.append({"object_type":"text", "text":msg}) # a "normal" (non-package) file was uploaded try: output_objects.append({'object_type': 'text', 'text' : 'File saved: %s' % remote_filename}) except Exception, err: output_objects.append({'object_type': 'error_text', 'text' : 'File seems to be saved, but could not get file size %s' % err}) return (output_objects, returnvalues.SYSTEM_ERROR) fileuploadobj['size'] = os.path.getsize(local_filename) fileuploadobj['name'] = remote_filename # Check if the extension is .mRSL if local_filename.upper().endswith('.MRSL')\ and submit_mrslfiles: # A .mrsl file was uploaded! # output_objects.append({"object_type":"text", "text": # "File name on MiG server: %s" # % (remote_filename)}) mrslfiles_to_parse.append(local_filename) else: # mrsl file created by html controls. create filename. Loop until # a filename that do not exits is created html_generated_mrsl_dir = base_dir + 'html_generated_mrsl' if os.path.exists(html_generated_mrsl_dir)\ and not os.path.isdir(html_generated_mrsl_dir): # oops, user might have created a file with the same name output_objects.append( {'object_type': 'error_text', 'text' : 'Please make sure %s does not exist or is a directory!' % 'html_generated_mrsl/'}) return (output_objects, returnvalues.CLIENT_ERROR) if not os.path.isdir(html_generated_mrsl_dir): os.mkdir(html_generated_mrsl_dir) while True: time_c = time.gmtime() timestamp = '%s_%s_%s__%s_%s_%s' % ( time_c[1], time_c[2], time_c[0], time_c[3], time_c[4], time_c[5], ) local_filename = html_generated_mrsl_dir\ + '/TextAreaAt_' + timestamp + '.mRSL' if not os.path.isfile(local_filename): break # A new filename was created, write content to file if not write_file(content, local_filename, logger): output_objects.append( {'object_type': 'error_text', 'text': 'Could not write: %s' % local_filename}) return (output_objects, returnvalues.SYSTEM_ERROR) fileuploadobj['name'] = os.sep\ + 'html_generated_mrsl/TextAreaAt_' + timestamp\ + '.mRSL' fileuploadobj['size'] = os.path.getsize(local_filename) mrslfiles_to_parse.append(local_filename) fileuploadobjs.append(fileuploadobj) # Submit selected file(s) for mrslfile in mrslfiles_to_parse: # do not reveal full path of mrsl file to client relative_filename = os.sep + mrslfile.replace(base_dir, '') submitstatus = {'object_type': 'submitstatus', 'name': relative_filename} (status, newmsg, job_id) = new_job(mrslfile, client_id, configuration, False, True) if not status: # output_objects.append({"object_type":"error_text", "text":"%s" # % newmsg}) submitstatus['status'] = False submitstatus['message'] = newmsg else: # return (output_objects, returnvalues.CLIENT_ERROR) submitstatus['status'] = True submitstatus['job_id'] = job_id # output_objects.append({"object_type":"text", "text":"%s" # % newmsg}) submitstatuslist.append(submitstatus) # prepare next loop filenumber += 1
def main(client_id, user_arguments_dict): """Main function used by front end""" (configuration, logger, output_objects, op_name) = \ initialize_main_variables(client_id, op_title=True, op_header=False) client_dir = client_id_dir(client_id) status = returnvalues.OK defaults = signature()[1] # TODO: do we need to cover more non-file fields? # All non-file fields must be validated validate_args = dict([(key, user_arguments_dict.get(key, val)) for \ (key, val) in defaults.items()]) # IMPORTANT: we must explicitly inlude CSRF token validate_args[csrf_field] = user_arguments_dict.get( csrf_field, ['AllowMe']) (validate_status, accepted) = validate_input_and_cert( validate_args, defaults, output_objects, client_id, configuration, allow_rejects=False, ) if not validate_status: return (accepted, returnvalues.CLIENT_ERROR) output_objects.append({ 'object_type': 'header', 'text': '%s file handling' % configuration.short_title }) submitstatuslist = [] fileuploadobjs = [] filenumber = 0 file_fields = int(accepted.get('file_fields', -1)[-1]) save_as_default = (accepted['save_as_default'][-1] != 'False') if not safe_handler(configuration, 'post', op_name, client_id, get_csrf_limit(configuration), accepted): output_objects.append({ 'object_type': 'error_text', 'text': '''Only accepting CSRF-filtered POST requests to prevent unintended updates''' }) return (output_objects, returnvalues.CLIENT_ERROR) # Please note that base_dir must end in slash to avoid access to other # user dirs when own name is a prefix of another user name base_dir = os.path.abspath( os.path.join(configuration.user_home, client_dir)) + os.sep mrsl = '' while True: (content, file_type) = handle_form_input(filenumber, user_arguments_dict, configuration) if not content: if filenumber < file_fields: # blank field but file_fields indicates more fields filenumber += 1 continue # no field count and no data for filenumber found break # always append mrsltextarea if available! try: mrsl = user_arguments_dict['mrsltextarea_%s' % filenumber][0] content += mrsl except: pass content += '\n' mrslfiles_to_parse = [] submit_mrslfiles = False submitmrsl_key = 'submitmrsl_%s' % filenumber if configuration.site_enable_jobs and \ user_arguments_dict.has_key(submitmrsl_key): val = str(user_arguments_dict[submitmrsl_key][0]).upper() if val == 'ON' or val == 'TRUE': submit_mrslfiles = True fileuploadobj = { 'object_type': 'fileuploadobj', 'submitmrsl': submit_mrslfiles } if file_type == 'plain': # get filename filename_key = 'FILENAME_%s' % filenumber if not user_arguments_dict.has_key(filename_key): output_objects.append( {'object_type': 'error_text','text' : ("The specified file_type is 'plain', but a filename" \ "value was not found. The missing control should be " \ "named %s") % filename_key}) return (output_objects, returnvalues.CLIENT_ERROR) filename_val = convert_control_value_to_line( filename_key, user_arguments_dict) if not filename_val: if filenumber < file_fields: # blank field but file_fields indicates more fields filenumber += 1 continue output_objects.append({ 'object_type': 'error_text', 'text': 'No filename found - please make sure you provide a " \ "file to upload' }) return (output_objects, returnvalues.CLIENT_ERROR) local_filename = base_dir + filename_val valid_status, valid_err = valid_user_path_name( filename_val, local_filename, base_dir) if not valid_status: output_objects.append({ 'object_type': 'error_text', 'text': valid_err }) return (output_objects, returnvalues.CLIENT_ERROR) # A new filename was created, write content to file if not write_file(content, local_filename, logger): logger.error("%s failed to write plain file %s" % \ (op_name, local_filename)) output_objects.append({ 'object_type': 'error_text', 'text': 'Could not write: %s' % local_filename }) return (output_objects, returnvalues.SYSTEM_ERROR) logger.info("%s wrote plain file %s" % (op_name, local_filename)) fileuploadobj['saved'] = True # msg += "%s created!" % local_filename fileuploadobj['name'] = os.sep\ + convert_control_value_to_line(filename_key, user_arguments_dict) if local_filename.upper().endswith('.MRSL')\ and submit_mrslfiles: mrslfiles_to_parse.append(local_filename) elif file_type == 'fileupload': # An input type=file was found fileupload_key = 'fileupload_%s_0_0' % filenumber # if not fileitem.filename: if not user_arguments_dict.has_key(fileupload_key + 'filename'): output_objects.append({ 'object_type': 'error_text', 'text': 'NO FILENAME error' }) return (output_objects, returnvalues.CLIENT_ERROR) base_name = strip_dir(user_arguments_dict[fileupload_key + 'filename']) if not base_name: if filenumber < file_fields: # blank field but file_fields indicates more fields # output_objects.append({'object_type': 'text', 'text': # 'skip item %d' % filenumber}) filenumber += 1 continue output_objects.append({ 'object_type': 'error_text', 'text': 'No filename found - please make sure you provide a " \ "file to upload' }) return (output_objects, returnvalues.CLIENT_ERROR) extract_packages = False extract_key = 'extract_%s' % filenumber if user_arguments_dict.has_key(extract_key): val = str(user_arguments_dict[extract_key][0]).upper() if val == 'ON' or val == 'TRUE': extract_packages = True remote_filename = '' default_remotefilename_key = 'default_remotefilename_%s'\ % filenumber if user_arguments_dict.has_key(default_remotefilename_key): remote_filename = \ user_arguments_dict[default_remotefilename_key][0] # remotefilename overwrites default_remotefilename if it exists remotefilename_key = 'remotefilename_%s' % filenumber if user_arguments_dict.has_key(remotefilename_key): remote_filename = \ user_arguments_dict[remotefilename_key][0] if not remote_filename: remote_filename = base_name # if remote_filename is a directory, use client's local filename # for the last part of the filename if remote_filename.strip().endswith(os.sep): remote_filename += base_name if not user_arguments_dict.has_key(fileupload_key): output_objects.append({ 'object_type': 'error_text', 'text': 'File content not found!' }) return (output_objects, returnvalues.CLIENT_ERROR) local_filename = os.path.abspath(base_dir + remote_filename) valid_status, valid_err = valid_user_path_name( remote_filename, local_filename, base_dir) if not valid_status: output_objects.append({ 'object_type': 'error_text', 'text': valid_err }) return (output_objects, returnvalues.CLIENT_ERROR) if not os.path.isdir(os.path.dirname(local_filename)): try: os.makedirs(os.path.dirname(local_filename), 0775) except Exception: fileuploadobj['message'] = \ {'object_type': 'error_text', 'text': 'Exception creating dirs %s'\ % os.path.dirname(local_filename)} fileuploadobj['name'] = remote_filename # reads uploaded file into memory binary = user_arguments_dict.has_key('%s_is_encoded' % fileupload_key) if binary: data = user_arguments_dict[fileupload_key][-1] data = str(base64.decodestring(data)) else: data = user_arguments_dict[fileupload_key][-1] # write file in memory to disk if not write_file(data, local_filename, configuration.logger): logger.error("%s failed to write upload file %s" % \ (op_name, local_filename)) output_objects.append({ 'object_type': 'error_text', 'text': 'Error writing file in memory to disk' }) return (output_objects, returnvalues.SYSTEM_ERROR) logger.info("%s wrote upload file %s" % (op_name, local_filename)) fileuploadobj['saved'] = True # Tell the client about the current settings (extract and submit) # extract_str = "Extract files from packages (.zip, .tar.gz, .tgz, .tar.bz2): " # if extract_packages: # extract_str += "ON" # else: # extract_str += "OFF" # output_objects.append({"object_type":"text", "text":extract_str}) fileuploadobj['extract_packages'] = extract_packages # submit_str = "Submit mRSL files to parser (including .mRSL files in packages!): " # if submit_mrslfiles: # submit_str += "ON" # else: # submit_str += "OFF" # output_objects.append({"object_type":"text", "text":submit_str}) # handle file package if extract_packages\ and (local_filename.upper().endswith('.ZIP') or local_filename.upper().endswith('.TAR.GZ') or local_filename.upper().endswith('.TGZ') or local_filename.upper().endswith('.TAR.BZ2')): (upload_status, msg) = handle_package_upload(local_filename, remote_filename, client_id, configuration, submit_mrslfiles, os.path.dirname(local_filename)) if upload_status: if submit_mrslfiles: if isinstance(msg, basestring): output_objects.append({ 'object_type': 'error_text', 'text': 'Error in submit: %s' % msg }) else: submitstatuslist = msg else: output_objects.append({ 'object_type': 'text', 'text': msg }) else: if submit_mrslfiles: if isinstance(msg, basestring): output_objects.append({ 'object_type': 'error_text', 'text': 'Error in unpack: %s' % msg }) else: submitstatuslist = msg else: output_objects.append({ 'object_type': 'error_text', 'text': 'Problems unpacking: %s' % msg }) else: # output_objects.append({"object_type":"text", "text":msg}) # a "normal" (non-package) file was uploaded try: output_objects.append({ 'object_type': 'text', 'text': 'File saved: %s' % remote_filename }) except Exception, err: output_objects.append({ 'object_type': 'error_text', 'text': 'File seems to be saved, but could not get file size %s' % err }) return (output_objects, returnvalues.SYSTEM_ERROR) fileuploadobj['size'] = os.path.getsize(local_filename) fileuploadobj['name'] = remote_filename # Check if the extension is .mRSL if local_filename.upper().endswith('.MRSL')\ and submit_mrslfiles: # A .mrsl file was uploaded! # output_objects.append({"object_type":"text", "text": # "File name on MiG server: %s" # % (remote_filename)}) mrslfiles_to_parse.append(local_filename) else: # mrsl file created by html controls. create filename. Loop until # a filename that do not exits is created html_generated_mrsl_dir = base_dir + 'html_generated_mrsl' if os.path.exists(html_generated_mrsl_dir)\ and not os.path.isdir(html_generated_mrsl_dir): # oops, user might have created a file with the same name output_objects.append({ 'object_type': 'error_text', 'text': 'Please make sure %s does not exist or is a directory!' % 'html_generated_mrsl/' }) return (output_objects, returnvalues.CLIENT_ERROR) if not os.path.isdir(html_generated_mrsl_dir): os.mkdir(html_generated_mrsl_dir) while True: time_c = time.gmtime() timestamp = '%s_%s_%s__%s_%s_%s' % ( time_c[1], time_c[2], time_c[0], time_c[3], time_c[4], time_c[5], ) local_filename = html_generated_mrsl_dir\ + '/TextAreaAt_' + timestamp + '.mRSL' if not os.path.isfile(local_filename): break # A new filename was created, write content to file if not write_file(content, local_filename, logger): output_objects.append({ 'object_type': 'error_text', 'text': 'Could not write: %s' % local_filename }) return (output_objects, returnvalues.SYSTEM_ERROR) fileuploadobj['name'] = os.sep\ + 'html_generated_mrsl/TextAreaAt_' + timestamp\ + '.mRSL' fileuploadobj['size'] = os.path.getsize(local_filename) mrslfiles_to_parse.append(local_filename) fileuploadobjs.append(fileuploadobj) # Submit selected file(s) for mrslfile in mrslfiles_to_parse: # do not reveal full path of mrsl file to client relative_filename = os.sep + mrslfile.replace(base_dir, '') submitstatus = { 'object_type': 'submitstatus', 'name': relative_filename } (new_status, newmsg, job_id) = new_job(mrslfile, client_id, configuration, False, True) if not new_status: # output_objects.append({"object_type":"error_text", "text":"%s" # % newmsg}) submitstatus['status'] = False submitstatus['message'] = newmsg else: # return (output_objects, returnvalues.CLIENT_ERROR) submitstatus['status'] = True submitstatus['job_id'] = job_id # output_objects.append({"object_type":"text", "text":"%s" # % newmsg}) submitstatuslist.append(submitstatus) # prepare next loop filenumber += 1