def xls_write_workbook(file, data, request=None, image_keys=None, title_function=None, list_brackets=None): ''' ***WARNING*** xlsx files load fully into memory on display - if there are >~ 2000 images, this will cause performance issues on the client.*** @param sheet_rows iterable of dicts, one per row ''' if not isinstance(data, dict): raise BadRequest( 'unknown data for generic xls serialization: %r' % type(data)) wb = xlsxwriter.Workbook(file, {'constant_memory': True}) logger.info('xls_write_workbook for data: %r', data.keys()) for key, sheet_rows in data.items(): logger.info('type sheet_rows: %r', type(sheet_rows)) if isinstance(sheet_rows, (dict, OrderedDict)): sheet_name = default_converter(key) logger.info('writing sheet %r...', sheet_name) sheet = wb.add_worksheet(sheet_name) for i, row in enumerate(csvutils.dict_to_rows(sheet_rows)): sheet.write_row(i,0,row) elif isinstance(sheet_rows, basestring): sheet_name = default_converter(key) logger.info('writing single string sheet %r...', sheet_name) sheet = wb.add_worksheet(sheet_name) sheet.write_string(0,0,sheet_rows) else: sheet_name = default_converter(key) logger.info('writing sheets for base name %r...', sheet_name) write_rows_to_sheet(wb, sheet_rows, sheet_name, request=request, image_keys=image_keys, title_function=title_function, list_brackets=list_brackets) wb.close()
def to_html(self, data, options=None): ''' For error reporting ''' if isinstance(data, six.string_types): return data else: return '<br>'.join([str(row) for row in dict_to_rows(data)])
def generic_xls_write_workbook(file, data): ''' Writes a dict of iterables to a workbook, where key=sheet name, value=iterable that is ready to write (e.g. a list) ''' wb = xlsxwriter.Workbook(file, {'constant_memory': True}) if isinstance(data, dict): logger.info('generic_xls_write_workbook for data: %r', data.keys()) for key, sheet_rows in data.items(): sheet_name = default_converter(key) logger.info('writing sheet %r...', sheet_name) sheet = wb.add_worksheet(sheet_name) if isinstance(sheet_rows, dict): for i, row in enumerate(csvutils.dict_to_rows(sheet_rows)): sheet.write_row(i,0,row) elif isinstance(sheet_rows, basestring): sheet.write_string(0,0,sheet_rows) else: generic_write_rows_to_sheet(sheet_rows, sheet) else: raise BadRequest( 'unknown data for generic xls serialization: %r' % type(data)) logger.info('save to file; %r', file.name) wb.close()
def generic_xls_write_workbook(file, data): ''' Writes a dict of iterables to a workbook, where key=sheet name, value=iterable that is ready to write (e.g. a list) ''' wb = xlsxwriter.Workbook(file, {'constant_memory': True}) if isinstance(data, dict): logger.info('generic_xls_write_workbook for data: %r', data.keys()) for key, sheet_rows in data.items(): sheet_name = default_converter(key) logger.info('writing sheet %r...', sheet_name) sheet = wb.add_worksheet(sheet_name) if isinstance(sheet_rows, dict): for i, row in enumerate(csvutils.dict_to_rows(sheet_rows)): sheet.write_row(i,0,row) elif isinstance(sheet_rows, basestring): sheet.write_string(0,0,sheet_rows) else: generic_write_rows_to_sheet(sheet_rows, sheet) else: raise Exception( 'unknown data for generic xls serialization: %r' % type(data)) logger.info('save to file; %r', file.name) wb.close()
def to_csv(self, data, root='objects', options=None): ''' Note: csv.py doesn't do Unicode; encode values as UTF-8 byte strings ''' # Note: cStringIO ok because csv writer will write only bytes raw_data = cStringIO.StringIO() # raw_data = StringIO.StringIO() writer = unicodecsv.writer(raw_data) if 'error' in data: for row in dict_to_rows(data['error']): writer.writerow(row) # writer.writerow(['error']) # writer.writerow([data['error']]) # logger.warn(str(('error', data))) return raw_data.getvalue() if 'objects' in data: data = data['objects'] if isinstance(data, dict): # usually, this happens when the data is actually an error message; # but also, it could be just one item being returned data = dict_to_rows(data) for item in data: writer.writerow(item) else: i = 0 keys = None for item in data: ### Removed, handled in convert_list_vals, ### data = to_simple(data) if i == 0: keys = item.keys() # writer.writerow([smart_text(key).encode('utf-8') for key in keys]) writer.writerow([smart_text(key) for key in keys]) i += 1 writer.writerow( [csvutils.convert_list_vals(val) for val in item.values()]) return raw_data.getvalue()
def to_csv(self, data, root='objects', options=None): ''' Note: csv.py doesn't do Unicode; encode values as UTF-8 byte strings ''' # Note: cStringIO ok because csv writer will write only bytes raw_data = cStringIO.StringIO() # raw_data = StringIO.StringIO() writer = unicodecsv.writer(raw_data) if 'error' in data: for row in dict_to_rows(data['error']): writer.writerow(row) # writer.writerow(['error']) # writer.writerow([data['error']]) # logger.warn(str(('error', data))) return raw_data.getvalue() if 'objects' in data: data = data['objects'] if isinstance(data, dict): # usually, this happens when the data is actually an error message; # but also, it could be just one item being returned data = dict_to_rows(data) for item in data: writer.writerow(item) else: i = 0 keys = None for item in data: ### Removed, handled in convert_list_vals, ### data = to_simple(data) if i == 0: keys = item.keys() # writer.writerow([smart_text(key).encode('utf-8') for key in keys]) writer.writerow([smart_text(key) for key in keys]) i += 1 writer.writerow([csvutils.convert_list_vals(val) for val in item.values()]) return raw_data.getvalue()
def to_csv(self, data, root='objects', options=None): data = to_simple(data) raw_data = cStringIO.StringIO() writer = csv.writer(raw_data) if 'error' in data: for row in dict_to_rows(data['error']): writer.writerow(row) # writer.writerow(['error']) # writer.writerow([data['error']]) # logger.warn(str(('error', data))) return raw_data.getvalue() if 'objects' in data: data = data['objects'] if len(data) == 0: return data if isinstance(data, dict): # usually, this happens when the data is actually an error message; # but also, it could be just one item being returned data = dict_to_rows(data) for item in data: writer.writerow(item) else: i = 0 keys = None for item in data: if i == 0: keys = item.keys() writer.writerow([smart_text(key) for key in keys]) i += 1 writer.writerow([csvutils.csv_convert(val) for val in item.values()]) return raw_data.getvalue()
def xls_write_workbook(file, data, request=None, image_keys=None, title_function=None, list_brackets=None): ''' ***WARNING*** xlsx files load fully into memory on display - if there are >~ 2000 images, this will cause performance issues on the client.*** @param sheet_rows iterable of dicts, one per row ''' if not isinstance(data, dict): raise BadRequest('unknown data for generic xls serialization: %r' % type(data)) wb = xlsxwriter.Workbook(file, {'constant_memory': True}) logger.info('xls_write_workbook for data: %r', data.keys()) for key, sheet_rows in data.items(): logger.info('type sheet_rows: %r', type(sheet_rows)) if isinstance(sheet_rows, (dict, OrderedDict)): sheet_name = default_converter(key) logger.info('writing sheet %r...', sheet_name) sheet = wb.add_worksheet(sheet_name) for i, row in enumerate(csvutils.dict_to_rows(sheet_rows)): sheet.write_row(i, 0, row) elif isinstance(sheet_rows, basestring): sheet_name = default_converter(key) logger.info('writing single string sheet %r...', sheet_name) sheet = wb.add_worksheet(sheet_name) sheet.write_string(0, 0, sheet_rows) else: sheet_name = default_converter(key) logger.info('writing sheets for base name %r...', sheet_name) write_rows_to_sheet(wb, sheet_rows, sheet_name, request=request, image_keys=image_keys, title_function=title_function, list_brackets=list_brackets) wb.close()
def get_xls_response( data, output_filename,request=None,image_keys=None, title_function=None, list_brackets=None): ''' Create an xlsx file that will be streamed through the StreamingHttpResponse. - if length exceeds MAX_ROWS_PER_XLS_FILE, create multiple files and zip them. - TODO: when using xlsx, can simply add extra sheets to the file. @param output_filename - for naming temp files FIXME: wrap cursor with cursorgenerator; pass in the image columns as arg FIXME: rework this using the generic_xlsx_response as a template: - this method is used for all xlsx serialization at this time, except for in testing, and in ScreenResultSerializer - 20160419. ''' if not isinstance(data, dict): raise BadRequest( 'unknown data for xls serialization: %r, must be a dict of ' 'sheet_row entries' % type(data)) # create a temp dir # with TemporaryFile() as f: temp_dir = os.path.join( settings.TEMP_FILE_DIR, str(time.clock()).replace('.', '_')) os.mkdir(temp_dir) try: # Create an new Excel file and add a worksheet. filename = '%s.xlsx' % (output_filename) temp_file = os.path.join(temp_dir, filename) file_names_to_zip = [temp_file] if DEBUG_STREAMING: logger.info('temp file: %r', temp_file) workbook = xlsxwriter.Workbook(temp_file, {'constant_memory': True}) for key, sheet_rows in data.items(): logger.info('type sheet_rows: %r', type(sheet_rows)) if isinstance(sheet_rows, (dict, OrderedDict)): sheet_name = default_converter(key) logger.info('writing sheet %r...', sheet_name) sheet = workbook.add_worksheet(sheet_name) for i, row in enumerate(csvutils.dict_to_rows(sheet_rows)): sheet.write_row(i,0,row) elif isinstance(sheet_rows, basestring): sheet_name = default_converter(key) logger.info('writing single string sheet %r...', sheet_name) sheet = workbook.add_worksheet(sheet_name) sheet.write_string(0,0,sheet_rows) else: sheet_name = default_converter(key) logger.info('writing sheets for base name %r...', sheet_name) max_rows_per_sheet = 2**20 sheet = workbook.add_worksheet(sheet_name) filerow = 0 sheets = 1 for row,values in enumerate(sheet_rows): if filerow == 0: for i,(key,val) in enumerate(values.items()): title = key if title_function: title = title_function(key) sheet.write_string(filerow,i,title) filerow += 1 for i, (key,val) in enumerate(values.items()): val = csvutils.csv_convert( val, delimiter=LIST_DELIMITER_XLS, list_brackets=list_brackets) if val is not None: if len(val) > 32767: logger.error('warn, row too long, %d, key: %r, len: %d', row,key,len(val) ) if image_keys and key in image_keys: max_rows_per_sheet = MAX_IMAGE_ROWS_PER_XLS_FILE if not request: raise Exception( 'must specify the request parameter for image export') # hack to speed things up: if ( key == 'structure_image' and 'library_well_type' in values and values['library_well_type'].lower() == 'empty' ): continue write_xls_image(sheet, filerow, i, val, request) else: sheet.write_string(filerow,i,val) filerow += 1 if row % 10000 == 0: logger.info('wrote %d rows to temp file', row) if filerow > max_rows_per_sheet: workbook.close() logger.info('wrote file: %r', temp_file) # Create an new Excel file and add a worksheet. filename = '%s_%s.xlsx' % (output_filename, filerow) temp_file = os.path.join(temp_dir, filename) workbook = xlsxwriter.Workbook(temp_file, {'constant_memory': True}) sheet = workbook.add_worksheet(sheet_name) file_names_to_zip.append(temp_file) filerow = 0 workbook.close() logger.info('wrote file: %r', temp_file) content_type = '%s; charset=utf-8' % XLSX_MIMETYPE if len(file_names_to_zip) >1: # create a temp zip file content_type='application/zip; charset=utf-8' temp_file = os.path.join('/tmp',str(time.clock())) logger.info('temp ZIP file: %r', temp_file) with ZipFile(temp_file, 'w') as zip_file: for _file in file_names_to_zip: zip_file.write(_file, os.path.basename(_file)) logger.info('wrote file %r', temp_file) filename = '%s.zip' % output_filename _file = file(temp_file) logger.info('download tmp file: %r, %r',temp_file,_file) wrapper = FileWrapper(_file) response = StreamingHttpResponse( wrapper, content_type=content_type) response['Content-Length'] = os.path.getsize(temp_file) response['Content-Disposition'] = \ 'attachment; filename=%s' % filename return response except Exception, e: logger.exception('xls streaming error') raise e