def upload_complete(self, request, filename, **kwargs):
self._pool.close()
self._pool.join()
self._mp.complete_upload()
# filename is a file at s3. Get it.
f = default_storage.open(filename, 'r')
# parse the file.
s = SpreadsheetAbstraction(request.account, f, request.import_type, filename=filename)
f.close()
# get the number of rows
num_rows = s.num_rows
# see if it has a header
header_row = []
has_header = s.has_header
if s.has_header:
header_row = s.header_row
# get the first five columns
first_rows = s.get_rows(0,8)
return_dict = {
'num_rows': num_rows,
'first_rows': first_rows,
'header_row': header_row,
'has_header': has_header,
'filename':filename,
}
return return_dict
def test_that_importing_a_csv_and_an_excel_file_with_identical_data_produce_an_identical_spreadsheet_object(self):
csv_filename = "nameemail.csv"
excel_filename = "nameemail.xls"
fh1 = open(os.path.join(settings.PROJECT_ROOT, TEST_SPREADSHEET_PATH, csv_filename ), 'r')
s = SpreadsheetAbstraction(self.a1, fh1, "people", filename=csv_filename)
csv_data = s.get_rows(0,s.num_rows)
fh1.close()
fh2 = open(os.path.join(settings.PROJECT_ROOT, TEST_SPREADSHEET_PATH, excel_filename ), 'r')
s = SpreadsheetAbstraction(self.a1, fh2, "people", filename=excel_filename)
excel_data = s.get_rows(0,s.num_rows)
fh2.close()
self.assertEqual(csv_data, excel_data)
def queue_data_import(acct_id, import_record):
account = Account.objects.using("default").get(pk=acct_id)
r = DataImport.raw_objects.using("default").get(pk=import_record.pk)
# print "Starting data import for %s" % import_record
try:
# Grab the spreadsheet, parse it, prep for import.
fh = default_storage.open(r.source_filename, 'r')
s = SpreadsheetAbstraction(account, fh, r.import_type, filename=r.source_filename, cache_key_pct_complete=DataImport.cache_key_for_import_id_percent_imported(r.pk))
r.num_source_rows = s.num_rows
r.save()
# invalidate johnny cache
# jcache.invalidate(DataImport)
# print "Parsing complete. Starting import.."
# Do the import
results = s.do_import(fields=r.fields)
cache.set(DataImport.cache_key_for_import_id_percent_imported(r.pk),99)
# print "Done, saving results."
# Save the results
for row in results:
model_key = [v.model_key for k,v in s.import_row_class.fields.items()][0]
ResultsRow.objects.using("default").create(
account=account,
data_import=r,
successfully_imported=row["success"],
new_record_created=row["created"],
targets=row["targets"],
primary_target_id=row["targets"][model_key].id, # TODO: this breaks when we go to multiple models
)
r.finish_time = datetime.datetime.now()
cache.delete(DataImport.cache_key_for_import_id_percent_imported(r.pk))
r.save()
# for m in r.import_row_class_instance.get_target_models():
# jcache.invalidate(m)
# print "Results saved."
except:
r.failed = True
r.save()
def generate_spreadsheet(account_id, useraccount_id, spreadsheet_id, downloaded_spreadsheet_id, file_type):
# print "starting.."
from spreadsheets.models import Spreadsheet, DownloadedSpreadsheet
from spreadsheets.spreadsheet import SpreadsheetAbstraction
from accounts.models import Account
account = Account.objects.get(pk=account_id)
spreadsheet = Spreadsheet.objects.get(pk=spreadsheet_id, account=account)
f_write = cStringIO.StringIO()
SpreadsheetAbstraction.create_spreadsheet(spreadsheet.members, spreadsheet.template_obj, file_type, file_handler=f_write)
extension = SpreadsheetAbstraction.extension_from_file_type(file_type)
downloaded_spreadsheet = DownloadedSpreadsheet.objects.get(pk=downloaded_spreadsheet_id)
downloaded_spreadsheet.downloaded_file.save("%s.%s.%s" % (spreadsheet.full_name, downloaded_spreadsheet.pk, extension), ContentFile(f_write.getvalue()), save=False)
downloaded_spreadsheet.generation_finished = True
downloaded_spreadsheet.save()
# print "done"
def create_and_save_200_person_spreadsheet(self, fields=["first_name","last_name","email","phone_number"], spreadsheet_filename=None):
if not spreadsheet_filename:
spreadsheet_filename = "test.xls"
full_filename = os.path.join(settings.PROJECT_ROOT, TEST_SPREADSHEET_PATH, spreadsheet_filename)
if not os.path.exists(full_filename):
Person.objects_by_account(self.account).all().delete()
[Factory.person(self.account) for f in range(0,200)]
fh = open(full_filename, 'w')
q = Person.objects_by_account(self.account).all()
SpreadsheetAbstraction.create_spreadsheet(q, fields, EXCEL_TYPE, file_handler=fh)
fh.flush()
fh.close()
Person.objects_by_account(self.account).all().delete()
return spreadsheet_filename
def people_mailing_list_spreadsheet_file(cls, account, file_type=None, template=SPREADSHEET_TEMPLATE_CHOICES[0][0], **kwargs):
f_write = cStringIO.StringIO()
s = cls.spreadsheet(account, template, file_type)
SpreadsheetAbstraction.create_spreadsheet(s.members, s.template_obj, s.default_filetype, file_handler=f_write, with_header=False)
f_read = cStringIO.StringIO(f_write.getvalue())
return f_read
