def get_labels(task, org, labelstring): """ Gets a list of label objects from a comma-separated string of the label codes, eg. "TB, aids" """ labels = set() labelstrings = parse_csv(labelstring) for labelstring in labelstrings: labelstring = labelstring.strip() try: label = Label.objects.get( org=org, name__iexact=labelstring) # iexact removes case sensitivity labels.add(label) except Exception as e: task.log("Label %s does not exist" % labelstring) raise e return list(labels)
def csv_import(task_id): # pragma: no cover from django.db import transaction # there is a possible race condition between this task starting # so we have a bit of loop here to fetch the task tries = 0 task = None while tries < 5 and not task: try: task = ImportTask.objects.get(pk=task_id) except Exception as e: # this object just doesn't exist yet, sleep a bit then try again tries+=1 if tries >= 5: raise e else: sleep(1) log = StringIO() if StrictVersion(django.get_version()) < StrictVersion('1.6'): transaction.enter_transaction_management() transaction.managed() try: task.task_id = csv_import.request.id task.log("Started import at %s" % timezone.now()) task.log("--------------------------------") task.save() transaction.commit() model = class_from_string(task.model_class) records = model.import_csv(task, log) task.save() task.log(log.getvalue()) task.log("Import finished at %s" % timezone.now()) task.log("%d record(s) added." % len(records)) transaction.commit() except Exception as e: transaction.rollback() import traceback traceback.print_exc(e) task.log("\nError: %s\n" % e) task.log(log.getvalue()) transaction.commit() raise e finally: transaction.leave_transaction_management() else: task.task_id = csv_import.request.id task.log("Started import at %s" % timezone.now()) task.log("--------------------------------") task.save() try: with transaction.atomic(): model = class_from_string(task.model_class) records = model.import_csv(task, log) task.save() task.log(log.getvalue()) task.log("Import finished at %s" % timezone.now()) task.log("%d record(s) added." % len(records)) except Exception as e: import traceback traceback.print_exc(e) task.log("\nError: %s\n" % e) task.log(log.getvalue()) raise e return task
def faq_csv_import(org_id, task_id): # pragma: no cover task = ImportTask.objects.get(id=task_id) org = Org.objects.get(id=org_id) task.task_id = faq_csv_import.request.id task.log("Started import at %s" % timezone.now()) task.log("--------------------------------") task.save() try: with transaction.atomic() and open( task.csv_file.path ) as csv_file: # transaction prevents partial csv import # Load csv into Dict records = csv.DictReader(csv_file) lines = 0 for line in records: lines += 1 # Get or create parent Language object parent_lang = line["Parent Language"] # Get label objects labels = get_labels(task, org, line["Labels"]) # Create parent FAQ parent_faq = FAQ.create(org, line["Parent Question"], line["Parent Answer"], parent_lang, None, labels) # Start creation of translation FAQs # get a list of the csv keys keys = list(line) # remove non-translation keys parent_keys = [ "Parent Question", "Parent Language", "Parent Answer", "Parent ID", "Labels" ] [keys.remove(parent_key) for parent_key in parent_keys] # get a set of unique translation language codes lang_codes = set() for key in keys: lang_code, name = key.split(" ") lang_codes.add(lang_code) # Loop through for each translation for lang_code in lang_codes: # Create translation FAQ FAQ.create( org, line["%s Question" % lang_code], line["%s Answer" % lang_code], lang_code, parent_faq, labels, ) task.save() task.log("Import finished at %s" % timezone.now()) task.log("%d FAQ(s) added." % lines) except Exception as e: if not settings.TESTING: traceback.print_exc(e) task.log("\nError: %s\n" % e) raise e return task
def csv_import(task): #pragma: no cover from django.db import transaction transaction.enter_transaction_management() transaction.managed() log = StringIO.StringIO() try: task.task_id = csv_import.request.id task.log("Started import at %s" % datetime.now()) task.log("--------------------------------") task.save() transaction.commit() model = class_from_string(task.model_class) records = model.import_csv(task.csv_file.file, task.created_by, log) task.log(log.getvalue()) task.log("Import finished at %s" % datetime.now()) task.log("%d record(s) added." % len(records)) transaction.commit() except Exception as e: transaction.rollback() import traceback traceback.print_exc(e) task.log("\nError: %s\n" % e) task.log(log.getvalue()) transaction.commit() raise e finally: transaction.leave_transaction_management() return task
def csv_import(task_id): # pragma: no cover from django.db import transaction # there is a possible race condition between this task starting # so we have a bit of loop here to fetch the task tries = 0 task = None while tries < 5 and not task: try: task = ImportTask.objects.get(pk=task_id) except Exception as e: # this object just doesn't exist yet, sleep a bit then try again tries += 1 if tries >= 5: raise e else: sleep(1) log = StringIO() if StrictVersion(django.get_version()) < StrictVersion('1.6'): transaction.enter_transaction_management() transaction.managed() try: task.task_id = csv_import.request.id task.log("Started import at %s" % timezone.now()) task.log("--------------------------------") task.save() transaction.commit() model = class_from_string(task.model_class) records = model.import_csv(task, log) task.save() task.log(log.getvalue()) task.log("Import finished at %s" % timezone.now()) task.log("%d record(s) added." % len(records)) transaction.commit() except Exception as e: transaction.rollback() import traceback traceback.print_exc(e) task.log("\nError: %s\n" % e) task.log(log.getvalue()) transaction.commit() raise e finally: transaction.leave_transaction_management() else: task.task_id = csv_import.request.id task.log("Started import at %s" % timezone.now()) task.log("--------------------------------") task.save() try: with transaction.atomic(): model = class_from_string(task.model_class) records = model.import_csv(task, log) task.save() task.log(log.getvalue()) task.log("Import finished at %s" % timezone.now()) task.log("%d record(s) added." % len(records)) except Exception as e: import traceback traceback.print_exc(e) task.log("\nError: %s\n" % e) task.log(log.getvalue()) raise e return task
def csv_import(task_id): # pragma: no cover task = ImportTask.objects.get(pk=task_id) log = StringIO() task.task_id = csv_import.request.id task.log("Started import at %s" % timezone.now()) task.log("--------------------------------") task.save() try: with transaction.atomic(): model = class_from_string(task.model_class) records = model.import_csv(task, log) task.save() task.log(log.getvalue()) task.log("Import finished at %s" % timezone.now()) task.log("%d record(s) added." % len(records)) except Exception as e: import traceback traceback.print_exc(e) task.log("\nError: %s\n" % e) task.log(log.getvalue()) raise e return task