def get_labels(task, org, labelstring):
"""
Gets a list of label objects from a comma-separated string of the label codes, eg. "TB, aids"
"""
labels = set()
labelstrings = parse_csv(labelstring)
for labelstring in labelstrings:
labelstring = labelstring.strip()
try:
label = Label.objects.get(
org=org,
name__iexact=labelstring) # iexact removes case sensitivity
labels.add(label)
except Exception as e:
task.log("Label %s does not exist" % labelstring)
raise e
return list(labels)
def csv_import(task_id): # pragma: no cover
from django.db import transaction
# there is a possible race condition between this task starting
# so we have a bit of loop here to fetch the task
tries = 0
task = None
while tries < 5 and not task:
try:
task = ImportTask.objects.get(pk=task_id)
except Exception as e:
# this object just doesn't exist yet, sleep a bit then try again
tries+=1
if tries >= 5:
raise e
else:
sleep(1)
log = StringIO()
if StrictVersion(django.get_version()) < StrictVersion('1.6'):
transaction.enter_transaction_management()
transaction.managed()
try:
task.task_id = csv_import.request.id
task.log("Started import at %s" % timezone.now())
task.log("--------------------------------")
task.save()
transaction.commit()
model = class_from_string(task.model_class)
records = model.import_csv(task, log)
task.save()
task.log(log.getvalue())
task.log("Import finished at %s" % timezone.now())
task.log("%d record(s) added." % len(records))
transaction.commit()
except Exception as e:
transaction.rollback()
import traceback
traceback.print_exc(e)
task.log("\nError: %s\n" % e)
task.log(log.getvalue())
transaction.commit()
raise e
finally:
transaction.leave_transaction_management()
else:
task.task_id = csv_import.request.id
task.log("Started import at %s" % timezone.now())
task.log("--------------------------------")
task.save()
try:
with transaction.atomic():
model = class_from_string(task.model_class)
records = model.import_csv(task, log)
task.save()
task.log(log.getvalue())
task.log("Import finished at %s" % timezone.now())
task.log("%d record(s) added." % len(records))
except Exception as e:
import traceback
traceback.print_exc(e)
task.log("\nError: %s\n" % e)
task.log(log.getvalue())
raise e
return task
def faq_csv_import(org_id, task_id): # pragma: no cover
task = ImportTask.objects.get(id=task_id)
org = Org.objects.get(id=org_id)
task.task_id = faq_csv_import.request.id
task.log("Started import at %s" % timezone.now())
task.log("--------------------------------")
task.save()
try:
with transaction.atomic() and open(
task.csv_file.path
) as csv_file: # transaction prevents partial csv import
# Load csv into Dict
records = csv.DictReader(csv_file)
lines = 0
for line in records:
lines += 1
# Get or create parent Language object
parent_lang = line["Parent Language"]
# Get label objects
labels = get_labels(task, org, line["Labels"])
# Create parent FAQ
parent_faq = FAQ.create(org, line["Parent Question"],
line["Parent Answer"], parent_lang,
None, labels)
# Start creation of translation FAQs
# get a list of the csv keys
keys = list(line)
# remove non-translation keys
parent_keys = [
"Parent Question", "Parent Language", "Parent Answer",
"Parent ID", "Labels"
]
[keys.remove(parent_key) for parent_key in parent_keys]
# get a set of unique translation language codes
lang_codes = set()
for key in keys:
lang_code, name = key.split(" ")
lang_codes.add(lang_code)
# Loop through for each translation
for lang_code in lang_codes:
# Create translation FAQ
FAQ.create(
org,
line["%s Question" % lang_code],
line["%s Answer" % lang_code],
lang_code,
parent_faq,
labels,
)
task.save()
task.log("Import finished at %s" % timezone.now())
task.log("%d FAQ(s) added." % lines)
except Exception as e:
if not settings.TESTING:
traceback.print_exc(e)
task.log("\nError: %s\n" % e)
raise e
return task
def csv_import(task): #pragma: no cover
from django.db import transaction
transaction.enter_transaction_management()
transaction.managed()
log = StringIO.StringIO()
try:
task.task_id = csv_import.request.id
task.log("Started import at %s" % datetime.now())
task.log("--------------------------------")
task.save()
transaction.commit()
model = class_from_string(task.model_class)
records = model.import_csv(task.csv_file.file, task.created_by, log)
task.log(log.getvalue())
task.log("Import finished at %s" % datetime.now())
task.log("%d record(s) added." % len(records))
transaction.commit()
except Exception as e:
transaction.rollback()
import traceback
traceback.print_exc(e)
task.log("\nError: %s\n" % e)
task.log(log.getvalue())
transaction.commit()
raise e
finally:
transaction.leave_transaction_management()
return task
def csv_import(task): #pragma: no cover
from django.db import transaction
transaction.enter_transaction_management()
transaction.managed()
log = StringIO.StringIO()
try:
task.task_id = csv_import.request.id
task.log("Started import at %s" % datetime.now())
task.log("--------------------------------")
task.save()
transaction.commit()
model = class_from_string(task.model_class)
records = model.import_csv(task.csv_file.file, task.created_by, log)
task.log(log.getvalue())
task.log("Import finished at %s" % datetime.now())
task.log("%d record(s) added." % len(records))
transaction.commit()
except Exception as e:
transaction.rollback()
import traceback
traceback.print_exc(e)
task.log("\nError: %s\n" % e)
task.log(log.getvalue())
transaction.commit()
raise e
finally:
transaction.leave_transaction_management()
return task
def csv_import(task_id): # pragma: no cover
from django.db import transaction
# there is a possible race condition between this task starting
# so we have a bit of loop here to fetch the task
tries = 0
task = None
while tries < 5 and not task:
try:
task = ImportTask.objects.get(pk=task_id)
except Exception as e:
# this object just doesn't exist yet, sleep a bit then try again
tries += 1
if tries >= 5:
raise e
else:
sleep(1)
log = StringIO()
if StrictVersion(django.get_version()) < StrictVersion('1.6'):
transaction.enter_transaction_management()
transaction.managed()
try:
task.task_id = csv_import.request.id
task.log("Started import at %s" % timezone.now())
task.log("--------------------------------")
task.save()
transaction.commit()
model = class_from_string(task.model_class)
records = model.import_csv(task, log)
task.save()
task.log(log.getvalue())
task.log("Import finished at %s" % timezone.now())
task.log("%d record(s) added." % len(records))
transaction.commit()
except Exception as e:
transaction.rollback()
import traceback
traceback.print_exc(e)
task.log("\nError: %s\n" % e)
task.log(log.getvalue())
transaction.commit()
raise e
finally:
transaction.leave_transaction_management()
else:
task.task_id = csv_import.request.id
task.log("Started import at %s" % timezone.now())
task.log("--------------------------------")
task.save()
try:
with transaction.atomic():
model = class_from_string(task.model_class)
records = model.import_csv(task, log)
task.save()
task.log(log.getvalue())
task.log("Import finished at %s" % timezone.now())
task.log("%d record(s) added." % len(records))
except Exception as e:
import traceback
traceback.print_exc(e)
task.log("\nError: %s\n" % e)
task.log(log.getvalue())
raise e
return task
def csv_import(task_id): # pragma: no cover
task = ImportTask.objects.get(pk=task_id)
log = StringIO()
task.task_id = csv_import.request.id
task.log("Started import at %s" % timezone.now())
task.log("--------------------------------")
task.save()
try:
with transaction.atomic():
model = class_from_string(task.model_class)
records = model.import_csv(task, log)
task.save()
task.log(log.getvalue())
task.log("Import finished at %s" % timezone.now())
task.log("%d record(s) added." % len(records))
except Exception as e:
import traceback
traceback.print_exc(e)
task.log("\nError: %s\n" % e)
task.log(log.getvalue())
raise e
return task
