def import_content(self):
thread_ids = zendesk_models.Post.objects.values_list(
'entry_id',
flat = True
).distinct()
threads_posted = 0
for thread_id in thread_ids:
thread_entries = zendesk_models.Post.objects.filter(
entry_id = thread_id
).order_by('created_at')
question_post = thread_entries[0]
question = post_question(question_post)
question_post.is_processed = True
question_post.save()
transaction.commit()
entry_count = thread_entries.count()
threads_posted += 1
console.print_action(str(threads_posted))
if entry_count > 1:
for answer_post in thread_entries[1:]:
post_answer(answer_post, question = question)
answer_post.is_processed = True
answer_post.save()
transaction.commit()
console.print_action(str(threads_posted), nowipe = True)
def import_content(self):
thread_ids = zendesk_models.Post.objects.values_list(
'entry_id',
flat = True
).distinct()
threads_posted = 0
for thread_id in thread_ids:
thread_entries = zendesk_models.Post.objects.filter(
entry_id = thread_id
).order_by('created_at')
question_post = thread_entries[0]
question = post_question(question_post)
question_post.is_processed = True
question_post.save()
transaction.commit()
entry_count = thread_entries.count()
threads_posted += 1
console.print_action(str(threads_posted))
if entry_count > 1:
for answer_post in thread_entries[1:]:
post_answer(answer_post, question = question)
answer_post.is_processed = True
answer_post.save()
transaction.commit()
console.print_action(str(threads_posted), nowipe = True)
def import_content(self):
thread_ids = zendesk_models.Post.objects.values_list(
'entry_id',
flat = True
).distinct()
threads_posted = 0
for thread_id in thread_ids:
thread_entries = zendesk_models.Post.objects.filter(
entry_id = thread_id
).order_by('created_at')
exercise_post = thread_entries[0]
exercise = post_exercise(exercise_post)
exercise_post.is_processed = True
exercise_post.save()
transaction.commit()
entry_count = thread_entries.count()
threads_posted += 1
console.print_action(str(threads_posted))
if entry_count > 1:
for problem_post in thread_entries[1:]:
post_problem(problem_post, exercise = exercise)
problem_post.is_processed = True
problem_post.save()
transaction.commit()
console.print_action(str(threads_posted), nowipe = True)
def read_xml_file(self,
file_name = None,
entry_name = None,
model = None,
fields = None,
extra_field_mappings = None
):
"""
* file_name - is name of xml file,
* entry_name - name of entries to read from the xml file
* model - model, which is to receive data
* fields - list of field names in xml that will be translated to model fields
by simple substitiution of '-' with '_'
* extra field mappings - list of two tuples where xml field names are
translated to model fields in a special way
"""
xml = self.get_file(file_name)
items_saved = 0
for xml_entry in xml.findall(entry_name):
instance = model()
for field in fields:
value = get_val(xml_entry, field)
model_field_name = field.replace('-', '_')
setattr(instance, model_field_name, value)
if extra_field_mappings:
for (field, model_field_name) in extra_field_mappings:
value = get_val(xml_entry, field)
setattr(instance, model_field_name, value)
instance.save()
transaction.commit()
items_saved += 1
console.print_action('%d items' % items_saved)
console.print_action('%d items' % items_saved, nowipe = True)
def read_xml_file(self,
file_name = None,
entry_name = None,
model = None,
fields = None,
extra_field_mappings = None
):
"""
* file_name - is name of xml file,
* entry_name - name of entries to read from the xml file
* model - model, which is to receive data
* fields - list of field names in xml that will be translated to model fields
by simple substitiution of '-' with '_'
* extra field mappings - list of two tuples where xml field names are
translated to model fields in a special way
"""
xml = self.get_file(file_name)
items_saved = 0
for xml_entry in xml.findall(entry_name):
instance = model()
for field in fields:
value = get_val(xml_entry, field)
model_field_name = field.replace('-', '_')
setattr(instance, model_field_name, value)
if extra_field_mappings:
for (field, model_field_name) in extra_field_mappings:
value = get_val(xml_entry, field)
setattr(instance, model_field_name, value)
instance.save()
transaction.commit()
items_saved += 1
console.print_action('%d items' % items_saved)
console.print_action('%d items' % items_saved, nowipe = True)
def forwards(self, orm):
"Write your forwards methods here."
print 'Migrating users to new avatar field'
for user in orm['auth.user'].objects.all():
print_action('migrating user: %s' % user.username)
if user.has_custom_avatar == True:
user.avatar_type = 'a'
else:
user.avatar_type = 'n'
user.save()
print_action('user %s migrated avatar_type: %s' % (user.username, user.avatar_type))
def forwards(self, orm):
"Write your forwards methods here."
print 'Migrating users to new avatar field'
for user in orm['auth.user'].objects.all():
print_action('migrating user: %s' % unidecode(user.username))
if user.has_custom_avatar == True:
user.avatar_type = 'a'
else:
user.avatar_type = 'n'
user.save()
print_action(
'user %s migrated avatar_type: %s' % \
(unidecode(user.username), user.avatar_type)
)
def forwards(self, orm):
"Write your forwards methods here."
print 'Migrating users to new avatar field'
for user in orm[AUTH_USER_MODEL].objects.all():
print_action('migrating user: %s' % unidecode(user.username))
if user.has_custom_avatar == True:
user.avatar_type = 'a'
else:
user.avatar_type = 'n'
user.save()
print_action(
'user %s migrated avatar_type: %s' % \
(unidecode(user.username), user.avatar_type)
)
def handle_noargs(self, **options):
users = User.objects.all()
has_avatar = User.objects.exclude(askbot_profile__avatar_type='n').count()
total_users = users.count()
print '%s users in total, %s have valid avatar' \
% (total_users, has_avatar)
for count, user in enumerate(users):
users_left = total_users - count
print_action(
'Updating %s (%d users left)' % (user.username, users_left)
)
user.update_avatar_type()
print 'Updated all the users'
has_avatar = User.objects.exclude(askbot_profile__avatar_type='n').count()
print '%s users in total, %s have real avatar image' \
% (total_users, has_avatar)
def handle(self, **options):
users = User.objects.all()
has_avatar = User.objects.exclude(askbot_profile__avatar_type='n').count()
total_users = users.count()
print('%s users in total, %s have valid avatar' \
% (total_users, has_avatar))
for count, user in enumerate(users):
users_left = total_users - count
print_action(
'Updating %s (%d users left)' % (user.username, users_left)
)
user.update_avatar_type()
print('Updated all the users')
has_avatar = User.objects.exclude(askbot_profile__avatar_type='n').count()
print('%s users in total, %s have real avatar image' \
% (total_users, has_avatar))
def handle_noargs(self, **options):
users = User.objects.all()
has_avatar = User.objects.exclude(avatar_type='n').count()
total_users = users.count()
print '%s users in total, %s have valid avatar' \
% (total_users, has_avatar)
for count, user in enumerate(users):
users_left = total_users - count
print_action('Updating %s (%d users left)' %
(user.username, users_left))
user.update_avatar_type()
transaction.commit()
print 'Updated all the users'
has_avatar = User.objects.exclude(avatar_type='n').count()
transaction.commit()
print '%s users in total, %s have valid avatar' \
% (total_users, has_avatar)
def import_tickets(self, tags, date_filter):
"""Import Zendesk Tickets into Askbot as questions.
:param tags: (list) tags (str) to filter Zendesk Tickets by.
Tickets that match ANY of the tags will be posted as questions. Tags
are case-insensitive in this import regardless of your settings in
Askbot.
:param date_filter: (tuple) two-element tuple representing the start
date and end date to filter Zendesk Tickets by date range. The
tuple values are datetime objects or None. The date_filter is matched
against Ticket.created_at.
"""
# todo: optimmize with smart query
# Ticket.objects.get(
# Q(created_at__gt=date_filter[0]),
# Q(created_at__lt=date_filter[1]),
# Q(tags__icontains='foo') | Q(tags__icontains='bar')
# )
if tags:
print("Filtering tickets by tags: %s" % tags)
if date_filter:
print("Filtering tickets by dates between %s and %s" %
(date_filter[0], date_filter[1]))
sys.stdout.write("Importing tickets: ")
ticket_count = 0
for ticket in zendesk_models.Ticket.objects.all():
# filters
# if provided, only post entries matching ANY of the tags
if not self._matches_tag_filter(ticket.current_tags, tags):
continue
if not self._matches_date_filter(ticket.created_at, date_filter):
continue
question = post_question_from_ticket(ticket)
if not question:
continue
ticket.ab_id = question.id
ticket.save()
self._import_comments(question, ticket)
ticket_count += 1
console.print_action("%d tickets" % ticket_count)
console.print_action("%d total tickets" % ticket_count, nowipe=True)
def import_tickets(self, tags, date_filter):
"""Import Zendesk Tickets into Askbot as questions.
:param tags: (list) tags (str) to filter Zendesk Tickets by.
Tickets that match ANY of the tags will be posted as questions. Tags
are case-insensitive in this import regardless of your settings in
Askbot.
:param date_filter: (tuple) two-element tuple representing the start
date and end date to filter Zendesk Tickets by date range. The
tuple values are datetime objects or None. The date_filter is matched
against Ticket.created_at.
"""
# todo: optimmize with smart query
# Ticket.objects.get(
# Q(created_at__gt=date_filter[0]),
# Q(created_at__lt=date_filter[1]),
# Q(tags__icontains='foo') | Q(tags__icontains='bar')
# )
if tags:
print "Filtering tickets by tags: %s" % tags
if date_filter:
print "Filtering tickets by dates between %s and %s" % (date_filter[0], date_filter[1])
sys.stdout.write("Importing tickets: ")
ticket_count = 0
for ticket in zendesk_models.Ticket.objects.all():
# filters
# if provided, only post entries matching ANY of the tags
if not self._matches_tag_filter(ticket.current_tags, tags):
continue
if not self._matches_date_filter(ticket.created_at, date_filter):
continue
question = post_question_from_ticket(ticket)
if not question:
continue
ticket.ab_id = question.id
ticket.save()
self._import_comments(question, ticket)
ticket_count += 1
console.print_action("%d tickets" % ticket_count)
console.print_action("%d total tickets" % ticket_count, nowipe = True)
def import_users(self):
"""Creates new Askbot users for each zendesk_models.User.
For each Zendesk user, see if there are any matching Askbot users
with the same email. If not, create a new Askbot user and copy
over any openauth id info as well.
See create_askbot_user() for a full list of fields that are copied over
from Zendesk.
"""
added_users = 0
for zd_user in zendesk_models.User.objects.all():
# if email is blank, just create a new user
if zd_user.email == '':
ab_user = create_askbot_user(zd_user)
# todo: check for failure?
if ab_user is None:
continue
added_users += 1
console.print_action(ab_user.username)
else:
# create new user if no matching user email was found
try:
ab_user = askbot_models.User.objects.get(
email=zd_user.email)
except askbot_models.User.DoesNotExist:
ab_user = create_askbot_user(zd_user)
if ab_user is None:
continue
added_users += 1
console.print_action("%d %s" %
(added_users, ab_user.username))
zd_user.askbot_user_id = ab_user.id
zd_user.save()
# save open auth info as well.
if zd_user.openid_url != None and \
'askbot.deps.django_authopenid' in settings.INSTALLED_APPS:
from askbot.deps.django_authopenid.models import UserAssociation
from askbot.deps.django_authopenid.util import get_provider_name
try:
assoc = UserAssociation(user=ab_user,
openid_url=zd_user.openid_url,
provider_name=get_provider_name(
zd_user.openid_url))
assoc.save()
except:
# unsupported provider
pass
console.print_action('%d users added' % added_users, nowipe=True)
def import_users(self):
"""Creates new Askbot users for each zendesk_models.User.
For each Zendesk user, see if there are any matching Askbot users
with the same email. If not, create a new Askbot user and copy
over any openauth id info as well.
See create_askbot_user() for a full list of fields that are copied over
from Zendesk.
"""
added_users = 0
for zd_user in zendesk_models.User.objects.all():
# if email is blank, just create a new user
if zd_user.email == '':
ab_user = create_askbot_user(zd_user)
# todo: check for failure?
if ab_user is None:
continue
added_users += 1
console.print_action(ab_user.username)
else:
# create new user if no matching user email was found
try:
ab_user = askbot_models.User.objects.get(email = zd_user.email)
except askbot_models.User.DoesNotExist:
ab_user = create_askbot_user(zd_user)
if ab_user is None:
continue
added_users += 1
console.print_action("%d %s" % (added_users, ab_user.username))
zd_user.askbot_user_id = ab_user.id
zd_user.save()
# save open auth info as well.
if zd_user.openid_url != None and \
'askbot.deps.django_authopenid' in settings.INSTALLED_APPS:
from askbot.deps.django_authopenid.models import UserAssociation
from askbot.deps.django_authopenid.util import get_provider_name
try:
assoc = UserAssociation(
user = ab_user,
openid_url = zd_user.openid_url,
provider_name = get_provider_name(zd_user.openid_url)
)
assoc.save()
except:
# unsupported provider
pass
console.print_action('%d users added' % added_users, nowipe = True)
def import_users(self):
added_users = 0
for zd_user in zendesk_models.User.objects.all():
#a whole bunch of fields are actually dropped now
#see what's available in users.xml meanings of some
#values there is not clear
#if email is blank, just create a new user
if zd_user.email == '':
ab_user = create_askbot_user(zd_user)
if ab_user in None:
print 'Warning: could not create user %s ' % zd_user.name
continue
console.print_action(ab_user.username)
else:
#else see if user with the same email already exists
#and only create new askbot user if email is not yet in the
#database
try:
ab_user = askbot_models.User.objects.get(email = zd_user.email)
except askbot_models.User.DoesNotExist:
ab_user = create_askbot_user(zd_user)
if ab_user is None:
continue
console.print_action(ab_user.username, nowipe = True)
added_users += 1
zd_user.askbot_user_id = ab_user.id
zd_user.save()
if zd_user.openid_url != None and \
'askbot.deps.django_authopenid' in settings.INSTALLED_APPS:
from askbot.deps.django_authopenid.models import UserAssociation
from askbot.deps.django_authopenid.util import get_provider_name
try:
assoc = UserAssociation(
user = ab_user,
openid_url = zd_user.openid_url,
provider_name = get_provider_name(zd_user.openid_url)
)
assoc.save()
except:
#drop user association
pass
transaction.commit()
console.print_action('%d users added' % added_users, nowipe = True)
def import_users(self):
added_users = 0
for zd_user in zendesk_models.User.objects.all():
#a whole bunch of fields are actually dropped now
#see what's available in users.xml meanings of some
#values there is not clear
#if email is blank, just create a new user
if zd_user.email == '':
ab_user = create_askbot_user(zd_user)
if ab_user in None:
print 'Warning: could not create user %s ' % zd_user.name
continue
console.print_action(ab_user.username)
else:
#else see if user with the same email already exists
#and only create new askbot user if email is not yet in the
#database
try:
ab_user = askbot_models.User.objects.get(email = zd_user.email)
except askbot_models.User.DoesNotExist:
ab_user = create_askbot_user(zd_user)
if ab_user is None:
continue
console.print_action(ab_user.username, nowipe = True)
added_users += 1
zd_user.askbot_user_id = ab_user.id
zd_user.save()
if zd_user.openid_url != None and \
'askbot.deps.django_authopenid' in settings.INSTALLED_APPS:
from askbot.deps.django_authopenid.models import UserAssociation
from askbot.deps.django_authopenid.util import get_provider_name
try:
assoc = UserAssociation(
user = ab_user,
openid_url = zd_user.openid_url,
provider_name = get_provider_name(zd_user.openid_url)
)
assoc.save()
except:
#drop user association
pass
transaction.commit()
console.print_action('%d users added' % added_users, nowipe = True)
def import_forums(self, forums, tags, date_filter):
"""Import Zendesk forums into Askbot. Create questions from Zendesk
Entries and answers from Zendesk Posts.
:param forums: (list) zendesk_models.Forum objects to import
:param tags: (list) tags (str) to filter Zendesk Forum Entries by.
Entries that match ANY of the tags will be posted as questions. Tags
are case-insensitive in this import regardless of your settings in
Askbot.
:param date_filter: (tuple) two-element tuple representing the start
date and end date to filter Zendesk Forum Entries by date range. The
tuple values are datetime objects or None.
"""
if tags:
print("Filtering forum posts by tags: %s" % tags)
if date_filter:
print("Filtering forum post by dates between %s and %s" %
(date_filter[0], date_filter[1]))
print("Importing forums... ")
print("=" * 64)
for forum in forums:
thread_count = 0
# don't import private forums, forums restricted to organizations
# or forums that require login (comment this out if you don't care,
# or modify the viewable_to_public() method for zendesk_models.Forum)
if not forum.viewable_to_public():
console.print_action("Skipping private forum \"%s\"" %
forum.name,
nowipe=True)
continue
sys.stdout.write("[#%d] %s: " % (forum.forum_id, forum.name))
for entry in zendesk_models.Entry.objects.filter(
forum_id=forum.forum_id):
# filters
# if provided, only post entries matching ANY of the tags
if not self._matches_tag_filter(entry.tags, tags):
continue
if not self._matches_date_filter(entry.created_at,
date_filter):
continue
if self._import_entry(entry):
thread_count += 1
console.print_action("%d threads" % thread_count)
console.print_action("%d total threads" % thread_count,
nowipe=True)
def import_forums(self, forums, tags, date_filter):
"""Import Zendesk forums into Askbot. Create questions from Zendesk
Entries and answers from Zendesk Posts.
:param forums: (list) zendesk_models.Forum objects to import
:param tags: (list) tags (str) to filter Zendesk Forum Entries by.
Entries that match ANY of the tags will be posted as questions. Tags
are case-insensitive in this import regardless of your settings in
Askbot.
:param date_filter: (tuple) two-element tuple representing the start
date and end date to filter Zendesk Forum Entries by date range. The
tuple values are datetime objects or None.
"""
if tags:
print "Filtering forum posts by tags: %s" % tags
if date_filter:
print "Filtering forum post by dates between %s and %s" % (date_filter[0], date_filter[1])
print "Importing forums... "
print "="*64
for forum in forums:
thread_count = 0
# don't import private forums, forums restricted to organizations
# or forums that require login (comment this out if you don't care,
# or modify the viewable_to_public() method for zendesk_models.Forum)
if not forum.viewable_to_public():
console.print_action("Skipping private forum \"%s\"" % forum.name,
nowipe = True)
continue
sys.stdout.write("[#%d] %s: " % (forum.forum_id, forum.name))
for entry in zendesk_models.Entry.objects.filter(forum_id=forum.forum_id):
# filters
# if provided, only post entries matching ANY of the tags
if not self._matches_tag_filter(entry.tags, tags):
continue
if not self._matches_date_filter(entry.created_at, date_filter):
continue
if self._import_entry(entry):
thread_count += 1
console.print_action("%d threads" % thread_count)
console.print_action("%d total threads" % thread_count, nowipe = True)
def read_xml_file(self,
file_name = None,
entry_name = None,
model = None,
fields = None,
extra_field_mappings = None,
sub_entities = []
):
"""Reads xml file, parses entries into Zendesk model objects, and saves
them to the database.
Values are cast to their correct data types.
Sub-entities are used for extracting an embedded structure from the
element tree into a separate model and table.
:param file_name: (str) name of xml file,
:param entry_name: (str) name of entries to read from the xml file
:param model: (obj) model where data will be stored
:param fields: (list) field names (str) in xml that will be translated
to model fields by simple substitiution of '_' for '-'
:param extra_field_mappings (tuple) list of two tuples for xml field
names have specific translation that doesn't follow the standard for
the fields parameter
:param sub_entities: (list) of dicts describing fields that should be
treated as separate models (like Ticket.comments). The structure is
similar to this method. Each dict key is the field name to be treated
as a sub-entity. The value is a tuple with (model, [sub-entity fields],
(sub-entity extra_field_mappings)).
[{'comments': (
zendesk_models.Comment,
['author-id', 'created-at', 'is-public', 'type',
'value', 'via-id', 'ticket-id'],
(),)
}]
todo: support blank values vs. nulls for strings
"""
cursor = connection.cursor()
cursor.execute('TRUNCATE TABLE "{0}" CASCADE'.format(model._meta.db_table))
xml = self.get_file(file_name)
items_saved = 0
for xml_entry in xml.findall(entry_name):
instance = model()
for field in fields:
value = get_xml_element_val(xml_entry, field)
model_field_name = field.replace('-', '_')
max_length = instance._meta.get_field(model_field_name).max_length
if value and max_length:
value = value[:max_length]
setattr(instance, model_field_name, value)
if extra_field_mappings:
for (field, model_field_name) in extra_field_mappings:
value = get_xml_element_val(xml_entry, field)
setattr(instance, model_field_name, value)
sub_instances = []
for sub_entity in sub_entities:
for sub_field_name, sub_def in sub_entity.iteritems():
sub_list = get_xml_element_val(xml_entry, sub_field_name)
sub_model, sub_fields, sub_extra_field_mappings = sub_def
for child in sub_list:
sub_instance = sub_model()
for sub_field in sub_fields:
sub_value = get_xml_element_val(child, sub_field)
sub_model_field_name = sub_field.replace('-', '_')
sub_max_length = sub_instance._meta.get_field(sub_model_field_name).max_length
if sub_value and sub_max_length:
sub_value = sub_value[:sub_max_length]
setattr(sub_instance, sub_model_field_name, sub_value)
sub_instances.append(sub_instance)
instance.save()
for si in sub_instances:
# set the parent id
setattr(si, "%s_id" % entry_name, instance.id)
si.save()
items_saved += 1
console.print_action('%d' % items_saved)
console.print_action('%d total' % items_saved, nowipe = True)
