def import_group(self):
"""
{"id":"15","topic_id":"0","name":"Stone Installation","url":"stone-installer","about":"Natural stone companies specialized in the installtion of granite, marble and similar natural stones. Installation of natural stone as flooring, wall cladding, counter tops or as other application.","description":"Companies specialized in the installtion of granite, marble and related natural stones, as flooring, wall cladding, counter tops or other applications.","keywords":"installation of stone, granite, marble, limestone, flooring, wall cladding","title_foto":"305","title_foto_ext":"jpg","count_members":"554","created_time":"2007-10-27 23:12:54","created_ip":"127.0.0.1","created_user":"******","time":"2008-08-18 06:27:47","ip":"84.137.125.24","is_invite_only":"0","is_private":"0","is_blocked":"0","is_deleted":"0"},
"""
print('Import groups: ', end='', flush=True)
Group.objects.all().delete()
for row in self.walkjsondata('groups'):
group = Group(id=row['id'])
group.name = row['name'][:30]
group.slug = row['url'][:30] # url
group.about = row['about']
group.description = row['description'][:255]
group.keywords = row['keywords'][:255]
group.title_foto = '/static/group/{}.jpg'.format(group.slug)
group.count_members = row['count_members']
group.created = parse_iso_datetime(row['created_time'])
group.save()
print('g', end='', flush=True)
# add members of this group
"""
{"user_id":"4","group_id":"13","applied_time":"2008-08-17 04:40:27","confirmed_time":"2008-08-17 04:40:27"}
"""
for row2 in self.walkjsondata('groups_members'):
if row2['group_id'] == group.id:
try:
user = User.objects.get(pk=row2['user_id'])
group.members.add(user)
print('.', end='', flush=True)
except User.DoesNotExist:
# print('Could not add user {} to group {}: User does '
# 'not exist.'.format(row2['user_id'], group.name))
print('X', end='', flush=True)
print('all groups done')
def import_user(self):
"""
--- user --------------------------------------------------------------
{"user_id":"22","nick":"YYY","pass":"******","email":"*****@*****.**",
"name":"Blah Corp.","type":"company","title_foto":"12","title_foto_ext"
:"jpg","time_offset":"0","datetime_format":"Y-m-d H:i:s",
"date_short":"Y-m-d","date_long":"l, d F Y","signup_ip":"0.0.0.0",
"signup_time":"2005-06-12 08:48:07","lastlogin_ip":"0.0.0.0",
"lastlogin_time":"2008-08-17 02:40:28","lastlogout_time":
"2008-08-17 02:40:28","is_blocked":"0","is_deleted":"0","is_readonly":
"0","is_admin":"0","is_mod_global":"0","is_mod_forum":"0",
"is_mod_fotos":"0","is_mod_stones":"0","is_mod_pages":"0",
"is_mod_groups":"0","is_mod_tradeshows":"0"}
"""
print('Import users and profiles', end='', flush=True)
User.objects.all().delete()
UserProfile.objects.all().delete()
for row in self.walkjsondata('user'):
if not row['user_id'] or \
not row['nick'] or row['type'] != 'company':
continue
if int(row['is_deleted']) or int(row['is_blocked']):
print('D', end='', flush=True)
continue
try:
user = User(id=row['user_id'])
user.username = row['nick'][:30]
user.set_password(row['pass'])
user.email = row['email']
user.last_login = parse_iso_datetime(row['lastlogin_time'])
user.date_joined = parse_iso_datetime(row['signup_time'])
user.save()
except IntegrityError:
print('E', end='', flush=True)
continue
user.profile.name = row['name']
user.profile.title_foto = row['title_foto']
user.profile.title_foto_ext = row['title_foto_ext']
user.profile.signup_ip = row['signup_ip']
user.profile.lastlogin_ip = row['lastlogin_ip']
user.profile.save()
print('.', end='', flush=True)
print('done! All users created.')
def import_projects(self):
i = 0
print('Import project', end='', flush=True)
Project.objects.all().delete()
for row in self.walkjsondata('stones_projects'):
try:
stone = Stone.objects.get(pk=row['stone_id'])
except Stone.DoesNotExist:
# print('Stone not found for proj. {}'.format(row['stone_id']))
print('SE', end='', flush=True)
continue
try:
user = User.objects.get(pk=row['user_id'])
except User.DoesNotExist:
# print('User not found for project {}'.format(row['user_id']))
print('UE', end='', flush=True)
continue
item = Project(id=row['id'])
item.user = user
item.created = parse_iso_datetime(row['time'])
if not item.created: # skip if no created time
continue
item.description = row['description']
item.is_blocked = bool(force_int(row['is_blocked']))
item.is_deleted = bool(force_int(row['is_deleted']))
if item.is_blocked or item.is_deleted: # skip deleted items
continue
item.is_recommended = bool(force_int(row['is_recommended']))
item.count_views = force_int(row['count_views'])
item.save()
item.stones.add(stone)
item.save()
i += 1
print('.', end='', flush=True)
if (i % 1000) == 0:
print(i, end='', flush=True)
print('done. {} project items imported.'.format(i))
def import_pages(self):
# pages_topics
# pages
Article.objects.all().delete()
Author.objects.all().delete()
Keyword.objects.all().delete()
Topic.objects.all().delete()
print('Import topics ', end='', flush=True)
for row in self.walkjsondata('pages_topics'):
item = Topic(id=row['id'])
item.title = row['title']
item.slug = row['url'][:50]
item.description = row['description']
item.save()
print('.', end='', flush=True)
print(' done!')
print('Import authors', end='', flush=True)
for row in self.walkjsondata('pages'):
a = row['author_name']
b = {'about': row['author_about'], 'url': row['author_url']}
item, created = Author.objects.get_or_create(name=a, defaults=b)
if created:
print('.', end='', flush=True)
print(' done!')
print('Import keywords', end='', flush=True)
for row in self.walkjsondata('pages'):
pass
print(' NOT IMPLEMENTED!')
print('Import articles', end='', flush=True)
user = User.objects.get(pk=1)
for row in self.walkjsondata('pages'):
item = Article(id=row['id'])
item.title = row['title']
item.slug = row['url'][:50]
item.created = parse_iso_datetime(row['time'])
item.user = user
try:
item.author = Author.objects.get(name=row['author_name'])
except Author.DoesNotExist:
item.author = None
item.is_published = bool(int(row['is_published']))
item.is_stickied = False
item.is_frontpage = True
item.topic = Topic.objects.get(pk=int(row['topic_id']))
item.teaser = row['teaser']
item.description = row['description']
item.text = row['text']
item.save()
for k in row['keywords'].split(', '):
kslug = slugify(k)[:50]
if not kslug:
continue
kobj, created = Keyword.objects.get_or_create(slug=kslug)
item.keywords.add(kobj)
print('.', end='', flush=True)
print('.', end='', flush=True)
print(' done!')
def import_pics(self):
"""
{"id":"1","user_id":"4","module":"profile","module_id":"4","folder":"0","time":"2005-06-12 08:48:06","ip":"0.0.0.0","size":"26860","width":"397","height":"297","ext":"jpg","title":"","caption":"","is_blocked":"0","is_deleted":"0","is_sticky":"0","is_comments":"0","is_approved":"1","is_title":"0"},
companydb.modules.Pic.MODULE_CHOICES = (
('profile', 'Profile'), ('projects', 'Projects'),
('stones', 'Stones'), ('stock', 'Stock'), ('groups', 'Groups'),
('pages', 'Pages'))
{'groups': 17, 'stock': 11754, 'stones': 2450,
'projects': 1770, 'pages': 29, 'profile': 11006}
"""
"""
print('Count pic modules:')
m = dict()
for row in self.walkjsondata('fotos'):
if row['module'] in m.keys():
m[row['module']] += 1
else:
m[row['module']] = 0
print('done counting:')
print(m)
input('Press ENTER to continue with pics import...')
"""
print('Import pics', end='', flush=True)
Pic.objects.all().delete()
i = 0
for row in self.walkjsondata('fotos'):
try:
user = User.objects.get(pk=row['user_id'])
except User.DoesNotExist:
print('U', end='', flush=True)
continue
pic = Pic(id=row['id'])
pic.user = user
# check integrity: find the object in the referenced model
pic.module = row['module'] # related model
pic.module_id = row['module_id'] # object id within related model
if pic.module == 'profile':
try:
User.objects.get(pk=pic.module_id)
except UserProfile.DoesNotExist:
print('U', end='', flush=True)
continue # user not found!
elif pic.module == 'projects':
try:
Project.objects.get(pk=pic.module_id)
except Project.DoesNotExist:
print('P', end='', flush=True)
continue # project item not found!
elif pic.module == 'stock':
try:
Stock.objects.get(pk=pic.module_id)
except Stock.DoesNotExist:
print('K', end='', flush=True)
continue # stock item not found!
elif pic.module == 'stones':
try:
Stone.objects.get(pk=pic.module_id)
except Stone.DoesNotExist:
print('S', end='', flush=True)
continue # stone item not found!
elif pic.module == 'pages':
try:
Article.objects.get(pk=pic.module_id)
except Article.DoesNotExist:
print('A', end='', flush=True)
continue # article item not found!
elif pic.module == 'groups':
try:
Group.objects.get(pk=pic.module_id)
except Group.DoesNotExist:
print('G', end='', flush=True)
continue # group item not found!
else:
continue # not a valid module
pic.created = parse_iso_datetime(row['time'])
if not pic.created: # skip if no timestamp
continue
pic.size = force_int(row['size'])
pic.width = force_int(row['width'])
pic.height = force_int(row['height'])
pic.ext = row['ext']
pic.title = row['title']
pic.caption = row['caption']
pic.is_blocked = bool(force_int(row['is_blocked']))
pic.is_deleted = bool(force_int(row['is_deleted']))
if pic.is_blocked or pic.is_deleted: # skip if deleted
continue
pic.is_sticky = bool(force_int(row['is_sticky']))
pic.is_comments = bool(force_int(row['is_comments']))
pic.is_approved = bool(force_int(row['is_approved']))
pic.is_title = bool(force_int(row['is_title']))
pic.save()
i += 1
print('.', end='', flush=True)
if (i % 1000) == 0:
print(i, end='', flush=True)
print('done. {} pics imported.'.format(i))