def survey_gatherer(cursor, cursor2):
cursor.execute('''SELECT * FROM survey
WHERE is_psychological_test = 0
''')
for record in cursor_generator(cursor):
survey_id = record['id']
cursor2.execute('''
SELECT user_uid
FROM survey_participant
WHERE survey_id=%d
'''%survey_id)
answered_user = filter(lambda x: x is not None,
[filter_user_uid(r['user_uid']) for r in cursor2.fetchall()])
article_uid = record['article_uid']
if article_uid is not None:
cursor2.execute('''
SELECT time,
uid AS article_uid
FROM article
WHERE
article.uid = %d
'''%(article_uid))
misc_info = cursor2.fetchone()
if not misc_info:
time = None
else:
time = misc_info['time']
else:
time = None
record['answered_user'] = answered_user
record['time'] = time
yield record
def panorama_converter(_iter):
# FIXME: type is incompatible
#
for idx, wing in enumerate(_iter):
if idx % 1000 == 0:
print "NOW MOVING %d-th record of panorama table..."%idx
'''
type
--
"article" -> "article"
"cafe" -> "gruop"
"dolblog" -> x
"user" -> x
"comment" -> "comment"
"commented" -> x
"tag" -> "tag"
'''
type=wing["type"]
'''
"addrel" -> "add_rel"
"invalid" -> x
"app" -> x
"modify" -> "modify"
"survey_start" -> "survey_start"
"removerel" -> "remove_rel"
"article" -> "article"
"cafe" -> "group"
"file" -> "file"
"remove" -> "remove"
"move" -> "move"
"dolblog" -> x
"add_link" -> "add_menu"
"add" -> "add"
"user" -> "user"
"remove_link" -> "remove_menu"
'''
event=wing["event"]
from_group_id=wing["cafe_uid"]
from_board_id=wing["board_uid"]
user_uid=filter_user_uid(wing["user_uid"])
target_uid=wing["target_uid"]
created_at=wing["time"]
class PanoramaSaver(BaseCustomSaver):
def save(self, db):
class FakeSession:
def __init__(self):
self.result = []
def add(self, elem):
self.result.append(elem)
fake_session = FakeSession()
yield PanoramaSaver()
def save(self, db):
max_id = db.query(func.max(RelatedTag.columns.get('id'))).scalar()
if max_id is None:
max_id = 0
max_id += 1
ins = RelatedTag.insert().values(id=max_id,
tag_1_id=wing["parent_id"],
tag_2_id=wing["id"],
related_by_id=filter_user_uid(wing["user_uid"]),
related_at=wing["time"])
db.execute(ins)
db.commit()
def tag_converter(_iter):
for idx, wing in enumerate(_iter):
if idx % 1000 == 0:
print "NOW MOVING %d-th record of tag table..."%idx
creator_id = filter_user_uid(wing["user_uid"])
if creator_id < 0 or creator_id == 3027: # SUPER HACK!
creator_id = None
tag = Tag(name=wing["name"],
creator_id=creator_id)
tag.id = wing["id"]
yield tag
def comment_converter(_iter):
for idx, wing in enumerate(_iter):
if idx % 1000 == 0:
print "NOW MOVING %d-th record of comment table..."%(wing["id"] - 1)
if wing["relate"] is None or wing["relate"] == wing["id"]:
parent_comment_id = None
else:
parent_comment_id = get_comment_uid(wing["relate"])
comment = Comment(recommend_count=wing["recommendation_count"],
author_id=filter_user_uid(wing["user_uid"]),
anonymous=None,
parent_id=wing["parent_uid"],
parent_comment_id=parent_comment_id,
display_order=wing["position"],
reply_display_order=wing["depth_position"],
content=wing["content"],
created_at=wing["time"])
if filter_user_uid(wing["user_uid"]) is None:
comment.anonymous_name = wing["user_name"] or None
# password may be NULL
comment.anonymous["password"] = _hexdigest(wing["user_password"])
comment.uid = get_comment_uid(wing["id"])
yield comment
def survey_question_converter(_iter):
for idx, wing in enumerate(_iter):
if idx % 1000 == 0:
print "NOW MOVING %d-th record of survey question table..."%idx
sq = SurveyQuestion(content=wing["question"],
min_answers=wing["lower_bound"],
max_answers=wing["upper_bound"],
_examples=json.dumps(wing["examples"]),
survey_id=wing["survey_id"],
order=wing["number"])
sq.id = wing["id"]
yield sq
for p in wing["survey_selection"]:
yield SurveyAnswer(survey_question_id=p["survey_question_id"],
user_id=filter_user_uid(p["user_uid"]),
answer=p["answer"])
def attached_tag_converter(_iter):
for idx, wing in enumerate(_iter):
if idx % 1000 == 0:
print "NOW MOVING %d-th record of attached-tag table..."%idx
area = None if wing["extra"] is None else json.loads(wing["extra"])
if is_orphaned_file_uid(wing["parent_uid"]):
print wing["parent_uid"], "is orphaned"
continue
attached_tag = AttachedTag(tag_id=wing["tag_id"],
target_id=wing["parent_uid"],
creator_id=filter_user_uid(wing["user_uid"]),
created_at=wing["time"],
area=area)
yield attached_tag
def survey_converter(_iter):
for idx, wing in enumerate(_iter):
if idx % 1000 == 0:
print "NOW MOVING %d-th record of survey table..."%idx
due_date = wing["due"] or (datetime.utcnow() + timedelta(days=36500))
survey = Survey(name=wing["subject"],
due_date=due_date,
parent_id=wing["article_uid"],
owner_id=filter_user_uid(wing["user_uid"]),
is_anonymous=wing["is_anonymous"],
permission_type=wing["permission_type"],
permission_value=wing["permission"] or "",
expose_level=wing["expose_level"],
min_vote_num=wing["minimum_number_of_votes"],
_answered_user=json.dumps(wing["answered_user"]),
created_at=wing["time"])
survey.id = wing["id"]
yield survey
def file_gatherer(cursor, cursor2):
'''
Type of Author Id of file
--
user
article -> author of article
cafe -> NULL
app - TODO
'''
cursor.execute('''
SELECT "file".*,
uid_list.type AS parent_uid_type
FROM "file"
JOIN uid_list ON
"file".parent_uid = uid_list.uid
''')
for record in cursor_generator(cursor):
parent_uid_type = record["parent_uid_type"]
parent_uid = record["parent_uid"]
if parent_uid_type == "user":
record["author_id"] = parent_uid
elif parent_uid_type == "article":
cursor2.execute('''
SELECT user_uid FROM article WHERE uid = %d
'''%(parent_uid))
record["author_id"] = filter_user_uid(cursor2.fetchone()["user_uid"])
elif parent_uid_type == "cafe":
record["author_id"] = None
elif parent_uid_type == "app":
print "App(%d) file won't be migrated in this phase"%parent_uid
continue
'''
# FIXME: author of app
record["author_id"] = None
'''
else:
raise Exception("NOT REACHABLE: %s"%parent_uid_type)
yield record
def favorite_converter(_iter):
for idx, wing in enumerate(_iter):
if idx % 1000 == 0:
print "NOW MOVING %d-th record of favorite table..."%idx
#FIXME: order in cloud.db
if is_invalid_favorite_target_uid(wing["target_uid"]):
print wing["target_uid"], "is target_uid!"
continue
if is_orphaned_file_uid(wing["target_uid"]):
print wing["target_uid"], "is orphaned!"
continue
user_uid = filter_user_uid(wing["user_uid"])
if user_uid is None:
print "user_uid cannot be NULL!"
continue
favorite = Favorite(user_id=user_uid,
target_id=wing["target_uid"],
name=wing["name"],
order=wing["position"])
favorite.id = wing["id"]
yield favorite
def article_gatherer(cursor, cursor2):
cursor.execute('''SELECT * FROM article''')
for wing in cursor_generator(cursor):
wing["user_uid"] = filter_user_uid(wing["user_uid"])
yield wing
