Exemplo n.º 1
0
def csvize_deleted_unique(csv_filename, error_code=-1, exclude_error_code=False):

	nowdatetime = weibo_module.get_current_chinatime()

	deleted_post_ids = weibo_module.get_deleted_postids(error_code, exclude_error_code)
	num_dead_posts = len(deleted_post_ids)

	#if we're not tracking any posts, get out of there
	if (num_dead_posts <= 0):
		return

	## OPEN A FILE
	with codecs.open(csv_filename, "wb", "utf-16") as wf:

		#write csv header
		csv_header = weibo_module.get_csv_header()
		wf.write(csv_header + "\n")
		#print csv_header

		#iterate through posts
		for this_post_id in deleted_post_ids:

			this_post = weibo_module.merge_deleted_from_new_old(this_post_id)

			csvline = weibo_module.make_csvline_from_post(this_post)

		
			csvline = map((lambda x: unicode(x)), csvline)

			#not csv, this is our delimiter now
			csvline = weibo_settings.delim.join(csvline)

#			#print csvline
			wf.write(csvline + "\n")
Exemplo n.º 2
0
def csvize_repost_timeline(csv_filename, type="deleted", error_code=-1, exclude_error_code=False):

	nowdatetime = weibo_module.get_current_chinatime()

	if type == "deleted":
		query_post_ids = weibo_module.get_deleted_postids(error_code, exclude_error_code)
	else:
		query_post_ids = weibo_module.get_all_postids()
#		query_post_ids = query_post_ids[:10]
		print query_post_ids

	num_query_posts = len(query_post_ids)

	#if we're not tracking any posts, get out of there
	if (num_query_posts <= 0):
		return

	## OPEN A FILE
	with codecs.open(csv_filename, "wb", "utf-16") as wf:

		#write csv header
		csv_header = weibo_module.get_csv_header()
		wf.write(csv_header + "\n")

		#iterate through posts
		for this_post_id in query_post_ids:

			print "\n==WRITING POST #=========", this_post_id

			# okay first we get the initial post
			this_post = weibo_module.merge_deleted_from_new_old(this_post_id)

			# and then we scan the rest 
			this_post_all_logs =  weibo_module.get_all_posts(this_post_id)

			# and then we amass a logline -- a csv file inside a csv file!
			this_log_line = []
			for this_log in this_post_all_logs:
				if 'post_repost_count' in this_log and this_log["post_repost_count"] <> None:
					this_log_line.append(str(this_log["post_repost_count"]))
					this_log_line.append(str(this_log["checked_at"]))
			this_log_line = weibo_settings.delim_log.join(this_log_line)

			#get csvline array
			csvline = weibo_module.make_csvline_from_post(this_post)

			#make each element in array unicode
			csvline = map((lambda x: unicode(x)), csvline)

			#join with delimiter
			csvline = weibo_settings.delim.join(csvline)

			#add logline
			csvline += weibo_settings.delim
			csvline += this_log_line

			#write  csvline
			wf.write(csvline + "\n")
Exemplo n.º 3
0
def csvize_deleted_unique(csv_filename,
                          error_code=-1,
                          exclude_error_code=False):

    nowdatetime = weibo_module.get_current_chinatime()

    deleted_post_ids = weibo_module.get_deleted_postids(
        error_code, exclude_error_code)
    num_dead_posts = len(deleted_post_ids)

    #if we're not tracking any posts, get out of there
    if (num_dead_posts <= 0):
        return

    ## OPEN A FILE
    with codecs.open(csv_filename, "wb", "utf-16") as wf:

        #write csv header
        csv_header = weibo_module.get_csv_header()
        wf.write(csv_header + "\n")
        #print csv_header

        #iterate through posts
        for this_post_id in deleted_post_ids:

            this_post = weibo_module.merge_deleted_from_new_old(this_post_id)

            csvline = weibo_module.make_csvline_from_post(this_post)

            csvline = map((lambda x: unicode(x)), csvline)

            #not csv, this is our delimiter now
            csvline = weibo_settings.delim.join(csvline)

            #			#print csvline
            wf.write(csvline + "\n")
Exemplo n.º 4
0
def csvize_repost_timeline(csv_filename,
                           type="deleted",
                           error_code=-1,
                           exclude_error_code=False):

    nowdatetime = weibo_module.get_current_chinatime()

    if type == "deleted":
        query_post_ids = weibo_module.get_deleted_postids(
            error_code, exclude_error_code)
    else:
        query_post_ids = weibo_module.get_all_postids()
        #		query_post_ids = query_post_ids[:10]
        print query_post_ids

    num_query_posts = len(query_post_ids)

    #if we're not tracking any posts, get out of there
    if (num_query_posts <= 0):
        return

    ## OPEN A FILE
    with codecs.open(csv_filename, "wb", "utf-16") as wf:

        #write csv header
        csv_header = weibo_module.get_csv_header()
        wf.write(csv_header + "\n")

        #iterate through posts
        for this_post_id in query_post_ids:

            print "\n==WRITING POST #=========", this_post_id

            # okay first we get the initial post
            this_post = weibo_module.merge_deleted_from_new_old(this_post_id)

            # and then we scan the rest
            this_post_all_logs = weibo_module.get_all_posts(this_post_id)

            # and then we amass a logline -- a csv file inside a csv file!
            this_log_line = []
            for this_log in this_post_all_logs:
                if 'post_repost_count' in this_log and this_log[
                        "post_repost_count"] <> None:
                    this_log_line.append(str(this_log["post_repost_count"]))
                    this_log_line.append(str(this_log["checked_at"]))
            this_log_line = weibo_settings.delim_log.join(this_log_line)

            #get csvline array
            csvline = weibo_module.make_csvline_from_post(this_post)

            #make each element in array unicode
            csvline = map((lambda x: unicode(x)), csvline)

            #join with delimiter
            csvline = weibo_settings.delim.join(csvline)

            #add logline
            csvline += weibo_settings.delim
            csvline += this_log_line

            #write  csvline
            wf.write(csvline + "\n")