def dump_submission_info(sub_path):

	start = time.time()

	# Log useful info
	num_submissions = 7333071

	counter = 0
	num_dumps = 0
	num_urls_found = 0
	num_xposts_found = 0

	# Will store chunks of data to write and cleared at certain intervals
	# to avoid too much I/O.
	data_buffer = []

	# Collect submissions
	with open(sub_path) as submission_file:
		for submission_raw in submission_file:

			submission = json.loads(submission_raw)

			# Get year, month, day
			tmstp = submission.get("created")
			if type(tmstp) == int:
				date = datetime.fromtimestamp(tmstp)
			elif type(tmstp) == datetime:
				date = tmstp
			else:
				print "WARNING::::different date format found", type(tmstp), tmstp
				continue

			try:
				sid = submission.get("subreddit_id").values()[0]
				subreddit = subreddit_ids[sid]
			except KeyError:
				print "WARNING::::subreddit id not found:", submission
				continue

			year = str(date.year)
			month = str(date.month)
			day = str(date.day)

			path = os.path.join(subreddit[0].lower(), subreddit, year, month, day)
			make_dir(path)

			# Extract relevant pieces of information
			submission_id = str(submission.get("_id").values()[0])
			submission_title = submission.get("submission_title") or ''
			submission_text = submission.get("submission_text") or ''
			submission_prawid = submission.get("praw_id") or '-1'

			# Create information to write to file
			submission_info = submission_id + "\t" + submission_prawid + "\t" + submission_title + " " + submission_text
			url_info = submission.get("url") or ''
			xpost_info = utils.extract_subreddit_xpost(submission_title) or utils.get_internal_link(url_info)

			# Create data object and store in buffer
			data = {'submissions': (submission_info + '\n').encode("utf8")}
			if url_info:
				data['urls'] = (submission_id + '\t' + url_info + '\n').encode("utf8")
			if xpost_info:
				data['xposts'] = (submission_id + '\t' + xpost_info + '\n').encode("utf8")

			data_buffer.append((path, data))

			# Write data and clear buffer every <chunk_size> submissions
			if len(data_buffer) >= chunk_size:

				filehandlers = {}
				for path, data in data_buffer:
					try:
						# See if file handlers have already been initialized
						fhs = filehandlers[path]
						for fh_type, fh in fhs.iteritems():
							if fh_type in data:
								fh.write(data[fh_type])
					except KeyError:
						# Otherwise create the file handlers and write to them
						fhs = {'submissions': open_file(path, 'submissions.txt')}
						fhs['submissions'].write(data['submissions'])
						if data.get('urls'):
							fhs['urls'] = open_file(path, 'urls.txt')
							fhs['urls'].write(data['urls'])
						if data.get('xposts'):
							fhs['xposts'] = open_file(path, 'xposts.txt')
							fhs['xposts'].write(data['xposts'])
						filehandlers[path] = fhs

				# Now close all file handlers
				for path, fhs in filehandlers.iteritems():
					for fh_type, fh in fhs.iteritems():
						fh.close()

				# Clear buffer
				data_buffer = []
				num_dumps += 1

			if counter % log_interval == 0:
				print "Progress:", counter, "submissions dumped out of", num_submissions, (counter / float(num_submissions)) * 100, "%"
				print "\tNumber of URLs found so far:", num_urls_found
				print "\tNumber of xposts found so far:", num_xposts_found
				print "\tNumber of dumps to file so far:", num_dumps
				print "\tTime spent:", (time.time() - start) / 60.0, "minutes"
				print ""
			counter += 1
					subdomains[subdomain] = {subreddit: 1}

				num_subdomains += 1

		# Attempt to extrapolate domain
		except AttributeError:
			domain = submission.get("domain")
			commenturl = submission.get("comment_url")
			url = submission.get("url")

			# Try looking at praw's domain extraction
			if domain and domain.startswith("self."):
				subreddit = domain.split(".")[1]
				num_extrapolated += 1
			elif domain == "reddit.com" and url:
				subreddit = utils.get_internal_link(url)
				num_extrapolated += 1

			# Otherwise look at comment url for hints
			elif commenturl:
				subreddit = utils.get_internal_link(commenturl)
				num_extrapolated += 1

			else:
				subreddit = None
				num_not_found += 1

			# If domain was found, can't infer subdomain, just try to find
			# xpost in title
			xpost = utils.extract_subreddit_xpost(submission.get("submission_title"))
def dump_wayback_submission_info(sub_path):
	start = time.time()

	# Log useful info
	num_submissions = 18609213
	counter = 0
	num_dumps = 0
	num_urls_found = 0
	num_xposts_found = 0
	num_domains_found = 0
	num_no_date = 0

	# Will store chunks of data to write, cleared at certain intervals
	# to avoid too much I/O.
	data_buffer = {}

	# Will store chunks of data specifically for comment file, used to
	# scrape later comments.
	comment_file = open(os.path.join('..', 'data', 'reddit', 'wayback_comments_to_scrape.txt'), 'w')
	comment_buffer = []

	# Collect submissions
	with open(sub_path) as submission_file:
		for submission_raw in submission_file:

			submission = json.loads(submission_raw)

			# Get year, month, day
			try:
				tmstp = submission.get("created").values()[0]
			except AttributeError:
				print "WARNING::::no timestamp?", submission.get("created")
				continue
			if type(tmstp) == int:
				date = datetime.fromtimestamp(tmstp / 1000)
			elif type(tmstp) == datetime:
				date = tmstp
			else:
				num_no_date += 1
				continue

			if submission.get("comment_url"):
				subreddit = utils.get_internal_link(submission.get("comment_url"))
				if not subreddit:
					continue
			# Ignore ones that don't have associated subreddit
			else:
				continue

			year = str(date.year)
			month = str(date.month)
			day = str(date.day)

			path = os.path.join(subreddit[0].lower(), subreddit, year, month, day)
			make_dir(path)

			# Extract relevant pieces of information
			submission_id = str(submission.get("_id").values()[0])
			submission_title = submission.get("submission_title") or ''
			submission_prawid = submission.get("reddit_id") or '-1'

			# Create information to write to file
			submission_info = submission_id + "\t" + submission_prawid + "\t" + submission_title
			url_info = submission.get("url") or ''
			domain_info = submission.get("domain")

			# Get potential xpost info
			xpost_info = utils.extract_subreddit_xpost(submission_title)
			if not xpost_info:
				if domain_info.startswith("self."):
					internal = domain_info.split(".")[1]
					if internal != subreddit:
						xpost_info = internal

			# Create data object and store in buffer
			data = {'submissions': (submission_info + '\n').encode("utf8")}
			if url_info:
				data['urls'] = (submission_id + '\t' + url_info + '\n').encode("utf8")
				num_urls_found += 1
			if xpost_info:
				data['xposts'] = (submission_id + '\t' + xpost_info + '\n').encode("utf8")
				num_xposts_found += 1
			if domain_info:
				data['domains'] = (submission_id + '\t' + domain_info + '\n').encode("utf8")
				num_domains_found += 1

			try:
				data_buffer[path].append(data)
			except KeyError:
				data_buffer[path] = [data]

			# Store information about comments for later scraping
			if submission_prawid != '-1':
				comment_buffer.append((path, submission_prawid))

			# Write data and clear buffer every <chunk_size> submissions
			if len(data_buffer) >= chunk_size:

				# Only ever open 5 files at a time
				for path, data in data_buffer.iteritems():
					# Open files associated with this path
					filehandlers = {'submissions': open_file(path, 'submissions.txt'),
									'urls': open_file(path, 'urls.txt'),
									'xposts': open_file(path, 'xposts.txt'),
									'domains': open_file(path, 'domains.txt')}

					# Write data to files
					for data_item in data:
						if data_item.get('urls'):
							filehandlers['urls'].write(data_item['urls'])
						if data_item.get('xposts'):
							filehandlers['xposts'].write(data_item['xposts'])
						if data_item.get('domains'):
							filehandlers['domains'].write(data_item['domains'])

					# Now close files
					filehandlers['submissions'].close()
					filehandlers['urls'].close()
					filehandlers['xposts'].close()
					filehandlers['domains'].close()

				# Update comment file
				for path, pid in comment_buffer:
					comment_file.write(path + '\t' + pid + '\n')

				# Clear buffer
				data_buffer = {}
				comment_buffer = []
				num_dumps += 1

			counter += 1
			if counter % log_interval == 0:
				print "Wayback Progress:", counter, "submissions dumped out of", num_submissions, (counter / float(num_submissions)) * 100, "%"
				print "\tNumber of URLs found so far:", num_urls_found
				print "\tNumber of xposts found so far:", num_xposts_found
				print "\tNumber of dumps to file so far:", num_dumps
				print "\tNumber with no date:", num_no_date
				time_spent = (time.time() - start) / 60.0
				print "\tTime spent:", time_spent, "minutes"
				print "\tExpected time remaining:", (num_submissions - counter) / (counter / time_spent)
				print ""

	comment_file.close()