def test_base_36_conversion(self):
self.assertEqual(helpers.convert_numeric_id_to_id36(295), '87')
self.assertEqual(helpers.convert_id36_to_numeric_id('87'), 295)
self.assertEqual(helpers.convert_numeric_id_to_id36(275492), '5wkk')
self.assertEqual(helpers.convert_id36_to_numeric_id('5wkk'), 275492)
self.assertRaises(TypeError, helpers.convert_numeric_id_to_id36)
self.assertRaises(ValueError, helpers.convert_numeric_id_to_id36, '1')
self.assertRaises(ValueError, helpers.convert_numeric_id_to_id36, -1)
self.assertRaises(TypeError, helpers.convert_id36_to_numeric_id)
self.assertRaises(ValueError, helpers.convert_id36_to_numeric_id,
't3_87')
self.assertRaises(ValueError, helpers.convert_id36_to_numeric_id, 87)
def test_base_36_conversion(self):
self.assertEqual(helpers.convert_numeric_id_to_id36(295), '87')
self.assertEqual(helpers.convert_id36_to_numeric_id('87'), 295)
self.assertEqual(helpers.convert_numeric_id_to_id36(275492), '5wkk')
self.assertEqual(helpers.convert_id36_to_numeric_id('5wkk'), 275492)
self.assertRaises(TypeError, helpers.convert_numeric_id_to_id36)
self.assertRaises(ValueError, helpers.convert_numeric_id_to_id36, '1')
self.assertRaises(ValueError, helpers.convert_numeric_id_to_id36, -1)
self.assertRaises(TypeError, helpers.convert_id36_to_numeric_id)
self.assertRaises(ValueError, helpers.convert_id36_to_numeric_id,
't3_87')
self.assertRaises(ValueError, helpers.convert_id36_to_numeric_id, 87)
df.dropna(inplace=True)
df["created_utc"] = df.created.apply(
lambda x: datetime.utcfromtimestamp(x))
df["crawled_utc"] = df.crawled.apply(
lambda x: datetime.utcfromtimestamp(x))
print "crawl started:", df.crawled_utc.min()
print "crawl ended:", df.crawled_utc.max()
print "crawl lasted for:", df.crawled_utc.max() - df.crawled_utc.min()
print "comment ids collected:", len(df)
print "avg comment rate: %2.4f comments per second" % (
len(df) /
(df.crawled_utc.max() - df.crawled_utc.min()).total_seconds())
# plt.show()
# print df.id.apply(lambda x: convert_id36_to_numeric_id(str(x[3:])))
df["num_ids"] = df.id.apply(
lambda x: convert_id36_to_numeric_id(str(x[3:])))
num_ids = df["num_ids"]
# print df.describe()
df.set_index("num_ids", inplace=True)
df["crawl_lag"] = df.crawled - df.created
ax = df.crawl_lag.plot(kind="area", zorder=300, alpha=.8, lw=0)
# plt.savefig("crawl lag in seconds.pdf")
# print "missing ids:", num_ids.max()-num_ids.min() - len(num_ids), "/", len(num_ids)
missing_ids = pd.Series(
sorted(set(range(num_ids.min(),
num_ids.max() + 1)) - set(num_ids))
) #.apply(lambda x: u"t1_"+convert_numeric_id_to_id36(int(x)))
# print df.index.min()
for i in missing_ids:
plt.axvline(i, color='k', alpha=.2, zorder=0)
# pd.DataFrame(data = zip(missing_ids.values, np.ones_like(missing_ids.values))).plot(ls="^")
