import sqlite3
import re, json
from utils.pyutils import database
from os import environ
from time import time
createsql = """
create table stories(
storyid integer primary key,
json text,
updated integer,
timezone text
)"""
outputfn = '%s/fimfic_story_api.sqlite3' % environ['datadir']
db = database.sqlite3db(outputfn, createsql)
timezone = environ['timezone']
client = httplib.HTTPSConnection('www.fimfiction.net')
def updatefic(storyid):
requesttime = int(time())
client.request('GET', '/api/story.php?story=%s' % storyid)
response = client.getresponse().read()
if json.loads(response).get('error') != None:
return
db.execute('insert or replace into stories values(?,?,?,?)',
import json, re
createsql = """
create table visited(
postid integer primary key
);
create table postsforword(
word text,
postid integer,
foreign key(postid) references visited(postid)
);
"""
inputfn = '%s/mootchan_mlp_thread.sqlite3' % environ['datadir']
indb = database.sqlite3db(inputfn, None)
outputfn = '%s/mlp_post_words.sqlite3' % environ['datadir']
outdb = database.sqlite3db(outputfn, createsql)
known = outdb.execute('select postid from visited')
known = map(lambda x: x[0], known)
known = set(known)
posts = indb.execute('select postid,json from posts')
for p in posts:
postid = p[0]
if postid in known:
continue
storylink text,
title text,
description text,
likes integer,
dislikes integer,
comments integer,
views integer,
wordcount integer,
author text,
characters text,
categories text
);
"""
inputapifn = '%s/fimfic_story_api.sqlite3' % environ['datadir']
inapidb = database.sqlite3db(inputapifn, None)
inputsearchfn = '%s/fimfic_search_results.sqlite3' % environ['datadir']
insearchdb = database.sqlite3db(inputsearchfn, None)
outputfn = '%s/fimfic_story_summary.sqlite3' % environ['datadir']
remove(outputfn)
outdb = database.sqlite3db(outputfn, createsql)
apidata = dict(inapidb.execute('select storyid, json from stories'))
searchdata = dict(insearchdb.execute('select storyid, html from stories'))
for storyid in searchdata.keys():
if not apidata.has_key(storyid):
continue
#!/usr/bin/python
from utils import pyutils
from utils.pyutils import database
from os import environ
from random import random
import json
inlocationfn = '%s/flatmanifold.cpickle' % environ['datadir']
storyindexes, locations = database.cpickleload(inlocationfn)
storyids = dict(zip(storyindexes.values(), storyindexes.keys()))
instorydatafn = '%s/fimfic_story_summary.sqlite3' % environ['datadir']
storydata = database.sqlite3db(instorydatafn, None)
storydata = dict([(x[0], list(x)) for x in storydata.execute('select * from stories')])
bootstrapcss = open('%s/bootstrap.css' % environ['extdir']).read()
bootstrapjs = open('%s/bootstrap.js' % environ['extdir']).read()
jquery = open('%s/jquery.js' % environ['extdir']).read()
highchartsjs = open('%s/highcharts.js' % environ['extdir']).read()
jsdata = []
dimensions = locations.max(axis=0) + 0.0
for i, location in enumerate(locations):
storyid = storyids[i]
pointdata = {
'x': (location[0] + 2*random() - 1)/dimensions[0],
'y': (location[1] + 2*random() - 1)/dimensions[1],
'p': [storydata[storyid]]
}