def setUp(self):
try:
os.remove(DB)
except OSError:
pass
self.dt = dumptruck.DumpTruck(dbname=DB, adapt_and_convert=False)
def get_db():
"""
Gets Database if it has not already been connected.
"""
dt = getattr(g, '_database', None)
if dt is None and os.path.isfile(DATABASE):
dt = g._database = dumptruck.DumpTruck(DATABASE,
adapt_and_convert=False)
return dt
def setUp(self):
try:
os.remove(DB)
except OSError:
pass
if not os.path.isdir(JACK):
os.makedirs(JACK)
self.dt = dumptruck.DumpTruck(dbname=DB)
def save(data):
dt = dumptruck.DumpTruck(dbname="scraperwiki.sqlite")
for table_name, rows in data.iteritems():
dt.insert(rows, table_name)
dt.create_index(['type'], 'tracks')
dt.create_index(['date_added'], 'tracks')
dt.create_index(['date_modified'], 'tracks')
dt.create_index(['date_last_played'], 'tracks')
dt.create_index(['play_count'], 'tracks')
dt.create_index(['rating'], 'tracks')
def switch_from_dt_to_peewee():
if os.path.isfile(os.path.expanduser("~/.bakthat.dt")):
import dumptruck
import time
dt = dumptruck.DumpTruck(dbname=os.path.expanduser("~/.bakthat.dt"), vars_table="config")
for backup in dt.dump("backups"):
try:
backup["tags"] = " ".join(backup.get("tags", []))
Backups.upsert(**backup)
time.sleep(0.1)
except Exception, exc:
print exc
for ivt in dt.dump("inventory"):
try:
Inventory.create(filename=ivt["filename"],
archive_id=ivt["archive_id"])
except Exception, exc:
print exc
def _q(self, dbname, p, output_check=None, code_check=None):
"""For testing box.json database file configuration. Runs
some query on the database *dbname*, using *p* as a parameter.
Normally the output will be inspected to see if it contains *p*
but if *output_check* is specified, then that
value will be checked for in the output instead.
*code_check* is used to check the status code.
"""
try:
os.remove(dbname)
except OSError:
pass
if not os.path.isdir(JACK):
os.makedirs(JACK)
dbname = os.path.join(JACK, os.path.expanduser(dbname))
dt = dumptruck.DumpTruck(dbname)
dt.drop('bacon', if_exists=True)
dt.insert({'p': p}, 'bacon')
os.environ['QUERY_STRING'] = 'q=SELECT+p+FROM+bacon&box=jack-in-a'
os.environ['REQUEST_METHOD'] = 'GET'
http = sql_helper()
if output_check:
self.assertIn(unicode(output_check), http)
if code_check:
self.assertIn(unicode(code_check), http.split('\n')[0])
# The body should be valid JSON
body = '\n\n'.join(http.split('\n\n')[1:])
json.loads(body)
try:
os.remove(dbname)
except OSError:
pass
import datetime
now = datetime.datetime.now()
parser = optparse.OptionParser()
(options, args) = parser.parse_args()
# Settings
user = args[0] if args else 'zarino'
api_key = '12b5aaf2b0f27b4b9402b391c956a88a'
per_page = 200
tracks_to_scrape = 0
tracks_scraped = 0
dt = dumptruck.DumpTruck(dbname="scraperwiki.sqlite")
def main():
status("%s - Scraping %s's history..." %
(now.strftime("%Y-%m-%d %H:%M"), user))
setUpDatabase()
getInfo()
getRecentTracks()
def status(message, type='ok'):
requests.post("https://x.scraperwiki.com/api/status",
data={
'type': type,
'message': message
#!/usr/bin/env python2
import os
import json
import dumptruck
dt = dumptruck.DumpTruck(dbname='/tmp/nyc-open-data.db')
DIR = os.path.join('downloads', 'views')
views = filter(lambda view: '.json' == view[-5:], os.listdir(DIR))
for view in views:
f = open(os.path.join(DIR, view))
data = json.load(f)
f.close()
dt.insert([{
'column': column['name'],
'dataset': data['id']
} for column in data['columns']], 'c')
#!/usr/bin/env python2
import datetime
import sys, os
import dumptruck
from boto.s3.connection import S3Connection
from boto.s3.key import Key
conn = S3Connection(sys.argv[1], sys.argv[2])
b = conn.get_bucket('www.deadpeoplebornonmybirthday.com')
#os.system('rm /tmp/s3_copy_log.sqlite')
dt = dumptruck.DumpTruck(dbname='/tmp/s3_copy_log.sqlite')
HEADERS = {
'Content-Type': 'application/json',
'Content-Encoding': 'gzip',
'Cache-Control': 'max-age=1234567890',
}
def upload(filename, birthday):
# Test that it's a date.
datetime.date(*map(int, birthday.replace('18', '19').split('-')))
k = Key(b)
k.key = os.path.join('data', 'bornon', birthday + '.json.gz')
k.storage_class = 'REDUCED_REDUNDANCY'
f = open(filename)