def execute(sqlquery, data=None, verbose=1):
# helpfully wrap singletons into a list, as is done for the % operator, because singleton tuples can be hard to understand
if data is not None and "?" in sqlquery and type(data) not in [list, tuple]:
data = [data]
global attachlist
result = jrequest({"maincommand":'sqliteexecute', "sqlquery":sqlquery, "data":data, "attachlist":attachlist})
if "error" in result:
raise databaseexception(result)
if "status" not in result and "keys" not in result:
raise Exception("possible signal timeout: "+str(result))
# list type for second field in message dump
if verbose:
if data == None:
ldata = [ ]
elif type(data) in [tuple, list]:
ldata = [ ifsencode_trunc(v, 50) for v in data ]
elif type(data) == dict:
ldata = [ ifsencode_trunc(v, 50) for v in data.values() ]
else:
ldata = [ str(data) ]
scraperwiki.dumpMessage({'message_type':'sqlitecall', 'command':'sqliteexecute', "val1":sqlquery, "lval2":ldata})
return result
def execute(sqlquery, data=None, verbose=1):
# helpfully wrap singletons into a list, as is done for the % operator, because singleton tuples can be hard to understand
if data is not None and "?" in sqlquery and type(data) not in [list, tuple]:
data = [data]
global attachlist
m = {"maincommand":'sqliteexecute', "sqlquery":sqlquery, "data":data, "attachlist":attachlist}
result = scraperwiki.datastore.make_request(m, attachlist)
if "Error" in result:
print 'Sent ', m
raise databaseexception(result)
if "status" not in result and "keys" not in result:
raise Exception(str(result))
# list type for second field in message dump
if verbose:
if data == None:
ldata = [ ]
elif type(data) in [tuple, list]:
ldata = [ ifsencode_trunc(v, 50) for v in data ]
elif type(data) == dict:
ldata = [ ifsencode_trunc(v, 50) for v in data.values() ]
else:
ldata = [ str(data) ]
scraperwiki.dumpMessage({'message_type':'sqlitecall', 'command':'sqliteexecute', "val1":sqlquery, "lval2":ldata})
return result
def log(message=""):
"""send message to console and the firebox logfile with a piece of the stack trace"""
stack = traceback.extract_stack()
tail = len(stack) >= 3 and ", %s() line %d" % (stack[-3][2], stack[-3][1]) or "" # go 2 levels up if poss
now = datetime.datetime.now()
str_now = now.strftime("%Y-%m-%d %H:%M:%S")
logmessage = "log( %s )\t\t %s() line %d%s : %s" % (str(message), stack[-2][2], stack[-2][1], tail, str_now)
scraperwiki.dumpMessage({"message_type": "console", "content": logmessage})
def httpresponseheader(headerkey, headervalue):
"""Set (certain) HTTP Response Header. For example
scraperwiki.utils.httpresponseheader('Content-Type', 'text/plain')
"""
# Messages of this type are picked up and specially interpreted by
# viewsrpc.rpcexecute
scraperwiki.dumpMessage({"message_type": "httpresponseheader", "headerkey": headerkey, "headervalue": headervalue})
def httpresponseheader(headerkey, headervalue):
'''Set (certain) HTTP Response Header. For example
scraperwiki.utils.httpresponseheader('Content-Type', 'text/plain')
'''
# Messages of this type are picked up and specially interpreted by
# viewsrpc.rpcexecute
scraperwiki.dumpMessage({'message_type': 'httpresponseheader', 'headerkey': headerkey, 'headervalue': headervalue})
def log(message=""):
'''send message to console and the firebox logfile with a piece of the stack trace'''
stack = traceback.extract_stack()
tail = len(stack) >= 3 and ", %s() line %d" % (stack[-3][2], stack[-3][1]) or "" # go 2 levels up if poss
now = datetime.datetime.now()
str_now = now.strftime("%Y-%m-%d %H:%M:%S")
logmessage = "log( %s )\t\t %s() line %d%s : %s" % (str(message), stack[-2][2], stack[-2][1], tail, str_now)
scraperwiki.dumpMessage({'message_type': 'console', 'content': logmessage})
def httpstatuscode(statuscode):
"""Experimental and Internal. Sets the HTTP Status Code to be
*statuscode* which should be an int.
"""
# Messages of this type are picked up and specially interpreted by
# viewsrpc.rpcexecute
scraperwiki.dumpMessage({"message_type": "httpstatuscode", "statuscode": statuscode})
def commit(verbose=1):
result = scraperwiki.datastore.request({"maincommand":'sqlitecommand', "command":"commit"})
if "error" in result:
raise databaseexception(result)
if "status" not in result:
raise Exception("possible signal timeout: "+str(result))
scraperwiki.dumpMessage({'message_type':'data', 'content': pdata})
return result
def httpstatuscode(statuscode):
"""Experimental and Internal. Sets the HTTP Status Code to be
*statuscode* which should be an int.
"""
# Messages of this type are picked up and specially interpreted by
# viewsrpc.rpcexecute
scraperwiki.dumpMessage(
{'message_type': 'httpstatuscode', 'statuscode': statuscode})
def attach(name, asname=None, verbose=1):
global attachlist
attachlist.append({"name":name, "asname":asname})
result = scraperwiki.datastore.request({"maincommand":'sqlitecommand', "command":"attach", "name":name, "asname":asname})
if "error" in result:
raise databaseexception(result)
if "status" not in result:
raise Exception("possible signal timeout: "+str(result))
scraperwiki.dumpMessage({'message_type':'data', 'content': pdata})
return result
def commit(verbose=1):
result = scraperwiki.datastore.request({
"maincommand": 'sqlitecommand',
"command": "commit"
})
if "error" in result:
raise databaseexception(result)
if "status" not in result:
raise Exception("possible signal timeout: " + str(result))
scraperwiki.dumpMessage({'message_type': 'data', 'content': pdata})
return result
def receiveoneline(socket):
sbuffer = [ ]
while True:
srec = socket.recv(1024)
if not srec:
scraperwiki.dumpMessage({'message_type': 'chat', 'message':"socket from dataproxy has unfortunately closed"})
break
ssrec = srec.split("\n") # multiple strings if a "\n" exists
sbuffer.append(ssrec.pop(0))
if ssrec:
break
line = "".join(sbuffer)
return line
def attach(name, asname=None, verbose=1):
global attachlist
attachlist.append({"name": name, "asname": asname})
result = scraperwiki.datastore.request({
"maincommand": 'sqlitecommand',
"command": "attach",
"name": name,
"asname": asname
})
if "error" in result:
raise databaseexception(result)
if "status" not in result:
raise Exception("possible signal timeout: " + str(result))
scraperwiki.dumpMessage({'message_type': 'data', 'content': pdata})
return result
def receiveoneline(socket):
# poor implementation. But previous longstanding version even had a bug on it if records concattenated
global sbuffer
if len(sbuffer) >= 2:
return sbuffer.pop(0)
while True:
srec = socket.recv(1024)
if not srec:
scraperwiki.dumpMessage({'message_type': 'chat', 'message':"socket from dataproxy has unfortunately closed"})
return None
ssrec = srec.split("\n") # multiple strings if a "\n" exists
sbuffer.append(ssrec[0])
if len(ssrec) >= 2:
line = "".join(sbuffer)
sbuffer = ssrec[1:]
return line
def jrequest(request):
request["scrapername"] = scraperwiki.datastore.m_scrapername
request["runid"] = scraperwiki.datastore.m_runid
request["attachables"] = " ".join(scraperwiki.datastore.m_attachables)
request["verify"] = scraperwiki.datastore.m_verification_key
url = 'http://%s:%s/scrapercall' % (scraperwiki.datastore.m_host, scraperwiki.datastore.m_port)
req = urllib2.Request(url, json.dumps(request))
response = urllib2.urlopen(req)
while True:
jres = response.readline()
if not jres:
break
res = json.loads(jres.strip())
if "progresstick" in res:
scraperwiki.dumpMessage({'message_type':'sqlitecall', 'command':'progresstick', "val1":res.get("progresstick"), "lval2":res.get("timeseconds")})
else:
return res
def receiveoneline(socket):
sbuffer = []
while True:
srec = socket.recv(1024)
if not srec:
scraperwiki.dumpMessage({
'message_type':
'chat',
'message':
"socket from dataproxy has unfortunately closed"
})
break
ssrec = srec.split("\n") # multiple strings if a "\n" exists
sbuffer.append(ssrec.pop(0))
if ssrec:
break
line = "".join(sbuffer)
return line
def receiveoneline(socket):
# poor implementation. But previous longstanding version even had a bug on it if records concattenated
global sbuffer
if len(sbuffer) >= 2:
return sbuffer.pop(0)
while True:
srec = socket.recv(1024)
if not srec:
scraperwiki.dumpMessage({
'message_type':
'chat',
'message':
"socket from dataproxy has unfortunately closed"
})
return None
ssrec = srec.split("\n") # multiple strings if a "\n" exists
sbuffer.append(ssrec[0])
if len(ssrec) >= 2:
line = "".join(sbuffer)
sbuffer = ssrec[1:]
return line
def execute(sqlquery, data=None, verbose=1):
# helpfully wrap singletons into a list, as is done for the % operator, because singleton tuples can be hard to understand
if data is not None and "?" in sqlquery and type(data) not in [list, tuple]:
data = [data]
global attachlist
result = scraperwiki.datastore.request({"maincommand":'sqliteexecute', "sqlquery":sqlquery, "data":data, "attachlist":attachlist})
while "progresstick" in result:
if verbose:
scraperwiki.dumpMessage({'message_type':'sqlitecall', 'command':'progresstick', "val1":result.get("progresstick"), "lval2":result.get("timeseconds")})
result = scraperwiki.datastore.request(None) # goes back and waits for next line
while result.get("stillproducing") == "yes":
dataresult = scraperwiki.datastore.request(None) # goes back and waits for next line
if verbose and dataresult.get("stillproducing"):
scraperwiki.dumpMessage({'message_type':'sqlitecall', 'command':'stillproducing', "val1":dataresult.get("producedbatches"), "lval2":dataresult.get("producedrows")})
result["data"].extend(dataresult["data"])
result["stillproducing"] = dataresult.get("stillproducing")
#result["nchunks"] += 1
if "error" in result:
raise databaseexception(result)
if "status" not in result and "keys" not in result:
raise Exception("possible signal timeout: "+str(result))
# list type for second field in message dump
if verbose:
if data == None:
ldata = [ ]
elif type(data) in [tuple, list]:
ldata = [ ifsencode_trunc(v, 50) for v in data ]
elif type(data) == dict:
ldata = [ ifsencode_trunc(v, 50) for v in data.values() ]
else:
ldata = [ str(data) ]
scraperwiki.dumpMessage({'message_type':'sqlitecall', 'command':'sqliteexecute', "val1":sqlquery, "lval2":ldata})
return result
try:
userquery = scraperwiki.utils.GET()
except:
userquery = {}
if userquery.get('dbimg'):
db_data = util.get_table_vals('status', 'name,date_plot',
"name = '%(dbimg)s' or sw_name like '%%%(dbimg)s%%' or ol_name like '%%%(dbimg)s%%' or snac_id = '%(dbimg)s' or authority_id = '%(dbimg)s'" % userquery)
#print db_data
if db_data and len(db_data) == 1 and db_data[0]['date_plot']:
scraperwiki.utils.httpresponseheader("Content-Type", "image/png")
scraperwiki.dumpMessage({"content":db_data[0]['date_plot'], "message_type":"console", "encoding":"base64"})
elif not db_data or (len(db_data) == 1 and not db_data[0]['date_plot']):
print "No date plot image found for '%(dbimg)s'" % userquery
else:
auth_list = util.get_list(db_data, 'name')
print "More than 1 scraper found when searching for date plot '%(dbimg)s': " % userquery + ', '.join(auth_list)
sys.exit()
header = """
<html><head>
<style type="text/css">%s</style>
import scraperwiki
scraperwiki.sqlite.attach('atoc_routeing_guide')
import cgi
import os
args = dict(cgi.parse_qsl(os.getenv("QUERY_STRING", "")))
map = scraperwiki.sqlite.select("data FROM maps WHERE pageno = ?",
(int(args.get('pageno', 1))))[0]['data']
scraperwiki.utils.httpresponseheader("Content-Type", "image/png")
scraperwiki.dumpMessage({
"content": map,
"message_type": "console",
"encoding": "base64"
})
import scraperwiki
scraperwiki.sqlite.attach('atoc_routeing_guide')
import cgi
import os
args = dict(cgi.parse_qsl(os.getenv("QUERY_STRING", "")))
map = scraperwiki.sqlite.select("data FROM maps WHERE pageno = ?",
(int(args.get('pageno', 1))))[0]['data']
scraperwiki.utils.httpresponseheader("Content-Type", "image/png")
scraperwiki.dumpMessage({
"content": map,
"message_type": "console",
"encoding": "base64"
})
import scraperwiki
scraperwiki.sqlite.attach('atoc_routeing_guide')
import cgi
import os
args = dict(cgi.parse_qsl(os.getenv("QUERY_STRING", "")))
map = scraperwiki.sqlite.select("data FROM maps WHERE pageno = ?", (int(args.get('pageno', 1))))[0]['data']
scraperwiki.utils.httpresponseheader("Content-Type", "image/png")
scraperwiki.dumpMessage({"content":map, "message_type":"console", "encoding":"base64"})
import scraperwiki
scraperwiki.sqlite.attach('atoc_routeing_guide')
import cgi
import os
args = dict(cgi.parse_qsl(os.getenv("QUERY_STRING", "")))
map = scraperwiki.sqlite.select("data FROM maps WHERE pageno = ?", (int(args.get('pageno', 1))))[0]['data']
scraperwiki.utils.httpresponseheader("Content-Type", "image/png")
scraperwiki.dumpMessage({"content":map, "message_type":"console", "encoding":"base64"})
# generate the image
fig = plt.figure()
ax = fig.add_subplot(111) # don't know what this does
ax.plot(r.x, r.y, 'o-') # what is o-? [drj: o: circle marker; -: solid line]
labels = ax.get_xticklabels()
for label in labels:
label.set_rotation(30)
ax.set_title(title)
#produce the binary image. the decoding into binary is done out in django
cout = StringIO.StringIO()
plt.savefig(cout, format=format, dpi=dpi)
if format == "svg":
scraperwiki.utils.httpresponseheader("Content-Type", "image/svg+xml")
scraperwiki.dumpMessage({"content":cout.getvalue(), "message_type":"console"})
else:
scraperwiki.utils.httpresponseheader("Content-Type", "image/%s" % format)
scraperwiki.dumpMessage({"content":base64.encodestring(cout.getvalue()), "message_type":"console", "encoding":"base64"})
import numpy
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
import matplotlib.cbook as cbook
import matplotlib.ticker as ticker
import scraperwiki
import sys
import base64
import StringIO
def execute(sqlquery, data=None, verbose=1):
# helpfully wrap singletons into a list, as is done for the % operator, because singleton tuples can be hard to understand
if data is not None and "?" in sqlquery and type(data) not in [
list, tuple
]:
data = [data]
global attachlist
result = scraperwiki.datastore.request({
"maincommand": 'sqliteexecute',
"sqlquery": sqlquery,
"data": data,
"attachlist": attachlist
})
while "progresstick" in result:
if verbose:
scraperwiki.dumpMessage({
'message_type': 'sqlitecall',
'command': 'progresstick',
"val1": result.get("progresstick"),
"lval2": result.get("timeseconds")
})
result = scraperwiki.datastore.request(
None) # goes back and waits for next line
while result.get("stillproducing") == "yes":
dataresult = scraperwiki.datastore.request(
None) # goes back and waits for next line
if verbose and dataresult.get("stillproducing"):
scraperwiki.dumpMessage({
'message_type': 'sqlitecall',
'command': 'stillproducing',
"val1": dataresult.get("producedbatches"),
"lval2": dataresult.get("producedrows")
})
result["data"].extend(dataresult["data"])
result["stillproducing"] = dataresult.get("stillproducing")
#result["nchunks"] += 1
if "error" in result:
raise databaseexception(result)
if "status" not in result and "keys" not in result:
raise Exception("possible signal timeout: " + str(result))
# list type for second field in message dump
if verbose:
if data == None:
ldata = []
elif type(data) in [tuple, list]:
ldata = [ifsencode_trunc(v, 50) for v in data]
elif type(data) == dict:
ldata = [ifsencode_trunc(v, 50) for v in data.values()]
else:
ldata = [str(data)]
scraperwiki.dumpMessage({
'message_type': 'sqlitecall',
'command': 'sqliteexecute',
"val1": sqlquery,
"lval2": ldata
})
return result
import scraperwiki
import urllib
import base64
scraperwiki.utils.httpresponseheader("Content-Type", "image/PNG")
url = 'http://scraperwiki.com/media/images/intro_slides/intro4.png'
pngbin = urllib.urlopen(url).read()
scraperwiki.dumpMessage({"content":base64.encodestring(pngbin), "message_type":"console", "encoding":"base64"})
import scraperwiki
import urllib
import base64
scraperwiki.utils.httpresponseheader("Content-Type", "image/PNG")
url = 'http://scraperwiki.com/media/images/intro_slides/intro4.png'
pngbin = urllib.urlopen(url).read()
scraperwiki.dumpMessage({"content":base64.encodestring(pngbin), "message_type":"console", "encoding":"base64"})
import scraperwiki
import urllib
import base64
scraperwiki.utils.httpresponseheader("Content-Type", "image/PNG")
url = 'http://scraperwiki.com/media/images/intro_slides/intro4.png'
pngbin = urllib.urlopen(url).read()
scraperwiki.dumpMessage({"content":base64.encodestring(pngbin), "message_type":"console", "encoding":"base64"})
import scraperwiki
def flush(self) :
if self.m_text:
scraperwiki.dumpMessage({'message_type': 'console', 'content': self.m_text})
self.m_text = ''
self.m_fd.flush()
# The CPU time limit handler simply throws a Python exception
# so it can be handled cleanly before the hard limit is reached.
def sigXCPU(signum_, frame_):
raise scraperwiki.CPUTimeExceededError("ScraperWiki CPU time exceeded")
signal.signal(signal.SIGXCPU, sigXCPU)
with open(options.script) as codef:
code = codef.read()
try:
import imp
mod = imp.new_module('scraper')
exec code.rstrip() + "\n" in mod.__dict__
except Exception, e:
etb = swutil.getExceptionTraceback(code)
assert etb.get('message_type') == 'exception'
scraperwiki.dumpMessage(etb)
except SystemExit, se:
sys.stdout.flush()
sys.stderr.flush()
# If we do not temporarily yield a slice of the CPU here then the
# launching process will not be able to read from stderr before
# we exit.
import time
time.sleep(0)
raise se
sys.stdout.flush()
sys.stderr.flush()
plt.ylabel("Time (min:sec)")
ax.plot_date(x, winners, 'ro', label="Winning time")
ax.errorbar(x, medians, yerr=spreads, label="Median time (and spread)")
fig.autofmt_xdate()
legend = plt.legend()
format = "png"
imagedata = StringIO()
plt.savefig(imagedata, format=format, dpi=96)
utils.httpresponseheader("Content-Type", "image/%s" % format)
dumpMessage({
"content": imagedata.getvalue().encode("base64"),
"message_type": "console",
"encoding": "base64"
})
import datetime as dt
import dateutil.parser
from StringIO import StringIO
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
import matplotlib as mpl
from matplotlib.ticker import FuncFormatter as ff
import numpy
import scraperwiki
from scraperwiki import dumpMessage, sqlite, utils
# Blank Python
import scraperwiki,urllib2
scraperwiki.utils.httpresponseheader("Content-Type", "image/jpeg")
lazycache=scraperwiki.swimport('lazycache')
html=lazycache.lazycache('http://placekitten.com/g/200/305')
html=lazycache.lazycache('http://placekitten.com/g/200/305')
#print html
html2=urllib2.urlopen('http://placekitten.com/g/200/305').read()
assert html==html2
import base64
scraperwiki.dumpMessage({"content":base64.encodestring(html2), "message_type":"console", "encoding":"base64"})
# Blank Python
import scraperwiki,urllib2
scraperwiki.utils.httpresponseheader("Content-Type", "image/jpeg")
lazycache=scraperwiki.swimport('lazycache')
html=lazycache.lazycache('http://placekitten.com/g/200/305')
html=lazycache.lazycache('http://placekitten.com/g/200/305')
#print html
html2=urllib2.urlopen('http://placekitten.com/g/200/305').read()
assert html==html2
import base64
scraperwiki.dumpMessage({"content":base64.encodestring(html2), "message_type":"console", "encoding":"base64"})
def save(unique_keys, data, table_name="swdata", verbose=2, date=None):
if unique_keys != None and type(unique_keys) not in [list, tuple]:
raise databaseexception({
"error": 'unique_keys must a list or tuple',
"unique_keys_type": str(type(unique_keys))
})
def convdata(unique_keys, scraper_data):
if unique_keys:
for key in unique_keys:
if key not in scraper_data:
return {
"error": 'unique_keys must be a subset of data',
"bad_key": key
}
if scraper_data[key] == None:
return {
"error": 'unique_key value should not be None',
"bad_key": key
}
jdata = {}
for key, value in scraper_data.items():
if not key:
return {"error": 'key must not be blank', "bad_key": key}
if type(key) not in [unicode, str]:
return {"error": 'key must be string type', "bad_key": key}
if not re.match("[a-zA-Z0-9_\- ]+$", key):
return {"error": 'key must be simple text', "bad_key": key}
if type(value) == datetime.date:
value = value.isoformat()
elif type(value) == datetime.datetime:
if value.tzinfo is None:
value = value.isoformat()
else:
value = value.astimezone(pytz.timezone('UTC')).isoformat()
assert "+00:00" in value
value = value.replace("+00:00", "")
elif value == None:
pass
elif isinstance(value, SqliteError):
return {"error": str(value)}
elif type(value) == str:
try:
value = value.decode("utf-8")
except:
return {"error": "Binary strings must be utf-8 encoded"}
elif type(value) not in [int, bool, float, unicode, str]:
value = unicode(value)
jdata[key] = value
return jdata
if type(data) == dict:
rjdata = convdata(unique_keys, data)
if rjdata.get("error"):
raise databaseexception(rjdata)
if date:
rjdata["date"] = date
else:
rjdata = []
for ldata in data:
ljdata = convdata(unique_keys, ldata)
if ljdata.get("error"):
raise databaseexception(ljdata)
rjdata.append(ljdata)
result = scraperwiki.datastore.request({
"maincommand": 'save_sqlite',
"unique_keys": unique_keys,
"data": rjdata,
"swdatatblname": table_name
})
if "error" in result:
raise databaseexception(result)
if verbose >= 2:
pdata = {}
if type(data) == dict:
for key, value in data.items():
pdata[strencode_trunc(key, 50)] = strencode_trunc(value, 50)
elif data:
for key, value in data[0].items():
pdata[strencode_trunc(key, 50)] = strencode_trunc(value, 50)
pdata["number_records"] = "Number Records: %d" % len(data)
scraperwiki.dumpMessage({'message_type': 'data', 'content': pdata})
return result
def save(unique_keys, data, table_name="swdata", verbose=2, date=None):
if unique_keys != None and type(unique_keys) not in [ list, tuple ]:
raise databaseexception({ "error":'unique_keys must a list or tuple', "unique_keys_type":str(type(unique_keys)) })
def convdata(unique_keys, scraper_data):
if unique_keys:
for key in unique_keys:
if key not in scraper_data:
return { "error":'unique_keys must be a subset of data', "bad_key":key }
if scraper_data[key] == None:
return { "error":'unique_key value should not be None', "bad_key":key }
jdata = { }
for key, value in scraper_data.items():
if not key:
return { "error": 'key must not be blank', "bad_key":key }
if type(key) not in [unicode, str]:
return { "error":'key must be string type', "bad_key":key }
if not re.match("[a-zA-Z0-9_\- ]+$", key):
return { "error":'key must be simple text', "bad_key":key }
if type(value) == datetime.date:
value = value.isoformat()
elif type(value) == datetime.datetime:
if value.tzinfo is None:
value = value.isoformat()
else:
value = value.astimezone(pytz.timezone('UTC')).isoformat()
assert "+00:00" in value
value = value.replace("+00:00", "")
elif value == None:
pass
elif isinstance(value, SqliteError):
return {"error": str(value)}
elif type(value) == str:
try:
value = value.decode("utf-8")
except:
return {"error": "Binary strings must be utf-8 encoded"}
elif type(value) not in [int, bool, float, unicode, str]:
value = unicode(value)
jdata[key] = value
return jdata
if type(data) == dict:
rjdata = convdata(unique_keys, data)
if rjdata.get("error"):
raise databaseexception(rjdata)
if date:
rjdata["date"] = date
else:
rjdata = [ ]
for ldata in data:
ljdata = convdata(unique_keys, ldata)
if ljdata.get("error"):
raise databaseexception(ljdata)
rjdata.append(ljdata)
result = scraperwiki.datastore.request({"maincommand":'save_sqlite', "unique_keys":unique_keys, "data":rjdata, "swdatatblname":table_name})
if "error" in result:
raise databaseexception(result)
if verbose >= 2:
pdata = {}
if type(data) == dict:
for key, value in data.items():
pdata[strencode_trunc(key, 50)] = strencode_trunc(value, 50)
elif data:
for key, value in data[0].items():
pdata[strencode_trunc(key, 50)] = strencode_trunc(value, 50)
pdata["number_records"] = "Number Records: %d" % len(data)
scraperwiki.dumpMessage({'message_type':'data', 'content': pdata})
return result
plt.xlabel("Date of Parkrun")
plt.ylabel("Time (min:sec)")
ax.plot_date(x, winners, 'ro', label="Winning time")
ax.errorbar(x, medians, yerr=spreads, label="Median time (and spread)")
fig.autofmt_xdate()
legend = plt.legend()
format = "png"
imagedata = StringIO()
plt.savefig(imagedata, format=format, dpi=96)
utils.httpresponseheader("Content-Type", "image/%s" % format)
dumpMessage({"content": imagedata.getvalue().encode("base64"), "message_type": "console", "encoding":"base64"})
import datetime as dt
import dateutil.parser
from StringIO import StringIO
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
import matplotlib as mpl
from matplotlib.ticker import FuncFormatter as ff
import numpy
import scraperwiki
from scraperwiki import dumpMessage, sqlite, utils
# generate the image
fig = plt.figure()
ax = fig.add_subplot(111) # don't know what this does
ax.plot(r.x, r.y, 'o-') # what is o-? [drj: o: circle marker; -: solid line]
labels = ax.get_xticklabels()
for label in labels:
label.set_rotation(30)
ax.set_title(title)
#produce the binary image. the decoding into binary is done out in django
cout = StringIO.StringIO()
plt.savefig(cout, format=format, dpi=dpi)
if format == "svg":
scraperwiki.utils.httpresponseheader("Content-Type", "image/svg+xml")
scraperwiki.dumpMessage({
"content": cout.getvalue(),
"message_type": "console"
})
else:
scraperwiki.utils.httpresponseheader("Content-Type", "image/%s" % format)
scraperwiki.dumpMessage({
"content": base64.encodestring(cout.getvalue()),
"message_type": "console",
"encoding": "base64"
})
import numpy
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
import matplotlib.cbook as cbook
import matplotlib.ticker as ticker
from matplotlib.pyplot import show
fig, ax = plt.subplots(1,1)
dendrogram=cluster.hierarchy.dendrogram(linkageMatrix)
ax.plot(dendrogram)
# Save the generated figure to a file
stream = cStringIO.StringIO()
fig.savefig(stream,format='png')
# Some http stuff..
scraperwiki.utils.httpresponseheader("Content-Type", "image/png")
scraperwiki.dumpMessage({"content":base64.encodestring(stream.getvalue()), "message_type":"console", "encoding":"base64"})# Blank Python
import scraperwiki, random, math
import numpy as np
import scipy as sp
import pylab
import cStringIO
from scipy import spatial
from scipy import cluster
sourcescraper = 'clustertenderpy'
scraperwiki.sqlite.attach('clustertenderpy')
lm = scraperwiki.sqlite.select("* from swdata")
print lm[0]['linkageMatrix']
linkageMatrix = np.array(lm[0]['linkageMatrix'],dtype=np.float64)