コード例 #1
0
def execute(sqlquery, data=None, verbose=1):
        # helpfully wrap singletons into a list, as is done for the % operator, because singleton tuples can be hard to understand
    if data is not None and "?" in sqlquery and type(data) not in [list, tuple]:
        data = [data]
        
    global attachlist
    result = jrequest({"maincommand":'sqliteexecute', "sqlquery":sqlquery, "data":data, "attachlist":attachlist})
    if "error" in result:
        raise databaseexception(result)
    if "status" not in result and "keys" not in result:
        raise Exception("possible signal timeout: "+str(result))
    
    # list type for second field in message dump
    if verbose:
        if data == None:
            ldata = [ ]
        elif type(data) in [tuple, list]:
            ldata = [ ifsencode_trunc(v, 50)  for v in data ]
        elif type(data) == dict:
            ldata = [ ifsencode_trunc(v, 50)  for v in data.values() ]
        else:
            ldata = [ str(data) ]
        scraperwiki.dumpMessage({'message_type':'sqlitecall', 'command':'sqliteexecute', "val1":sqlquery, "lval2":ldata})
    
    return result
コード例 #2
0
def execute(sqlquery, data=None, verbose=1):
        # helpfully wrap singletons into a list, as is done for the % operator, because singleton tuples can be hard to understand
    if data is not None and "?" in sqlquery and type(data) not in [list, tuple]:
        data = [data]
        
    global attachlist
    
    m = {"maincommand":'sqliteexecute', "sqlquery":sqlquery, "data":data, "attachlist":attachlist}
    result = scraperwiki.datastore.make_request(m, attachlist)
    if "Error" in result:
        print 'Sent ', m
        raise databaseexception(result)
    if "status" not in result and "keys" not in result:
        raise Exception(str(result))
    
    # list type for second field in message dump
    if verbose:
        if data == None:
            ldata = [ ]
        elif type(data) in [tuple, list]:
            ldata = [ ifsencode_trunc(v, 50)  for v in data ]
        elif type(data) == dict:
            ldata = [ ifsencode_trunc(v, 50)  for v in data.values() ]
        else:
            ldata = [ str(data) ]
        scraperwiki.dumpMessage({'message_type':'sqlitecall', 'command':'sqliteexecute', "val1":sqlquery, "lval2":ldata})
    
    return result
コード例 #3
0
def log(message=""):
    """send message to console and the firebox logfile with a piece of the stack trace"""
    stack = traceback.extract_stack()
    tail = len(stack) >= 3 and ", %s() line %d" % (stack[-3][2], stack[-3][1]) or ""  # go 2 levels up if poss
    now = datetime.datetime.now()
    str_now = now.strftime("%Y-%m-%d %H:%M:%S")
    logmessage = "log( %s )\t\t %s() line %d%s : %s" % (str(message), stack[-2][2], stack[-2][1], tail, str_now)
    scraperwiki.dumpMessage({"message_type": "console", "content": logmessage})
コード例 #4
0
def httpresponseheader(headerkey, headervalue):
    """Set (certain) HTTP Response Header.  For example
    scraperwiki.utils.httpresponseheader('Content-Type', 'text/plain')
    """

    # Messages of this type are picked up and specially interpreted by
    # viewsrpc.rpcexecute
    scraperwiki.dumpMessage({"message_type": "httpresponseheader", "headerkey": headerkey, "headervalue": headervalue})
コード例 #5
0
def httpresponseheader(headerkey, headervalue):
    '''Set (certain) HTTP Response Header.  For example
    scraperwiki.utils.httpresponseheader('Content-Type', 'text/plain')
    '''

    # Messages of this type are picked up and specially interpreted by
    # viewsrpc.rpcexecute
    scraperwiki.dumpMessage({'message_type': 'httpresponseheader', 'headerkey': headerkey, 'headervalue': headervalue})
コード例 #6
0
def log(message=""):
    '''send message to console and the firebox logfile with a piece of the stack trace'''
    stack = traceback.extract_stack()
    tail = len(stack) >= 3 and ", %s() line %d" % (stack[-3][2], stack[-3][1]) or ""  # go 2 levels up if poss
    now = datetime.datetime.now()
    str_now = now.strftime("%Y-%m-%d %H:%M:%S")
    logmessage = "log( %s )\t\t %s() line %d%s : %s" % (str(message), stack[-2][2], stack[-2][1], tail, str_now)
    scraperwiki.dumpMessage({'message_type': 'console', 'content': logmessage})
コード例 #7
0
def httpstatuscode(statuscode):
    """Experimental and Internal.  Sets the HTTP Status Code to be
    *statuscode* which should be an int.
    """

    # Messages of this type are picked up and specially interpreted by
    # viewsrpc.rpcexecute
    scraperwiki.dumpMessage({"message_type": "httpstatuscode", "statuscode": statuscode})
コード例 #8
0
def commit(verbose=1):
    result = scraperwiki.datastore.request({"maincommand":'sqlitecommand', "command":"commit"})
    if "error" in result:
        raise databaseexception(result)
    if "status" not in result:
        raise Exception("possible signal timeout: "+str(result))
        scraperwiki.dumpMessage({'message_type':'data', 'content': pdata})
    return result    
コード例 #9
0
def httpstatuscode(statuscode):
    """Experimental and Internal.  Sets the HTTP Status Code to be
    *statuscode* which should be an int.
    """

    # Messages of this type are picked up and specially interpreted by
    # viewsrpc.rpcexecute
    scraperwiki.dumpMessage(
      {'message_type': 'httpstatuscode', 'statuscode': statuscode})
コード例 #10
0
def attach(name, asname=None, verbose=1):
    global attachlist
    attachlist.append({"name":name, "asname":asname})
    result = scraperwiki.datastore.request({"maincommand":'sqlitecommand', "command":"attach", "name":name, "asname":asname})
    if "error" in result:
        raise databaseexception(result)
    if "status" not in result:
        raise Exception("possible signal timeout: "+str(result))
        scraperwiki.dumpMessage({'message_type':'data', 'content': pdata})
    return result
コード例 #11
0
def commit(verbose=1):
    result = scraperwiki.datastore.request({
        "maincommand": 'sqlitecommand',
        "command": "commit"
    })
    if "error" in result:
        raise databaseexception(result)
    if "status" not in result:
        raise Exception("possible signal timeout: " + str(result))
        scraperwiki.dumpMessage({'message_type': 'data', 'content': pdata})
    return result
コード例 #12
0
def receiveoneline(socket):
    sbuffer = [ ]
    while True:
        srec = socket.recv(1024)
        if not srec:
            scraperwiki.dumpMessage({'message_type': 'chat', 'message':"socket from dataproxy has unfortunately closed"})
            break
        ssrec = srec.split("\n")  # multiple strings if a "\n" exists
        sbuffer.append(ssrec.pop(0))
        if ssrec:
            break
    line = "".join(sbuffer)
    return line
コード例 #13
0
def attach(name, asname=None, verbose=1):
    global attachlist
    attachlist.append({"name": name, "asname": asname})
    result = scraperwiki.datastore.request({
        "maincommand": 'sqlitecommand',
        "command": "attach",
        "name": name,
        "asname": asname
    })
    if "error" in result:
        raise databaseexception(result)
    if "status" not in result:
        raise Exception("possible signal timeout: " + str(result))
        scraperwiki.dumpMessage({'message_type': 'data', 'content': pdata})
    return result
コード例 #14
0
def receiveoneline(socket):
    # poor implementation.  But previous longstanding version even had a bug on it if records concattenated
    global sbuffer
    if len(sbuffer) >= 2:
        return sbuffer.pop(0)
    while True:
        srec = socket.recv(1024)
        if not srec:
            scraperwiki.dumpMessage({'message_type': 'chat', 'message':"socket from dataproxy has unfortunately closed"})
            return None
        ssrec = srec.split("\n")  # multiple strings if a "\n" exists
        sbuffer.append(ssrec[0])
        if len(ssrec) >= 2:
            line = "".join(sbuffer)
            sbuffer = ssrec[1:]
            return line
コード例 #15
0
def jrequest(request):
    request["scrapername"] = scraperwiki.datastore.m_scrapername
    request["runid"] = scraperwiki.datastore.m_runid
    request["attachables"] = " ".join(scraperwiki.datastore.m_attachables)
    request["verify"] = scraperwiki.datastore.m_verification_key
    url = 'http://%s:%s/scrapercall' % (scraperwiki.datastore.m_host, scraperwiki.datastore.m_port)
    req = urllib2.Request(url, json.dumps(request))
    response = urllib2.urlopen(req)
    while True:
        jres = response.readline()
        if not jres:
            break
        res = json.loads(jres.strip())
        if "progresstick" in res:
            scraperwiki.dumpMessage({'message_type':'sqlitecall', 'command':'progresstick', "val1":res.get("progresstick"), "lval2":res.get("timeseconds")})
        else:
            return res
コード例 #16
0
def receiveoneline(socket):
    sbuffer = []
    while True:
        srec = socket.recv(1024)
        if not srec:
            scraperwiki.dumpMessage({
                'message_type':
                'chat',
                'message':
                "socket from dataproxy has unfortunately closed"
            })
            break
        ssrec = srec.split("\n")  # multiple strings if a "\n" exists
        sbuffer.append(ssrec.pop(0))
        if ssrec:
            break
    line = "".join(sbuffer)
    return line
コード例 #17
0
def receiveoneline(socket):
    # poor implementation.  But previous longstanding version even had a bug on it if records concattenated
    global sbuffer
    if len(sbuffer) >= 2:
        return sbuffer.pop(0)
    while True:
        srec = socket.recv(1024)
        if not srec:
            scraperwiki.dumpMessage({
                'message_type':
                'chat',
                'message':
                "socket from dataproxy has unfortunately closed"
            })
            return None
        ssrec = srec.split("\n")  # multiple strings if a "\n" exists
        sbuffer.append(ssrec[0])
        if len(ssrec) >= 2:
            line = "".join(sbuffer)
            sbuffer = ssrec[1:]
            return line
コード例 #18
0
def execute(sqlquery, data=None, verbose=1):
        # helpfully wrap singletons into a list, as is done for the % operator, because singleton tuples can be hard to understand
    if data is not None and "?" in sqlquery and type(data) not in [list, tuple]:
        data = [data]
        
    global attachlist
    result = scraperwiki.datastore.request({"maincommand":'sqliteexecute', "sqlquery":sqlquery, "data":data, "attachlist":attachlist})
    while "progresstick" in result:
        if verbose:
            scraperwiki.dumpMessage({'message_type':'sqlitecall', 'command':'progresstick', "val1":result.get("progresstick"), "lval2":result.get("timeseconds")})
        result = scraperwiki.datastore.request(None)  # goes back and waits for next line

    while result.get("stillproducing") == "yes":
        dataresult = scraperwiki.datastore.request(None)  # goes back and waits for next line
        if verbose and dataresult.get("stillproducing"):
            scraperwiki.dumpMessage({'message_type':'sqlitecall', 'command':'stillproducing', "val1":dataresult.get("producedbatches"), "lval2":dataresult.get("producedrows")})
        result["data"].extend(dataresult["data"])
        result["stillproducing"] = dataresult.get("stillproducing")
        #result["nchunks"] += 1
        
    if "error" in result:
        raise databaseexception(result)
    if "status" not in result and "keys" not in result:
        raise Exception("possible signal timeout: "+str(result))
    
    # list type for second field in message dump
    if verbose:
        if data == None:
            ldata = [ ]
        elif type(data) in [tuple, list]:
            ldata = [ ifsencode_trunc(v, 50)  for v in data ]
        elif type(data) == dict:
            ldata = [ ifsencode_trunc(v, 50)  for v in data.values() ]
        else:
            ldata = [ str(data) ]
        scraperwiki.dumpMessage({'message_type':'sqlitecall', 'command':'sqliteexecute', "val1":sqlquery, "lval2":ldata})
    
    return result
コード例 #19
0
try:
    userquery = scraperwiki.utils.GET()
except:
    userquery = {}

if userquery.get('dbimg'):

    db_data = util.get_table_vals('status', 'name,date_plot', 
        "name = '%(dbimg)s' or sw_name like '%%%(dbimg)s%%' or ol_name like '%%%(dbimg)s%%' or snac_id = '%(dbimg)s' or authority_id = '%(dbimg)s'" % userquery)
    #print db_data

    if db_data and len(db_data) == 1 and db_data[0]['date_plot']:

        scraperwiki.utils.httpresponseheader("Content-Type", "image/png")
        scraperwiki.dumpMessage({"content":db_data[0]['date_plot'], "message_type":"console", "encoding":"base64"})

    elif not db_data or (len(db_data) == 1 and not db_data[0]['date_plot']):

        print "No date plot image found for '%(dbimg)s'" % userquery

    else:

        auth_list = util.get_list(db_data, 'name')
        print "More than 1 scraper found when searching for date plot '%(dbimg)s': " % userquery + ', '.join(auth_list)

    sys.exit()
    
header = """
<html><head>
<style type="text/css">%s</style>
コード例 #20
0
import scraperwiki
scraperwiki.sqlite.attach('atoc_routeing_guide')
import cgi
import os

args = dict(cgi.parse_qsl(os.getenv("QUERY_STRING", "")))
map = scraperwiki.sqlite.select("data FROM maps WHERE pageno = ?",
                                (int(args.get('pageno', 1))))[0]['data']
scraperwiki.utils.httpresponseheader("Content-Type", "image/png")
scraperwiki.dumpMessage({
    "content": map,
    "message_type": "console",
    "encoding": "base64"
})
import scraperwiki
scraperwiki.sqlite.attach('atoc_routeing_guide')
import cgi
import os

args = dict(cgi.parse_qsl(os.getenv("QUERY_STRING", "")))
map = scraperwiki.sqlite.select("data FROM maps WHERE pageno = ?",
                                (int(args.get('pageno', 1))))[0]['data']
scraperwiki.utils.httpresponseheader("Content-Type", "image/png")
scraperwiki.dumpMessage({
    "content": map,
    "message_type": "console",
    "encoding": "base64"
})
コード例 #21
0
import scraperwiki
scraperwiki.sqlite.attach('atoc_routeing_guide')
import cgi
import os

args = dict(cgi.parse_qsl(os.getenv("QUERY_STRING", "")))
map = scraperwiki.sqlite.select("data FROM maps WHERE pageno = ?", (int(args.get('pageno', 1))))[0]['data']
scraperwiki.utils.httpresponseheader("Content-Type", "image/png")
scraperwiki.dumpMessage({"content":map, "message_type":"console", "encoding":"base64"})
import scraperwiki
scraperwiki.sqlite.attach('atoc_routeing_guide')
import cgi
import os

args = dict(cgi.parse_qsl(os.getenv("QUERY_STRING", "")))
map = scraperwiki.sqlite.select("data FROM maps WHERE pageno = ?", (int(args.get('pageno', 1))))[0]['data']
scraperwiki.utils.httpresponseheader("Content-Type", "image/png")
scraperwiki.dumpMessage({"content":map, "message_type":"console", "encoding":"base64"})
コード例 #22
0
# generate the image
fig = plt.figure()
ax = fig.add_subplot(111)     # don't know what this does
ax.plot(r.x, r.y, 'o-')  # what is o-? [drj: o: circle marker; -: solid line]
labels = ax.get_xticklabels() 
for label in labels: 
    label.set_rotation(30) 
ax.set_title(title)


#produce the binary image.  the decoding into binary is done out in django
cout = StringIO.StringIO()
plt.savefig(cout, format=format, dpi=dpi)
if format == "svg":
    scraperwiki.utils.httpresponseheader("Content-Type", "image/svg+xml")
    scraperwiki.dumpMessage({"content":cout.getvalue(), "message_type":"console"})
else:
    scraperwiki.utils.httpresponseheader("Content-Type", "image/%s" % format)
    scraperwiki.dumpMessage({"content":base64.encodestring(cout.getvalue()), "message_type":"console", "encoding":"base64"})


import numpy
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
import matplotlib.cbook as cbook
import matplotlib.ticker as ticker

import scraperwiki
import sys
import base64
import StringIO
コード例 #23
0
def execute(sqlquery, data=None, verbose=1):
    # helpfully wrap singletons into a list, as is done for the % operator, because singleton tuples can be hard to understand
    if data is not None and "?" in sqlquery and type(data) not in [
            list, tuple
    ]:
        data = [data]

    global attachlist
    result = scraperwiki.datastore.request({
        "maincommand": 'sqliteexecute',
        "sqlquery": sqlquery,
        "data": data,
        "attachlist": attachlist
    })
    while "progresstick" in result:
        if verbose:
            scraperwiki.dumpMessage({
                'message_type': 'sqlitecall',
                'command': 'progresstick',
                "val1": result.get("progresstick"),
                "lval2": result.get("timeseconds")
            })
        result = scraperwiki.datastore.request(
            None)  # goes back and waits for next line

    while result.get("stillproducing") == "yes":
        dataresult = scraperwiki.datastore.request(
            None)  # goes back and waits for next line
        if verbose and dataresult.get("stillproducing"):
            scraperwiki.dumpMessage({
                'message_type': 'sqlitecall',
                'command': 'stillproducing',
                "val1": dataresult.get("producedbatches"),
                "lval2": dataresult.get("producedrows")
            })
        result["data"].extend(dataresult["data"])
        result["stillproducing"] = dataresult.get("stillproducing")
        #result["nchunks"] += 1

    if "error" in result:
        raise databaseexception(result)
    if "status" not in result and "keys" not in result:
        raise Exception("possible signal timeout: " + str(result))

    # list type for second field in message dump
    if verbose:
        if data == None:
            ldata = []
        elif type(data) in [tuple, list]:
            ldata = [ifsencode_trunc(v, 50) for v in data]
        elif type(data) == dict:
            ldata = [ifsencode_trunc(v, 50) for v in data.values()]
        else:
            ldata = [str(data)]
        scraperwiki.dumpMessage({
            'message_type': 'sqlitecall',
            'command': 'sqliteexecute',
            "val1": sqlquery,
            "lval2": ldata
        })

    return result
コード例 #24
0
import scraperwiki
import urllib
import base64

scraperwiki.utils.httpresponseheader("Content-Type", "image/PNG")
url = 'http://scraperwiki.com/media/images/intro_slides/intro4.png'
pngbin = urllib.urlopen(url).read()
scraperwiki.dumpMessage({"content":base64.encodestring(pngbin), "message_type":"console", "encoding":"base64"})


import scraperwiki
import urllib
import base64

scraperwiki.utils.httpresponseheader("Content-Type", "image/PNG")
url = 'http://scraperwiki.com/media/images/intro_slides/intro4.png'
pngbin = urllib.urlopen(url).read()
scraperwiki.dumpMessage({"content":base64.encodestring(pngbin), "message_type":"console", "encoding":"base64"})


import scraperwiki
import urllib
import base64

scraperwiki.utils.httpresponseheader("Content-Type", "image/PNG")
url = 'http://scraperwiki.com/media/images/intro_slides/intro4.png'
pngbin = urllib.urlopen(url).read()
scraperwiki.dumpMessage({"content":base64.encodestring(pngbin), "message_type":"console", "encoding":"base64"})


import scraperwiki
コード例 #25
0
 def flush(self) :
     if self.m_text:
         scraperwiki.dumpMessage({'message_type': 'console', 'content': self.m_text})
         self.m_text = ''
         self.m_fd.flush()
コード例 #26
0
# The CPU time limit handler simply throws a Python exception
# so it can be handled cleanly before the hard limit is reached.
def sigXCPU(signum_, frame_):
    raise scraperwiki.CPUTimeExceededError("ScraperWiki CPU time exceeded")
signal.signal(signal.SIGXCPU, sigXCPU)

with open(options.script) as codef:
    code = codef.read()
try:
    import imp
    mod = imp.new_module('scraper')
    exec code.rstrip() + "\n" in mod.__dict__
except Exception, e:
    etb = swutil.getExceptionTraceback(code)  
    assert etb.get('message_type') == 'exception'
    scraperwiki.dumpMessage(etb)
except SystemExit, se:
    sys.stdout.flush()
    sys.stderr.flush()

    # If we do not temporarily yield a slice of the CPU here then the
    # launching process will not be able to read from stderr before
    # we exit.
    import time 
    time.sleep(0)

    raise se

sys.stdout.flush()
sys.stderr.flush()
コード例 #27
0
plt.ylabel("Time (min:sec)")

ax.plot_date(x, winners, 'ro', label="Winning time")
ax.errorbar(x, medians, yerr=spreads, label="Median time (and spread)")

fig.autofmt_xdate()

legend = plt.legend()

format = "png"
imagedata = StringIO()
plt.savefig(imagedata, format=format, dpi=96)
utils.httpresponseheader("Content-Type", "image/%s" % format)
dumpMessage({
    "content": imagedata.getvalue().encode("base64"),
    "message_type": "console",
    "encoding": "base64"
})

import datetime as dt
import dateutil.parser
from StringIO import StringIO

import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
import matplotlib as mpl
from matplotlib.ticker import FuncFormatter as ff
import numpy

import scraperwiki
from scraperwiki import dumpMessage, sqlite, utils
コード例 #28
0
# Blank Python
import scraperwiki,urllib2
scraperwiki.utils.httpresponseheader("Content-Type", "image/jpeg")

lazycache=scraperwiki.swimport('lazycache')
html=lazycache.lazycache('http://placekitten.com/g/200/305')
html=lazycache.lazycache('http://placekitten.com/g/200/305')
#print html

html2=urllib2.urlopen('http://placekitten.com/g/200/305').read()

assert html==html2
import base64
scraperwiki.dumpMessage({"content":base64.encodestring(html2), "message_type":"console", "encoding":"base64"})

# Blank Python
import scraperwiki,urllib2
scraperwiki.utils.httpresponseheader("Content-Type", "image/jpeg")

lazycache=scraperwiki.swimport('lazycache')
html=lazycache.lazycache('http://placekitten.com/g/200/305')
html=lazycache.lazycache('http://placekitten.com/g/200/305')
#print html

html2=urllib2.urlopen('http://placekitten.com/g/200/305').read()

assert html==html2
import base64
scraperwiki.dumpMessage({"content":base64.encodestring(html2), "message_type":"console", "encoding":"base64"})

コード例 #29
0
def save(unique_keys, data, table_name="swdata", verbose=2, date=None):
    if unique_keys != None and type(unique_keys) not in [list, tuple]:
        raise databaseexception({
            "error": 'unique_keys must a list or tuple',
            "unique_keys_type": str(type(unique_keys))
        })

    def convdata(unique_keys, scraper_data):
        if unique_keys:
            for key in unique_keys:
                if key not in scraper_data:
                    return {
                        "error": 'unique_keys must be a subset of data',
                        "bad_key": key
                    }
                if scraper_data[key] == None:
                    return {
                        "error": 'unique_key value should not be None',
                        "bad_key": key
                    }
        jdata = {}
        for key, value in scraper_data.items():
            if not key:
                return {"error": 'key must not be blank', "bad_key": key}
            if type(key) not in [unicode, str]:
                return {"error": 'key must be string type', "bad_key": key}
            if not re.match("[a-zA-Z0-9_\- ]+$", key):
                return {"error": 'key must be simple text', "bad_key": key}

            if type(value) == datetime.date:
                value = value.isoformat()
            elif type(value) == datetime.datetime:
                if value.tzinfo is None:
                    value = value.isoformat()
                else:
                    value = value.astimezone(pytz.timezone('UTC')).isoformat()
                    assert "+00:00" in value
                    value = value.replace("+00:00", "")
            elif value == None:
                pass
            elif isinstance(value, SqliteError):
                return {"error": str(value)}
            elif type(value) == str:
                try:
                    value = value.decode("utf-8")
                except:
                    return {"error": "Binary strings must be utf-8 encoded"}
            elif type(value) not in [int, bool, float, unicode, str]:
                value = unicode(value)
            jdata[key] = value
        return jdata

    if type(data) == dict:
        rjdata = convdata(unique_keys, data)
        if rjdata.get("error"):
            raise databaseexception(rjdata)
        if date:
            rjdata["date"] = date
    else:
        rjdata = []
        for ldata in data:
            ljdata = convdata(unique_keys, ldata)
            if ljdata.get("error"):
                raise databaseexception(ljdata)
            rjdata.append(ljdata)
    result = scraperwiki.datastore.request({
        "maincommand": 'save_sqlite',
        "unique_keys": unique_keys,
        "data": rjdata,
        "swdatatblname": table_name
    })

    if "error" in result:
        raise databaseexception(result)

    if verbose >= 2:
        pdata = {}
        if type(data) == dict:
            for key, value in data.items():
                pdata[strencode_trunc(key, 50)] = strencode_trunc(value, 50)
        elif data:
            for key, value in data[0].items():
                pdata[strencode_trunc(key, 50)] = strencode_trunc(value, 50)
            pdata["number_records"] = "Number Records: %d" % len(data)
        scraperwiki.dumpMessage({'message_type': 'data', 'content': pdata})
    return result
コード例 #30
0
def save(unique_keys, data, table_name="swdata", verbose=2, date=None):
    if unique_keys != None and type(unique_keys) not in [ list, tuple ]:
        raise databaseexception({ "error":'unique_keys must a list or tuple', "unique_keys_type":str(type(unique_keys)) })

    def convdata(unique_keys, scraper_data):
        if unique_keys:
            for key in unique_keys:
                if key not in scraper_data:
                    return { "error":'unique_keys must be a subset of data', "bad_key":key }
                if scraper_data[key] == None:
                    return { "error":'unique_key value should not be None', "bad_key":key }
        jdata = { }
        for key, value in scraper_data.items():
            if not key:
                return { "error": 'key must not be blank', "bad_key":key }
            if type(key) not in [unicode, str]:
                return { "error":'key must be string type', "bad_key":key }
            if not re.match("[a-zA-Z0-9_\- ]+$", key):
                return { "error":'key must be simple text', "bad_key":key }
            
            if type(value) == datetime.date:
                value = value.isoformat()
            elif type(value) == datetime.datetime:
                if value.tzinfo is None:
                    value = value.isoformat()
                else:
                    value = value.astimezone(pytz.timezone('UTC')).isoformat()
                    assert "+00:00" in value
                    value = value.replace("+00:00", "")
            elif value == None:
                pass
            elif isinstance(value, SqliteError):
                return {"error": str(value)}
            elif type(value) == str:
                try:
                    value = value.decode("utf-8")
                except:
                    return {"error": "Binary strings must be utf-8 encoded"}
            elif type(value) not in [int, bool, float, unicode, str]:
                value = unicode(value)
            jdata[key] = value
        return jdata
            

    if type(data) == dict:
        rjdata = convdata(unique_keys, data)
        if rjdata.get("error"):
            raise databaseexception(rjdata)
        if date:
            rjdata["date"] = date
    else:
        rjdata = [ ]
        for ldata in data:
            ljdata = convdata(unique_keys, ldata)
            if ljdata.get("error"):
                raise databaseexception(ljdata)
            rjdata.append(ljdata)
    result = scraperwiki.datastore.request({"maincommand":'save_sqlite', "unique_keys":unique_keys, "data":rjdata, "swdatatblname":table_name})

    if "error" in result:
        raise databaseexception(result)

    if verbose >= 2:
        pdata = {}
        if type(data) == dict:
            for key, value in data.items():
                pdata[strencode_trunc(key, 50)] = strencode_trunc(value, 50)
        elif data:
            for key, value in data[0].items():
                pdata[strencode_trunc(key, 50)] = strencode_trunc(value, 50)
            pdata["number_records"] = "Number Records: %d" % len(data)
        scraperwiki.dumpMessage({'message_type':'data', 'content': pdata})
    return result
コード例 #31
0
plt.xlabel("Date of Parkrun")
plt.ylabel("Time (min:sec)")

ax.plot_date(x, winners, 'ro', label="Winning time")
ax.errorbar(x, medians, yerr=spreads, label="Median time (and spread)")

fig.autofmt_xdate()

legend = plt.legend()


format = "png"
imagedata = StringIO()
plt.savefig(imagedata, format=format, dpi=96)
utils.httpresponseheader("Content-Type", "image/%s" % format)
dumpMessage({"content": imagedata.getvalue().encode("base64"), "message_type": "console", "encoding":"base64"})


import datetime as dt
import dateutil.parser
from StringIO import StringIO

import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
import matplotlib as mpl
from matplotlib.ticker import FuncFormatter as ff
import numpy

import scraperwiki
from scraperwiki import dumpMessage, sqlite, utils
コード例 #32
0
# generate the image
fig = plt.figure()
ax = fig.add_subplot(111)  # don't know what this does
ax.plot(r.x, r.y, 'o-')  # what is o-? [drj: o: circle marker; -: solid line]
labels = ax.get_xticklabels()
for label in labels:
    label.set_rotation(30)
ax.set_title(title)

#produce the binary image.  the decoding into binary is done out in django
cout = StringIO.StringIO()
plt.savefig(cout, format=format, dpi=dpi)
if format == "svg":
    scraperwiki.utils.httpresponseheader("Content-Type", "image/svg+xml")
    scraperwiki.dumpMessage({
        "content": cout.getvalue(),
        "message_type": "console"
    })
else:
    scraperwiki.utils.httpresponseheader("Content-Type", "image/%s" % format)
    scraperwiki.dumpMessage({
        "content": base64.encodestring(cout.getvalue()),
        "message_type": "console",
        "encoding": "base64"
    })

import numpy
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
import matplotlib.cbook as cbook
import matplotlib.ticker as ticker
コード例 #33
0
from matplotlib.pyplot import show


fig, ax = plt.subplots(1,1)

dendrogram=cluster.hierarchy.dendrogram(linkageMatrix)

ax.plot(dendrogram)

# Save the generated figure to a file
stream = cStringIO.StringIO()
fig.savefig(stream,format='png')

# Some http stuff..
scraperwiki.utils.httpresponseheader("Content-Type", "image/png")
scraperwiki.dumpMessage({"content":base64.encodestring(stream.getvalue()), "message_type":"console", "encoding":"base64"})# Blank Python
import scraperwiki, random, math
import numpy as np
import scipy as sp
import pylab
import cStringIO
from scipy import spatial
from scipy import cluster

sourcescraper = 'clustertenderpy'
scraperwiki.sqlite.attach('clustertenderpy')

lm = scraperwiki.sqlite.select("* from swdata")
print lm[0]['linkageMatrix']

linkageMatrix = np.array(lm[0]['linkageMatrix'],dtype=np.float64)