Beispiel #1
0
def getResults(response, table):
    if Common.exists(response, table):
        return response[table]
    elif Common.exists(response, 'results'):
        return response['results']
    else:
        return response
def rip_multi_page(schema, table_key):
    page = count = 1

    #Get one response, process it, and if it has pages then do below
    params = schema['static_params'] + [('page', page)]
    url = Common.buildUrl(schema['endpoints'][table_key], params)
    response = json.loads(urllib.request.urlopen(url).read())

    num_pages = math.ceil(int(response['count']) / int(response['perPage']))
    for page in range(1, num_pages):
        params = schema['static_params'] + [('page', page)]
        yield Common.buildUrl(schema['endpoints'][table_key], params)
Beispiel #3
0
 def escape(string):
     # escape(x) if isNumber(x) else "'" + escape(x) + "'"
     if Common.isString(string):
         if (string == 'desc'):
             return '"desc"'
         else:
             result = str(string).split("'")
             return "'" + "''".join(result) + "'"
     else:
         return str(string)
def rip_aggregates(schema, table_key):
    from datetime import date
    date_format = '%Y-%m-%d'
    date_from = datetime.datetime.strptime('2000-01-01', date_format)
    date_to = datetime.datetime.strptime(date.today().strftime(date_format),
                                         date_format)

    num_days = (date_to - date_from).days + 1
    tickers = getAllTickers(schema['db_connection_str'])

    for ticker in tickers:
        date = date_from
        while date < date_to:
            _date_to = date + datetime.timedelta(days=1)
            params = schema['static_params'] + [
                ('asset', ticker), ('date-from', date.strftime(date_format)),
                ('date-to', _date_to.strftime(date_format))
            ]
            yield Common.buildUrl(schema['endpoints'][table_key], params)

            date += datetime.timedelta(days=1)
def rip_ticker_detail(schema, table_key):
    tickers = getAllTickers(schema['db_connection_str'])

    for ticker in tickers:
        params = schema['static_params'] + [('asset', ticker)]
        yield Common.buildUrl(schema['endpoints'][table_key], params)
def rip_single_page(schema, table_key):
    yield Common.buildUrl(schema['endpoints'][table_key],
                          schema['static_params'])
Beispiel #7
0
    ripper = schema['endpoint_rippers']['default']
    if table_key in schema['endpoint_rippers'].keys():
        ripper = schema['endpoint_rippers'][table_key]

    urls = []
    for url in ripper(schema, table_key):
        if task_limit >= 0 and len(urls) >= task_limit:
            break
        urls.append(url)

    with alive_bar(len(urls)) as bar:
        for url in urls:
            if multithreaded:
                tm.do_task(Common.get_response, [schema_file, table_key, url])
            else:
                r = Common.get_response([schema_file, table_key, url])
                if r is not None:
                    db_helper.execute(r)
                bar()

        tasks_complete = 0
        while not tm.all_tasks_complete():
            time.sleep(0.1)

            for i in range(0, tm.num_completed_tasks() - tasks_complete):
                bar()
            tasks_complete = tm.num_completed_tasks()
    tm.reset()

exit()