Python DataCollector Examples

Example #1

def retrieve_data(filename, parameter, indexmax):

    pricedata = DataCollector(filename=filename, overwrite=False)
    prices = pricedata.retrieve_data(parameter=parameter, indexes=(0, indexmax))
    price_times = pricedata.retrieve_data(parameter=parameter+'_time', indexes=(0, indexmax))
    return prices, price_times

Example #2

File: bittrex.py Project: jwhite510/multivariable_rnn_btc

        # print('price: ', str(price))
        # print('price_datetime: ', str(price_datetime))

        return str(price), str(price_datetime)

    except Exception as e:
        print(e)
        print('request failed')


if __name__ == "__main__":

    parameters = []
    parameters.append({'name': 'btcprice', 'maxlength': 100})
    bittrexbtcprices = DataCollector(filename='bittrexbtcprices',
                                     parameters=parameters,
                                     overwrite=False,
                                     checklength=True)

    parameters = []
    parameters.append({'name': 'tetherprice', 'maxlength': 100})
    bittrextetherprices = DataCollector(filename='bittrextetherprices',
                                        parameters=parameters,
                                        overwrite=False,
                                        checklength=True)

    parameters = []
    parameters.append({'name': 'ethereumprice', 'maxlength': 100})
    ethereumprices = DataCollector(filename='ethereumprices',
                                   parameters=parameters,
                                   overwrite=False,
                                   checklength=True)

Example #3

File: twitter.py Project: jwhite510/multivariable_rnn_btc

                        twitter_credentials.CONSUMER_SECRET)

    auth.set_access_token(twitter_credentials.ACCESS_TOKEN,
                          twitter_credentials.ACCESS_TOKEN_SECRET)

    stream = Stream(auth, listener)

    stream.filter(track=['bitcoin'])


if __name__ == "__main__":

    parameters = []
    parameters.append({'name': 'tweet', 'maxlength': 4000})
    twittertweets = DataCollector(filename='twittertweets',
                                  parameters=parameters,
                                  overwrite=False,
                                  checklength=True)

    while True:

        try:

            initialize_and_run(twittertweets)

        except Exception as e:

            print('Error Occured')
            print(e)
            time.sleep(10)

Example #4

File: view_reddit_data.py Project: jwhite510/multivariable_rnn_btc

from datastorage import DataCollector



indexmax = 999999

# retrieve the data
redditsubmissions = DataCollector(filename='redditsubmissions', overwrite=False)
submissions = redditsubmissions.retrieve_data(parameter='submission', indexes=(0, indexmax))
submission_times = redditsubmissions.retrieve_data(parameter='submission_time', indexes=(0, indexmax))


redditcomments = DataCollector(filename='redditcomments', overwrite=False)
comments = redditcomments.retrieve_data(parameter='comment', indexes=(0, indexmax))
comment_times = redditcomments.retrieve_data(parameter='comment_time', indexes=(0, indexmax))


print(len(comments))
print(len(submissions))
exit(0)
index = -1

# view the comments
print(comments[index])
print(comment_times[index])

# view the submissions
print(submissions[index])
print(submission_times[index])

exit(0)

Example #5

from datastorage import DataCollector

indexmax = 9999999

# retrieve the data
twittertweets = DataCollector(filename='twittertweets', overwrite=False)
tweets = twittertweets.retrieve_data(parameter='tweet', indexes=(0, indexmax))
tweet_times = twittertweets.retrieve_data(parameter='tweet_time',
                                          indexes=(0, indexmax))

index = -1
print(len(tweets))
print(tweets[index])
print(tweet_times[index])

Example #6

File: process_data.py Project: jwhite510/multivariable_rnn_btc

def convert_time_to_unix(dtstring):

    datetime_object = datetime.strptime(dtstring, '%Y-%m-%d %H:%M:%S')
    time_s = time.mktime(datetime_object.timetuple())

    return time_s


if __name__ == '__main__':

    indexmax = 999999

    # bitcoin data
    bittrexbtcpricedata = DataCollector(filename='bittrexbtcprices',
                                        overwrite=False)
    btcprices = bittrexbtcpricedata.retrieve_data(parameter='btcprice',
                                                  indexes=(0, indexmax))
    btcprice_times = bittrexbtcpricedata.retrieve_data(
        parameter='btcprice_time', indexes=(0, indexmax))
    btcprice_times = [fix_btc_time(btctime) for btctime in btcprice_times]

    # reddit submissions
    redditsubmissions = DataCollector(filename='redditsubmissions',
                                      overwrite=False)
    submissions = redditsubmissions.retrieve_data(parameter='submission',
                                                  indexes=(0, indexmax))
    submission_times = redditsubmissions.retrieve_data(
        parameter='submission_time', indexes=(0, indexmax))

    # reddit comments

Example #7

File: reddit.py Project: jwhite510/multivariable_rnn_btc

        except Exception as e:
            print('Error occured collecting comment')
            print(e)







if __name__ == "__main__":

    # the data types that this file will contain
    parameters = []
    parameters.append({'name': 'submission', 'maxlength': 10000})
    redditsubmissions = DataCollector(filename='redditsubmissions', parameters=parameters, overwrite=False,
                            checklength=True)


    parameters = []
    parameters.append({'name': 'comment', 'maxlength': 4000})
    redditcomments = DataCollector(filename='redditcomments', parameters=parameters, overwrite=False,
                                   checklength=True)

    # start the reddit instance
    reddit = praw.Reddit(client_id=reddit_credentials.CLIENT_ID,
                         client_secret=reddit_credentials.CLIENT_SECRET,
                         user_agent='agent')

    subreddit = reddit.subreddit('bitcoin')

Example #8

File: view_bittrex_data.py Project: jwhite510/multivariable_rnn_btc

from datastorage import DataCollector

indexmax = 999999

# retrieve the data
bittrexbtcpricedata = DataCollector(filename='bittrexbtcprices',
                                    overwrite=False)
btcprices = bittrexbtcpricedata.retrieve_data(parameter='btcprice',
                                              indexes=(0, indexmax))
btcprice_times = bittrexbtcpricedata.retrieve_data(parameter='btcprice_time',
                                                   indexes=(0, indexmax))

print(len(btcprices))
index = -1

# view the submissions
print(btcprices[index])
print(btcprice_times[index])