def buildCryptoChangepointEvents(fromTime, toTime, currency): validateMongoEnvironment() client = getMongoClient() collection = client.cryptoposts.changepoints query = { "coin": currency, "changepoint": { "$gte": fromTime, "$lte": toTime } } returnedData = queryDatabase(collection, query) changepointset = set() fileredData = [] for data in returnedData: data["_id"] = "null" if (data["changepoint"] not in changepointset): changepointset.add(data["changepoint"]) fileredData.append(data) with open('cryptoApp/timelinePlotter/static/cryptoChangepoints.json', 'w+') as outputFile: json.dump(fileredData, outputFile) client.close()
def runEventDetector(timelineId, peakWindowSize, sensitivity, startTime, endTime): validateMongoEnvironment() client = getMongoClient() timeline = getTimeline(timelineId, client, startTime, endTime) events = findEvents(timeline, peakWindowSize, sensitivity) postEventsToDatabase(client, timelineId, events, peakWindowSize, sensitivity) client.close()
def runAggregator(startTime, endTime, tag, granularity=HOUR, submissionWeight=3, submissionScoreWeight=1, commentWeight=2, commentScoreWeight=1): """ Gets data from mongoDB, aggregates it, and posts the results back to mongodb. Returns an unique identifier for the timeseries. The aggregation values are calculated based on the provided weights. Arguments: startTime {float} -- Unix time, start of range to query. endTime {float} -- Unix time, end of range to query. tag {string} -- The tag of the data. E.g 'bitcoin' Keyword Arguments: granularity {int} -- The step size in the time dimension to aggregate on (default: {HOUR}) submissionWeight {int} -- The weight given to the existence of a submission (default: {3}) submissionScoreWeight {int} -- The weight given to the score of a submission (default: {1}) commentWeight {int} -- The weight given to the existence of a comment (default: {2}) commentScoreWeight {int} -- The weight given to the score of a comment (default: {1}) Returns: string -- uuid used to tag the timeseries. Corresponds to the seriesId field in mongoDB. """ validateMongoEnvironment() client = getMongoClient() print("Aggregating data about {} in the time interval [{},{}]".format( tag, startTime, endTime)) (aggregationId, aggregation) = getAggregation(client, startTime, endTime, tag, granularity=granularity, submissionWeight=submissionWeight, submissionScoreWeight=submissionScoreWeight, commentWeight=commentWeight, commentScoreWeight=commentScoreWeight) aggregationCollection = client.reddit_data.aggregation bulkPostToDatabase(aggregationCollection, aggregation) postTimelineIndexToDatabase(client, aggregationId, startTime, endTime, tag, granularity, submissionWeight, submissionScoreWeight, commentWeight, commentScoreWeight) client.close() return aggregationId
def buildCryptoDataSeries(fromTime, toTime, currency): validateMongoEnvironment() client = getMongoClient() collection = client.cryptoposts.crypto query = {"coin": currency, "time": {"$gte": fromTime, "$lt": toTime}} returnedData = queryDatabase(collection, query) for data in returnedData: data["_id"] = "null" with open('cryptoApp/timelinePlotter/static/crypto.json', 'w+') as outputFile: json.dump(returnedData, outputFile) client.close() return
def validateEnvironments(): """Check if all environment variables for reddit and mongodb is set. The program terminates if this is not the case """ if (not validateRedditEnvironment()): quit() if (not validateMongoEnvironment()): quit()
def buildSocialMediaSeries(seriesId): validateMongoEnvironment() client = getMongoClient() buildTimeline(client, seriesId) buildSocialMediaEvents(client, seriesId) client.close()
from cryptoApp.mongoService.setup import validateMongoEnvironment, getMongoClient from cryptoApp.constants.unixTime import DAY from time import mktime from datetime import datetime def deleteInRange(collection, startTime, endTime): query = {"timestamp": {"$gte": startTime, "$lte": endTime}} return collection.delete_many(query) # YYYY MM DD HH MM SS days = 10 startTime = int(mktime(datetime(2017, 12, 11, 00, 00, 00).timetuple())) endTime = startTime + days * DAY validateMongoEnvironment() client = getMongoClient() print( "deleted: ", deleteInRange(client.reddit_data.comments, startTime, endTime).deleted_count) print( "deleted: ", deleteInRange(client.reddit_data.submissions, startTime, endTime).deleted_count) client.close()