def json2dataset(self, jsonFile):
result_array = []
#init min and max dates
self.minDate = utils.convertDate(jsonFile[0]["date"])
self.maxDate = utils.convertDate(jsonFile[0]["date"])
for i in range(0, len(jsonFile)-1):
result_dict = {}
jobj = jsonFile[i]
jdata = jobj["data"]
jobj_type = jobj["type"]
if jobj_type == 'kill':
event_type = 'kill'
event_actor = jdata["actor"]["playerId"]
if event_actor not in self.actors:
self.actors.append(event_actor)
event_victim = jdata["victim"]["playerId"]
if jdata["headshot"] == True:
event_type = event_type + '_hs'
if jdata["penetrated"] == True:
event_type = event_type + '_pnt'
result_dict["event_type"] = event_type
result_dict["date"] = utils.convertDate(jobj["date"]) #use python datetime format for dates
#update min and max dates of session
if result_dict["date"] < self.minDate:
self.minDate = result_dict["date"]
elif result_dict["date"] > self.maxDate:
self.maxDate = result_dict["date"]
result_dict["actor"] = event_actor
result_dict["victim"] = event_victim
result_dict["event_id"] = i
result_array.append(result_dict)
self.dataset = result_array
def loadDict(self, data):
self.name = data['name']
for role in data['roles']:
self.roles.append(role)
if data['ip'] != '0.0.0.0':
self.ip.append(data['ip'])
self.lastUsed = utils.convertDate(data['last used'])
self.registered = utils.convertDate(data['registered'])
if data['token']:
self.token = data['token']
def stockScrape(self,
stockCode,
source="YAHOOFINANCE",
minDate=DEFAULT_STARTDATE):
# Initialize pandas dataframe to hold stock data
stockDataFrame = pd.DataFrame({
self.DATE: [],
self.TICKER: [],
self.PRICE: []
})
if source == "YAHOOFINANCE":
YahooCode = self.getYahooCode(stockCode)
stock = yf.Ticker(YahooCode)
sdate = utils.convertDate(minDate) # start date
dowloaded_data = stock.history(interval="1d", start=sdate)
# Manipulate the output
Dates = dowloaded_data.index.to_frame()
Dates = Dates.reset_index(drop=True)
Price = dowloaded_data['Close'].reset_index(drop=True)
Ticker = pd.DataFrame([stockCode] * len(dowloaded_data['Close']),
columns=['Ticker'])
if source == "GOOGLEFINANCE":
googleticker = self.getGoogleCode(stockCode)
data = self.googledatatable.copy()
sdate = utils.convertDate(minDate) # start date
edate = datetime.now()
edate = edate.replace(hour=0, minute=0, second=0,
microsecond=0) # end date
data.DATE = pd.to_datetime(data.DATE, format='%d/%m/%Y')
dates_mask = ((data.DATE >= sdate) & (data.DATE <= edate))
data = data.loc[dates_mask, [self.DATE, googleticker]]
data = data.dropna()
data = data[(data[googleticker] != 'NA')]
Dates = pd.DataFrame(data.DATE,
columns=['DATE']).reset_index(drop=True)
Ticker = pd.DataFrame([stockCode] * len(data.DATE),
columns=['Ticker'])
Price = pd.DataFrame(data[googleticker],
columns=[googleticker]).reset_index(drop=True)
stockDataFrame = pd.concat([Dates, Ticker, Price], axis=1)
stockDataFrame.columns = self.HISTORICAL_COLUMNS
stockDataFrame.ignore_index = True
# Add to SQL database
self.addToDatabase(stockDataFrame, self.HISTORICAL_TABLE_NAME)
def fastSync(self):
lastSync = settings('LastIncrementalSync')
if not lastSync:
lastSync = "2010-01-01T00:00:00Z"
lastSyncTime = utils.convertDate(lastSync)
log.info("Last sync run: %s" % lastSyncTime)
# get server RetentionDateTime
try:
result = self.doUtils(
"{server}/emby/Emby.Kodi.SyncQueue/GetServerDateTime?format=json"
)
retention_time = result['RetentionDateTime']
except Exception as error:
log.error(error)
retention_time = "2010-01-01T00:00:00Z"
retention_time = utils.convertDate(retention_time)
log.info("RetentionDateTime: %s" % retention_time)
# if last sync before retention time do a full sync
if retention_time > lastSyncTime:
log.info(
"Fast sync server retention insufficient, fall back to full sync"
)
return False
params = {'LastUpdateDT': lastSync}
if settings('enableMusic') != "true":
params['filter'] = "music"
url = "{server}/emby/Emby.Kodi.SyncQueue/{UserId}/GetItems?format=json"
try:
result = self.doUtils(url, parameters=params)
processlist = {
'added': result['ItemsAdded'],
'update': result['ItemsUpdated'],
'userdata': result['UserDataChanged'],
'remove': result['ItemsRemoved']
}
except Exception as error: # To be reviewed to only catch specific errors.
log.error(error)
log.error("Failed to retrieve latest updates using fast sync.")
xbmcgui.Dialog().ok(lang(29999), lang(33095))
return False
else:
log.info("Fast sync changes: %s" % result)
for action in processlist:
self.triage_items(action, processlist[action])
return True
def fastSync(self):
lastSync = settings('LastIncrementalSync')
if not lastSync:
lastSync = "2010-01-01T00:00:00Z"
lastSyncTime = utils.convertDate(lastSync)
log.info("Last sync run: %s" % lastSyncTime)
# get server RetentionDateTime
try:
result = self.doUtils("{server}/emby/Emby.Kodi.SyncQueue/GetServerDateTime?format=json")
retention_time = result['RetentionDateTime']
except Exception as error:
log.error(error)
retention_time = "2010-01-01T00:00:00Z"
retention_time = utils.convertDate(retention_time)
log.info("RetentionDateTime: %s" % retention_time)
# if last sync before retention time do a full sync
if retention_time > lastSyncTime:
log.info("Fast sync server retention insufficient, fall back to full sync")
return False
params = {'LastUpdateDT': lastSync}
if settings('enableMusic') != "true":
params['filter'] = "music"
url = "{server}/emby/Emby.Kodi.SyncQueue/{UserId}/GetItems?format=json"
try:
result = self.doUtils(url, parameters=params)
processlist = {
'added': result['ItemsAdded'],
'update': result['ItemsUpdated'],
'userdata': result['UserDataChanged'],
'remove': result['ItemsRemoved']
}
except Exception as error: # To be reviewed to only catch specific errors.
log.error(error)
log.error("Failed to retrieve latest updates using fast sync.")
xbmcgui.Dialog().ok(lang(29999), lang(33095))
return False
else:
log.info("Fast sync changes: %s" % result)
for action in processlist:
self.triage_items(action, processlist[action])
return True
def getHistoricalData(ticker, daterange):
'''
Will download the historical data for the given ticker and return it as a pandas dataframe.
Daterange must be a 2 element list, in the following format: [[<start date>], [<end date>]], date format = int YYYYMMDD
'''
startYear, startMonth, startDay = utils.convertDate(daterange[0])
endYear, endMonth, endDay = utils.convertDate(daterange[1])
start = datetime(startYear, startMonth, startDay)
end = datetime(endYear, endMonth, endDay)
dataframe = iex.get_historical_data(ticker, start=start, end=end, output_format='pandas')
dataframe.head()
return dataframe
def saveLastSync(self):
# Save last sync time
overlap = 2
try: # datetime fails when used more than once, TypeError
if self.isFastSync:
result = self.doUtils("{server}/emby/Emby.Kodi.SyncQueue/GetServerDateTime?format=json")
server_time = result['ServerDateTime']
server_time = utils.convertDate(server_time)
else:
raise Exception("Fast sync server plugin is not enabled.")
except Exception as e:
# If the server plugin is not installed or an error happened.
log.debug("An exception occurred: %s" % e)
time_now = datetime.utcnow()-timedelta(minutes=overlap)
lastSync = time_now.strftime('%Y-%m-%dT%H:%M:%SZ')
log.info("New sync time: client time -%s min: %s" % (overlap, lastSync))
else:
lastSync = (server_time - timedelta(minutes=overlap)).strftime('%Y-%m-%dT%H:%M:%SZ')
log.info("New sync time: server time -%s min: %s" % (overlap, lastSync))
finally:
settings('LastIncrementalSync', value=lastSync)
def saveLastSync(self):
# Save last sync time
overlap = 2
try: # datetime fails when used more than once, TypeError
if self.isFastSync:
result = self.doUtils("{server}/emby/Emby.Kodi.SyncQueue/GetServerDateTime?format=json")
server_time = result['ServerDateTime']
server_time = utils.convertDate(server_time)
else:
raise Exception("Fast sync server plugin is not enabled.")
except Exception as e:
# If the server plugin is not installed or an error happened.
log.debug("An exception occurred: %s" % e)
time_now = datetime.utcnow()-timedelta(minutes=overlap)
lastSync = time_now.strftime('%Y-%m-%dT%H:%M:%SZ')
log.info("New sync time: client time -%s min: %s" % (overlap, lastSync))
else:
lastSync = (server_time - timedelta(minutes=overlap)).strftime('%Y-%m-%dT%H:%M:%SZ')
log.info("New sync time: server time -%s min: %s" % (overlap, lastSync))
finally:
settings('LastIncrementalSync', value=lastSync)
def updatePosts(self, posts):
if not posts:
return
if utils.convertDate(posts[0]['date']) > utils.convertDate(
posts[-1]['date']):
posts.reverse()
for i in range(len(posts) - 1):
if utils.convertDate(posts[i]['date']) > utils.convertDate(
posts[i + 1]['date']):
print 'ERROR: database.updatePosts incoming posts not in order'
print '-', utils.convertDate(
posts[i]['date']), utils.convertDate(posts[i + 1]['date'])
return
newPostCount = 0
newUserCount = 0
dateLookup = {}
for post in self.posts:
if post.date in dateLookup:
dateLookup[post.date].append(post)
else:
dateLookup[post.date] = [post]
for data in posts:
newPost = Post(data)
isNew = True
if newPost.date in dateLookup:
if newPost in dateLookup[newPost.date]:
isNew = False
dateLookup[newPost.date].append(newPost)
else:
dateLookup[newPost.date] = [newPost]
if isNew:
if newPost.name not in self.users:
user = User()
user.name = newPost.name
self.users[user.name] = user
newUserCount += 1
self.users[newPost.name].addIp(newPost.ip)
self.posts.append(newPost)
newPostCount += 1
if newPostCount > 0:
print newPostCount, 'posts inserted'
self.posts.sort()
self.savePosts()
if newUserCount > 0:
print newUserCount, 'users inserted'
self.saveUsers()
def __init__(self, data):
self.id = int(data[0]) # Message id
self.time = int(data[1]) # Unix timestamp
self.date = utils.convertDate(data[2].split(',')[0]) # Date string
self.name = data[3] # Author's name
self.level = self.levels[int(data[4]) - 1] # Privileges
self.exturl = data[5] # Email/url
self.content = BeautifulSoup(data[6],
"html.parser").text # Message text
self.imgurl = data[7] # Image url
self.badFlags = int(data[8]) # Message type flags
self.userid = data[9] # Author's id
self.flaghtml = data[10]
self.localId = data[11]
self.isRedirected = bool(self.badFlags & 16)
self.isPrivate = bool(self.badFlags & 32)
self.isSticky = self.id == -1
self.isTemp = self.id == 0
def updateStockData(self, stockCode, source="YAHOOFINANCE"):
# Reads database
sqlQuery = """SELECT {} FROM {} WHERE {} = '{}'; """ \
.format(self.TICKER, self.HISTORICAL_TABLE_NAME, self.TICKER, stockCode)
#print(sqlQuery)
stockData = self.readDatabase(sqlQuery)
# Checks whether any previous data has been added for the particular stock code
# if not then run initialStockScrape to get all past data
if stockData.empty:
print('Running stockScrape() on {} using {}. --First run.'.format(
stockCode, source))
self.stockScrape(stockCode, source)
else:
#access database to get latestDate
print('Running stockScrape() on {} using {}. --Updating data.'.
format(stockCode, source))
# Performs SQL query to get the latest stock data date in database
sqlQuery = """SELECT {}, max({}) AS Date FROM {} WHERE {} = '{}' GROUP BY {}""" \
.format(self.TICKER, self.DATE, self.HISTORICAL_TABLE_NAME, self.TICKER, stockCode, self.TICKER)
y = self.readDatabase(sqlQuery)
minDate = y.Date[
0] # minDate is the earliest data of data that the program needs to download
# Increment date by 1 day
minDate = utils.incrementDate(minDate)
today = datetime.now()
today = today.replace(hour=0, minute=0, second=0,
microsecond=0) # end date
if utils.convertDate(minDate) < today:
# Updates stock data
self.stockScrape(stockCode, source, minDate)
else:
# Data are already up to date
print(
"Data for {} are already up to date. Module: updateStockData."
.format(stockCode))
def loadDict(self, data):
self.date = utils.convertDate(data['date'])
self.name = data['name']
self.content = data['content']
self.ip = data['ip']
self.email = data['email']
def find_frame(event, start_date, vid):
date_str = event['date']
date = utils.convertDate(date_str)
delta = get_time_delta(start_date, date)
return calculate_frames(delta, vid.get(cv2.CAP_PROP_FPS))
