def __init__(self, mydb=None):
self.__childTask = None
self.data = queue.Queue()
self.mydb = mydb
self.EDDNimport = EDDNimport(self.mydb)
def __init__(self, mydb=None):
self.__childTask = None
self.data = queue.Queue()
self.mydb = mydb
self.EDDNimport = EDDNimport(self.mydb)
class loader(object):
def __init__(self, mydb=None):
self.__childTask = None
self.data = queue.Queue()
self.mydb = mydb
self.EDDNimport = EDDNimport(self.mydb)
def start(self):
if self.__childTask:
self.__childTask.stop()
self.__childTask = _child(self.data)
self.__childTask.daemon = True
self.__childTask.setName("EDDNchildThread")
self.__childTask.start()
def stop(self):
if self.__childTask:
self.__childTask.stop()
def isRunning(self):
if self.__childTask:
return self.__childTask.is_alive()
def get(self):
return self.data.get()
def update(self):
if not self.data.empty():
while not self.data.empty():
data = self.data.get()
if data:
data = data.decode(encoding='utf-8', errors='replace')
#print("import", data)
self.EDDNimport.importData(data)
self.mydb.con.commit()
if self.__childTask and self.isRunning() == False:
print("restart child")
self.start()
class loader(object):
def __init__(self, mydb=None):
self.__childTask = None
self.data = queue.Queue()
self.mydb = mydb
self.EDDNimport = EDDNimport(self.mydb)
def start(self):
if self.__childTask:
self.__childTask.stop()
self.__childTask = _child(self.data)
self.__childTask.daemon = True
self.__childTask.setName("EDDNchildThread")
self.__childTask.start()
def stop(self):
if self.__childTask:
self.__childTask.stop()
def isRunning(self):
if self.__childTask:
return self.__childTask.is_alive()
def get(self):
return self.data.get()
def update(self):
if not self.data.empty():
while not self.data.empty():
data = self.data.get()
if data:
data = data.decode(encoding='utf-8', errors='replace')
#print("import", data)
self.EDDNimport.importData(data)
self.mydb.con.commit()
if self.__childTask and self.isRunning() == False:
print("restart child")
self.start()
def __init__(self, mydb):
self.mydb = mydb
self.EDDNimport = EDDNimport(self.mydb)
class loader(object):
'''
EDDN on DynamoDB loader
'''
def __init__(self, mydb):
self.mydb = mydb
self.EDDNimport = EDDNimport(self.mydb)
def update(self):
self.updateCommoditie()
self.updateShipyard()
self.updateOutfitting()
def updateCommoditie(self):
lastUpdateTime = self.mydb.getConfig( 'last_EDDN_DynamoDB_Update' )
correntUpdateTime = datetime.now()
if lastUpdateTime:
lastUpdateTime = datetime.strptime(lastUpdateTime, "%Y-%m-%d %H:%M:%S")
if lastUpdateTime < correntUpdateTime - timedelta(hours=24):
lastUpdateTime = correntUpdateTime - timedelta(hours=24)
else:
lastUpdateTime = datetime.now().replace(hour=0, minute=0, second=0)
if lastUpdateTime < correntUpdateTime - timedelta(minutes=10):
print("update EDDN on DynamoDB")
epochFrom = datetime2epochMicroseconds( lastUpdateTime )
lastGatewayTime = None
'''
commoditie data
'''
activeDonwload = True
while activeDonwload:
# activeDonwload = False
getRequest = {"from": epochFrom, }
correntUpdateTime = datetime.now()
apiurl = "%s/commodities/%s" % (__APIUrl__, epochMicroseconds2datetime(epochFrom).strftime("%Y-%m-%d") )
josnData = self.sendAPIRequest(apiurl, getRequest)
if josnData:
if 'Count' in josnData and int(josnData['Count']) > 0:
for item in josnData['Items']:
self.EDDNimport.importData(item)
lastGTime = self.EDDNimport.convertStrptimeToDatetimeUTC(item["header"]["gatewayTimestamp"])
if not lastGatewayTime or lastGatewayTime < lastGTime:
lastGatewayTime = lastGTime
else:
activeDonwload = False
if 'LastEvaluatedTimestamp' in josnData:
epochFrom = int(josnData['LastEvaluatedTimestamp']) + 1
time.sleep(5)
else:
activeDonwload = False
else:
if epochFrom:
correntUpdateTime = epochMicroseconds2datetime(epochFrom)
activeDonwload = False
'''
save last update time
'''
if correntUpdateTime:
self.mydb.setConfig('last_EDDN_DynamoDB_Update', correntUpdateTime.strftime("%Y-%m-%d %H:%M:%S") )
self.mydb.con.commit()
def updateShipyard(self):
lastUpdateTime = self.mydb.getConfig( 'last_EDDN_DynamoDB_Shipyard' )
correntUpdateTime = datetime.now()
if lastUpdateTime:
lastUpdateTime = datetime.strptime(lastUpdateTime, "%Y-%m-%d %H:%M:%S")
if lastUpdateTime < correntUpdateTime - timedelta(hours=24):
lastUpdateTime = correntUpdateTime - timedelta(hours=24)
else:
lastUpdateTime = datetime.now().replace(hour=0, minute=0, second=0)
if lastUpdateTime < correntUpdateTime - timedelta(minutes=10):
'''
shipyard data
'''
epochFrom = datetime2epochMicroseconds( lastUpdateTime )
activeDonwload = True
while activeDonwload:
getRequest = {"from": epochFrom, }
apiurl = "%s/shipyards/%s" % (__APIUrl__, epochMicroseconds2datetime(epochFrom).strftime("%Y-%m-%d") )
josnData = self.sendAPIRequest(apiurl, getRequest)
if josnData:
if 'Count' in josnData and int(josnData['Count']) > 0:
for item in josnData['Items']:
self.EDDNimport.importData(item)
else:
activeDonwload = False
if 'LastEvaluatedTimestamp' in josnData:
epochFrom = int(josnData['LastEvaluatedTimestamp']) + 1
time.sleep(5)
else:
activeDonwload = False
else:
if epochFrom:
correntUpdateTime = epochMicroseconds2datetime(epochFrom)
activeDonwload = False
if correntUpdateTime:
self.mydb.setConfig('last_EDDN_DynamoDB_Shipyard', correntUpdateTime.strftime("%Y-%m-%d %H:%M:%S") )
self.mydb.con.commit()
def updateOutfitting(self):
lastUpdateTime = self.mydb.getConfig( 'last_EDDN_DynamoDB_Outfitting' )
correntUpdateTime = datetime.now()
if lastUpdateTime:
lastUpdateTime = datetime.strptime(lastUpdateTime, "%Y-%m-%d %H:%M:%S")
if lastUpdateTime < correntUpdateTime - timedelta(hours=24):
lastUpdateTime = correntUpdateTime - timedelta(hours=24)
else:
lastUpdateTime = datetime.now().replace(hour=0, minute=0, second=0)
if lastUpdateTime < correntUpdateTime - timedelta(minutes=10):
'''
outfitting data
'''
epochFrom = datetime2epochMicroseconds( lastUpdateTime )
activeDonwload = True
while activeDonwload:
getRequest = {"from": epochFrom, }
apiurl = "%s/outfitting/%s" % (__APIUrl__, epochMicroseconds2datetime(epochFrom).strftime("%Y-%m-%d") )
josnData = self.sendAPIRequest(apiurl, getRequest)
if josnData:
if 'Count' in josnData and int(josnData['Count']) > 0:
for item in josnData['Items']:
self.EDDNimport.importData(item)
else:
activeDonwload = False
if 'LastEvaluatedTimestamp' in josnData:
epochFrom = int(josnData['LastEvaluatedTimestamp']) + 1
time.sleep(5)
else:
activeDonwload = False
else:
if epochFrom:
correntUpdateTime = epochMicroseconds2datetime(epochFrom)
activeDonwload = False
if correntUpdateTime:
self.mydb.setConfig('last_EDDN_DynamoDB_Outfitting', correntUpdateTime.strftime("%Y-%m-%d %H:%M:%S") )
self.mydb.con.commit()
def sendAPIRequest(self, apiurl, getRequest=None, postRequest=None):
url = apiurl
if postRequest:
postRequest = str( json.dumps(postRequest) ).encode('utf-8')
if getRequest:
getRequest = urlencode(getRequest)
url = apiurl + "?" + getRequest
print("download %s" % url)
request = urllib2.Request(url, postRequest)
request.add_header('User-Agent', __useragent__)
if postRequest:
request.add_header('Content-Type', 'application/json; charset=utf-8')
request.add_header('Accept', 'application/json; charset=utf-8')
if postRequest:
request.add_header('Content-Length', len(postRequest))
request.add_header('Accept-encoding', 'gzip')
# print(apiurl, getRequest, postRequest)
try:
response = urllib2.urlopen(request)
except:
traceback.print_exc()
return
if response.info().get('Content-Encoding') == 'gzip':
# print("gzip ok")
buf = io.BytesIO(response.read())
f = gzip.GzipFile(fileobj=buf)
else:
# print("none")
f = response
result = f.read()
# print(result)
if result:
try:
josnData = json.loads(result.decode('utf-8', 'replace'))
except:
traceback.print_exc()
return
return josnData
