def get_json_data(url):
"""
Fetch JSON data from a URL and return the resulting dictionary.
Displays a progress bar as it downloads.
"""
requests = import_requests()
req = requests.get(url, stream=True)
totalLength = req.headers.get('content-length')
if totalLength is None:
compression = req.headers.get('content-encoding')
compression = (compression + "'ed") if compression else "uncompressed"
print("Downloading {}: {}...".format(compression, url))
jsData = req.content
else:
totalLength = int(totalLength)
progBar = pbar.Progress(totalLength, 25)
jsData = bytes()
for data in req.iter_content():
jsData += data
progBar.increment(
len(data),
postfix=lambda value, goal: \
" {}/{}".format(
makeUnit(value),
makeUnit(goal),
))
progBar.clear()
return json.loads(jsData.decode())
def importListings(self, listings_file):
"""
Updates the market data (AKA the StationItem table) using listings.csv
Writes directly to database.
"""
tdb, tdenv = self.tdb, self.tdenv
tdenv.NOTE("Processing market data from {}: Start time = {}", listings_file, datetime.datetime.now())
if not (self.dataPath / listings_file).exists():
tdenv.NOTE("File not found, aborting: {}", (self.dataPath / listings_file))
return
progress = 0
total = 1
if listings_file == LISTINGS:
from_live = 0
else:
from_live = 1
def blocks(f, size = 65536):
while True:
b = f.read(size)
if not b: break
yield b
with open(str(self.dataPath / listings_file), "r",encoding = "utf-8",errors = 'ignore') as f:
total += (sum(bl.count("\n") for bl in blocks(f)))
with open(str(self.dataPath / listings_file), "rU") as fh:
if self.getOption("progbar"):
prog = pbar.Progress(total, 50)
listings = csv.DictReader(fh)
cur_station = -1
station_items = dict()
for listing in listings:
if self.getOption("progbar"):
prog.increment(1, postfix=lambda value, goal: " " + str(round(value / total * 100)) + "%")
else:
progress += 1
print("\rProgress: (" + str(progress) + "/" + str(total) + ") " + str(round(progress / total * 100, 2)) + "%\t\t", end = "\r")
station_id = int(listing['station_id'])
item_id = int(listing['commodity_id'])
modified = datetime.datetime.utcfromtimestamp(int(listing['collected_at'])).strftime('%Y-%m-%d %H:%M:%S')
demand_price = int(listing['sell_price'])
demand_units = int(listing['demand'])
demand_level = int(listing['demand_bracket']) if listing['demand_bracket'] != '' else -1
supply_price = int(listing['buy_price'])
supply_units = int(listing['supply'])
supply_level = int(listing['supply_bracket']) if listing['supply_bracket'] != '' else -1
if station_id != cur_station:
for item in station_items:
if not item:
self.execute("DELETE from StationItem WHERE station_id = ? and item_id = ?", (station_id, item))
del station_items, cur_station
cur_station = station_id
station_items = dict()
cursor = self.execute("SELECT item_id from StationItem WHERE station_id = ?", (station_id,))
for item in cursor:
station_items[item] = False
del cursor
station_items[item_id] = True
result = self.execute("SELECT modified FROM StationItem WHERE station_id = ? AND item_id = ?", (station_id, item_id)).fetchone()
if result:
updated = timegm(datetime.datetime.strptime(result[0],'%Y-%m-%d %H:%M:%S').timetuple())
# When the dump file data matches the database, update to make from_live == 0.
if int(listing['collected_at']) == updated and listings_file == LISTINGS:
self.execute("""UPDATE StationItem
SET from_live = 0
WHERE station_id = ? AND item_id = ?""",
(station_id, item_id))
if int(listing['collected_at']) > updated:
tdenv.DEBUG1("Updating:{}, {}, {}, {}, {}, {}, {}, {}, {}",
station_id, item_id, modified,
demand_price, demand_units, demand_level,
supply_price, supply_units, supply_level)
try:
self.execute("""UPDATE StationItem
SET modified = ?,
demand_price = ?, demand_units = ?, demand_level = ?,
supply_price = ?, supply_units = ?, supply_level = ?,
from_live = ?
WHERE station_id = ? AND item_id = ?""",
(modified, demand_price, demand_units, demand_level, supply_price, supply_units, supply_level, from_live,
station_id, item_id))
except sqlite3.IntegrityError:
tdenv.DEBUG1("Error on update.")
else:
tdenv.DEBUG1("Inserting:{}, {}, {}, {}, {}, {}, {}, {}, {}",
station_id, item_id, modified,
demand_price, demand_units, demand_level,
supply_price, supply_units, supply_level)
try:
self.execute("""INSERT INTO StationItem
(station_id, item_id, modified,
demand_price, demand_units, demand_level,
supply_price, supply_units, supply_level, from_live)
VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )""",
(station_id, item_id, modified,
demand_price, demand_units, demand_level,
supply_price, supply_units, supply_level, from_live))
except sqlite3.IntegrityError:
tdenv.DEBUG1("Error on insert.")
if self.getOption("progbar"):
while prog.value < prog.maxValue:
prog.increment(1, postfix=lambda value, goal: " " + str(round(value / total * 100)) + "%")
prog.clear()
del from_live
self.updated['Listings'] = True
tdenv.NOTE("Finished processing market data. End time = {}", datetime.datetime.now())
def importStations(self):
"""
Populate the Station table using stations.jsonl
Also populates the ShipVendor table if the option is set.
Writes directly to database.
"""
tdb, tdenv = self.tdb, self.tdenv
tdenv.NOTE("Processing Stations, this may take a bit: Start time = {}", datetime.datetime.now())
if self.getOption('shipvend'):
tdenv.NOTE("Simultaneously processing ShipVendors.")
if self.getOption('upvend'):
tdenv.NOTE("Simultaneously processing UpgradeVendors, this will take quite a while.")
progress = 0
total = 1
def blocks(f, size = 65536):
while True:
b = f.read(size)
if not b: break
yield b
with open(str(self.dataPath / self.stationsPath), "r",encoding = "utf-8",errors = 'ignore') as f:
total += (sum(bl.count("\n") for bl in blocks(f)))
with open(str(self.dataPath / self.stationsPath), "rU") as fh:
if self.getOption("progbar"):
prog = pbar.Progress(total, 50)
for line in fh:
if self.getOption("progbar"):
prog.increment(1, postfix=lambda value, goal: " " + str(round(value / total * 100)) + "%")
else:
progress += 1
print("\rProgress: (" + str(progress) + "/" + str(total) + ") " + str(round(progress / total * 100, 2)) + "%\t\t", end = "\r")
station = json.loads(line)
# Import Stations
station_id = station['id']
name = station['name']
system_id = station['system_id']
ls_from_star = station['distance_to_star'] if station['distance_to_star'] else 0
blackmarket = 'Y' if station['has_blackmarket'] else 'N'
max_pad_size = station['max_landing_pad_size'] if station['max_landing_pad_size'] and station['max_landing_pad_size'] != 'None' else '?'
market = 'Y' if station['has_market'] else 'N'
shipyard = 'Y' if station['has_shipyard'] else 'N'
modified = datetime.datetime.utcfromtimestamp(station['updated_at']).strftime('%Y-%m-%d %H:%M:%S')
outfitting = 'Y' if station['has_outfitting'] else 'N'
rearm = 'Y' if station['has_rearm'] else 'N'
refuel = 'Y' if station['has_refuel'] else 'N'
repair = 'Y' if station['has_repair'] else 'N'
planetary = 'Y' if station['is_planetary'] else 'N'
type_id = station['type_id'] if station['type_id'] else 0
system = self.execute("SELECT System.name FROM System WHERE System.system_id = ?", (system_id,)).fetchone()[0].upper()
result = self.execute("SELECT modified FROM Station WHERE station_id = ?", (station_id,)).fetchone()
if result:
updated = timegm(datetime.datetime.strptime(result[0],'%Y-%m-%d %H:%M:%S').timetuple())
if station['updated_at'] > updated:
tdenv.DEBUG0("{}/{} has been updated: {} vs {}",
system ,name, modified, result[0])
tdenv.DEBUG1("Updating: {}, {}, {}, {}, {}, {}, {},"
" {}, {}, {}, {}, {}, {}, {}, {}",
station_id, name, system_id, ls_from_star, blackmarket,
max_pad_size, market, shipyard, modified, outfitting,
rearm, refuel, repair, planetary, type_id)
self.execute("""UPDATE Station
SET name = ?, system_id = ?, ls_from_star = ?, blackmarket = ?,
max_pad_size = ?, market = ?, shipyard = ?, modified = ?,
outfitting = ?, rearm = ?, refuel = ?, repair = ?, planetary = ?, type_id = ?
WHERE station_id = ?""",
(name, system_id, ls_from_star, blackmarket,
max_pad_size, market, shipyard, modified,
outfitting, rearm, refuel, repair, planetary, type_id,
station_id))
self.updated['Station'] = True
else:
tdenv.DEBUG0("{}/{} has been added:", system ,name)
tdenv.DEBUG1("Inserting: {}, {}, {}, {}, {}, {}, {},"
" {}, {}, {}, {}, {}, {}, {}, {}",
station_id, name, system_id, ls_from_star, blackmarket,
max_pad_size, market, shipyard, modified, outfitting,
rearm, refuel, repair, planetary, type_id)
self.execute("""INSERT INTO Station (
station_id,name,system_id,ls_from_star,
blackmarket,max_pad_size,market,shipyard,
modified,outfitting,rearm,refuel,
repair,planetary,type_id ) VALUES
( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) """,
(station_id,name,system_id,ls_from_star,
blackmarket,max_pad_size,market,shipyard,
modified,outfitting,rearm,refuel,
repair,planetary,type_id))
self.updated['Station'] = True
#Import shipyards into ShipVendors if shipvend is set.
if station['has_shipyard'] and self.getOption('shipvend'):
if not station['shipyard_updated_at']:
station['shipyard_updated_at'] = station['updated_at']
modified = datetime.datetime.utcfromtimestamp(station['shipyard_updated_at']).strftime('%Y-%m-%d %H:%M:%S')
result = self.execute("SELECT modified FROM ShipVendor WHERE station_id = ?", (station_id,)).fetchone()
if result:
updated = timegm(datetime.datetime.strptime(result[0],'%Y-%m-%d %H:%M:%S').timetuple())
else:
updated = 0
if station['shipyard_updated_at'] > updated:
self.execute("DELETE FROM ShipVendor WHERE station_id = ?", (station_id,))
tdenv.DEBUG1("{}/{} has shipyard, updating ships sold.", system, name)
for ship in station['selling_ships']:
# Make sure all the 'Mark N' ship names abbreviate 'Mark' as '<Name> Mk. <Number>'.
# Fix capitalization.
ship = ship.replace('MK', 'Mk').replace('mk','Mk').replace('mK','Mk')
# Fix no '.' in abbreviation.
if "Mk" in ship and "Mk." not in ship:
ship = ship.replace('Mk', 'Mk.')
# Fix no trailing space.
if "Mk." in ship and "Mk. " not in ship:
ship = ship.replace("Mk.", "Mk. ")
# Fix no leading space.
if "Mk." in ship and " Mk." not in ship:
ship = ship.replace("Mk.", " Mk.")
tdenv.DEBUG2("ship_id:{},station_id:{},modified:{}",
ship,
station_id,
modified)
try:
self.execute("""INSERT INTO ShipVendor
( ship_id,station_id,modified ) VALUES
( (SELECT Ship.ship_id FROM Ship WHERE Ship.name = ?), ?, ? ) """,
(ship,
station_id,
modified))
except sqlite3.IntegrityError:
continue
self.updated['ShipVendor'] = True
#Import Outfitters into UpgradeVendors if upvend is set.
if station['has_outfitting'] and self.getOption('upvend'):
if not station['outfitting_updated_at']:
station['outfitting_updated_at'] = station['updated_at']
modified = datetime.datetime.utcfromtimestamp(station['outfitting_updated_at']).strftime('%Y-%m-%d %H:%M:%S')
result = self.execute("SELECT modified FROM UpgradeVendor WHERE station_id = ?", (station_id,)).fetchone()
if result:
updated = timegm(datetime.datetime.strptime(result[0],'%Y-%m-%d %H:%M:%S').timetuple())
else:
updated = 0
if station['outfitting_updated_at'] > updated:
self.execute("DELETE FROM UpgradeVendor WHERE station_id = ?", (station_id,))
tdenv.DEBUG1("{}/{} has outfitting, updating modules sold.", system, name)
for upgrade in station['selling_modules']:
tdenv.DEBUG2("upgrade_id:{},station_id:{},modified:{}",
upgrade,
station['id'],
modified)
try:
self.execute("""INSERT INTO UpgradeVendor
( upgrade_id,station_id,cost,modified ) VALUES
( ?, ?, (SELECT Upgrade.cost FROM Upgrade WHERE Upgrade.upgrade_id = ?), ? ) """,
(upgrade,
station_id,
upgrade,
modified))
except sqlite3.IntegrityError:
continue
self.updated['UpgradeVendor'] = True
if self.getOption("progbar"):
while prog.value < prog.maxValue:
prog.increment(1, postfix=lambda value, goal: " " + str(round(value / total * 100)) + "%")
prog.clear()
tdenv.NOTE("Finished processing Stations. End time = {}", datetime.datetime.now())
def importSystems(self):
"""
Populate the System table using systems_populated.jsonl
Writes directly to database.
"""
tdb, tdenv = self.tdb, self.tdenv
tdenv.NOTE("Processing Systems: Start time = {}", datetime.datetime.now())
progress = 0
total = 1
def blocks(f, size = 65536):
while True:
b = f.read(size)
if not b: break
yield b
with open(str(self.dataPath / self.systemsPath), "r",encoding = "utf-8",errors = 'ignore') as f:
total += (sum(bl.count("\n") for bl in blocks(f)))
with open(str(self.dataPath / self.systemsPath), "rU") as fh:
if self.getOption("progbar"):
prog = pbar.Progress(total, 50)
for line in fh:
if self.getOption("progbar"):
prog.increment(1, postfix=lambda value, goal: " " + str(round(value / total * 100)) + "%")
else:
progress += 1
print("\rProgress: (" + str(progress) + "/" + str(total) + ") " + str(round(progress / total * 100, 2)) + "%\t\t", end = "\r")
system = json.loads(line)
system_id = system['id']
name = system['name']
pos_x = system['x']
pos_y = system['y']
pos_z = system['z']
modified = datetime.datetime.utcfromtimestamp(system['updated_at']).strftime('%Y-%m-%d %H:%M:%S')
result = self.execute("SELECT modified FROM System WHERE system_id = ?", (system_id,)).fetchone()
if result:
updated = timegm(datetime.datetime.strptime(result[0],'%Y-%m-%d %H:%M:%S').timetuple())
if system['updated_at'] > updated:
tdenv.DEBUG0("System '{}' has been updated: '{}' vs '{}'", name, modified, result[0])
tdenv.DEBUG1("Updating: {}, {}, {}, {}, {}, {}", system_id, name, pos_x, pos_y, pos_z, modified)
self.execute("""UPDATE System
SET name = ?,pos_x = ?,pos_y = ?,pos_z = ?,modified = ?
WHERE system_id = ?""",
(name, pos_x, pos_y, pos_z, modified,
system_id))
self.updated['System'] = True
else:
tdenv.DEBUG0("System '{}' has been added.", name)
tdenv.DEBUG1("Inserting: {}, {}, {}, {}, {}, {}", system_id, name, pos_x, pos_y, pos_z, modified)
self.execute("""INSERT INTO System
( system_id,name,pos_x,pos_y,pos_z,modified ) VALUES
( ?, ?, ?, ?, ?, ? ) """,
(system_id, name, pos_x, pos_y, pos_z, modified))
self.updated['System'] = True
if self.getOption("progbar"):
while prog.value < prog.maxValue:
prog.increment(1, postfix=lambda value, goal: " " + str(round(value / total * 100)) + "%")
prog.clear()
tdenv.NOTE("Finished processing Systems. End time = {}", datetime.datetime.now())
def getBestHops(self, routes, restrictTo=None):
"""
Given a list of routes, try all available next hops from each
route.
Store the results by destination so that we pick the
best route-to-point for each destination at each step.
If we have two routes: A->B->D, A->C->D and A->B->D produces
more profit, there's no point continuing the A->C->D path.
"""
tdb = self.tdb
tdenv = self.tdenv
avoidPlaces = getattr(tdenv, 'avoidPlaces', None) or ()
assert not restrictTo or isinstance(restrictTo, set)
maxJumpsPer = tdenv.maxJumpsPer
maxLyPer = tdenv.maxLyPer
maxPadSize = tdenv.padSize
planetary = tdenv.planetary
noPlanet = tdenv.noPlanet
maxLsFromStar = tdenv.maxLs or float('inf')
reqBlackMarket = getattr(tdenv, 'blackMarket', False) or False
maxAge = getattr(tdenv, 'maxAge') or 0
credits = tdenv.credits - (getattr(tdenv, 'insurance', 0) or 0)
fitFunction = self.defaultFit
capacity = tdenv.capacity
maxUnits = getattr(tdenv, 'limit') or capacity
bestToDest = {}
safetyMargin = 1.0 - tdenv.margin
unique = tdenv.unique
loopInt = getattr(tdenv, 'loopInt', 0) or None
# Penalty is expressed as percentage, reduce it to a multiplier
if tdenv.lsPenalty:
lsPenalty = tdenv.lsPenalty / 100
else:
lsPenalty = 0
goalSystem = tdenv.goalSystem
uniquePath = None
restrictStations = set()
if restrictTo:
for place in restrictTo:
if isinstance(place, Station):
restrictStations.add(place)
elif isinstance(place, System) and place.stations:
restrictStations.update(place.stations)
# Are we doing direct routes?
if tdenv.direct:
if goalSystem and not restrictTo:
restrictTo = (goalSystem, )
restrictStations = set(goalSystem.stations)
if avoidPlaces:
restrictStations = set(
stn for stn in restrictStations
if stn not in avoidPlaces and \
stn.system not in avoidPlaces
)
def station_iterator(srcStation):
srcSys = srcStation.system
srcDist = srcSys.distanceTo
for stn in restrictStations:
stnSys = stn.system
yield Destination(stnSys, stn, (srcSys, stnSys),
srcDist(stnSys))
else:
getDestinations = tdb.getDestinations
def station_iterator(srcStation):
yield from getDestinations(
srcStation,
maxJumps=maxJumpsPer,
maxLyPer=maxLyPer,
avoidPlaces=avoidPlaces,
maxPadSize=maxPadSize,
maxLsFromStar=maxLsFromStar,
noPlanet=noPlanet,
planetary=planetary,
)
prog = pbar.Progress(len(routes), 25)
connections = 0
getSelling = self.stationsSelling.get
for route in routes:
if tdenv.progress:
prog.increment(1)
tdenv.DEBUG1("Route = {}", route.str())
srcStation = route.lastStation
startCr = credits + int(route.gainCr * safetyMargin)
routeJumps = len(route.jumps)
srcSelling = getSelling(srcStation.ID, None)
srcSelling = tuple(values for values in srcSelling
if values[1] <= startCr)
if not srcSelling:
tdenv.DEBUG1("Nothing sold/affordable - next.")
continue
if goalSystem:
origSystem = route.firstSystem
srcSystem = srcStation.system
srcDistTo = srcSystem.distanceTo
goalDistTo = goalSystem.distanceTo
origDistTo = origSystem.distanceTo
srcGoalDist = srcDistTo(goalSystem)
srcOrigDist = srcDistTo(origSystem)
origGoalDist = origDistTo(goalSystem)
if unique:
uniquePath = route.route
elif loopInt:
uniquePath = route.route[-loopInt:-1]
stations = (dest for dest in station_iterator(srcStation)
if dest.station != srcStation)
if reqBlackMarket:
stations = (d for d in stations
if d.station.blackMarket == 'Y')
if uniquePath:
stations = (d for d in stations if d.station not in uniquePath)
if restrictStations:
stations = (d for d in stations
if d.station in restrictStations)
if maxAge:
stations = (d for d in stations if d.station.dataAge)
stations = (d for d in stations if d.station.dataAge <= maxAge)
if goalSystem:
if bool(tdenv.unique):
stations = (d for d in stations
if d.system is not srcSystem)
stations = (
d for d in stations
if d.system is goalSystem or d.distLy < srcGoalDist)
if tdenv.debug >= 1:
def annotate(dest):
tdenv.DEBUG1("destSys {}, destStn {}, jumps {}, distLy {}",
dest.system.dbname, dest.station.dbname,
"->".join(jump.str()
for jump in dest.via), dest.distLy)
return True
stations = (d for d in stations if annotate(d))
for dest in stations:
dstStation = dest.station
connections += 1
items = self.getTrades(srcStation, dstStation, srcSelling)
if not items:
continue
trade = fitFunction(items, startCr, capacity, maxUnits)
multiplier = 1.0
# Calculate total K-lightseconds supercruise time.
# This will amortize for the start/end stations
dstSys = dest.system
if goalSystem and dstSys is not goalSystem:
dstGoalDist = goalDistTo(dstSys)
# Biggest reward for shortening distance to goal
score = 5000 * origGoalDist / dstGoalDist
# bias towards bigger reductions
score += 50 * srcGoalDist / dstGoalDist
# discourage moving back towards origin
if dstSys is not origSystem:
score += 10 * (origDistTo(dstSys) - srcOrigDist)
# Gain per unit pays a small part
score += (trade.gainCr / trade.units) / 25
else:
score = trade.gainCr
if lsPenalty:
# Only want 1dp
cruiseKls = int(dstStation.lsFromStar / 100) / 10
# Produce a curve that favors distances under 1kls
# positively, starts to penalize distances over 1k,
# and after 4kls starts to penalize aggresively
# http://goo.gl/Otj2XP
penalty = ((cruiseKls**2) - cruiseKls) / 3
penalty *= lsPenalty
multiplier *= (1 - penalty)
score *= multiplier
dstID = dstStation.ID
try:
# See if there is already a candidate for this destination
btd = bestToDest[dstID]
except KeyError:
# No existing candidate, we win by default
pass
else:
bestRoute = btd[1]
bestScore = btd[5]
# Check if it is a better option than we just produced
bestTradeScore = bestRoute.score + bestScore
newTradeScore = route.score + score
if bestTradeScore > newTradeScore:
continue
if bestTradeScore == newTradeScore:
bestLy = btd[4]
if bestLy <= dest.distLy:
continue
bestToDest[dstID] = (dstStation, route, trade, dest.via,
dest.distLy, score)
prog.clear()
if connections == 0:
raise NoHopsError(
"No destinations could be reached within the constraints.")
result = []
for (dst, route, trade, jumps, ly, score) in bestToDest.values():
result.append(route.plus(dst, trade, jumps, score))
return result
def download(
tdenv,
url,
localFile,
headers=None,
backup=False,
shebang=None,
chunkSize=4096,
):
"""
Fetch data from a URL and save the output
to a local file. Returns the response headers.
tdenv:
TradeEnv we're working under
url:
URL we're fetching (http, https or ftp)
localFile:
Name of the local file to open.
headers:
dict() of additional HTTP headers to send
shebang:
function to call on the first line
"""
requests = import_requests()
tdenv.NOTE("Requesting {}".format(url))
req = requests.get(url, headers=headers or None, stream=True)
req.raise_for_status()
encoding = req.headers.get('content-encoding', 'uncompress')
length = req.headers.get('content-length', None)
transfer = req.headers.get('transfer-encoding', None)
if transfer != 'chunked':
# chunked transfer-encoding doesn't need a content-length
if length is None:
raise Exception(
"Remote server replied with invalid content-length.")
length = int(length)
if length <= 0:
raise TradeException(
"Remote server gave an empty response. Please try again later."
)
if tdenv.detail > 1:
if length:
tdenv.NOTE("Downloading {} {}ed data", makeUnit(length), encoding)
else:
tdenv.NOTE("Downloading {} {}ed data", transfer, encoding)
tdenv.DEBUG0(str(req.headers).replace("{", "{{").replace("}", "}}"))
# Figure out how much data we have
if length and not tdenv.quiet:
progBar = pbar.Progress(length, 20)
else:
progBar = None
actPath = Path(localFile)
tmpPath = Path("tmp/{}.dl".format(actPath.name))
histogram = deque()
fetched = 0
lastTime = started = time.time()
spinner, spinners = 0, [
'. ', '.. ', '... ', ' ... ', ' ...', ' ..', ' .'
]
with tmpPath.open("wb") as fh:
for data in req.iter_content(chunk_size=chunkSize):
fh.write(data)
fetched += len(data)
if shebang:
bangLine = data.decode().partition("\n")[0]
tdenv.DEBUG0("Checking shebang of {}", bangLine)
shebang(bangLine)
shebang = None
if progBar:
now = time.time()
deltaT = max(now - lastTime, 0.001)
lastTime = now
if len(histogram) >= 15:
histogram.popleft()
histogram.append(len(data) / deltaT)
progBar.increment(
len(data),
postfix=lambda value, goal: \
" {:>7s} [{:>7s}/s] {:>3.0f}% {:1s}".format(
makeUnit(value),
makeUnit(sum(histogram) / len(histogram)),
(fetched * 100. / length),
spinners[spinner]
)
)
if deltaT > 0.200:
spinner = (spinner + 1) % len(spinners)
tdenv.DEBUG0("End of data")
if not tdenv.quiet:
if progBar:
progBar.clear()
elapsed = (time.time() - started) or 1
tdenv.NOTE("Downloaded {} of {}ed data {}/s", makeUnit(fetched),
encoding, makeUnit(fetched / elapsed))
# Swap the file into place
if backup:
bakPath = Path(localFile + ".bak")
if bakPath.exists():
bakPath.unlink()
if actPath.exists():
actPath.rename(localFile + ".bak")
if actPath.exists():
actPath.unlink()
tmpPath.rename(actPath)
req.close()
return req.headers
def getBestHops(self, routes, restrictTo=None):
"""
Given a list of routes, try all available next hops from each
route.
Store the results by destination so that we pick the
best route-to-point for each destination at each step.
If we have two routes: A->B->D, A->C->D and A->B->D produces
more profit, there's no point continuing the A->C->D path.
"""
tdb = self.tdb
tdenv = self.tdenv
avoidPlaces = getattr(tdenv, 'avoidPlaces', None) or ()
assert not restrictTo or isinstance(restrictTo, set)
maxJumpsPer = tdenv.maxJumpsPer
maxLyPer = tdenv.maxLyPer
maxPadSize = tdenv.padSize
planetary = tdenv.planetary
noPlanet = tdenv.noPlanet
maxLsFromStar = tdenv.maxLs or float('inf')
reqBlackMarket = getattr(tdenv, 'blackMarket', False) or False
maxAge = getattr(tdenv, 'maxAge') or 0
credits = tdenv.credits - (getattr(tdenv, 'insurance', 0) or 0)
fitFunction = self.defaultFit
capacity = tdenv.capacity
maxUnits = getattr(tdenv, 'limit') or capacity
bestToDest = {}
safetyMargin = 1.0 - tdenv.margin
unique = tdenv.unique
loopInt = getattr(tdenv, 'loopInt', 0) or None
# Penalty is expressed as percentage, reduce it to a multiplier
if tdenv.lsPenalty:
lsPenalty = max(min(tdenv.lsPenalty / 100, 1), 0)
else:
lsPenalty = 0
goalSystem = tdenv.goalSystem
uniquePath = None
restrictStations = set()
if restrictTo:
for place in restrictTo:
if isinstance(place, Station):
restrictStations.add(place)
elif isinstance(place, System) and place.stations:
restrictStations.update(place.stations)
# Are we doing direct routes?
if tdenv.direct:
if goalSystem and not restrictTo:
restrictTo = (goalSystem, )
restrictStations = set(goalSystem.stations)
if avoidPlaces:
restrictStations = set(
stn for stn in restrictStations
if stn not in avoidPlaces and \
stn.system not in avoidPlaces
)
def station_iterator(srcStation):
srcSys = srcStation.system
srcDist = srcSys.distanceTo
for stn in restrictStations:
stnSys = stn.system
yield Destination(stnSys, stn, (srcSys, stnSys),
srcDist(stnSys))
else:
getDestinations = tdb.getDestinations
def station_iterator(srcStation):
yield from getDestinations(
srcStation,
maxJumps=maxJumpsPer,
maxLyPer=maxLyPer,
avoidPlaces=avoidPlaces,
maxPadSize=maxPadSize,
maxLsFromStar=maxLsFromStar,
noPlanet=noPlanet,
planetary=planetary,
)
prog = pbar.Progress(len(routes), 25)
connections = 0
getSelling = self.stationsSelling.get
for route in routes:
if tdenv.progress:
prog.increment(1)
tdenv.DEBUG1("Route = {}", route.str())
srcStation = route.lastStation
startCr = credits + int(route.gainCr * safetyMargin)
routeJumps = len(route.jumps)
srcSelling = getSelling(srcStation.ID, None)
srcSelling = tuple(values for values in srcSelling
if values[1] <= startCr)
if not srcSelling:
tdenv.DEBUG1("Nothing sold/affordable - next.")
continue
if goalSystem:
origSystem = route.firstSystem
srcSystem = srcStation.system
srcDistTo = srcSystem.distanceTo
goalDistTo = goalSystem.distanceTo
origDistTo = origSystem.distanceTo
srcGoalDist = srcDistTo(goalSystem)
srcOrigDist = srcDistTo(origSystem)
origGoalDist = origDistTo(goalSystem)
if unique:
uniquePath = route.route
elif loopInt:
uniquePath = route.route[-loopInt:-1]
stations = (dest for dest in station_iterator(srcStation)
if dest.station != srcStation)
if reqBlackMarket:
stations = (d for d in stations
if d.station.blackMarket == 'Y')
if uniquePath:
stations = (d for d in stations if d.station not in uniquePath)
if restrictStations:
stations = (d for d in stations
if d.station in restrictStations)
if maxAge:
stations = (d for d in stations if d.station.dataAge)
stations = (d for d in stations if d.station.dataAge <= maxAge)
if goalSystem:
if bool(tdenv.unique):
stations = (d for d in stations
if d.system is not srcSystem)
stations = (
d for d in stations
if d.system is goalSystem or d.distLy < srcGoalDist)
if tdenv.debug >= 1:
def annotate(dest):
tdenv.DEBUG1("destSys {}, destStn {}, jumps {}, distLy {}",
dest.system.dbname, dest.station.dbname,
"->".join(jump.str()
for jump in dest.via), dest.distLy)
return True
stations = (d for d in stations if annotate(d))
for dest in stations:
dstStation = dest.station
connections += 1
items = self.getTrades(srcStation, dstStation, srcSelling)
if not items:
continue
trade = fitFunction(items, startCr, capacity, maxUnits)
multiplier = 1.0
# Calculate total K-lightseconds supercruise time.
# This will amortize for the start/end stations
dstSys = dest.system
if goalSystem and dstSys is not goalSystem:
dstGoalDist = goalDistTo(dstSys)
# Biggest reward for shortening distance to goal
score = 5000 * origGoalDist / dstGoalDist
# bias towards bigger reductions
score += 50 * srcGoalDist / dstGoalDist
# discourage moving back towards origin
if dstSys is not origSystem:
score += 10 * (origDistTo(dstSys) - srcOrigDist)
# Gain per unit pays a small part
score += (trade.gainCr / trade.units) / 25
else:
score = trade.gainCr
if lsPenalty:
# [kfsone] Only want 1dp
cruiseKls = int(dstStation.lsFromStar / 100) / 10
# Produce a curve that favors distances under 1kls
# positively, starts to penalize distances over 1k,
# and after 4kls starts to penalize aggresively
# http://goo.gl/Otj2XP
# [eyeonus] As aadler pointed out, this goes into negative
# numbers, which causes problems.
#penalty = ((cruiseKls ** 2) - cruiseKls) / 3
#penalty *= lsPenalty
#multiplier *= (1 - penalty)
# [eyeonus]:
# (Keep in mind all this ignores values of x<0.)
# The sigmoid: (1-(25(x-1))/(1+abs(25(x-1))))/4
# ranges between 0.5 and 0 with a drop around x=1,
# which makes it great for giving a boost to distances < 1Kls.
#
# The sigmoid: (-1-(50(x-4))/(1+abs(50(x-4))))/4
# ranges between 0 and -0.5 with a drop around x=4,
# making it great for penalizing distances > 4Kls.
#
# The curve: (-1+1/(x+1)^((x+1)/4))/2
# ranges between 0 and -0.5 in a smooth arc,
# which will be used for making distances
# closer to 4Kls get a slightly higher penalty
# then distances closer to 1Kls.
#
# Adding the three together creates a doubly-kinked curve
# that ranges from ~0.5 to -1.0, with drops around x=1 and x=4,
# which closely matches ksfone's intention without going into
# negative numbers and causing problems when we add it to
# the multiplier variable. ( 1 + -1 = 0 )
#
# You can see a graph of the formula here:
# https://goo.gl/sn1PqQ
# NOTE: The black curve is at a penalty of 0%,
# the red curve at a penalty of 100%, with intermediates at
# 25%, 50%, and 75%.
# The other colored lines show the penalty curves individually
# and the teal composite of all three.
def sigmoid(x):
return x / (1 + abs(x))
boost = (1 - sigmoid(25 * (cruiseKls - 1))) / 4
drop = (-1 - sigmoid(50 * (cruiseKls - 4))) / 4
try:
penalty = (-1 + 1 /
(cruiseKls + 1)**((cruiseKls + 1) / 4)) / 2
except OverflowError:
penalty = -0.5
multiplier += (penalty + boost + drop) * lsPenalty
score *= multiplier
dstID = dstStation.ID
try:
# See if there is already a candidate for this destination
btd = bestToDest[dstID]
except KeyError:
# No existing candidate, we win by default
pass
else:
bestRoute = btd[1]
bestScore = btd[5]
# Check if it is a better option than we just produced
bestTradeScore = bestRoute.score + bestScore
newTradeScore = route.score + score
if bestTradeScore > newTradeScore:
continue
if bestTradeScore == newTradeScore:
bestLy = btd[4]
if bestLy <= dest.distLy:
continue
bestToDest[dstID] = (dstStation, route, trade, dest.via,
dest.distLy, score)
prog.clear()
if connections == 0:
raise NoHopsError(
"No destinations could be reached within the constraints.")
result = []
for (dst, route, trade, jumps, ly, score) in bestToDest.values():
result.append(route.plus(dst, trade, jumps, score))
return result