def main(argv):
cmdIndex = commands.CommandIndex()
cmdenv = cmdIndex.parse(argv)
tdb = tradedb.TradeDB(cmdenv, load=cmdenv.wantsTradeDB)
if cmdenv.usesTradeData:
tsc = tdb.tradingStationCount
if tsc == 0:
raise exceptions.NoDataError(
"There is no trading data for ANY station in "
"the local database. Please enter or import "
"price data."
)
if tsc == 1:
raise exceptions.NoDataError(
"The local database only contains trading data "
"for one station. Please enter or import data "
"for additional stations."
)
if tsc < 8:
cmdenv.NOTE(
"The local database only contains trading data "
"for {} stations. Please enter or import data "
"for additional stations.".format(
tsc
)
)
results = cmdenv.run(tdb)
if results:
tdb.close()
results.render()
def test_derp(tdb=None, tdenv=None):
"""
Test whether the station names in a trade database are free of derp.
Examples:
import tradedb
tdb = tradedb.TradeDB()
test_derp(tdb)
python -i cache.py
>>> test_derp()
"""
tdb = tdb or tradedb.TradeDB()
tdenv = tdenv or tdb.tdenv
matches = 0
for stn in tdb.stationByID.values():
m = checkForOcrDerp(tdenv, stn.system.dbname, stn.dbname)
if m:
print("Match", m.groups(1))
matches += 1
if not matches:
print("Current data is free of known derp")
def main():
argv = parse_arguments()
tdenv = tradeenv.TradeEnv(properties=argv)
tdb = tradedb.TradeDB(tdenv)
system, tdbSys = get_system(argv, tdb)
if argv.pick:
argv.destinations.extend(pick_destinations(argv, tdb))
if argv.destinations:
process_destinations(argv, tdb)
return
print("Add EDSC Star Distances for \"{}\"".format(system))
print()
print("You will now be prompted for distances to various stars.")
print()
print(
"At each prompt, the star name will be copied into your paste buffer. "
"You should alt-tab into the game and paste the name into the Galaxy "
"Map's search box. Then alt-tab back and enter the distance value.")
print()
print("At each prompt enter a ly distance (e.g. 123.45), q to stop, "
"or leave the line empty if you don't want to enter data for "
"this star.")
print("5 distances are required for EDSC to make a first guess at a "
"star's location. You can submit more to increase the accuracy "
"but the only time you need to submit more than 10 is when you "
"are trying to submit corrections.")
print()
clip = SystemNameClip()
print()
print("""
===================================================
STANDARD STARS: (q to skip to the next section)
These are stars with well-known positions.
===================================================
""")
distances, term = get_distances(argv, clip, standardStars)
if distances:
send_and_check_distances(argv, tdb, clip, distances)
outliers = get_outliers(argv)
if outliers:
print("""
===================================================
EXTRA STARS: (q to skip to the next section)
Stars from {}.
===================================================
""".format(argv.extraFile))
distances, term = get_distances(argv, clip, outliers)
if distances:
send_and_check_distances(argv, tdb, clip, distances)
print("""
===================================================
CHOOSE YOUR OWN: (q to stop)
Specify additional stars.
Prefix names with a '+' to add them to
{}.
===================================================
""".format(argv.extraFile))
distances = []
newOutliers = []
while True:
star = input("Enter star name: ")
star = star.strip().upper()
if not star or star == 'Q':
break
# Remove surrounding quotes
save = False
if star.startswith('+'):
save = True
star = star[1:].strip()
star = re.sub(r'\s+', ' ', star)
star = re.sub(r"''+", "'", star)
star = re.sub(r'^("|\')+\s*(.*)\s*\1+', r'\2', star)
if star.find('"') >= 0:
print("Invalid star name")
continue
if star.startswith('+'):
save = True
star = star[1:].strip()
for ref in distances:
if ref['name'] == star:
print("'{}' is already listed.")
continue
check_system(argv, tdb, tdbSys, star)
extras, term = get_distances(argv, clip, [star])
if term != 'q' and len(extras) > 0:
distances.extend(extras)
if save and star not in outliers and star not in newOutliers:
newOutliers.append(star)
if send_and_check_distances(argv, tdb, clip, distances):
add_extra_stars(argv, newOutliers)
def main():
doDeletions = False
parser = argparse.ArgumentParser(
description='Check for prices that are outside reasonable bounds.'
)
parser.add_argument(
'--percentile',
help='Set cutoff percentile',
type=float,
default=2,
)
parser.add_argument(
'--selling',
help='Check the StationSelling table instead of StationBuying',
action='store_true',
)
parser.add_argument(
'--delete',
help='Remove bad elements from the local .db immediately',
action='store_true',
default=False,
)
parser.add_argument(
'--db',
help='Specify location of the SQLite database.',
default=None,
dest='dbFilename',
type=str,
)
parser.add_argument(
'--debug', '-w',
help='Enable/raise level of diagnostic output.',
default=0,
required=False,
action='count',
)
parser.add_argument(
'--detail', '-v',
help='Increase level of detail in output.',
default=0,
required=False,
action='count',
)
parser.add_argument(
'--quiet', '-q',
help='Reduce level of detail in output.',
default=0,
required=False,
action='count',
)
parser.add_argument(
'--margin',
help='Adjust the error margin.',
type=int,
default=25,
)
parser.add_argument(
'--ignore',
help='Ignore items.',
action='append',
default=list(),
)
filters = parser.add_mutually_exclusive_group()
filters.add_argument(
'--dumb',
help='Limit to "DUMB" items (<11cr).',
dest='filters',
action='store_const',
const='DUMB',
)
filters.add_argument(
'--high',
help='Limit to "HIGH" items.',
dest='filters',
action='store_const',
const='HIGH',
)
filters.add_argument(
'--low',
help='Limit to "LOW" items.',
dest='filters',
action='store_const',
const='LOW',
)
argv = parser.parse_args(sys.argv[1:])
argv.ignore = [
ignore.upper() for ignore in argv.ignore
]
table = "StationSelling" if argv.selling else "StationBuying"
tdenv = tradeenv.TradeEnv(properties=argv)
tdb = tradedb.TradeDB(tdenv)
tdenv.NOTE(
"Checking {}, margin={}", table, argv.margin,
)
errorFilter = getattr(argv, "filters", None)
check_price_bounds(
tdb,
table,
margin=argv.margin,
doDeletions=argv.delete,
percentile=argv.percentile,
errorFilter=errorFilter,
)
def main():
argv = parse_arguments()
tdenv = tradeenv.TradeEnv(properties=argv)
tdenv.quiet = 1
tdb = tradedb.TradeDB(tdenv)
if not argv.summary:
try:
argv.startSys = tdb.lookupSystem(argv.refSystem)
except (LookupError, tradedb.AmbiguityError):
raise UsageError(
"Unrecognized system '{}'. Reference System must be an "
"*exact* name for a system that TD already knows.\n"
"Did you forget to put double-quotes around the reference "
"system name?".format(argv.refSystem))
if not argv.date:
argv.date = tdb.query("SELECT MAX(modified) FROM System").fetchone()[0]
dateRe = re.compile(
r'^20\d\d-(0[1-9]|1[012])-(0[1-9]|[12]\d|3[01]) ([01]\d|2[0123]):[0-5]\d:[0-5]\d$'
)
if not dateRe.match(argv.date):
raise UsageError(
"Invalid date: '{}', expecting YYYY-MM-DD HH:MM:SS format.".format(
argv.date))
print("start date: {}".format(argv.date))
edsq = misc.edsm.StarQueryMulti(
log=argv.logEDSM,
startDateTime=argv.date,
submitted=1,
showId=1,
)
data = edsq.fetch()
systems = data
print("{} results".format(len(systems)))
# Filter out systems we already know that match the EDSM data.
systems = [sysinfo for sysinfo in systems if is_change(tdb, sysinfo)]
print("{} deltas".format(len(systems)))
if argv.summary or len(systems) <= 0:
return
systems = [sysinfo for sysinfo in systems if 'coords' in sysinfo]
if argv.random:
num = min(len(systems), argv.maxSystems)
systems = random.sample(systems, num)
if argv.refSys:
refSys = tdb.lookupPlace(argv.refSys)
else:
refSys = None
startSys = argv.startSys
for sysinfo in systems:
x = sysinfo['coords']['x']
y = sysinfo['coords']['y']
z = sysinfo['coords']['z']
sysinfo['distance'] = get_distance(tdb, startSys, x, y, z)
if refSys:
sysinfo['refdist'] = get_distance(tdb, refSys, x, y, z)
else:
sysinfo['refdist'] = None
if argv.distance:
if argv.maxLy > 0:
if refSys:
systems = [
sysinfo for sysinfo in systems
if sysinfo['refdist'] <= argv.maxLy
]
else:
systems = [
sysinfo for sysinfo in systems
if sysinfo['distance'] <= argv.maxLy
]
else:
if refSys:
systems.sort(key=lambda sysinfo: sysinfo['refdist'])
else:
systems.sort(key=lambda sysinfo: sysinfo['distance'])
systems = systems[:argv.maxSystems]
if argv.splash and not argv.autoOK:
print(
"\n"
"===============================================================\n"
"\n"
" The tool will now take you through the stars returned by EDSM\n"
" that are new or different from your local System.csv.\n"
"\n"
" You will be prompted with the name and predicted distance from\n"
" your current system so you can check for mistakes.\n"
"\n"
" The name will be copied into your clipboard so you can alt-tab\n"
" into the game and paste the name into the Galaxy Map's SEARCH\n"
" box (under NAVIGATION). Let the map zoom to the system.\n"
"\n"
" Check the name and distance, then use the appropriate action.\n"
"\n"
" (Use the -NS option to skip this text in future)\n"
"\n"
"===============================================================\n"
"\n")
input("Hit enter to continue: ")
if not argv.autoOK:
print("""At the prompt enter:
q
to indicate you've suffered enough,
y
to accept the name/value,
n (or anything else)
to skip the name/value (no confirmation),
=name (e.g. =SOL)
to accept the distance but correct spelling,
""")
print()
extras = get_extras()
clip = misc.clipboard.SystemNameClip()
total = len(systems)
current = 0
with open("tmp/new.systems.csv", "w", encoding="utf-8") as output:
if argv.autoOK:
commit = False
else:
commit = True
for sysinfo in systems:
current += 1
name = sysinfo['name']
created = sysinfo['date']
x = sysinfo['coords']['x']
y = sysinfo['coords']['y']
z = sysinfo['coords']['z']
if not argv.autoOK:
print("\n"
"-----------------------------------------------\n"
"{syidlab:.<12}: {syid}\n"
"{crealab:.<12}: {crts}\n".format(
syidlab="ID",
crealab="Created",
syid=sysinfo['id'],
crts=created,
))
if refSys:
print("{reflab:.<12}: {refdist}ly\n".format(
reflab="Ref Dist",
refdist=sysinfo['refdist'],
))
check_database(tdb, name, x, y, z)
change = has_position_changed(sysinfo['place'], name, x, y, z)
if change:
oldDist = startSys.distanceTo(sysinfo['place'])
print("Old Distance: {:.2f}ly".format(oldDist))
distance = sysinfo['distance']
clip.copy_text(name)
prompt = "{}/{}: '{}': {:.2f}ly? ".format(
current,
total,
name,
distance,
)
if argv.autoOK:
if change:
ok = "n"
else:
ok = "y"
else:
ok = input(prompt)
if ok.lower() == 'q':
break
if ok.startswith('='):
name = ok[1:].strip().upper()
if not name in extras:
add_to_extras(argv, name)
ok = 'y'
if ok.lower() != 'y':
continue
if argv.add:
print("Add {:>6}: {:>12} {} {}".format(current, sysinfo['id'],
created, name))
tdb.addLocalSystem(name,
x,
y,
z,
added='EDSM',
modified=created,
commit=commit)
print("'{}',{},{},{},'EDSM','{}'".format(
name,
x,
y,
z,
created,
),
file=output)
if argv.add and not commit:
tdb.getDB().commit()
# not useful as data.
demandStr = defIQL if fromStn <= 0 else unkIQL
if supplyLevel == 0:
supplyStr = naIQL
elif supplyLevel < 0 and supply <= 0:
supplyStr = defIQL
else:
units = "?" if supply < 0 else str(supply)
level = levelDesc[supplyLevel + 1]
supplyStr = units + level
else:
if fromStn == 0 or demandLevel == 0:
demandStr = naIQL
elif demandLevel < 0 and demand <= 0:
demandStr = defIQL
else:
units = "?" if demand < 0 else str(demand)
level = levelDesc[demandLevel + 1]
demandStr = units + level
supplyStr = naIQL
output += outFmt.format(item, fromStn, toStn, demandStr, supplyStr,
modified)
file.write(output)
if __name__ == "__main__":
import tradedb
tdb = tradedb.TradeDB(load=False)
dumpPrices(tdb.dbPath, elementMask=Element.full)
def main():
argv = parse_arguments()
tdenv = tradeenv.TradeEnv(properties=argv)
tdb = tradedb.TradeDB(tdenv)
if not argv.summary:
try:
argv.startSys = tdb.lookupSystem(argv.refSystem)
except (LookupError, tradedb.AmbiguityError):
raise UsageError(
"Unrecognized system '{}'. Reference System must be an "
"*exact* name for a system that TD already knows.\n"
"Did you forget to put double-quotes around the reference "
"system name?".format(argv.refSystem))
print("start date: {}".format(argv.date))
edsq = misc.edsc.StarQuery(
test=argv.test,
confidence=argv.confidence,
date=argv.date,
)
data = edsq.fetch()
if edsq.status['statusnum'] != 0:
raise Exception("Query failed: {} ({})".format(
edsq.status['msg'],
edsq.status['statusnum'],
))
date = data['date']
systems = data['systems']
print("{} results".format(len(systems)))
# Filter out systems we already know that match the EDSC data.
systems = [sysinfo for sysinfo in systems if is_change(tdb, sysinfo)]
print("{} deltas".format(len(systems)))
if argv.summary or len(systems) <= 0:
return
systems = [sysinfo for sysinfo in systems if 'coord' in sysinfo]
if argv.random:
num = min(len(systems), argv.maxSystems)
systems = random.sample(systems, num)
if argv.refSys:
refSys = tdb.lookupPlace(argv.refSys)
else:
refSys = None
startSys = argv.startSys
for sysinfo in systems:
x, y, z = sysinfo['coord']
sysinfo['distance'] = get_distance(tdb, startSys, x, y, z)
if refSys:
sysinfo['refdist'] = get_distance(tdb, refSys, x, y, z)
else:
sysinfo['refdist'] = None
if argv.distance:
if refSys:
systems.sort(key=lambda sysinfo: sysinfo['refdist'])
else:
systems.sort(key=lambda sysinfo: sysinfo['distance'])
systems = systems[:argv.maxSystems]
if argv.splash:
print(
"\n"
"===============================================================\n"
"\n"
" The tool will now take you through the stars returned by EDSC\n"
" that are new or different from your local System.csv.\n"
"\n"
" You will be prompted with the name and predicted distance from\n"
" your current system so you can check for mistakes.\n"
"\n"
" The name will be copied into your clipboard so you can alt-tab\n"
" into the game and paste the name into the Galaxy Map's SEARCH\n"
" box (under NAVIGATION). Let the map zoom to the system.\n"
"\n"
" Check the name and distance, then use the appropriate action.\n"
"\n"
" (Use the -NS option to skip this text in future)\n"
"\n"
"===============================================================\n"
"\n")
input("Hit enter to continue: ")
print("""At the prompt enter:
q
to indicate you've suffered enough,
y
to accept the name/value and confirm with EDSC,
n (or anything else)
to skip the name/value (no confirmation),
=name (e.g. =SOL)
to accept the distance but correct spelling,
~dist (e.g. ~104.49)
to submit a distance correction for the system,
""")
print()
extras = get_extras()
clip = misc.clipboard.SystemNameClip()
total = len(systems)
current = 0
with open("tmp/new.systems.csv", "w", encoding="utf-8") as output:
for sysinfo in systems:
current += 1
name = sysinfo['name']
created = sysinfo['createdate']
x, y, z = sysinfo['coord']
print("\n"
"-----------------------------------------------\n"
"{syidlab:.<12}: {syid}\n"
"{conflab:.<12}: {conf}\n"
"{crealab:.<12}: {crcm} {crts}\n"
"{updtlab:.<12}: {upcm} {upts}\n".format(
syidlab="ID",
conflab="Confidence",
crealab="Created",
updtlab="Updated",
syid=sysinfo['id'],
conf=sysinfo['cr'],
crcm=sysinfo['commandercreate'],
crts=created,
upcm=sysinfo.get('commanderupdate', '[never]'),
upts=sysinfo.get('updatedate', ''),
))
if refSys:
print("{reflab:.<12}: {refdist}ly\n".format(
reflab="Ref Dist",
refdist=sysinfo['refdist'],
))
check_database(tdb, name, x, y, z)
change = has_position_changed(sysinfo['place'], name, x, y, z)
if change:
oldDist = startSys.distanceTo(sysinfo['place'])
print("Old Distance: {:.2f}ly".format(oldDist))
distance = sysinfo['distance']
clip.copy_text(name)
prompt = "{}/{}: '{}': {:.2f}ly? ".format(
current,
total,
name,
distance,
)
ok = input(prompt)
if ok.lower() == 'q':
break
if ok.startswith('~'):
correction = float(ok[1:])
submit_distance(argv, name, correction)
if not name.upper() in extras:
add_to_extras(argv, name)
continue
if ok.startswith('='):
name = ok[1:].strip().upper()
if not name in extras:
add_to_extras(argv, name)
ok = 'y'
if ok.lower() != 'y':
continue
if argv.add:
tdb.addLocalSystem(name,
x,
y,
z,
added='Release 1.00-EDStar',
modified=created,
commit=True)
print("'{}',{},{},{},'Release 1.00-EDStar','{}'".format(
name,
x,
y,
z,
created,
),
file=output)
submit_distance(argv, name, distance)
import csvexport
import misc.eddb
import tradedb
# Build an ID=>Name mapping for EDDB systems
tdb = tradedb.TradeDB()
systems = {}
tdSysLookup = tdb.systemByName.get
for s in misc.eddb.SystemsQuery():
tdSys = tdSysLookup(s['name'].upper(), None)
if tdSys and tdSys.stations:
systems[s['id']] = {stn.dbname.upper(): stn for stn in tdSys.stations}
def matching_stations():
# generator that searches the eddb station set for entries that
# match tdb entries and yields the tuple (tdStn, eddbStn)
for eddbStn in misc.eddb.StationsQuery():
stationList = systems.get(eddbStn['system_id'], None)
if not stationList:
continue
name = eddbStn['name'].upper()
tdStn = stationList.get(name, None)
if tdStn:
yield tdStn, eddbStn
updateStation = tdb.updateLocalStation
bool_trans = {None: '?', 0: 'N', 1: 'Y'}
def export_listings():
"""
Creates a "listings-live.csv" file in "export_path" every X seconds,
as defined in the configuration file.
Only runs when program configured as server.
"""
global export_ack, export_busy
if config['side'] == 'server':
# We want to perform some automatic DB maintenance when running as server.
maintenance_time = time.time() + (config['server_maint_every_x_hour'] *
3600)
tdb = tradedb.TradeDB(load=False)
db = tdb.getDB()
listings_file = (Path(config['export_path']).resolve() /
Path("listings-live.csv"))
listings_tmp = listings_file.with_suffix(".tmp")
print("Listings will be exported to: \n\t" + str(listings_file))
while go:
now = time.time()
# Wait until the time specified in the "export_every_x_sec" config
# before doing an export, watch for busy signal or shutdown signal
# while waiting.
while time.time() < now + config['export_every_x_sec']:
if not go:
break
if update_busy:
print("Listings exporter acknowledging busy signal.")
export_ack = True
while update_busy and go:
time.sleep(1)
export_ack = False
# Just in case we caught the shutdown command while waiting.
if not go:
break
print("Busy signal off, listings exporter resuming.")
now = time.time()
if time.time() >= maintenance_time:
start = datetime.datetime.now()
print("Performing server maintenance tasks." + str(start))
try:
db_execute(db, "VACUUM")
db_execute(db, "PRAGMA optimize")
except Error as e:
print("Error performing maintenance: " + str(e))
maintenance_time = time.time() + (
config['server_maint_every_x_hour'] * 3600)
complete = datetime.datetime.now()
print("Server maintenance tasks completed. " +
str(complete))
print("Maintenance cycle took " + str(complete - start) +
".")
time.sleep(1)
# We may be here because we broke out of the waiting loop,
# so we need to see if we lost go and quit the main loop if so.
if not go:
break
start = datetime.datetime.now()
print("Listings exporter sending busy signal. " + str(start))
export_busy = True
# We don't need to wait for acknowledgement from the update checker,
# because it waits for one from this, and this won't acknowledge
# until it's finished exporting.
while not (process_ack):
if not go:
break
print("Busy signal acknowledged, getting listings for export.")
try:
cursor = fetchIter(
db_execute(
db,
"SELECT * FROM StationItem WHERE from_live = 1 ORDER BY station_id, item_id"
))
results = list(cursor)
except sqlite3.DatabaseError as e:
print(e)
export_busy = False
continue
except AttributeError as e:
print("Got Attribute error trying to fetch StationItems: " + e)
print(cursor)
continue
export_busy = False
print("Exporting 'listings-live.csv'. (Got listings in " +
str(datetime.datetime.now() - start) + ")")
with open(str(listings_tmp), "w") as f:
f.write(
"id,station_id,commodity_id,supply,supply_bracket,buy_price,sell_price,demand,demand_bracket,collected_at\n"
)
lineNo = 1
for result in results:
# If we lose go during export, we need to abort.
if not go:
break
station_id = str(result[0])
commodity_id = str(result[1])
sell_price = str(result[2])
demand = str(result[3])
demand_bracket = str(result[4])
buy_price = str(result[5])
supply = str(result[6])
supply_bracket = str(result[7])
collected_at = str(
timegm(
datetime.datetime.strptime(
result[8], '%Y-%m-%d %H:%M:%S').timetuple()))
listing = station_id + "," + commodity_id + "," \
+supply + "," + supply_bracket + "," + buy_price + "," \
+sell_price + "," + demand + "," + demand_bracket + "," \
+collected_at
f.write(str(lineNo) + "," + listing + "\n")
lineNo += 1
del results
# If we aborted the export because we lost go, listings_tmp is broken and useless, so delete it.
if not go:
listings_tmp.unlink()
print("Export aborted, received shutdown signal.")
break
while listings_file.exists():
try:
listings_file.unlink()
except:
time.sleep(1)
listings_tmp.rename(listings_file)
print("Export completed in " +
str(datetime.datetime.now() - start))
print("Shutting down listings exporter.")
else:
export_ack = True
def process_messages():
global process_ack
tdb = tradedb.TradeDB(load=False)
conn = tdb.getDB()
# Place the database into autocommit mode to avoid issues with
# sqlite3 doing automatic transactions.
conn.isolation_level = None
curs = conn.cursor()
# same SQL every time
updStmt = "UPDATE Station SET system_id = ? WHERE station_id = ?"
delStmt = "DELETE FROM StationItem WHERE station_id = ?"
insStmt = ("INSERT OR IGNORE INTO StationItem("
" station_id, item_id, modified,"
" demand_price, demand_units, demand_level,"
" supply_price, supply_units, supply_level, from_live)"
" VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1)")
avgStmt = "UPDATE Item SET avg_price = ? WHERE item_id = ?"
while go:
# We don't want the threads interfering with each other,
# so pause this one if either the update checker or
# listings exporter report that they're active.
if update_busy or export_busy:
print("Message processor acknowledging busy signal.")
process_ack = True
while (update_busy or export_busy) and go:
time.sleep(1)
process_ack = False
# Just in case we caught the shutdown command while waiting.
if not go:
break
print("Busy signal off, message processor resuming.")
# Either get the first message in the queue,
# or go to sleep and wait if there aren't any.
try:
entry = q.popleft()
except IndexError:
time.sleep(1)
continue
# Get the station_is using the system and station names.
system = entry.system.upper()
station = entry.station.upper()
# And the software version used to upload the schema.
software = entry.software
swVersion = entry.version
station_id = station_ids.get(system + "/" + station)
if not station_id:
# Mobile stations are stored in the dict a bit differently.
station_id = station_ids.get("MEGASHIP/" + station)
system_id = system_ids.get(system)
if station_id and system_id:
print("Megaship station, updating system.", end=" ")
# Update the system the station is in, in case it has changed.
success = False
while not success:
try:
print(system_id, end='\r')
curs.execute("BEGIN IMMEDIATE")
curs.execute(updStmt, (system_id, station_id))
conn.commit()
success = True
except sqlite3.IntegrityError:
if config['verbose']:
print("ERROR: Not found in Systems: " + system +
"/" + station)
continue
except sqlite3.OperationalError:
print("Database is locked, waiting for access.",
end="\n")
time.sleep(1)
else:
if config['verbose']:
print("ERROR: Not found in Stations: " + system + "/" +
station)
continue
modified = entry.timestamp.replace('T', ' ').replace('Z', '')
commodities = entry.commodities
start_update = datetime.datetime.now()
if config['debug']:
with debugPath.open('a', encoding="utf-8") as fh:
fh.write(system + "/" + station + " with station_id '" +
str(station_id) + "' updated at " + modified +
" using " + software + swVersion + " ---\n")
itemList = []
avgList = []
for commodity in commodities:
if commodity['sellPrice'] == 0 and commodity['buyPrice'] == 0:
# Skip blank entries
continue
# Get fdev_id using commodity name from message.
item_edid = db_name.get(commodity['name'].lower())
if not item_edid:
if config['verbose']:
print("Ignoring item: " + commodity['name'])
continue
# Some items, mostly recently added items, are found in db_name but not in item_ids
# (This is entirely EDDB.io's fault.)
item_id = item_ids.get(item_edid)
if not item_id:
if config['verbose']:
print(
"EDDB.io's API does not include likely recently added item: '"
+ commodity['name'] +
"', using fdev_id as placeholder, please inform the current EDDB.io maintainer."
)
item_id = item_edid
itemList.append((
station_id,
item_id,
modified,
commodity['sellPrice'],
commodity['demand'],
commodity['demandBracket']
if commodity['demandBracket'] != '' else -1,
commodity['buyPrice'],
commodity['stock'],
commodity['stockBracket']
if commodity['stockBracket'] != '' else -1,
))
# We only "need" to update the avg_price for the few items not included in
# EDDB.io's API, but might as well do it for all of them.
avgList.append((commodity['meanPrice'], item_id))
success = False
while not success:
try:
curs.execute("BEGIN IMMEDIATE")
success = True
except sqlite3.OperationalError:
print("Database is locked, waiting for access.", end="\n")
time.sleep(1)
curs.execute(delStmt, (station_id, ))
try:
curs.executemany(insStmt, itemList)
curs.executemany(avgStmt, avgList)
except Exception as e:
if config['debug']:
with debugPath.open('a', encoding="utf-8") as fh:
fh.write("Error '" + str(e) +
"' when inserting message:\n" + str(itemList))
success = False
while not success:
try:
conn.commit()
success = True
except sqlite3.OperationalError:
print("Database is locked, waiting for access.", end="\n")
time.sleep(1)
if config['verbose']:
print("Market update for " + system + "/" + station \
+" finished in " + str(int((datetime.datetime.now() - start_update).total_seconds() * 1000) / 1000) + " seconds.")
else:
print("Updated " + system + "/" + station)
print("Shutting down message processor.")
def process_messages():
global process_ack
tdb = tradedb.TradeDB(load=False)
while go:
db = tdb.getDB()
# Place the database into autocommit mode to avoid issues with
# sqlite3 doing automatic transactions.
db.isolation_level = None
# We don't want the threads interfering with each other,
# so pause this one if either the update checker or
# listings exporter report that they're active.
if update_busy or export_busy:
print("Message processor acknowledging busy signal.")
process_ack = True
while (update_busy or export_busy) and go:
time.sleep(1)
process_ack = False
# Just in case we caught the shutdown command while waiting.
if not go:
# Make sure any changes are committed before shutting down.
#
# As we are using autocommit, bypass the db.commit() for now
# by setting success to "True"
success = True
while not success:
try:
db.commit()
success = True
except sqlite3.OperationalError as e:
if "locked" not in str(e):
success = True
raise sqlite3.OperationalError(e)
else:
print(
"(execute) Database is locked, waiting for access.",
end="\r")
time.sleep(1)
db.close()
break
print("Busy signal off, message processor resuming.")
# Either get the first message in the queue,
# or go to sleep and wait if there aren't any.
try:
entry = q.popleft()
except IndexError:
time.sleep(1)
continue
# Get the station_is using the system and station names.
system = entry.system.upper()
station = entry.station.upper()
# And the software version used to upload the schema.
software = entry.software
swVersion = entry.version
try:
station_id = station_ids[system + "/" + station]
except KeyError:
try:
# Mobile stations are stored in the dict a bit differently.
station_id = station_ids["MEGASHIP/" + station]
system_id = system_ids[system]
print("Megaship station, updating system.", end=" ")
# Update the system the station is in, in case it has changed.
try:
db_execute(
db, """UPDATE Station
SET system_id = ?
WHERE station_id = ?""",
(system_id, station_id))
except Exception as e:
print(e)
except KeyError as e:
if config['verbose']:
print("ERROR: Not found in Stations: " + system + "/" +
station)
continue
modified = entry.timestamp.replace('T', ' ').replace('Z', '')
commodities = entry.commodities
start_update = datetime.datetime.now()
items = dict()
if config['debug']:
with debugPath.open('a', encoding="utf-8") as fh:
fh.write(system + "/" + station + " with station_id '" +
str(station_id) + "' updated at " + modified +
" using " + software + swVersion + " ---\n")
for commodity in commodities:
# Get item_id using commodity name from message.
try:
name = db_name[commodity['name'].lower()]
except KeyError:
if config['verbose']:
print("Ignoring rare item: " + commodity['name'])
continue
# Some items, mostly salvage items, are found in db_name but not in item_ids
# (This is entirely EDDB.io's fault.)
try:
item_id = item_ids[name]
except KeyError:
if config['verbose']:
print("EDDB.io does not include likely salvage item: '" +
name + "'")
continue
items[name] = {
'item_id':
item_id,
'demand_price':
commodity['sellPrice'],
'demand_units':
commodity['demand'],
'demand_level':
commodity['demandBracket']
if commodity['demandBracket'] != '' else -1,
'supply_price':
commodity['buyPrice'],
'supply_units':
commodity['stock'],
'supply_level':
commodity['stockBracket']
if commodity['stockBracket'] != '' else -1,
}
for key in item_ids:
if key in items:
entry = items[key]
else:
entry = {
'item_id': item_ids[key],
'demand_price': 0,
'demand_units': 0,
'demand_level': 0,
'supply_price': 0,
'supply_units': 0,
'supply_level': 0,
}
if config['debug']:
with debugPath.open('a', encoding="utf-8") as fh:
fh.write("\t" + key + ": " + str(entry) + "\n")
try:
# Skip inserting blank entries so as to not bloat DB.
if entry['demand_price'] == 0 and entry['supply_price'] == 0:
raise sqlite3.IntegrityError
db_execute(
db, """INSERT INTO StationItem
(station_id, item_id, modified,
demand_price, demand_units, demand_level,
supply_price, supply_units, supply_level, from_live)
VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, 1 )""",
(station_id, entry['item_id'], modified,
entry['demand_price'], entry['demand_units'],
entry['demand_level'], entry['supply_price'],
entry['supply_units'], entry['supply_level']))
except sqlite3.IntegrityError:
try:
db_execute(
db, """UPDATE StationItem
SET modified = ?,
demand_price = ?, demand_units = ?, demand_level = ?,
supply_price = ?, supply_units = ?, supply_level = ?,
from_live = 1
WHERE station_id = ? AND item_id = ?""",
(modified, entry['demand_price'],
entry['demand_units'], entry['demand_level'],
entry['supply_price'], entry['supply_units'],
entry['supply_level'], station_id, entry['item_id']))
except sqlite3.IntegrityError as e:
if config['verbose']:
print("Unable to insert or update: '" + commodity +
"' Error: " + str(e))
del entry
# Don't try to commit if there are still messages waiting.
if len(q) == 0:
# As we are using autocommit, bypass the db.commit() for now
# by setting success to "True"
success = True
while not success:
try:
db.commit()
success = True
except sqlite3.OperationalError:
print("Database is locked, waiting for access.", end="\r")
time.sleep(1)
# Don't close DB until we've committed the changes.
db.close()
if config['verbose']:
print("Market update for " + system + "/" + station\
+ " finished in " + str(int((datetime.datetime.now() - start_update).total_seconds() * 1000) / 1000) + " seconds.")
else:
print("Updated " + system + "/" + station)
print("Shutting down message processor.")
