def downloadFile(self, urlTail, path):
"""
Fetch the latest dumpfile from the website if newer than local copy.
"""
tdb, tdenv = self.tdb, self.tdenv
tdenv.DEBUG0("Checking for update to '{}'.", path)
if urlTail == SHIPS_URL:
url = SHIPS_URL
else:
if not self.getOption('fallback'):
try:
url = BASE_URL + urlTail
response = request.urlopen(url)
except:
# If Tromador's server fails for whatever reason,
# fallback to download direct from EDDB.io
self.options["fallback"] = True
if self.getOption('fallback'):
# EDDB.io doesn't have live listings.
if urlTail == LIVE_LISTINGS:
return False
url = FALLBACK_URL + urlTail
response = request.urlopen(url)
dumpModded = 0
# The coriolis file is from github, so it doesn't have a "Last-Modified" metadata.
if url != SHIPS_URL:
# Stupid strptime() is locale-dependent, so we do it ourself.
# 'Last-Modified' is in the form "DDD, dd MMM yyyy hh:mm:ss GMT"
# dDL 0 1 2 3 4 5
# dTL 0 1 2
# We'll need to turn the 'MMM' into a number.
Months = {'Jan':1, 'Feb':2, 'Mar':3, 'Apr':4, 'May':5, 'Jun':6, 'Jul':7, 'Aug':8, 'Sep':9, 'Oct':10, 'Nov':11, 'Dec':12}
# We need to split the string twice, because the time is separated by ':', not ' '.
dDL = response.getheader("Last-Modified").split(' ')
dTL = dDL[4].split(':')
# Now we need to make a datetime object using the DateList and TimeList we just created,
# and then we can finally convert that to a Unix-epoch number.
dumpDT = datetime.datetime(int(dDL[3]), Months[dDL[2]], int(dDL[1]),\
hour=int(dTL[0]), minute=int(dTL[1]), second=int(dTL[2]),\
tzinfo=datetime.timezone.utc)
dumpModded = timegm(dumpDT.timetuple())
if Path.exists(self.dataPath / path):
localModded = (self.dataPath / path).stat().st_mtime
if localModded >= dumpModded and url != SHIPS_URL:
tdenv.DEBUG0("'{}': Dump is not more recent than Local.", path)
return False
tdenv.NOTE("Downloading file '{}'.", path)
transfers.download( self.tdenv, url, self.dataPath / path, )
return True
def download_fdevids(self):
"""
Download the current data from EDCD,
see https://github.com/EDCD/FDevIDs
"""
tdb, tdenv = self.tdb, self.tdenv
BASE_URL = "https://raw.githubusercontent.com/EDCD/FDevIDs/master/"
downloadList = []
if self.getOption("shipyard"):
downloadList.append(
('FDevShipyard', 'shipyard.csv', self.process_fdevids_table)
)
if self.getOption("outfitting"):
downloadList.append(
('FDevOutfitting', 'outfitting.csv', self.process_fdevids_table)
)
if self.getOption("commodity"):
downloadList.append(
('FDevCommodity', 'commodity.csv', self.process_fdevids_items)
)
if len(downloadList) == 0:
tdenv.NOTE("I don't know what do to, give me some options!")
return
optLocal = self.getOption("local")
for tableName, fileEDCD, callMeBack in downloadList:
localPath = tdb.dataPath / pathlib.Path(tableName).with_suffix(".csv")
if optLocal:
if not localPath.exists():
raise PluginException(
"CSV-file '{}' not found.".format(str(localPath))
)
else:
transfers.download(
tdenv,
BASE_URL + fileEDCD,
str(localPath),
)
if callMeBack:
callMeBack(localPath, tableName)
def download_prices(self, lastRunDays):
""" Figure out which file to download and fetch it. """
# Argument checking
use3h = 1 if self.getOption("use3h") else 0
use2d = 1 if self.getOption("use2d") else 0
usefull = 1 if self.getOption("usefull") else 0
if use3h + use2d + usefull > 1:
raise PluginException(
"Only one of use3h/use2d/usefull can be used at once."
)
if self.getOption("skipdl"):
if use3h or use2d or usefull:
raise PluginException(
"use3h/use2d/usefull has no effect with --opt=skipdl"
)
return
# Overrides
if use3h:
lastRunDays = 0.01
elif use2d:
lastRunDays = 1.0
elif usefull:
lastRunDays = 3.0
# Use age/options to determine which file
if lastRunDays < 3 / 24:
priceFile = "prices-3h.asp"
elif lastRunDays < 1.9:
priceFile = "prices-2d.asp"
else:
priceFile = "prices.asp"
# Fetch!
transfers.download(
self.tdenv,
BASE_URL + priceFile,
self.filename,
shebang=lambda line: self.check_shebang(line, True),
)
def download_prices(self, lastRunDays):
""" Figure out which file to download and fetch it. """
# Argument checking
use3h = 1 if self.getOption("use3h") else 0
use2d = 1 if self.getOption("use2d") else 0
usefull = 1 if self.getOption("usefull") else 0
if use3h + use2d + usefull > 1:
raise PluginException(
"Only one of use3h/use2d/usefull can be used at once.")
if self.getOption("skipdl"):
if (use3h or use2d or usefull):
raise PluginException(
"use3h/use2d/usefull has no effect with --opt=skipdl")
return
# Overrides
if use3h:
lastRunDays = 0.01
elif use2d:
lastRunDays = 1.0
elif usefull:
lastRunDays = 3.0
# Use age/options to determine which file
if lastRunDays < 3 / 24:
priceFile = "prices-3h.asp"
elif lastRunDays < 1.9:
priceFile = "prices-2d.asp"
else:
priceFile = "prices.asp"
# Fetch!
transfers.download(
self.tdenv,
BASE_URL + priceFile,
self.filename,
shebang=lambda line: self.check_shebang(line, True),
)
def run(results, cmdenv, tdb):
# If we're using a plugin, initialize that first.
if cmdenv.plug:
if cmdenv.pluginOptions:
cmdenv.pluginOptions = chain.from_iterable(
opt.split(',') for opt in cmdenv.pluginOptions)
try:
pluginClass = plugins.load(cmdenv.plug, "ImportPlugin")
except plugins.PluginException as e:
raise CommandLineError("Plugin Error: " + str(e))
# Initialize the plugin
plugin = pluginClass(tdb, cmdenv)
# Run the plugin. If it returns False, then it did everything
# that needs doing and we can stop now.
# If it returns True, it is returning control to the module.
if not plugin.run():
return None
tdb.reloadCache()
tdb.close()
if cmdenv.filename:
if re.match(r"^https?://", cmdenv.filename, re.IGNORECASE):
cmdenv.url, cmdenv.filename = cmdenv.filename, None
if cmdenv.url:
cmdenv.filename = cmdenv.filename or "import.prices"
transfers.download(cmdenv, cmdenv.url, cmdenv.filename)
if cmdenv.download:
return None
# If the filename specified was "-" or None, then go ahead
# and present the user with an open file dialog.
if not cmdenv.filename and hasTkInter:
tk = tkinter.Tk()
tk.withdraw()
filetypes = (
("TradeDangerous '.prices' Files", "*.prices"),
("All Files", "*.*"),
)
filename = tkfd.askopenfilename(
title="Select the file to import",
initialfile="TradeDangerous.prices",
filetypes=filetypes,
initialdir='.',
)
if not filename:
raise SystemExit("Aborted")
cmdenv.filename = filename
# check the file exists.
if cmdenv.filename != "-":
fh = None
filePath = Path(cmdenv.filename)
if not filePath.is_file():
raise CommandLineError("File not found: {}".format(str(filePath)))
else:
filePath = "stdin"
fh = sys.stdin
if cmdenv.plug:
if not plugin.finish():
cache.regeneratePricesFile()
return None
cache.importDataFromFile(tdb,
cmdenv,
filePath,
pricesFh=fh,
reset=cmdenv.reset)
return None
def run(results, cmdenv, tdb):
# If we're using a plugin, initialize that first.
if cmdenv.plug:
if cmdenv.pluginOptions:
cmdenv.pluginOptions = chain.from_iterable(
opt.split(',') for opt in cmdenv.pluginOptions
)
try:
pluginClass = plugins.load(cmdenv.plug, "ImportPlugin")
except plugins.PluginException as e:
raise CommandLineError("Plugin Error: "+str(e))
# Initialize the plugin
plugin = pluginClass(tdb, cmdenv)
# Run the plugin. If it returns False, then it did everything
# that needs doing and we can stop now.
# If it returns True, it is returning control to the module.
if not plugin.run():
return None
tdb.reloadCache()
tdb.close()
if cmdenv.filename:
if re.match(r"^https?://", cmdenv.filename, re.IGNORECASE):
cmdenv.url, cmdenv.filename = cmdenv.filename, None
if cmdenv.url:
cmdenv.filename = cmdenv.filename or "import.prices"
transfers.download(cmdenv, cmdenv.url, cmdenv.filename)
if cmdenv.download:
return None
# If the filename specified was "-" or None, then go ahead
# and present the user with an open file dialog.
if not cmdenv.filename and hasTkInter:
tk = tkinter.Tk()
tk.withdraw()
filetypes = (
("TradeDangerous '.prices' Files", "*.prices"),
("All Files", "*.*"),
)
filename = tkfd.askopenfilename(
title="Select the file to import",
initialfile="TradeDangerous.prices",
filetypes=filetypes,
initialdir='.',
)
if not filename:
raise SystemExit("Aborted")
cmdenv.filename = filename
# check the file exists.
if cmdenv.filename != "-":
fh = None
filePath = Path(cmdenv.filename)
if not filePath.is_file():
raise CommandLineError("File not found: {}".format(
str(filePath)
))
else:
filePath = "stdin"
fh = sys.stdin
if cmdenv.plug:
if not plugin.finish():
cache.regeneratePricesFile()
return None
cache.importDataFromFile(tdb, cmdenv, filePath, pricesFh=fh, reset=cmdenv.reset)
return None
