def testget_list_of_atels_still_to_download_function(self):
from atelParser import download
atels = download(log=log, settings=settings)
atelsToDownload = atels.get_list_of_atels_still_to_download()
atels.maxsleep = 7
atels.download_list_of_atels(atelsToDownload[:1])
def test_download_function_exception(self):
from atelParser import download
try:
this = download(log=log,
settings=settings,
fakeKey="break the code")
this.get()
assert False
except Exception as e:
assert True
print(str(e))
def download_new_atels(self):
"""*download new atel html files*
"""
self.log.debug('starting the ``download_new_atels`` method')
self.update_git_repo()
from atelParser import download
atels = download(log=self.log, settings=self.settings)
atelsToDownload = atels.get_list_of_atels_still_to_download()
if "test" in self.settings and self.settings["test"] == True:
atelsToDownload = [atelsToDownload[-1]]
atels.maxsleep = 1
else:
atels.maxsleep = 20
atels.download_list_of_atels(atelsToDownload)
self.update_git_repo()
self.log.debug('completed the ``download_new_atels`` method')
return None
def main(arguments=None):
"""
*The main function used when `cl_utils.py` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="atelParser",
defaultSettingsFile=True)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES
# AUTOMATICALLY
a = {}
for arg, val in list(arguments.items()):
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
a[varname] = val
if arg == "--dbConn":
dbConn = val
a["dbConn"] = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
# set options interactively if user requests
if "interactiveFlag" in a and a["interactiveFlag"]:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
if a["init"]:
from os.path import expanduser
home = expanduser("~")
filepath = home + "/.config/atelParser/atelParser.yaml"
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
return
count = a["count"]
download = a["download"]
parse = a["parse"]
reparseFlag = a["reparseFlag"]
# CALL FUNCTIONS/OBJECTS
if download:
from atelParser import download
atels = download(log=log, settings=settings)
atelsToDownload = atels.get_list_of_atels_still_to_download()
atels.download_list_of_atels(atelsToDownload)
if count:
from atelParser import download
atels = download(log=log, settings=settings)
latestNumber = atels.get_latest_atel_number()
from datetime import datetime, date, time
now = datetime.now()
now = now.strftime("%Y/%m/%d %H:%M:%Ss")
print("%(latestNumber)s ATels have been reported as of %(now)s" %
locals())
if parse:
from atelParser import mysql
parser = mysql(log=log, settings=settings, reParse=reparseFlag)
parser.atels_to_database()
parser.parse_atels()
parser.populate_htm_columns()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
def test_get_latest_atel_number_function(self):
from atelParser import download
atels = download(log=log, settings=settings)
latestNumber = atels.get_latest_atel_number()