def cli(ctx, ed_dir, log_level):
"""
Command Line Interface for the Energy Dashboard. This tooling
collects information from a number of data feeds, imports that data,
transforms it, and inserts it into a database.
"""
# pass this logger as a child logger to the edl methods
log.configure_logging()
logger = logging.getLogger(__name__)
logger.setLevel(log_level)
log.debug(
logger, {
"name": __name__,
"method": "cli",
"ed_dir": ed_dir,
"log_level": "%s" % log_level
})
if ed_dir is None:
ed_dir = os.path.curdir
else:
if not os.path.exists(ed_dir):
log.critical(
logger, {
"name": __name__,
"method": "cli",
"ed_dir": ed_dir,
"CRITICAL": "ed_dir does not exist"
})
eddir = os.path.abspath(os.path.expanduser(ed_dir))
ctx.obj = {LOGGER: logger, EDDIR: eddir}
def prune(logger, feed, ed_path, stage):
chlogger = logger.getChild(__name__)
p = pre_prune(logger, feed, ed_path, stage)
ext = STAGE_DIRS[stage]
ending = ".%s" % ext
try:
files = filesystem.glob_dir(p, ending)
count = 0
for f in files:
os.remove(os.path.join(p, f))
count += 1
log.debug(
chlogger, {
"name": __name__,
"method": "prune",
"path": ed_path,
"feed": feed,
"target_dir": p,
"ending": ending,
"removed": count,
"message": "pruned target_dir",
})
except Exception as e:
log.critical(
chlogger, {
"name": __name__,
"method": "prune",
"path": ed_path,
"feed": feed,
"target_dir": p,
"ending": ending,
"ERROR": "failed to prune target_dir",
"exception": str(e)
})
def run(logger, manifest, config):
start_date = datetime.date(*manifest['start_date'])
resource_name = manifest['name']
resource_url = manifest['url']
delay = manifest['download_delay_secs']
download_dir = config['working_dir']
state_file = config['state_file']
# sleep for N seconds in between downloads to meet caiso expected use requirements
dates = xtime.range_pairs(xtime.day_range_to_today(start_date))
urls = list(web.generate_urls(logger, dates, resource_url))
log.debug(
logger, {
"name": __name__,
"method": "run",
"resource": resource_name,
"url": resource_url,
"delay": delay,
"download_dir": download_dir,
"state_file": state_file,
"start_date": str(start_date),
"urls_count": len(urls),
})
state.update(
web.download(logger, resource_name, delay, urls, state_file,
download_dir), state_file)
def insert(logger, resource_name, sql_dir, db_dir, new_files):
chlogger = logger.getChild(__name__)
with DbMgr(chlogger, resource_name) as dbmgr:
new_files_count = len(new_files)
if not os.path.exists(db_dir):
os.makedirs(db_dir)
if not os.path.exists(db_dir):
raise Exception("Failed to create db_dir: %s" % db_dir)
log.info(chlogger, {
"name" : __name__,
"src" : resource_name,
"method" : "insert",
"sql_dir" : sql_dir,
"db_dir" : db_dir,
"new_files" : new_files_count,
})
for (idx, sql_file_name) in enumerate(new_files):
yield insert_file(logger, resource_name, dbmgr, sql_dir, db_dir, sql_file_name, idx, depth=0, max_depth=5)
save_dir = os.path.join(os.path.dirname(db_dir), "save")
save_state_file = os.path.join(save_dir, "state.txt")
db_files = filesystem.glob_dir(db_dir, ".db")
with open(save_state_file, 'w') as f:
for dbf in db_files:
f.write("%s\n" % dbf)
log.debug(logger, {
"name" : __name__,
"method" : "insert",
"resource" : resource_name,
"db_file" : dbf,
"state_file": save_state_file,
"message" : "added db_file to state file",
})
def archive_locally(logger, feed, ed_path, archivedir):
chlogger = logger.getChild(__name__)
try:
archivedir1 = os.path.expanduser(archivedir)
if archivedir1.startswith("/"):
archivedire2 = archivedir1
else:
archivedir2 = os.path.join(ed_path, archivedir1)
archive_name = os.path.join(archivedir2, feed)
root_dir = os.path.expanduser(os.path.join(ed_path, 'data', feed))
log.debug(
chlogger, {
"name": __name__,
"method": "archive_locally",
"path": ed_path,
"feed": feed,
"target_dir": archivedir2,
"archive_name": archive_name,
"root_dir": root_dir
})
return make_archive(archive_name, 'gztar', root_dir)
except Exception as e:
log.critical(
chlogger, {
"name": __name__,
"method": "archive_locally",
"path": ed_path,
"feed": feed,
"target_dir": archivedir2,
"archive_name": archive_name,
"root_dir": root_dir,
"ERROR": "make archive failed",
"exception": str(e)
})
def generate_urls(logger, date_pairs, url_template, date_format="%Y%m%d"):
"""
Generate download urls for the provided date_pairs.
date_pairs : list of tuples with (start, end) dates
url_template : contains a _START_ and _END_ strings which will be replaced
by (start,end) tuples formated by the date_format
date_format : format string for the start and end dates
TODO/BIKESHED : replace _X_ with Jinja mustache templates {{}}
"""
chlogger = logger.getChild(__name__)
for (start, end) in date_pairs:
s = start.strftime(date_format)
e = end.strftime(date_format)
url = url_template.replace("_START_", s).replace("_END_", e)
log.debug(
chlogger, {
"name": __name__,
"method": "generate_urls",
"start": s,
"end": e,
"url": url
})
yield url
def list(logger, energy_dashboard_path):
chlogger = logger.getChild(__name__)
log.debug(chlogger, {
"name": __name__,
"method": "list",
"path": energy_dashboard_path
})
return os.listdir(os.path.join(energy_dashboard_path, "data"))
def clone(logger, ed_path):
cmd = "git clone https://github.com/energy-analytics-project/energy-dashboard.git"
log.debug(logger, {
"name": __name__,
"method": "init",
"path": ed_path,
"cmd": cmd,
})
return runyield(cmd, ed_path)
def update(logger, ed_path):
cmd = "git submodule update --init --recursive"
log.debug(logger, {
"name": __name__,
"method": "update",
"path": ed_path,
"cmd": cmd,
})
return runyield(cmd, ed_path)
def __exit__(self, type, value, traceback):
for k,v in self.dbs.items():
v.close()
log.debug(self.logger, {
"name" : __name__,
"src" : self.resource_name,
"method" : "DbMgr.__exit__",
"db_path" : k,
"message" : "Closed in memory db",
})
def reset(logger, feed, ed_path, stage):
chlogger = logger.getChild(__name__)
p = pre_reset(logger, feed, ed_path, stage)
try:
if os.path.exists(p):
shutil.rmtree(p)
log.debug(
chlogger, {
"name": __name__,
"method": "reset",
"path": ed_path,
"feed": feed,
"target_dir": p,
"message": "removed target_dir",
})
except Exception as e:
log.critical(
chlogger, {
"name": __name__,
"method": "reset",
"path": ed_path,
"feed": feed,
"target_dir": p,
"ERROR": "failed to remove target_dir",
"exception": str(e)
})
try:
if not os.path.exists(p):
os.makedirs(p)
log.debug(
chlogger, {
"name": __name__,
"method": "reset",
"path": ed_path,
"feed": feed,
"target_dir": p,
"message": "makedirs target_dir",
})
except Exception as e:
log.critical(
chlogger, {
"name": __name__,
"method": "reset",
"path": ed_path,
"feed": feed,
"target_dir": p,
"ERROR": "failed to makedirs target_dir",
"exception": str(e)
})
return p
def src_files(logger, feed, ed_path):
chlogger = logger.getChild(__name__)
feed_dir = os.path.join(ed_path, 'data', feed)
src_dir = os.path.join(feed_dir, 'src')
src_files = sorted(os.listdir(src_dir))
log.debug(
chlogger, {
"name": __name__,
"method": "src_files",
"path": ed_path,
"feed": feed,
"feed_dir": feed_dir,
"src_dir": src_dir,
"src_files": src_files
})
return src_files
def process_file(logger, feed, ed_path, src_file):
chlogger = logger.getChild(__name__)
feed_dir = os.path.join(ed_path, 'data', feed)
rel_path = os.path.join("src", src_file)
cmd = "%s %s" % (rel_path,
log.LOGGING_LEVEL_STRINGS[chlogger.getEffectiveLevel()])
log.debug(
chlogger, {
"name": __name__,
"method": "process_file",
"path": ed_path,
"feed": feed,
"cmd": cmd
})
return runyield(cmd, feed_dir)
def process_stages(logger, feed, ed_path, stages):
chlogger = logger.getChild(__name__)
stage_files = [STAGE_PROCS[s] for s in stages]
for sf in stage_files:
if sf in src_files(logger, feed, ed_path):
yield process_file(logger, feed, ed_path, sf)
else:
log.debug(
chlogger, {
"name": __name__,
"method": "process_stages",
"path": ed_path,
"feed": feed,
"stage_file": sf,
"src_files": src_files,
"ERROR": "stage_file not in src_files"
})
def get(self, db_path):
log.debug(self.logger, {
"name" : __name__,
"src" : self.resource_name,
"method" : "DbMgr.get",
"db_path" : db_path,
})
if db_path not in self.dbs:
db = MemDb(db_path)
cnx = db.open()
self.dbs[db_path] = db
log.debug(self.logger, {
"name" : __name__,
"src" : self.resource_name,
"method" : "DbMgr.get",
"db_path" : db_path,
"message" : "Created in memory db",
})
return self.dbs[db_path].db()
def parse_text_file(logger, resource_name, txt_dir, sql_dir, f):
input_file = os.path.join(txt_dir, f)
(dict_renewable, dict_total) = read_file_name(input_file)
renewable_sql = gen_renewable_sql(dict_renewable)
total_sql = gen_total_sql(dict_total)
(f_name, f_ext) = os.path.splitext(f)
output_file = os.path.join(sql_dir, "%s.sql" % f_name)
with open(output_file, 'w') as sqlfile:
[sqlfile.write("%s\n" % line) for line in renewable_sql]
[sqlfile.write("%s\n" % line) for line in total_sql]
log.debug(
logger, {
"name": __name__,
"method": "parse_text_file",
"src": "30_pars.py",
"input": input_file,
"output": output_file,
})
return f
def run(logger, manifest, config):
resource_name = manifest['name']
resource_url = manifest['url']
xml_dir = config['source_dir']
sql_dir = config['working_dir']
state_file = config['state_file']
new_files = state.new_files(resource_name, state_file, xml_dir, '.xml')
log.debug(logger, {
"name" : __name__,
"method" : "run",
"resource" : resource_name,
"url" : resource_url,
"xml_dir" : xml_dir,
"sql_dir" : sql_dir,
"state_file": state_file,
"new_files_count" : len(new_files),
})
state.update(
xmlparser.parse(logger, resource_name, new_files, xml_dir, sql_dir),
state_file)
def run(logger, manifest, config):
resource_name = manifest['name']
resource_url = manifest['url']
txt_dir = config['source_dir']
sql_dir = config['working_dir']
state_file = config['state_file']
new_files = state.new_files(resource_name, state_file, txt_dir,
'DailyRenewablesWatch.txt')
log.debug(
logger, {
"name": __name__,
"method": "run",
"resource": resource_name,
"url": resource_url,
"txt_dir": txt_dir,
"sql_dir": sql_dir,
"state_file": state_file,
"new_files_count": len(new_files),
})
state.update(
parse_text_files(logger, resource_name, new_files, txt_dir, sql_dir),
state_file)
def invoke(logger, feed, ed_path, command):
chlogger = logger.getChild(__name__)
target_dir = os.path.join(ed_path, 'data', feed)
log.debug(
chlogger, {
"name": __name__,
"method": "invoke",
"path": ed_path,
"feed": feed,
"command": command
})
if not os.path.exists(target_dir):
log.critical(
chlogger, {
"name": __name__,
"method": "invoke",
"path": ed_path,
"feed": feed,
"command": command,
"target_dir": target_dir,
"ERROR": "target_dir does not exist"
})
else:
return runyield([command], target_dir)
def process_all_stages(logger, feed, ed_path):
chlogger = logger.getChild(__name__)
found_src_files = src_files(logger, feed, ed_path)
if len(found_src_files) < 1:
log.critical(
chlogger, {
"name": __name__,
"method": "process_all_stages",
"path": ed_path,
"feed": feed,
"src_files": found_src_files,
"ERROR": "No files found, nothing to process"
})
return
log.debug(
chlogger, {
"name": __name__,
"method": "process_all_stages",
"path": ed_path,
"feed": feed,
"src_files": found_src_files
})
for src_file in found_src_files:
yield process_file(logger, feed, ed_path, src_file)
})
for output in clifeed.archive_to_s3(logger, resource_name, ed_path, "digitalocean", digitalocean_bandwidth_limit):
log.info(logger, {
"name" : __name__,
"method" : "run",
"resource" : resource_name,
"service" : "digitalocean",
"stdout" : str(output),
})
shutil.rmtree(os.path.join(os.curdir, 'dist'))
# -----------------------------------------------------------------------------
# Main
# -----------------------------------------------------------------------------
if __name__ == "__main__":
if len(sys.argv) > 1:
loglevel = sys.argv[1]
else:
loglevel = "INFO"
log.configure_logging()
logger = logging.getLogger(__name__)
logger.setLevel(loglevel)
log.debug(logger, {
"name" : __name__,
"method" : "main",
"src" : "70_arch.py"
})
with open('manifest.json', 'r') as json_file:
m = json.load(json_file)
run(logger, m, config())
"name": __name__,
"method": "run",
"resource": resource_name,
"sql_dir": sql_dir,
"db_dir": db_dir,
"state_file": state_file,
"new_files_count": len(new_files),
"message": "finished processing sql files",
})
# -----------------------------------------------------------------------------
# Main
# -----------------------------------------------------------------------------
if __name__ == "__main__":
if len(sys.argv) > 1:
loglevel = sys.argv[1]
else:
loglevel = "INFO"
log.configure_logging()
logger = logging.getLogger(__name__)
logger.setLevel(loglevel)
log.debug(logger, {
"name": __name__,
"method": "main",
"src": "40_inse.py"
})
with open('manifest.json', 'r') as json_file:
m = json.load(json_file)
run(logger, m, config())
"url": resource_url,
"txt_dir": txt_dir,
"sql_dir": sql_dir,
"state_file": state_file,
"new_files_count": len(new_files),
})
state.update(
parse_text_files(logger, resource_name, new_files, txt_dir, sql_dir),
state_file)
# -----------------------------------------------------------------------------
# Main
# -----------------------------------------------------------------------------
if __name__ == "__main__":
if len(sys.argv) > 1:
loglevel = sys.argv[1]
else:
loglevel = "INFO"
log.configure_logging()
logger = logging.getLogger(__name__)
logger.setLevel(loglevel)
log.debug(logger, {
"name": __name__,
"method": "main",
"src": "30_pars.py"
})
with open('manifest.json', 'r') as json_file:
m = json.load(json_file)
run(logger, m, config())
logger, {
"name": __name__,
"method": "run",
"resource": resource_name,
"db_dir": db_dir,
"save_dir": db_dir,
"state_file": state_file,
"message": "finished saving state",
})
# -----------------------------------------------------------------------------
# Main
# -----------------------------------------------------------------------------
if __name__ == "__main__":
if len(sys.argv) > 1:
loglevel = sys.argv[1]
else:
loglevel = "INFO"
log.configure_logging()
logger = logging.getLogger(__name__)
logger.setLevel(loglevel)
log.debug(logger, {
"name": __name__,
"method": "main",
"src": "50_save.py"
})
with open('manifest.json', 'r') as json_file:
m = json.load(json_file)
run(logger, m, config())
def run(logger, manifest, config):
start_date = datetime.date(*manifest['start_date'])
resource_name = manifest['name']
resource_url = manifest['url']
delay = manifest['download_delay_secs']
download_dir = config['working_dir']
txt_dir = config['source_dir']
state_file = config['state_file']
# sleep for N seconds in between downloads to meet caiso expected use requirements
dates = xtime.range_pairs(xtime.day_range_to_today(start_date))
urls = list(web.generate_urls(logger, dates, resource_url))
log.debug(
logger, {
"name": __name__,
"method": "run",
"resource": resource_name,
"url": resource_url,
"delay": delay,
"download_dir": download_dir,
"state_file": state_file,
"start_date": str(start_date),
"urls_count": len(urls),
})
# download .txt files
downloaded_txt_urls = web.download(logger,
resource_name,
delay,
urls,
state_file,
download_dir,
ending='.txt')
# copy .txt files to ./text dir and then
# compress original .txt files to .zip files
if not os.path.exists(txt_dir):
log.debug(
logger, {
"name": __name__,
"method": "run",
"src": "10_down.py",
"message": "created target txt dir: %s" % txt_dir,
})
os.makedirs(txt_dir)
# process downloaded .txt files
data_files = glob.glob(
os.path.join(download_dir, "*DailyRenewablesWatch.txt"))
for tf in data_files:
try:
# remove write protections for .txt files
os.chmod(
os.path.join(download_dir, tf),
S_IWRITE | S_IWGRP | S_IWOTH | S_IREAD | S_IRGRP | S_IROTH)
# if the txt file is here, it needs to be copied to the ./txt dir
fqtf = os.path.join(download_dir, tf)
fqtf2 = os.path.join(txt_dir, tf)
fqtfzip = os.path.join(download_dir, '%s.zip' % tf)
if not os.path.exists(fqtf2):
shutil.copyfile(fqtf, fqtf2)
with zipfile.ZipFile(fqtfzip, 'w') as myzip:
myzip.write(fqtfzip)
# set .zip file to be read only
os.chmod(fqtfzip, S_IREAD | S_IRGRP | S_IROTH)
# remove the zip/.txt file as it's been copied to txt/.txt
if os.path.exists(fqtf2) and os.path.exists(fqtfzip):
os.remove(fqtf)
log.debug(
logger, {
"name": __name__,
"method": "run",
"src": "10_down.py",
"message": "zipped file: %s" % tf,
})
except Exception as e:
log.error(
logger, {
"name": __name__,
"method": "run",
"src": "10_down.py",
"file": tf,
"error": "failed to process file",
"exception": str(e),
})
# TODO: something is clobbering perms on the state file, so clobber it back
os.chmod(os.path.join(download_dir, 'state.txt'),
S_IWRITE | S_IWGRP | S_IWOTH | S_IREAD | S_IRGRP | S_IROTH)
# final step
state.update(downloaded_txt_urls, state_file)
def create(logger, ed_path, feed, maintainer, company, email, url, start_date,
delay):
"""
start_date : list of numbers : [2019,09,1]
"""
chlogger = logger.getChild(__name__)
new_feed_dir = os.path.join(ed_path, 'data', feed)
try:
os.mkdir(new_feed_dir)
log.debug(
chlogger, {
"name": __name__,
"method": "create",
"path": ed_path,
"feed": feed,
"dir": new_feed_dir,
"message": "created directory"
})
template_files = [
"LICENSE", "Makefile", "README.md", "src/10_down.py",
"src/20_unzp.py", "src/30_pars.py", "src/40_inse.py",
"src/50_save.py", "src/60_dist.sh", "src/70_arch.py",
"manifest.json"
]
env = Environment(loader=PackageLoader('edl', 'templates'),
autoescape=select_autoescape(['py']))
m = {
'NAME': feed,
'MAINTAINER': maintainer,
'COMPANY': company,
'EMAIL': email,
'DATA_URL': url,
'REPO_URL':
"https://github.com/energy-analytics-project/%s" % feed,
'START': start_date,
'DELAY': delay
}
for tf in template_files:
template = env.get_template(tf)
target = os.path.join(new_feed_dir, tf)
path = os.path.dirname(target)
if not os.path.exists(path):
os.makedirs(path)
with open(target, 'w') as f:
f.write(template.render(m))
log.debug(
chlogger, {
"name": __name__,
"method": "create",
"path": ed_path,
"feed": feed,
"target": target,
"message": "rendered target"
})
hidden_files = ['gitignore']
for hf in hidden_files:
template = env.get_template(hf)
target = os.path.join(new_feed_dir, ".%s" % hf)
with open(target, 'w') as f:
f.write(template.render(m))
log.debug(
chlogger, {
"name": __name__,
"method": "create",
"path": ed_path,
"feed": feed,
"target": target,
"message": "rendered target"
})
for src_file in os.listdir(os.path.join(new_feed_dir, 'src')):
fp = os.path.join(new_feed_dir, 'src', src_file)
f = Path(fp)
f.chmod(f.stat().st_mode | stat.S_IEXEC)
log.debug(
chlogger, {
"name": __name__,
"method": "create",
"path": ed_path,
"feed": feed,
"file": fp,
"message": "chmod +x"
})
for d in DIRS:
os.makedirs(os.path.join(new_feed_dir, d))
return feed
except Exception as e:
tb = traceback.format_exc()
log.critical(
chlogger, {
"name": __name__,
"method": "create",
"path": ed_path,
"feed": feed,
"ERROR": "FAILED to create feed",
"exception": str(e),
"trace": str(tb),
})
def download(logger,
resource_name,
delay,
urls,
state_file,
path,
ending=".zip"):
"""
urls : list of urls to download
state_file : list of urls that have already been downloaded
path : path to write downloaded files to
"""
chlogger = logger.getChild(__name__)
downloaded = []
prev_downloaded = set()
if os.path.exists(state_file):
with open(state_file, "r") as f:
prev_downloaded = set([line.rstrip() for line in f])
status = {'manifest': 0, 'filesystem': 0, 'downloaded': 0, 'error': 0}
for url in urls:
try:
filename = filesystem.url2filename(url, ending=ending)
if url in prev_downloaded:
log.debug(
chlogger, {
"src": resource_name,
"action": 'skip_download',
"url": url,
"file": filename,
"msg": 'url exists in download manifest'
})
status['manifest'] += 1
continue
target_file = os.path.join(path, filename)
if os.path.exists(target_file):
log.debug(
chlogger, {
"src": resource_name,
"action": 'skip_download',
"url": url,
"file": filename,
"msg": 'file exists locally, updating manifest'
})
# update the state_file with files that were found on disk
downloaded.append(url)
status['filesystem'] += 1
continue
r = requests.get(url)
if r.status_code == 200:
with open(target_file, 'wb') as fd:
for chunk in r.iter_content(chunk_size=128):
fd.write(chunk)
downloaded.append(url)
status['downloaded'] += 1
log.debug(
chlogger, {
"src": resource_name,
"action": 'download',
"url": url,
"file": filename
})
else:
log.error(
chlogger, {
"src": resource_name,
"action": 'download',
"url": url,
"file": filename,
"status_code": r.status_code,
"ERROR": 'http_request_failed'
})
except Exception as e:
log.error(
chlogger, {
"src": resource_name,
"action": 'download',
"url": url,
"ERROR": "http_request_failed",
"exception": str(e),
"traceback": str(tb=traceback.format_exc())
})
status['error'] += 1
# TODO: this is such a hack
time.sleep(delay)
# ensure that all files in the download directery are read only
for f in filesystem.glob_dir(path, ending):
os.chmod(os.path.join(path, f), S_IREAD | S_IRGRP | S_IROTH)
log.info(chlogger, { \
"src" : resource_name, \
"action" : 'download', \
"url" : url, \
'skipped_in_manifest' : status['manifest'], \
'skipped_in_filesystem' : status['filesystem'], \
'downloaded' : status['downloaded'], \
'error' : status['error'], \
})
return downloaded
def insert_file(logger, resource_name, dbmgr, sql_dir, db_dir, sql_file_name, idx, depth, max_depth):
chlogger = logger.getChild(__name__)
db_name = gen_db_name(resource_name, depth)
sql_file = os.path.join(sql_dir, sql_file_name)
db_file = os.path.join(db_dir, db_name)
if depth > max_depth:
log.error(chlogger, {
"name" : __name__,
"src" : resource_name,
"method" : "insert_file",
"db_file" : db_file,
"file_idx" : idx,
"sql_file" : sql_file,
"depth" : depth,
"max_depth" : max_depth,
"dbmgr" : str(dbmgr),
"ERROR" :"insert sql_file failed, max_depth exceeded",
})
return
log.info(chlogger, {
"name" : __name__,
"src" : resource_name,
"method" : "insert_file",
"db_file" : db_file,
"file_idx" : idx,
"sql_file" : sql_file,
"depth" : depth,
"dbmgr" : str(dbmgr),
"message" : "started",
})
cnx = dbmgr.get(db_file)
try:
with open(sql_file, 'r') as sf:
log.debug(chlogger, {
"name" : __name__,
"src" : resource_name,
"method" : "insert_file",
"db_file" : db_file,
"file_idx" : idx,
"sql_file" : sql_file,
"depth" : depth,
"dbmgr" : str(dbmgr),
"message" : "started",
})
cnx.executescript(sf.read())
log.debug(chlogger, {
"name" : __name__,
"src" : resource_name,
"method" : "insert_file",
"db_file" : db_file,
"file_idx" : idx,
"sql_file" : sql_file,
"depth" : depth,
"dbmgr" : str(dbmgr),
"message" : "completed",
})
return sql_file_name
except Exception as e:
log.error(chlogger, {
"name" : __name__,
"src" : resource_name,
"method" : "insert_file",
"file_idx" : idx,
"db_file" : db_file,
"sql_file" : sql_file,
"depth" : depth,
"dbmgr" : str(dbmgr),
"ERROR" : "insert sql_file failed",
"exception": str(e),
})
insert_file(logger, resource_name, dbmgr, sql_dir, db_dir, sql_file, idx, depth+1, max_depth)
# -----------------------------------------------------------------------------
# Entrypoint
# -----------------------------------------------------------------------------
def run(logger, manifest, config):
# nothing to do, text files have already been copied to ./txt by
# 10_down.py.
pass
# -----------------------------------------------------------------------------
# Main
# -----------------------------------------------------------------------------
if __name__ == "__main__":
if len(sys.argv) > 1:
loglevel = sys.argv[1]
else:
loglevel = "INFO"
log.configure_logging()
logger = logging.getLogger(__name__)
logger.setLevel(loglevel)
log.debug(logger, {
"name": __name__,
"method": "main",
"src": "20_unzp.py"
})
with open('manifest.json', 'r') as json_file:
m = json.load(json_file)
run(logger, m, config())
"delay": delay,
"download_dir": download_dir,
"state_file": state_file,
"start_date": str(start_date),
"urls_count": len(urls),
})
state.update(
web.download(logger, resource_name, delay, urls, state_file,
download_dir), state_file)
# -----------------------------------------------------------------------------
# Main
# -----------------------------------------------------------------------------
if __name__ == "__main__":
if len(sys.argv) > 1:
loglevel = sys.argv[1]
else:
loglevel = "INFO"
log.configure_logging()
logger = logging.getLogger(__name__)
logger.setLevel(loglevel)
log.debug(logger, {
"name": __name__,
"method": "main",
"src": "10_down.py"
})
with open('manifest.json', 'r') as json_file:
m = json.load(json_file)
run(logger, m, config())
