def download_source(use_cache, do_store, type, attributes):
storage = Storage()
timestamp = datetime.datetime.now()
try:
try:
source = attributes["class"](use_cache=use_cache)
if type == "meta":
data = source.download_meta_data()
else:
data = source.download_snapshot_data()
if not data:
raise ValueError(f"No data returned from {attributes['class'].__name__}.download_{type}_data()")
if do_store and data:
storage.store(attributes["source_id"], timestamp, data, type)
return data
except BaseException as e:
error_str = f"{attributes['source_id']}: {e.__class__.__name__}"
traceback_str = traceback.format_exc()
print(f"{error_str}\n{traceback_str}")
if do_store:
storage.store(
attributes["source_id"],
timestamp,
{
"error": error_str,
"traceback": traceback_str,
},
"error"
)
except BaseException as e:
print(f"{attributes['source_id']}: {e.__class__.__name__}: {e}\n{traceback.format_exc()}")
