def replaceFiles(ctx, table_id, payload_dir):
"""Reupload a given set of files to an existing table for reprocessing.
Parameters
----------
ctx : Context
A Click Context object.
table_id : int
The Id of the table to upload to.
payload_dir : str
The path of the payload directory containing the files.
"""
print "replaceFiles"
# Fetch the payload files
config = {}
for (dirpath, dirnames, filenames) in os.walk(payload_dir):
# config['files'] = [{'filename': f} for f in filenames if f != ".DS_Store"]
filepaths = [os.path.join(payload_dir, f) for f in filenames if f != ".DS_Store"]
break
# Upload the payload files in separate threads
pp(filepaths)
start_time = time.time()
upload_files_multithreaded(ctx, table_id, "vector", filepaths, chunk_size=20971520)
ctx.log("All uploads completed and took %s minutes" % (round((time.time() - start_time) / 60, 2)))
exit()
# # Hacky workaround that doesn't use multi-threading.
# # Force all asset files into "uploading" state by providing the first 256KB of each file.
# for i in config['files']:
# upload_file_init(ctx, table_id, "vector", os.path.join(payload_dir, i['filename']))
# ctx.log("Bailing!")
#
# # Upload the payload files
# start_time = time.time()
# for i in config['files']:
# upload_file(ctx, table_id, "vector", os.path.join(payload_dir, i['filename']), chunk_size=20971520)
# ctx.log("All uploads completed and took %s minutes" % (round((time.time() - start_time) / 60, 2)))
# Poll until asset has processed
poll_asset_processing(ctx, table_id, ctx.service().tables())
def reprocessAndRepublishLayers(ctx, layers):
"""Reprocess and republish the given set of layers.
Parameters
----------
ctx : Context
A Click Context object.
layers : list
The Ids of the layers to patch.
"""
@retries(100, delay=5)
def processLayer(ctx, layer_id):
try:
return ctx.service().layers().process(id=layer_id).execute()
except HttpError as e:
# "The resource is already up to date."
if e.resp.status == 409:
print e.content
ctx.log("### Layer was already processed successfully for some reason!")
return
else:
raise e
@retries(100, delay=5)
def publishLayer(ctx, layer_id):
return ctx.service().layers().publish(id=layer_id).execute()
for l in layers:
ctx.log("Layer %s processesing begun." % (l["id"]))
processLayer(ctx, l["id"])
poll_asset_processing(ctx, l["id"], ctx.service().layers())
ctx.log("Layer %s publishing begun." % (l["id"]))
publishLayer(ctx, l["id"])
poll_layer_publishing(ctx, l["id"])
