def start_upload_task(task_name: str):
create_table(task_name)
df = pandas.read_csv(os.path.join(settings.BASE_DIR, 'sample_data.csv'))
rows = [list(row) for row in df.values]
current_row_index = 0
task_status = {'status': 'run'}
if not redis_instance.get(task_name):
redis_instance.set(task_name, json.dumps(task_status))
else:
task_status = json.loads(redis_instance.get(task_name))
while task_status.get('status') == 'run' and current_row_index < len(rows):
row = rows[current_row_index]
process_row(task_name, row, current_row_index)
current_row_index += 1
update_task_progress(task_name, current_row_index, len(rows))
task_status = json.loads(redis_instance.get(task_name))
if task_status.get('status') == 'pause':
task_status = json.loads(redis_instance.get(task_name))
task_status['current_row_index'] = current_row_index
redis_instance.set(task_name, json.dumps(task_status))
return
# Upload finished successfully
upload_successful(task_name)
def start_download_task(task_name: str, filename: str):
with open(os.path.join(settings.BASE_DIR, 'static', f"{filename}.csv"),
"a+") as csv_file:
csvwriter = csv.writer(csv_file)
csvwriter.writerow([
'Sid', 'Region', 'Country', 'Item Type', 'Sales Channel',
'Order Priority', 'Order ID', 'Units Sold', 'Unit Price',
'Unit Cost', 'Total Revenue', 'Total Cost', 'Total Profit'
])
row_count = count_rows(task_name)
current_row = 0
task_status = {'status': 'run'}
if not redis_instance.get(task_name):
redis_instance.set(task_name, json.dumps(task_status))
else:
task_status = json.loads(redis_instance.get(task_name))
while task_status.get('status') == 'run' and current_row < row_count:
process_csv(filename, task_name, current_row)
current_row += 1
update_task_progress(task_name, current_row, row_count)
task_status = json.loads(redis_instance.get(task_name))
if task_status.get('status') == 'pause':
task_status = json.loads(redis_instance.get(task_name))
task_status['current_row'] = current_row
redis_instance.set(task_name, json.dumps(task_status))
return
# download finished successfully
download_successful(task_name)
def post(self, request, format=None):
'''
Get download progress. Returns download url if csv generated successfully
'''
try:
user_id = request.data.get('user_id')
task_name = request.data.get('task_name')
# Check if requested table exists
if not UploadStatus.objects.filter(user_id=user_id,
task_completed=True,
table_name=task_name).exists():
return Response(
{"message": "Upload pending or invalid task_name"},
status=status.HTTP_403_FORBIDDEN)
# TODO: If already generated CSV: Directly give download link
elif DownloadStatus.objects.filter(user_id=user_id,
table_name=task_name,
task_completed=True).exists():
return Response({
"download_url":
f"http://localhost:8000/static/{DownloadStatus.objects.get(user_id=user_id, table_name=task_name, task_completed=True).filename}.csv"
})
return Response(
{"progress": redis_instance.get(f"{task_name}__progress")})
except:
return Response(status=status.HTTP_400_BAD_REQUEST)
def post(self, request, format=None):
'''
Gets Upload progress.
'''
user_id = request.data.get('user_id')
task_name = request.data.get('task_name')
# Check if requested table exists
if not UploadStatus.objects.filter(user_id=user_id,
table_name=task_name).exists():
return Response({"message": "No task found"},
status=status.HTTP_404_NOT_FOUND)
if redis_instance.get(f"{task_name}__progress"):
return Response(
{"progress": redis_instance.get(f"{task_name}__progress")})
else: # progress removed from redis on task completion
return Response({"progress": "complete"})
def resume_download_task(task_name: str):
filename = DownloadStatus.objects.get(table_name=task_name).filename
task_status = json.loads(redis_instance.get(task_name))
row_count = count_rows(task_name)
current_row = task_status.get('current_row')
while task_status.get('status') == 'run' and current_row < row_count:
process_csv(filename, task_name, current_row)
current_row += 1
update_task_progress(task_name, current_row, row_count)
task_status = json.loads(redis_instance.get(task_name))
if task_status.get('status') == 'pause':
task_status = json.loads(redis_instance.get(task_name))
task_status['current_row'] = current_row
redis_instance.set(task_name, json.dumps(task_status))
return
# download finished successfully
download_successful(task_name)
def resume_upload_task(task_name: str):
task_status = json.loads(redis_instance.get(task_name))
df = pandas.read_csv(os.path.join(settings.BASE_DIR, 'sample_data.csv'))
rows = [list(row) for row in df.values]
current_row_index = task_status.get('current_row_index')
while task_status.get("status") == "run" and current_row_index < len(rows):
row = rows[current_row_index]
process_row(task_name, row, current_row_index)
current_row_index += 1
update_task_progress(task_name, current_row_index, len(rows))
task_status = json.loads(redis_instance.get(task_name))
if task_status.get("status") == "pause":
task_status = json.loads(redis_instance.get(task_name))
task_status['current_row_index'] = current_row_index
redis_instance.set(task_name, json.dumps(task_status))
return
# Upload finished successfully
upload_successful(task_name)
def post(self, request, format=None):
'''
Pause ongoing upload task
'''
task_name = request.data.get('task_name')
# set task status to paused
task_status = json.loads(redis_instance.get(task_name))
task_status['status'] = 'pause'
redis_instance.set(task_name, json.dumps(task_status))
return Response({"task_name": task_name})
def post(self, request, format=None):
'''
Terminate an ongoing upload task
'''
task_name = request.data.get('task_name')
# Set task status to terminate
task_status = json.loads(redis_instance.get(task_name))
task_status['status'] = 'terminate'
redis_instance.set(task_name, json.dumps(task_status))
# Run the background process for rolling back the changes
rollback_upload_task.delay(task_name)
return Response({"task_name": task_name})
def post(self, request, format=None):
'''
Resume a paused upload task
'''
task_name = request.data.get('task_name')
# Set task status to run
task_status = json.loads(redis_instance.get(task_name))
task_status['status'] = 'run'
redis_instance.set(task_name, json.dumps(task_status))
# Run the background process for resuming the task
resume_upload_task.delay(task_name)
return Response({"task_name": task_name})
def post(self, request, format=None):
'''
Pause ongoing download task
'''
try:
task_name = request.data.get('task_name')
# set task status to paused
task_status = json.loads(redis_instance.get(task_name))
task_status['status'] = 'pause'
redis_instance.set(task_name, json.dumps(task_status))
return Response({"task_name": task_name})
except:
return Response(status=status.HTTP_400_BAD_REQUEST)