def getresults3(param2):
global idenmatrix
#global iden_matrix_num
global listTest
#global Data
Data=session['DATA']
DataReceive=Data[param2]
scenarioName=str(param2)+"scenarioName"
Scenario=DataReceive[scenarioName]
testSourceName=str(param2)+"testSourceName"
SourceName=DataReceive[testSourceName]
#del Data[param2]
#Data.insert(param2,"")
#time.sleep(30)
task= celery.send_task('mytask.get', args=[Scenario,SourceName], kwargs={})
#iden_matrix_num=session['iden_matrix_num']
iden_matrix_num=session['iden_matrix_num']
iden_matrix_num+=1
idenmatrix[iden_matrix_num]=task.id
session['iden_matrix_num']=iden_matrix_num
#session['iden_matrix_num']=iden_matrix_num
Task={"request_id":str(iden_matrix_num)}
return json.dumps(Task,sort_keys=True,indent=4)
def getresults3(param2):
global idenmatrix
#global iden_matrix_num
global listTest
#global Data
Data=session['DATA']
DataReceive=Data[param2]
scenarioName=str(param2)+"scenarioName"
Scenario=DataReceive[scenarioName]
testSourceName=str(param2)+"testSourceName"
SourceName=DataReceive[testSourceName]
#del Data[param2]
#Data.insert(param2,"")
with open('data.json') as json_file:
listTest2=json.load(json_file)
Name=[]
param1=0
param3=0
i=session['index']
for cle in listTest2.keys():
Name.append(cle)
for K in range(1,i+1):
DataR=Data[K]
testSourceNames=str(K)+"testSourceName"
SourceNames=DataR[testSourceNames]
scenarioNames=str(K)+"scenarioName"
Scenarios=DataR[scenarioNames]
# if SourceNames==Name[1] and Scenarios==Scenario:
# param1+=1
# elif SourceNames==Name[0] and Scenarios==Scenario:
# param3+=1
#if SourceName==Name[1]:
# param2=param1
#elif SourceName==Name[0]:
#param2=param3
#if Scenarios==Scenario:
if Scenarios==Scenario:
number(SourceNames)
param2=getresult(SourceName)
initialisation()
#time.sleep(30)
task= celery.send_task('mytask.get', args=[Scenario,SourceName,param2], kwargs={})
#iden_matrix_num=session['iden_matrix_num']
iden_matrix_num=session['iden_matrix_num']
iden_matrix_num+=1
#idenmatrix=session['idenmatrix']
#idenmatrix=getattr(g,'idenmatrix', None)
#task= celery.send_task('mytask.get', args=[Scenario,SourceName,param2],kwargs={})
idenmatrix[iden_matrix_num]=task.id
session['iden_matrix_num']=session['iden_matrix_num']+1
#session['iden_matrix_num']=iden_matrix_num
Task={"request_id":str(iden_matrix_num),"param":param2}
return json.dumps(Task,sort_keys=True,indent=4)
def add(param1, param2):
#clientId = request.sid
task = celery.send_task('mytasks.add', args=[param1, param2], kwargs={})
# initiate progress
socketio.emit('progress', {'status': 10}, namespace='/cruncher')
time.sleep(1)
# check progress
res = celery.AsyncResult(task.id)
if res.state == states.PENDING:
socketio.emit('progress', {'status': 50}, namespace='/cruncher')
time.sleep(2)
#clientId.socketio.emit('progress', {'status': 100, 'result': str(res.result)}, namespace='/cruncher')
socketio.emit('progress', {'status': 100, 'result': str(res.result)}, namespace='/cruncher')
return jsonify(task_id=task.id, status_url=url_for('main.check_task', id=task.id, _external=True))
def getresultsmatrix2(Scenario):
global idenmatrix
global iden_matrix_num
global listTest
global Data
global i
DataReceive=Data[i]
tester=str(i)+"tester"
test=DataReceive[tester]
del Data[i]
i-=1
#time.sleep(30)
task= celery.send_task('mytask.get', args=[Scenario,test], kwargs={})
iden_matrix_num+=1
idenmatrix[iden_matrix_num]=task.id
return "Request Send:"+str(iden_matrix_num)+":"
def getresults2(Scenario):
global iden
global iden_num
global listTest
global DataVector
global k
DataReceive=DataVector[k]
tester=str(k)+"tester"
test=DataReceive[tester]
del DataVector[k]
k-=1
task= celery.send_task('mytask.get', args=[Scenario,test], kwargs={})
res= celery.AsyncResult(task.id)
iden_num+=1
iden[iden_num]=task.id
return "Request Send:"+str(iden_num)+":"
def getresultsmatrix2(param2):
global idenmatrix
global iden_matrix_num
global listTest
global Data
DataReceive=Data[param2]
scenario=str(param2)+"scenario"
Scenario=DataReceive[scenario]
tester=str(param2)+"tester"
test=DataReceive[tester]
del Data[param2]
Data.insert(param2,"")
#time.sleep(30)
task= celery.send_task('mytask.get', args=[Scenario,test], kwargs={})
iden_matrix_num+=1
idenmatrix[iden_matrix_num]=task.id
return "Request Send:"+str(iden_matrix_num)+":"
def getresults2(param4):
global iden
global iden_num
global listTest
global DataVector
global k
DataReceive=DataVector[param4]
tester=str(param4)+"tester"
scenario=str(param4)+"scenario"
test=DataReceive[tester]
Scenario=DataReceive[scenario]
del DataVector[param4]
DataVector.insert(param4,"")
task= celery.send_task('mytask.get', args=[Scenario,test], kwargs={})
res= celery.AsyncResult(task.id)
iden_num+=1
iden[iden_num]=task.id
return "Request Send:"+str(iden_num)+":"
def getresultsmatrix2(param2):
global idenmatrix
global iden_matrix_num
global listTest
global Data
DataReceive=Data[param2]
scenario=str(param2)+"scenario"
Scenario=DataReceive[scenario]
tester=str(param2)+"tester"
test=DataReceive[tester]
#del Data[param2]
#Data.insert(param2,"")
#time.sleep(30)
task= celery.send_task('mytask.get', args=[Scenario,test], kwargs={})
iden_matrix_num+=1
idenmatrix[iden_matrix_num]=task.id
Task={"request_id":str(iden_matrix_num)}
return json.dumps(Task,sort_keys=True,indent=4)
def getresults3(param2):
global idenmatrix
global iden_matrix_num
global listTest
global Data
DataReceive=Data[param2]
scenarioName=str(param2)+"scenarioName"
Scenario=DataReceive[scenarioName]
testSourceName=str(param2)+"testSourceName"
SourceName=DataReceive[testSourceName]
#del Data[param2]
#Data.insert(param2,"")
with open('data.json') as json_file:
listTest2=json.load(json_file)
Name=[]
param1=0
param3=0
for cle in listTest2.keys():
Name.append(cle)
#for i in range(0,len(Data)):
# i=+1
# DataR=Data[i]
# testSourceNames=str(i)+"testSourceName"
# SourceNames=DataR[testSourceNames]
# if SourceNames==Name[0]:
# param1+=1
# elif SourceNames==Name[1]:
# param3+=1
#param3=len(DataReceive)-param2-1
#time.sleep(30)
task= celery.send_task('mytask.get', args=[Scenario,SourceName,param2], kwargs={})
iden_matrix_num+=1
idenmatrix[iden_matrix_num]=task.id
Task={"request_id":str(iden_matrix_num)}
return json.dumps(Task,sort_keys=True,indent=4)
def getresults2(param4):
global iden
global iden_num
global listTest
global DataVector
global k
DataReceive=DataVector[param4]
testSourceName=str(param4)+"testSourceName"
scenarioName=str(param4)+"scenarioName"
SourceName=DataReceive[testSourceName]
Scenario=DataReceive[scenarioName]
#del DataVector[param4]
#DataVector.insert(param4,"")
task= celery.send_task('mytask.get', args=[Scenario,SourceName], kwargs={})
res= celery.AsyncResult(task.id)
iden_num+=1
iden[iden_num]=task.id
Task={"request_id":str(iden_num)}
return json.dumps(Task,sort_keys=True,indent=4)
def getresults3(param2):
global idenmatrix
global listTest
Data=session['DATA']
DataReceive=Data[param2]
scenarioName=str(param2)+"scenarioName"
Scenario=DataReceive[scenarioName]
testSourceName=str(param2)+"testSourceName"
SourceName=DataReceive[testSourceName]
with open('data.json') as json_file:
listTest2=json.load(json_file)
Name=[]
param1=0
param3=0
i=session['index']
for cle in listTest2.keys():
Name.append(cle)
for K in range(1,i+1):
DataR=Data[K]
testSourceNames=str(K)+"testSourceName"
SourceNames=DataR[testSourceNames]
scenarioNames=str(K)+"scenarioName"
Scenarios=DataR[scenarioNames]
if Scenarios==Scenario:
number(SourceNames)
param2=getresult(SourceName)
initialisation()
task= celery.send_task('mytask.get', args=[Scenario,SourceName,param2], kwargs={})
iden_matrix_num=session['iden_matrix_num']
iden_matrix_num+=1
idenmatrix[iden_matrix_num]=task.id
session['iden_matrix_num']=session['iden_matrix_num']+1
Task={"request_id":str(iden_matrix_num),"param":param2}
return json.dumps(Task,sort_keys=True,indent=4)
def get_translations():
task = celery.send_task('unbabel.get_unbabel_translation',
args=[],
kwargs={})
return "<a href='{url}'>check status of {id} </a>".format(
id=task.id, url=url_for('check_task', id=task.id, _external=True))
def multiply(param1: int, param2: int) -> str:
app.logger.info('[route] /multiply/' + str(param1) + '/' + str(param2))
task = celery.send_task('tasks.multiply', args=[param1, param2], kwargs={})
response = f"<a href='{url_for('check_task', task_id=task.id, external=True)}'>check status of {task.id} </a>"
return response
def post(self):
'''Create a new pod'''
task = celery.send_task('tasks.redeployall', args=[podnum], kwargs={})
response = "pod deploying"
return response
def run_pipeline(pipeline_id):
engine = create_engine(DATABASE_URI)
Session = sessionmaker(bind=engine)
session = Session()
Base.metadata.drop_all(engine, checkfirst=True)
CeleryTask.__table__.drop(engine, checkfirst=True)
CeleryTask.__table__.create(engine, checkfirst=True)
Base.metadata.create_all(engine)
if pipeline_id == 0:
for i in range(8):
session.add(Task(sleep=random.randint(2, 7))) # sleep for 1-7 secs
session.add(
Workflow(dag_adjacency_list=dict([(1, [3]), (2, [4]), (
3, [5]), (4, [5]), (5, [6, 7]), (6, [8]), (7, [8])])))
elif pipeline_id == 1:
for i in range(15):
session.add(Task(sleep=random.randint(2, 7))) # sleep for 1-7 secs
session.add(
Workflow(dag_adjacency_list=dict([(1, [2]), (2, [3]), (
3, [10]), (4, [5]), (5, [6]), (6, [10]), (7, [8]), (
8, [9]), (9, [12]), (10, [11]), (11, [14]), (
12, [13]), (13, [14]), (14, [15])])))
elif pipeline_id == 2:
for i in range(42):
session.add(Task(sleep=random.randint(2, 7))) # sleep for 1-7 secs
session.add(
Workflow(dag_adjacency_list=dict([(1, [2]), (2, [3]), (
3, [25]), (4, [5]), (5, [6]), (6, [25]), (7, [8]), (
8, [9]), (9, [25]), (10, [11]), (11, [12]), (
12, [26]), (13, [14]), (14, [15]), (
15, [26]), (16, [17]), (17, [18]), (
18, [27]), (19, [20]), (
20, [21]), (21, [27]), (
22, [23]), (23, [24]), (
24,
[27]), (25,
[28]), (26, [29, 30]),
(27, [31, 32, 33]), (
28,
[35]), (29,
[35]), (30,
[34]),
(31, [34]), (32,
[37]), (33,
[38]),
(34, [40]), (35,
[36]), (36,
[40]),
(38, [39]), (
39, [40]), (40,
[41, 42])])))
session.commit()
workflow = session.query(Workflow).all()[-1]
print("WORKFLOW", workflow.id)
session.flush()
session.close()
task = celery.send_task('mytasks.pipeline',
args=(workflow.id, ),
kwargs={})
task_info[task.id] = ["Task submitted"]
return task
def add(param1, param2):
task = celery.send_task('tasks.add', args=[param1, param2], kwargs={})
response = "<a href='{url}'>check status of {id} </a>".format(id=task.id,
url=url_for('check_task', task_id=task.id, external=True))
return response
def run_indeed_scrap():
task = celery.send_task('tasks.indeed_scrap_task',kwargs={})
response = f"<a href='{url_for('check_task', task_id=task.id, external=True)}'>check status of {task.id} </a>"
return response
def r_process(t):
task = celery.send_task('R_broker.R_script', args=[t], kwargs={})
id = task.id
t.setTaskID(id)
handleResults(t)
def add(param1,param2):
task = celery.send_task('mytasks.add', args=[param1, param2], kwargs={})
return "<a href='{url}'>check status of {id} </a>".format(id=task.id,
url=url_for('check_task',id=task.id,_external=True))
def run_indeed_scrap_api(city_set: str):
print ('city_set :', city_set)
task = celery.send_task('tasks.indeed_scrap_task_api',city_set,kwargs={})
response = f"<a href='{url_for('check_task', task_id=task.id, external=True)}'>check status of {task.id} </a>"
return response
def dataload():
stream = celery.send_task('tasks.stream')
processData = celery.send_task('tasks.getData')
progressBar = celery.send_task('tasks.long_task')
return jsonify({'dataLocation': url_for('load_data', task_id=processData.id)}), 202, {'progressLocation': url_for('taskstatus',
task_id=progressBar.id)}
def add(param1: int, param2: int) -> str:
task = celery.send_task('tasks.add', args=[param1, param2], kwargs={})
response = f"<a href='{url_for('check_task', task_id=task.id, external=True)}'>check status of {task.id} </a>"
return response
def add(param1, param2):
task = celery.send_task('tasks.add', args=[param1, param2], kwargs={})
res = celery.AsyncResult(task.id)
return jsonify({str(task.id): str(res.state)})
def add(param1, param2):
task = celery.send_task('ins00.add', args=[param1, param2], kwargs={})
return "<a href='{url}'>check status of {id} </a>".format(
id=task.id, url=url_for('check_task', id=task.id, _external=True))
def add(x, y):
task = celery.send_task('tasks.add', args=[x, y], kwargs={})
response = "check status of {}\n".format(task.id)
return response, 200
def fill():
task = celery.send_task('hackernews.update_topstories', args=[], kwargs={})
return "<a href='{url}'>check status of {id} </a>".format(
obj_id=task.id, url=url_for('check_task', id=task.id, _external=True))
def index():
methodlist = {
'Counts': 'counts',
'Tau': 'tau',
'Gini coefficient': 'gini',
'Simpson index': 'simpson',
'Shannon entropy specificity': 'shannon_specificity',
'ROKU specificity': 'roku_specificity',
'TSI': 'tsi',
'Z-score': 'zscore',
'SPM': 'spm',
'SPM DPM': 'spm_dpm',
'Jensen-Shannon distance specificity': 'js_specificity',
'Jensen-Shannon distance specificity DPM': 'js_specificity_dpm',
}
if request.method == 'POST':
# Get the "method" and "log" parameters
method = methodlist[request.form.get('selector')]
log = request.form.getlist('checklog')
# Check input file extension
input_file = request.files['file']
if not allowed_file(input_file.filename):
flash('Only TSV, CSV and Excel files are allowed.', 'danger')
return redirect(request.url)
if input_file and allowed_file(input_file.filename):
filename = secure_filename(input_file.filename)
# Add 5 random characters to the file name and save it to the uploads directory
filename = '{}_{}{}'.format(
os.path.splitext(filename)[0],
uuid.uuid4().hex[0:5],
os.path.splitext(filename)[1],
)
input_file.save(os.path.join(app.config['UPLOAD_FOLDER'],
filename))
input_file_path = os.path.join(app.config['UPLOAD_FOLDER'],
filename)
# Get the file extension and open it in pandas
input_extension = input_file_path.rsplit('.', 1)[1].lower()
if input_extension == 'tsv':
data = pd.read_csv(input_file_path,
sep='\t',
thousands=',',
index_col=0)
elif input_extension == 'csv':
data = pd.read_csv(input_file_path, thousands=',', index_col=0)
elif input_extension in ['xls', 'xlsx']:
data = pd.read_excel(input_file_path,
thousands=',',
index_col=0)
# Drop columns where all values are NAs
data = data.dropna(axis=1, how='all')
# Check if there are duplicated gene names
if data.index.duplicated().any():
flash(
'We detected duplicated gene names in your file. Ensure all names in the first column are different.',
'danger',
)
return redirect(request.url)
# Convert data to a JSON string
data_json = data.to_json()
# Delete input file
os.remove(input_file_path)
# Create a name for the output
output_file = '{}_{}.tsv'.format(
os.path.splitext(filename)[0], method)
output_file_path = os.path.join(app.config['UPLOAD_FOLDER'],
output_file)
# Start a Celery task and send user to the results page
celery.send_task(
'tasks.start_tspex',
args=[data_json, output_file_path, method, log],
kwargs={},
)
return submission_complete(output_file)
return render_template('index.html', methodlist=methodlist, title=TITLE)
def correct_addresses(number_of_tasks: int) -> str:
dataset = [
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catel",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
"Catels",
]
dataset = list(chunker_list(dataset, number_of_tasks))
celery.send_task('tasks.correct_addresses',
args=[number_of_tasks, dataset],
kwargs={})
return str(number_of_tasks) + " tasks started !"
def getresults2(Scenario):
global iden
task= celery.send_task('mytask.get', args=[Scenario], kwargs={})
res= celery.AsyncResult(task.id)
iden=task.id
return "Request Send"
async def create_item(item: Item):
task_name = "hello.task"
task = celery.send_task(task_name, args=[item.name])
return dict(id=task.id, url='localhost:5000/check_task/{}'.format(task.id))
def submit():
task = celery.send_task('tasks.get_remote_hash',
kwargs=request.args.to_dict())
return jsonify({'id': task.id})
async def create_item_bye(img: UploadFile = File(...)):
task_name = "classify.task"
task = celery.send_task(task_name, args=[img.filename])
#return task
return dict(id=task.id, url='localhost:5000/check_task/{}'.format(task.id))
def force_run() -> str:
force = True
task = celery.send_task('tasks.run', args=[force], kwargs={})
response = f"<a href='{url_for('check_task', task_id=task.id, external=True)}'>Check the current status of {task.id} </a>"
return response
