def AddNewProject(request):
if 'login' in request.session:
user=User.objects.get(login=request.session['login'])
f = forms.ProjectRegisterForm(prefix='project')
fr = forms.ProjectPerks(prefix='perk')
context = RequestContext(request, {'formset': f, 'form1': fr})
if request.method == 'POST':
f = forms.ProjectRegisterForm(request.POST, prefix='project')
p = Project()
if(f.is_valid()):
p.title = f.cleaned_data['title']
p.short_description = f.cleaned_data['short_description']
p.funding_goal = f.cleaned_data['funding_goal']
p.full_description = f.cleaned_data['description']
p.category = f.cleaned_data['category']
p.user =user
Project.save(p)
if (request.FILES.getlist('file')!=""):
if(not os.path.exists(p.title)):
os.mkdir(p.title)
for file in request.FILES.getlist('file'):
l = open(p.title+'\\'+file.name, 'wb+')
for chunk in file.chunks():
l.write(chunk)
l.close()
atachment=Atachment()
atachment.url=p.title+'\\'+file.name
atachment.project=p
Atachment.save(atachment)
for urlfile in request.POST.getlist('urlfile'):
atachment=Atachment()
atachment.url=urlfile
atachment.project=p
Atachment.save(atachment)
j=0
perkvalue=Perk()
for perk in request.POST.getlist('perk'):
if j==0:
perkvalue.project=p
perkvalue.title= perk
j+=1
elif j==1:
perkvalue.description=perk
j+=1
elif j==2:
j+=1
perkvalue.amount= int(perk)
else:
j=0
if int(perk)>0:
perkvalue.number_available=int(perk)
perkvalue.save()
perkvalue=Perk()
return redirect('/', request)
else:
return render_to_response('AddNewProject.html', context)
else:
return redirect('/logowanie', request)
def addKeyanResualt(request):
tempNews=Project()
tempNews.name= request.GET['name']
tempNews.project_type= request.GET['project_type']
tempNews.remarks= request.GET['remarks']
tempNews.start_date= request.GET['fromdate']
tempNews.end_date= request.GET['todate']
tempNews.valid=1
tempNews.save()
posts = tempNews.__class__.objects.all()
t =loader.get_template('keyangongzuo.html')
c =Context({'posts':posts})
return HttpResponse(t.render(c))
def create_project(request):
if request.method == 'POST':
name = request.POST['name']
pitch = request.POST['pitch']
description = request.POST.get('description', "")
status = request.POST['status']
developer = 'OK' if 'DEV' in request.POST else 'NO'
designer = 'OK' if 'DES' in request.POST else 'NO'
business = 'OK' if 'BUS' in request.POST else 'NO'
project = Project(user=request.user, name=name, tweet_pitch=pitch,\
status=status, description=description,\
developer=developer, designer=designer,\
business=business)
if 'icon' in request.FILES:
image = request.FILES['icon']
project.image = image
project.save()
return redirect('/project/' + str(project.id) + '/')
return render(request, 'create_project.html', {})
def connect_project(message, pk):
from main.models import Project
try:
project = Project.objects.get(pk=pk)
except Project.DoesNotExist:
message.reply_channel.send({
'text': json.dumps({'error': 'Project does not exist.'}),
'close': True,
})
return
group_name = Project.get_channels_group(pk)
Group(group_name).add(message.reply_channel)
def AddNewProject(request):
f = forms.ProjectRegisterForm(prefix='project')
fr = forms.ProjectPerks(prefix='perk')
context = RequestContext(request, {'formset': f, 'form1': fr})
if request.method == 'POST':
f = forms.ProjectRegisterForm(request.POST, prefix='project')
fr = forms.ProjectPerks(request.POST, prefix='perk')
if f.is_valid():
p = Project()
p.title = f.cleaned_data['title']
p.short_description = f.cleaned_data['short_description']
p.funding_goal = f.cleaned_data['funding_goal']
p.full_description = f.cleaned_data['description']
p.category = f.cleaned_data['category']
p.user_id = 1
Project.save(p)
return redirect('/', request)
else:
return render_to_response('AddNewProject.html', context)
def add_project():
form = AddProjectForm()
if request.method == "POST":
title = request.form['title']
image_link = request.form['image_link']
short_description = request.form['short_description']
content = request.form['content']
new_project = Project(title=title,
image_link=image_link,
short_description=short_description,
content=content)
db.session.add(new_project)
db.session.commit()
new_project_id = Project.query.filter_by(title=title).first().id
return redirect(url_for('page.project', id=new_project_id))
return render_template('add_project.html', form=form)
def cms_projects_new(request):
if request.method == "POST":
name = request.POST['name']
description = request.POST['description']
project = Project()
project.name = name
project.description = description
project.date = datetime.now()
project.save()
return redirect('/cms/projects/')
context = {'title': 'Новый проект'}
return render(request, 'cms/new-project.html', context)
def setUp(self):
u = User.objects.create_user("testuser_getnexttask", "*****@*****.**", "abc")
u.full_clean()
u.save()
g = Group(name="test group")
g.full_clean()
g.save()
u.groups.add(g)
u.full_clean()
u.save()
p = Project(
admin=u, title="Test Project", description="Testing project.", type="simple", annotator_count=1, priority=3
)
p.full_clean()
p.save()
p.annotators.add(g)
p.full_clean()
p.save()
t = SimpleTask(question="test question", project=p)
t.full_clean()
t.save()
self.t = t
self.user = u
def handle(self, *args, **options):
for projects in data['College_Projects']:
Name = projects['Name']
Description = projects['Description']
Link = projects['Link']
p = Project(project_name=Name,
description=Description,
project_type='College',
github_link=Link)
p.save()
for projects in data['Own_Projects']:
Name = projects['Name']
Description = projects['Description']
Link = projects['Link']
p = Project(project_name=Name,
description=Description,
project_type='Own',
github_link=Link)
p.save()
def create_project(self, client):
# List all current projects for this client to avoid clashes
projects = client.projects.all()
self.write("Current projects: %s" % ", ".join([p.name for p in projects]))
project_name = raw_input("Name of new project? ")
try:
existing_project = client.projects.get(name=project_name)
self.write("A project already exists with name '%s'" % project_name)
return create_project(client)
except Project.DoesNotExist:
pass
project_desc = raw_input("Description (optional)? ")
self.write(" Creating project '%s'" % project_name)
project = Project()
project.client = client
project.name = project_name
project.description = project_desc
project.save()
return project
def mutate(self, info, title, user_id):
user = User.objects.get(pk=user_id)
project = Project(title=title, user=user)
project.save()
ok = True
return CreateProject(project=project, ok=ok)
def new_project(request):
if request.method == "POST":
try:
name = request.POST.get('name')
slug = slugify(name).replace('-', '_')
if len(slug) < 1:
raise Exception("Project name should not be empty.")
project = Project.objects.filter(user=request.user, slug=slug)
if project:
if request.POST.get('delete-if-exists'):
project[0].delete_project_path()
project[0].delete()
else:
raise Exception("Project with same name already exists, please give another name.")
project = Project(name=name,
slug=slug,
user=request.user,
secret=get_random_string(length=32))
project.create_project_path()
fileTypes = ['tree.txt', 'data.txt', 'fasta.fa',
'samples-order.txt', 'samples-info.txt',
'additional-layers.txt', 'state.json',
'bins.txt', 'bins-info.txt', 'items-order.txt']
for fileType in fileTypes:
if fileType in request.FILES:
put_project_file(project.get_path(), fileType, request.FILES[fileType])
interactive = project.get_interactive()
profile_db_path = project.get_file_path('profile.db', default=None)
samples_info = project.get_file_path('samples-info.txt', default=None)
if samples_info:
miscdata.MiscDataTableFactory(argparse.Namespace(target_data_table='layers', profile_db=profile_db_path)).populate_from_file(samples_info)
samples_order = project.get_file_path('samples-order.txt', default=None)
if samples_order:
miscdata.MiscDataTableFactory(argparse.Namespace(target_data_table='layer_orders', profile_db=profile_db_path)).populate_from_file(samples_order)
state_file = project.get_file_path('state.json', default=None)
if state_file:
interactive.states_table.store_state('default', open(state_file, 'r').read(), datetime.datetime.now().strftime("%d.%m.%Y %H:%M:%S"))
project.num_states = 1
bins_file = project.get_file_path('bins.txt', default=None)
bins_info_file = project.get_file_path('bins-info.txt', default=None)
if bins_file and bins_info_file:
collections_table = collections.TablesForCollections(profile_db_path)
bins = get_TAB_delimited_file_as_dictionary(bins_file, no_header = True, column_names = ['split_id', 'bin_name'])
bins_info = get_TAB_delimited_file_as_dictionary(bins_info_file, no_header = True, column_names = ['bin_name', 'source', 'html_color'])
bin_data = {}
for split_name in bins:
bin_name = bins[split_name]['bin_name']
if not bin_name in bin_data:
bin_data[bin_name] = set([])
bin_data[bin_name].add(split_name)
collections_table.append('default', bin_data, bins_info)
project.num_collections = 1
# try to get number of leaves
try:
project.num_leaves = len(interactive.displayed_item_names_ordered)
except:
project.num_leaves = 0
# try to get number of layers
try:
project.num_layers = len(interactive.views['single'][0]) - 1 # <- -1 because first column is contigs
except:
project.num_layers = 0
# store description
dbops.update_description_in_db(profile_db_path, request.POST.get('description') or '')
project.save()
return JsonResponse({'status': 0})
except Exception as e:
try:
project.delete_project_path()
except:
# slug is not unique, so there is no project instance or direcotry created.
pass
message = str(e.clear_text()) if 'clear_text' in dir(e) else str(e)
# trim the full path from exception message, show only file name
message = re.sub(r"(\'.*\/)(.+?)(\.txt\')", r"'\2'", message)
return JsonResponse({
'status': 1,
'message': message
})
return render(request, 'projects/new.html')
def handle(self, *args, **options):
conn = sqlite3.connect(options['userdb_path'][0])
"""
CREATE TABLE users (
0 login TEXT PRIMARY KEY,
1 firstname TEXT,
2 lastname TEXT,
3 email TEXT,
4 password TEXT,
5 path TEXT,
6 token TEXT,
7 accepted INTEGER,
8 project TEXT,
9 affiliation TEXT,
10 ip TEXT,
11 clearance TEXT,
12 date TEXT,
13 visit TEXT)
"""
print("Migrating users table...")
user_paths = {}
for row in conn.execute('SELECT * FROM users;'):
username = sanitize_username(row[0])
password = "******" + row[4]
email = row[3]
is_active = True
is_superuser = True if row[11] == 'admin' else False
user_paths[row[5]] = username
date_joined = datetime.strptime(
row[12], "%Y-%m-%d").replace(tzinfo=timezone.utc)
newuser = User(username=username,
password=password,
email=email,
is_active=is_active,
is_superuser=is_superuser,
is_staff=is_superuser,
date_joined=date_joined)
newuser.save()
fullname = "%s %s" % (row[1], row[2])
institution = row[9]
if len(fullname) < 2:
fullname = None
newuser_profile = UserProfile(user=newuser,
fullname=fullname,
orcid=None,
institution=institution)
newuser_profile.save()
print(" - Successful.")
print("Moving project files... ")
for path in user_paths:
username = user_paths[path]
# old user project dir
src = os.path.join(options['userfiles_path'][0], path)
#new user project dir
dst = os.path.join(settings.USER_DATA_DIR, username)
try:
shutil.copytree(src, dst)
except FileNotFoundError:
# if user path does not exists create empty dir for user
os.makedirs(dst)
print(" - Successful")
print("Migratins project table... ")
"""
CREATE TABLE projects (
0 name TEXT PRIMARY KEY,
1 path TEXT,
2 user TEXT,
3 description TEXT)
"""
for row in conn.execute('SELECT * FROM projects;'):
name = row[0]
slug = slugify(name).replace('-', '_')
path = row[1]
username = sanitize_username(row[2])
description = row[3]
#rename project files
fileTypes_old = [
'treeFile', 'dataFile', 'fastaFile', 'samplesOrderFile',
'samplesInformationFile'
]
fileTypes_new = [
'tree.txt', 'data.txt', 'fasta.fa', 'samples-order.txt',
'samples-info.txt'
]
for i in range(5):
try:
os.rename(
os.path.join(settings.USER_DATA_DIR, username, path,
fileTypes_old[i]),
os.path.join(settings.USER_DATA_DIR, username, path,
fileTypes_new[i]))
except:
pass
try:
if not description or not len(description) > 0:
description = ""
project = Project(name=name,
slug=slug,
user=User.objects.get(username=username),
secret=path)
samples_info = project.get_file_path('samples-order.txt',
default=None)
samples_order = project.get_file_path('samples-info.txt',
default=None)
if (samples_info
or samples_order) and not project.get_file_path(
'samples.db', default=None):
s = dbops.SamplesInformationDatabase(project.get_file_path(
'samples.db', dont_check_exists=True),
quiet=True)
s.create(samples_order, samples_info)
interactive = project.get_interactive()
# try to get number of leaves
try:
leaves = get_names_order_from_newick_tree(
project.get_file_path('tree.txt', default=None))
project.num_leaves = len(leaves) if leaves != [''] else 0
except:
project.num_leaves = 0
# try to get number of layers
try:
project.num_layers = len(
interactive.views['single']
[0]) - 1 # <- -1 because first column is contigs
except:
project.num_layers = 0
# store description
dbops.update_description_in_db(
project.get_file_path('profile.db', default=None),
description or '')
project.synchronize_num_states()
project.synchronize_num_collections()
project.save()
# try to migrate old links.
for row_links in conn.execute(
'SELECT * FROM views WHERE project LIKE \'%s\';' %
(name)):
old_link = OldLinks(
name=row_links[0],
user=sanitize_username(row_links[1]),
project=Project.objects.filter(
name=row_links[2],
user__username=sanitize_username(row_links[1]))[0],
is_public=True if row_links[3] == 1 else False,
token=row_links[4])
old_link.save()
project_link = ProjectLink(project=old_link.project,
link=old_link.token)
project_link.save()
except Exception as e:
print(username + " " + name + " " + path +
" failed to create project, here is the exception " +
str(e))
shutil.rmtree(
os.path.join(settings.USER_DATA_DIR, username, path))
print(" - Successful")
class AnalysisEngine:
def __init__(self):
"""
Creates and saves a new project model to DB.
"""
self.id = uuid.uuid4()
self.project = Project(id=self.id)
self.project.save()
def infer_target_type(self) -> None:
"""
Attempts to infer the target columns data type.
"""
self.target_column = self.df.columns[-1]
if self.df[self.target_column].dtype == 'object':
self.output = 'classification'
else:
column_arr = self.df[self.target_column]
column_arr = column_arr.dropna()
points = 0
# Check if column only contains ints
if np.array_equal(column_arr, column_arr.astype(int)):
points += 0.7
# Check if column only contains floats
elif np.array_equal(column_arr, column_arr.astype(float)):
points += 0.8
# Check if column is normally distributed
k2, p = stats.normaltest(column_arr)
if p >= 0.1:
points += 0.3
# Check if number of unique values is low
unique_values = set(column_arr)
if len(unique_values) > len(column_arr) * 0.1:
points += 0.3
self.output = 'regression' if points > 1.0 else 'classification'
def save_columns(self):
"""
Analyzes each column in the data frame, and saves the meta data
in the DB.
"""
self.columns = []
for col in self.df.columns:
null_count = self.df[col].isna().sum()
fill_rate = round((len(self.df) - null_count) / len(self.df), 2)
# Numerical columns
if self.df[col].dtype in ('int', 'float'):
new_column = Column(name=col,
type='numerical',
filled=fill_rate,
min=round(self.df[col].min(), 2),
mean=round(self.df[col].mean(), 2),
median=round(self.df[col].median(), 2),
max=round(self.df[col].max(), 2),
unique=0,
target=(col == self.target_column),
project=self.project)
new_column.save()
# Categorical columns
else:
new_column = Column(name=col,
type='categorical',
filled=fill_rate,
min=0,
mean=0,
median=0,
max=0,
unique=self.df[col].nunique(),
target=(col == self.target_column),
project=self.project)
new_column.save()
self.columns.append(new_column)
# Mark as complete
self.project.analytics_complete = True
self.project.save()
def run_models(self) -> None:
"""
Performs model search, and saves the results of each machine learning
model.
"""
searcher = ModelSearcher(self.output)
searcher.fit(self.X, self.y, 1)
for model_name, scores in searcher.results.items():
new_model = Model(name=model_name,
accuracy=round(scores['acc'], 3),
roc=round(scores['roc'], 3),
f1=round(scores['f1'], 3),
explained_variance=round(
scores['explained_variance'], 3),
r2=round(scores['r2'], 3),
mae=round(scores['mae'], 3),
fit_time=round(scores['fit_time'], 3),
score_time=round(scores['score_time'], 3),
model_size=round(scores['model_size'], 3),
params=scores['params'],
confusion=scores['confusion'],
errors=scores['errors'],
project=self.project)
new_model.save()
# Save feature importances
importances = searcher.feature_importances
for column, importance in zip(self.columns, importances):
column.importance = round(importance, 3)
column.save()
# Mark as complete
self.project.models_complete = True
self.project.save()
def preprocess(self) -> None:
"""
Imputes missing values from data set, and converts strings to numbers.
"""
self.X = self.df.drop(self.target_column, axis=1)
self.y = self.df[self.target_column]
# Preprocess features
for col in self.X.columns:
if self.X[col].dtype in ('int', 'float'):
self.X[col] = self.X[col].fillna(value=self.X[col].mean())
else:
self.X[col] = self.X[col].fillna('None')
self.X[col] = LabelEncoder().fit_transform(self.X[col])
# Preprocess targets
if self.output == 'classification':
encoder = LabelEncoder()
self.y = encoder.fit_transform(self.y)
self.project.columns = list(encoder.classes_)
self.project.save()
else:
self.y = self.y.fillna(value=self.y.mean())
def run_engine(self) -> None:
"""
Entry point for analysis and model search.
"""
self.infer_target_type()
self.save_columns()
self.preprocess()
self.run_models()
def run(self, file_path: str) -> None:
"""
Opens CSV file and runs separate thread to start the data processing.
Args:
file_path: Path to CSV file.
"""
self.df = pd.read_csv(file_path)
Thread(target=self.run_engine).start()
def disconnect_project(message, pk):
from main.models import Project
group_name = Project.get_channels_group(pk)
Group(group_name).discard(message.reply_channel)
def submit(request, hackathon_id):
hackathon = Hackathon.objects.get(id=hackathon_id)
if request.method == "POST":
# Grabbing form POST data
title = request.POST.get("title")
description = request.POST.get("description")
technologies_used = request.POST.get("technologies-used").split(
",") # Splitting into array by commas
github_link = request.POST.get("github-link")
project_link = request.POST.get("project-link")
video_link = request.POST.get("video-link")
# Creating Project model
project = Project(title=title,
description=description,
technologies_used=technologies_used,
github_link=github_link,
project_link=project_link,
video_link=video_link)
# Checking to see if project image was provided
if "project-image" in request.FILES:
project.project_image = request.FILES["project-image"]
# Checking to see if extra files were provided
if "extra-files" in request.FILES:
project.project_image = request.FILES["extra-files"]
# Setting creator to currently signed in user
project.creator = Profile.objects.get(user=request.user)
project.save()
# Adding project to submissions
hackathon.submissions.add(project)
hackathon.save()
return HttpResponseRedirect(f"/{hackathon.id}/submissions/")
# Define user_already_submitted -> only allow submission from user if the user hasn't submitted yet
user_already_submitted = False
for submission in list(hackathon.submissions.all()):
if Profile.objects.get(
user=request.user
) == submission.creator: # This submission was submitted by current user
user_already_submitted = True
# Define too_early and too_late -> defines if the user is too early or too late to submit
too_early = hackathon.submissions_open_datetime.astimezone(
timezone('US/Pacific')).strftime("%m/%d/%Y %H:%M:%S") > datetime.now(
).astimezone(timezone('US/Pacific')).strftime("%m/%d/%Y %H:%M:%S")
too_late = hackathon.submissions_close_datetime.astimezone(
timezone('US/Pacific')).strftime("%m/%d/%Y %H:%M:%S") < datetime.now(
).astimezone(timezone('US/Pacific')).strftime("%m/%d/%Y %H:%M:%S")
# Getting opening and closing submissions datetimes
submissions_open_datetime = hackathon.submissions_open_datetime.astimezone(
timezone('US/Pacific')).strftime("%m/%d/%Y %H:%M:%S")
submissions_close_datetime = hackathon.submissions_close_datetime.astimezone(
timezone('US/Pacific')).strftime("%m/%d/%Y %H:%M:%S")
# Define profile and competitors
profile = Profile.objects.get(user=request.user)
competitors = list(hackathon.competitors.all())
return render(request,
"hackathon/submit.html",
context={
"hackathon": hackathon,
"too_early": too_early,
"too_late": too_late,
"user_already_submitted": user_already_submitted,
"profile": profile,
"competitors": competitors,
"submissions_open_datetime": submissions_open_datetime,
"submissions_close_datetime": submissions_close_datetime
})
def assign3TeamsToTutor(tutor, testCourse):
testProject1 = Project(id=1, course=testCourse, tutor=tutor)
testProject1.save()
testProject2 = Project(id=2, course=testCourse, tutor=tutor)
testProject2.save()
testProject3 = Project(id=3, course=testCourse, tutor=tutor)
testProject3.save()
testTeam1 = ProjectTeam(id=1, project=testProject1, course=testCourse, accepted=True)
testTeam1.save()
testTeam2 = ProjectTeam(id=2, project=testProject2, course=testCourse, accepted=True)
testTeam2.save()
testTeam3 = ProjectTeam(id=3, project=testProject3, course=testCourse, accepted=True)
testTeam3.save()
