Example #1
0
    def __buildClusters(self, task, results, result):
        """
        Build the cluster objects
        :param task:
        :param results:
        :param result:
        :return:
        """

        for tmp in results:
            cluster = Cluster()
            cluster.centerMean = []
            cluster.centerMedian = []
            cluster.result = result

            profiles = []
            for sequence in tmp:
                profiles.append(sequence.dna)

            profiles = np.array(profiles)
            mean = np.mean(profiles, axis = 0)
            median = np.median(profiles, axis = 0)

            cluster.centerMean = mean.tolist()
            cluster.centerMedian = median.tolist()

            cluster.save()

            for sequence in tmp:
                cluster.sequences.add(sequence)

            cluster.save()
Example #2
0
    def index_view(self):
        # Grab parameters from URL
        page, sort_idx, sort_desc, search = self._get_list_extra_args()

        page_size = 10
        count = Cluster._get_collection().count()
        num_pages = count // page_size
        if count % page_size != 0:
            num_pages += 1

        data = Cluster._get_collection().find(
            {},
            {}).skip(page * page_size).limit(page_size)

        # Various URL generation helpers
        def pager_url(p):
            # Do not add page number if it is first page
            if p == 0:
                p = None

            return self._get_url('.index_view', p, sort_idx, sort_desc,
                                 search)

        return self.render('admin/shreds.html', data=data, count=count,
                           pager_url=pager_url,
                           num_pages=num_pages,
                           page=page,)
Example #3
0
def next():
    if request.method == "POST":
        # TODO: helper
        tags = set(map(unicode.lower, request.form.getlist("tags")))

        Cluster.objects(pk=request.form["_id"]).update_one(
            push__tags=ShredTags(
                user=g.user.id,
                tags=list(tags),
                recognizable_chars=request.form.get("recognizable_chars", ""),
                angle=int(request.form.get("angle", 0))),
            inc__users_count=1,
            add_to_set__users_processed=g.user.id)

        User.objects(pk=g.user.id).update_one(
            inc__processed=1, inc__tags_count=len(tags),
            add_to_set__tags=list(tags))

        session["processed"] = session.get("processed", 0) + 1

        for tag in tags:
            Tags.objects(pk=tag).update_one(
                set_on_insert__is_base=False,
                set_on_insert__created_by=g.user.id,
                set_on_insert__created_at=Tags().created_at,
                inc__usages=1,
                add_to_set__shreds=request.form["_id"],
                upsert=True)

        start = datetime.strptime(request.form["tagging_start"],
                                  '%Y-%m-%d %H:%M:%S.%f')
        end = datetime.utcnow()
        TaggingSpeed.objects.create(
            user=g.user.id,
            cluster=request.form["_id"],
            tags_count=len(tags),
            msec=(end - start).total_seconds() * 1000)

    cluster = Cluster.next_for_user(g.user, app.config['USERS_PER_SHRED'])

    auto_tags = cluster and cluster.get_auto_tags() or []
    return render_template(
        "_shred.html",
        cluster=cluster,
        auto_tags=auto_tags,
        all_tags=get_tags(),
        tagging_start=datetime.utcnow(),

        # TODO: move to context processor
        processed_per_session=session.get("processed", 0),
        processed_total=User.objects(id=g.user.id).first()["processed"],
        rating=list(User.objects.order_by(
            "-processed").values_list("id")).index(g.user.id) + 1
    )
Example #4
0
def rest_clusters(request):
    if request.method == 'GET':
        clusters = Cluster.objects.filter(user_id=request.user.id)
        return JsonResponse(clusters, encoder=QuerySetJSONEncoder, safe=False)
    elif request.method == 'POST':
        data = json.loads(request.body)
        cluster = Cluster()
        cluster.id = data['id']
        cluster.name = data['name']
        cluster.systems = data['systems']['storage'] + data['systems']['compute']
        cluster.created = datetime.now()
        cluster.user = request.user
        cluster.save()

        data['config']['id'] = cluster.id
        response = requests.post(url=DEPLOY_URL, data=json.dumps(data['config']))

        if json.loads(response.text).get('result') == 'success':
            return JsonResponse(model_to_dict(cluster))
        else:
            return JsonResponse({
                'error': response['error_message']
            })
    else:
        return HttpResponse(status=404)
Example #5
0
def cluster_settings(request, username):
    state = "Change Settings"

    if request.method == "POST":
        form = ClusterForm(request.user, request.POST)
        if "delete" in request.POST:
            i = 0
            for i, cluster in enumerate(get_clusters_from_request(request)):
                cluster.delete()
            logger.info("%s deleted %d clusters(s)" % (username, i+1))
            state = "Settings Successfully Saved"
            form = ClusterForm(request.user)

        elif "save" in request.POST:
            if form.is_valid():
                obj = form.save(commit=False)
                obj.creator = request.user
                obj.save()
                state = "Settings Successfully Saved"
                form = ClusterForm(request.user)

    else:
        form = ClusterForm(request.user)

    c = {
        "pages": PAGES,
        "page": "clusters",
        "state": state,
        "form": form,
        "clusters": Cluster.get_clusters(request.user),
    }
    return render(request, "cluster/cluster_settings.html", c)
Example #6
0
def pages():
    if request.method == "POST":
        shreds = set(request.form.getlist("shreds"))
        page_name = request.form.get("page_name")
        page_id = request.form.get("page_id")

        if page_id:
            page = Pages.objects.get(pk=page_id)
        else:
            page, _ = Pages.objects.get_or_create(
                created_by=g.user.id, name=page_name)

        page.update(add_to_set__shreds=shreds)

        for shred in Cluster.objects(id__in=shreds):
            tags = shred.get_user_tags(g.user)
            if tags is not None:
                tags.pages = list(set(tags.pages + [page]))
            # TODO: else 404?

            shred.save()

    pages = Pages.objects(created_by=g.user.id)

    return render_template(
        "_pages.html",
        pages=pages)
Example #7
0
    def test_skipping(self):
        self.create_user_and_login("user")
        user = User.objects.get(username="******")
        self.assertEqual(user.skipped, 0)
        self.assertEqual(user.processed, 0)

        self.client.post(url_for("fixtures.create_shreds"))

        res = self.client.get(url_for("next"))
        self.assert200(res)
        body = res.get_data(as_text=True)

        current_shred_id = first_shred_id = self.parse_shred_id(body)
        seen_shreds = {current_shred_id}

        for i in xrange(9):
            res = self.client.post(url_for("skip"),
                                   data={"_id": current_shred_id},
                                   follow_redirects=True)

            body = res.get_data(as_text=True)
            self.assert200(res)

            current_shred_id = self.parse_shred_id(body)
            self.assertNotIn(current_shred_id, seen_shreds)
            seen_shreds.add(current_shred_id)

        self.assertEqual(
            len(Cluster.objects(id=first_shred_id).first().users_skipped), 1)

        res = self.client.post(url_for("skip"),
                               data={"_id": current_shred_id},
                               follow_redirects=True)

        body = res.get_data(as_text=True)
        self.assert200(res)

        current_shred_id = self.parse_shred_id(body)
        self.assertIn(current_shred_id, seen_shreds)

        self.assertEqual(
            len(Cluster.objects(id=current_shred_id).first().users_skipped), 0)

        user.reload()
        self.assertEqual(user.skipped, 10)
        self.assertEqual(user.processed, 0)
Example #8
0
def calc_clusters(request):
    if request.method == 'POST':
        form = CalcClusterForm(request.POST)
        if form.is_valid():
            compound_group = CompoundCollection.objects().with_id(str(request.POST["collection_id"]))
            compounds = Compound.objects(compound_group = str(request.POST["collection_id"]))

            #input_text = "<("
            #for compound in compounds :
            #    input_text += compound.mol + "\n"
            #input_text += ")"

            input_text = "/home/mjs/Test/approved.gspan.gz"


            Cluster.calcualte_clusters(compound_group,input_text,"EDEN")
#           handle_uploaded_file(request.FILES['file'])
            return HttpResponseRedirect('/cluster/')
    else:
        form = CalcClusterForm()
    return render_to_response('calc_clusters.html',{"view_titel":"Calculate Clusters","form":form},context_instance=RequestContext(request))  # Create your views here.
Example #9
0
def list_batches():
    batch_counts = Cluster._get_collection().aggregate([
        {
            '$group': {
                '_id': '$batch',
                'shreds_created': {
                    '$sum': 1}
                }
        },
        {
            '$sort': {
                'name': 1,
            },
        }])['result']
    return [{'name': item['_id'], 'shreds_created': item['shreds_created']}
            for item in batch_counts]
Example #10
0
def get_cluster(cluster_id=None):
    """Renders a single cluster as json.

    If the cluster id is not provided, picks a random one.
    """
    batch = request.args.get("batch")
    if cluster_id is None:
        cluster = Cluster.get_some(batch=batch)
    else:
        cluster = Cluster.objects.get_or_404(pk=cluster_id)

    return jsonify({
        "success": True,
        "data": {
            "cluster": json.loads(cluster.to_json()),
        },
    })
Example #11
0
def _fetch_normalized_shreds_tags(repeats):
    """Gets normalized tags for every cluster.

    Args:
        repeats: minimum number of tag occurences to be included in the result.

    Returns:
        Dict {obj_id: set(tags)} mapping cluster ids to sets of string tags.
    """

    shreds = Cluster.objects().timeout(False).only(
        'id', 'tags.tags', 'members.shred')[:SHREDS_CAP]
    shreds_tags = {}
    # TODO: on every iteration queries mongodb for
    # cluster->member->shred->auto_tags. That's too slow.
    for s in shreds:
        tags = s.get_repeated_tags(repeats)
        if tags:
            shreds_tags[s.id] = frozenset(tags)
    return shreds_tags
Example #12
0
def create_cluster():
    """Merges two clusters.

    Processes a POST request that should contain an object like:
    { "cluster": {
        "parents": ["parent1_id", "parent2_id"],
        "members": [{
            "shred": "shred1_id",
            "position": [100, 500],
            "angle": 35,
          },
          ...
        ]
    } }

    """

    req = request.get_json().get('cluster')

    parents = [Cluster.objects.get_or_404(pk=pk) for pk in req['parents']]

    if len(parents) != 2:
        response = jsonify({
            "success": False,
            "message": "Wrong number of good parents: %s" % req['parents']})
        response.status_code = 400
        return response

    if parents[0].batch != parents[1].batch:
        response = jsonify({
            "success": False,
            "message": "Parents are from different batches: %s %s" % (
                parents[0].batch, parents[1].batch)})
        response.status_code = 400
        return response

    member_fields = ['shred', 'position', 'angle']
    for member in req['members']:
        for field in member_fields:
            if field not in member:
                response = jsonify({
                    "success": False,
                    "message": "One of the members doesn't have all required "
                               "fields (%s): %s" % (member_fields, member),
                })
                response.status_code = 400
                return response

    cluster = Cluster(
            id=str(uuid.uuid1()),
            users_count=0,
            users_skipped=[],
            users_processed=[],
            batch=parents[0].batch,
            tags=[],
            parents=parents,
            members=[ClusterMember(
                shred=m['shred'], position=m['position'], angle=m['angle'])
                for m in req['members']],
    )
    cluster.save()

    return jsonify({
        "success": True,
        "id": cluster.id,
    })
Example #13
0
def skip():
    Cluster.objects(pk=request.form["_id"]).update_one(
        add_to_set__users_skipped=g.user.id)
    User.objects(pk=g.user.id).update_one(inc__skipped=1)

    return redirect(url_for("next"))
Example #14
0
def load_new_batch(fname_glob, batch):
    if app.config["S3_ENABLED"]:
        storage = S3Storage(app.config)
    else:
        storage = LocalFSStorage(app.config)

    pages_processed = 0
    shreds_created = 0

    out_dir = os.path.join(app.config["SPLIT_OUT_DIR"], "batch_%s" % batch)
    storage.clear(out_dir)
    Cluster.objects(batch=batch).delete()

    for src_key in storage.list(fname_glob):
        fname = storage.get_file(src_key)
        sheet_name = os.path.splitext(os.path.basename(fname))[0]

        echo("\n\nProcessing file %s from %s" % (fname, sheet_name))
        sheet = SheetIO(fname, sheet_name, [GeometryFeatures, ColourFeatures],
                        out_dir, "png")

        image_path_fields = ["piece_fname", "mask_fname",
                             "piece_in_context_fname"]

        # TODO: Remove when all field names match unshred's.
        field_name_map = {
            # Unshred-tag name: unshred name.
            "mask_fname": "features_fname",
        }
        drop_fields = ['simplified_contour', 'img_roi']
        drop_features = ['on_sheet_height', 'on_sheet_width', 'on_sheet_angle',
                'bottommost', 'topmost', 'on_sheet_x', 'on_sheet_y']


        pages_processed += 1

        for shred in sheet.get_shreds():
            shred = shred._asdict()
            shred["id"] = "%s:%s_%s" % (batch, shred["sheet"], shred["name"])
            shreds_created += 1

            def _convert_opencv_contour(contour):
                """Converts opencv contour to a list of pairs."""
                return contour.reshape((len(contour), 2)).tolist()
            shred["contour"] = _convert_opencv_contour(
                shred["simplified_contour"])
            shred['tags'] = shred.pop('tags_suggestions')

            for field in drop_fields:
                del shred[field]
            for field in drop_features:
                del shred['features'][field]

            cluster = {}
            cluster["id"] = shred["id"]
            cluster["users_count"] = 0
            cluster["batch"] = batch

            for model_field_name in image_path_fields:
                import_field_name = field_name_map.get(model_field_name,
                                                       model_field_name)
                image_path = shred.pop(import_field_name)
                res = storage.put_file(image_path)
                shred[model_field_name] = res

            cluster["parents"] = []
            try:
                shred_obj = Shred.objects.create(**shred)
                cluster_member = ClusterMember(shred=shred_obj, position=[0, 0],
                                               angle=0)
                cluster["members"] = [cluster_member]
                Cluster.objects.create(**cluster)
            except bson.errors.InvalidDocument:
                echo(shred)
                raise

    Cluster.ensure_index(["users_processed", "users_count", "batch"])
    Cluster.ensure_index(["users_skipped", "users_count", "batch"])