Exemplo n.º 1
0
 def setUp(self):
     from dal.mongo_util import get_collection_ref
     # empty all collections in test_db
     get_collection_ref('Profiles').drop()
     get_collection_ref('FigshareCollections').drop()
     get_collection_ref('OrcidCollections').drop()
     get_collection_ref('EnaCollections').drop()
     get_collection_ref('CollectionHeads').drop()
Exemplo n.º 2
0
 def __init__(self, profile_id=None, source_id=None):
     self.SampleCollection = get_collection_ref(SampleCollection)
     self.schema = DataSchemas("COPO").get_ui_template_as_obj().copo.sample.fields
     #self.schema_characeteristics = DataSchemas("COPO").get_ui_template_as_obj().copo.sample.characteristics
     #self.schema_factors = DataSchemas("COPO").get_ui_template_as_obj().copo.sample.factors
     self.profile_id = profile_id
     self.source_id = source_id
Exemplo n.º 3
0
def upload_to_dropbox(request):
    return_structure = {}
    # set up an Aspera collection handle
    AsperaCollection = get_collection_ref("AsperaCollections")
    transfer_token = ""

    task = request.POST['task']

    if task == "initiate_transfer":  # initiate the transfer process
        # get the target datafile and obtain the file reference
        study_id = request.POST["study_id"]
        ena_collection_id = request.POST["ena_collection_id"]
        data_file_id = request.POST["data_file_id"]
        data_file = EnaCollection().get_study_datafile(study_id, ena_collection_id, data_file_id)
        chunked_upload = ChunkedUpload.objects.get(id=int(data_file["fileId"]))

        # set a new document in the aspera collection,
        # thus obtaining a transfer token to orchestrate the transfer process
        db_template = d_utils.json_to_pytype(lkup.DB_TEMPLATES['ASPERA_COLLECTION'])
        transfer_token = AsperaCollection.insert(db_template)
        path_to_file = os.path.join(MEDIA_ROOT, chunked_upload.file.name)

        # update some initial fields
        # assume transfer_status is 'transferring' initially to allow the progress monitor to kick-start
        AsperaCollection.update({"_id": transfer_token},
                                {"$set": {"transfer_commenced": str(datetime.now()),
                                          "file_path": path_to_file,
                                          "transfer_status": "transferring",
                                          "pct_completed": 0}
                                 })

        # instantiate an aspera transfer process
        process = Thread(target=AsperaTransfer, args=(transfer_token,))
        process.start()

        return_structure['initiate_data'] = {"transfer_token": str(transfer_token)}

    elif task == "transfer_progress":
        tokens = ast.literal_eval(request.POST["tokens"])
        progress_list = []

        for key, value in tokens.items():
            doc = AsperaCollection.find_one({"_id": ObjectId(key)})
            if doc:
                progress = {"transfer_token": key, "pct_completed": doc["pct_completed"],
                            "transfer_status": doc["transfer_status"]}
                progress_list.append(progress)

        return_structure['progress_data'] = progress_list

    return_structure['exit_status'] = 'success'
    out = jsonpickle.encode(return_structure)
    return HttpResponse(out, content_type='json')
Exemplo n.º 4
0
 def __init__(self, profile_id=None):
     self.profile_id = profile_id
     self.PubCollection = get_collection_ref(PubCollection)
     self.PersonCollection = get_collection_ref(PersonCollection)
     self.DataCollection = get_collection_ref(DataCollection)
     self.SourceCollection = get_collection_ref(SourceCollection)
     self.SampleCollection = get_collection_ref(SampleCollection)
     self.SubmissionCollection = get_collection_ref(SubmissionCollection)
Exemplo n.º 5
0
    def do_aspera_transfer(self):
        # reference to the database collection
        AsperaCollection = get_collection_ref("AsperaCollections")


        # test start
        for i in range(21):
            if i > 0:
                time.sleep(1)
            print("executing token: " + str(self.transfer_token) + " " + str(i * 2))
            AsperaCollection.update(
                {"_id": self.transfer_token},
                {"$set": {"transfer_status": "transferring",
                          "pct_completed": str(i * 5)
                          }
                 })
        print("finished for: " + str(self.transfer_token))
        AsperaCollection.update(
            {"_id": self.transfer_token},
            {"$set": {"transfer_status": "completed"
                      }
             })
Exemplo n.º 6
0
 def __init__(self, profile_id=None):
     self.SourceCollection = get_collection_ref(SourceCollection)
     self.schema = DataSchemas("COPO").get_ui_template_as_obj().copo.source.fields
     #self.schema_chars = DataSchemas("COPO").get_ui_template_as_obj().copo.source.characteristics.fields
     self.profile_id = profile_id
Exemplo n.º 7
0
from datetime import date
import difflib
import string
import uuid
import ast
import re

import random
import web.apps.web_copo.schemas.utils.data_utils as d_utils
from chunked_upload.models import ChunkedUpload
from dal.mongo_util import get_collection_ref
from dal.base_resource import Resource
from dal import ObjectId
from dal.copo_base_da import DataSchemas

EnaCollections = get_collection_ref("EnaCollections")


class EnaCollection(Resource):
    def GET(self, id):
        doc = EnaCollections.find_one({"_id": ObjectId(id)})
        if not doc:
            pass
        return doc

    def PUT(self, doc):
        return EnaCollections.insert(doc)

    def add_ena_study(self, ena_collection_id, study_type_list):
        # get study template from the ENA db template
        study_template = d_utils.get_db_template("ENA")['studies'][0]
Exemplo n.º 8
0
 def __init__(self):
     self.OAuthToken = get_collection_ref('OAuthToken')
Exemplo n.º 9
0
 def __init__(self):
     super(Repository, self).__init__(None, "repository")
     self.Repository = get_collection_ref(RepositoryCollection)
Exemplo n.º 10
0
 def __init__(self, profile_id=None):
     self.DescriptionCollection = get_collection_ref(DescriptionCollection)
     self.profile_id = profile_id
Exemplo n.º 11
0
from web.apps.web_copo.schemas.utils import data_utils
from web.apps.web_copo.schemas.utils.data_utils import DecoupleFormSubmission

PubCollection = 'PublicationCollection'
PersonCollection = 'PersonCollection'
DataCollection = 'DataCollection'
SampleCollection = 'SampleCollection'
SubmissionCollection = 'SubmissionCollection'
SourceCollection = 'SourceCollection'
DataFileCollection = 'DataFileCollection'
RemoteFileCollection = 'RemoteFileCollection'
DescriptionCollection = 'DescriptionCollection'
ProfileCollection = 'Profiles'
AnnotationReference = 'AnnotationCollection'

handle_dict = dict(publication=get_collection_ref(PubCollection),
                   person=get_collection_ref(PersonCollection),
                   sample=get_collection_ref(SampleCollection),
                   source=get_collection_ref(SourceCollection),
                   profile=get_collection_ref(ProfileCollection),
                   submission=get_collection_ref(SubmissionCollection),
                   datafile=get_collection_ref(DataFileCollection),
                   annotation=get_collection_ref(AnnotationReference)
                   )


class ProfileInfo:
    def __init__(self, profile_id=None):
        self.profile_id = profile_id

    def get_counts(self):
Exemplo n.º 12
0
__author__ = '[email protected] - 24/06/15'

import json

from allauth.socialaccount.models import SocialAccount
from dal.base_resource import Resource
from dal.mongo_util import get_collection_ref

ORCID = get_collection_ref("OrcidCollections")


class Orcid(Resource):
    def store_orcid_profile(self, profile_data, user):

        social_account = SocialAccount.objects.get(user_id=user)
        profile_data = social_account.extra_data
        profile_data = json.dumps(profile_data).replace('-', '_')

        orcid_dict = {'user': user, 'op': json.loads(profile_data)}

        ORCID.update({'user': user}, orcid_dict, True)

    def get_orcid_profile(self, user):

        u_id = user.id
        orc = ORCID.find_one({'user': u_id})
        if (orc is not None):
            return orc
        else:
            return ''
Exemplo n.º 13
0
import os
import pandas as pd
from bson import ObjectId
from dal import cursor_to_list
from dal.copo_da import Sample, Source, CGCore
from dal.mongo_util import get_collection_ref
from web.apps.web_copo.lookup.resolver import RESOLVER
import web.apps.web_copo.schemas.utils.data_utils as d_utils
"""
class is a service for the resolution of search terms to local objects in COPO. 
Each resolver should provide a mechanism for:
1. resolving a search term to valid objects
2. resolving accessions (i.e., id, values, etc.) to obtain matching or corresponding objects
"""

Lookups = get_collection_ref("Lookups")


class COPOLookup:
    def __init__(self, **kwargs):
        self.param_dict = kwargs
        self.search_term = self.param_dict.get("search_term", str()).lower()
        self.accession = self.param_dict.get("accession", dict())
        self.data_source = self.param_dict.get("data_source", str())
        self.profile_id = self.param_dict.get("profile_id", str())
        self.referenced_field = self.param_dict.get("referenced_field", str())
        self.drop_downs_pth = RESOLVER['copo_drop_downs']

    def broker_component_search(self):
        dispatcher = {
            'agrovoclabels': self.get_lookup_type,
Exemplo n.º 14
0
 def __init__(self, profile_id=None):
     self.FigshareSubmission = get_collection_ref(FigshareSubmissionName)
     self.PartialSubmission = get_collection_ref(PartialSubmissionName)
     self.FigshareTokens = get_collection_ref(FigshareTokenCollection)
Exemplo n.º 15
0
__author__ = '[email protected] - 24/06/15'

import json

from allauth.socialaccount.models import SocialAccount
from dal.base_resource import Resource
from dal.mongo_util import get_collection_ref

ORCID = get_collection_ref("OrcidCollections")


class Orcid(Resource):

    def store_orcid_profile(self, profile_data, user):

        user_id = user.id
        social_account = SocialAccount.objects.get(user_id=user_id)
        profile_data = social_account.extra_data
        profile_data = json.dumps(profile_data).replace('-', '_')

        orcid_dict = {'user': user_id, 'op': json.loads(profile_data)}

        ORCID.update({'user': user_id},
                     orcid_dict,
                     True)


    def get_orcid_profile(self, user):

        u_id = user.id
        orc = ORCID.find_one({'user': u_id})
Exemplo n.º 16
0
 def __init__(self, profile_id=None):
     self.FigshareSubmission = get_collection_ref(FigshareSubmissionName)
     self.PartialSubmission = get_collection_ref(PartialSubmissionName)
     self.FigshareTokens = get_collection_ref(FigshareTokenCollection)
Exemplo n.º 17
0
 def __init__(self, profile_id=None):
     self.DescriptionCollection = get_collection_ref(DescriptionCollection)
     self.profile_id = profile_id
     self.component = str()
Exemplo n.º 18
0
 def __init__(self, profile_id=None):
     self.RemoteFileCollection = get_collection_ref(RemoteFileCollection)
     self.profile_id = profile_id
Exemplo n.º 19
0
SubmissionCollection = 'SubmissionCollection'
SourceCollection = 'SourceCollection'
DataFileCollection = 'DataFileCollection'
RemoteFileCollection = 'RemoteFileCollection'
DescriptionCollection = 'DescriptionCollection'
ProfileCollection = 'Profiles'
AnnotationReference = 'AnnotationCollection'
GroupCollection = 'GroupCollection'
RepositoryCollection = 'RepositoryCollection'
CGCoreCollection = 'CGCoreCollection'
TextAnnotationCollection = 'TextAnnotationCollection'
SubmissionQueueCollection = 'SubmissionQueueCollection'
MetadataTemplateCollection = 'MetadataTemplateCollection'

handle_dict = dict(
    publication=get_collection_ref(PubCollection),
    person=get_collection_ref(PersonCollection),
    sample=get_collection_ref(SampleCollection),
    source=get_collection_ref(SourceCollection),
    profile=get_collection_ref(ProfileCollection),
    submission=get_collection_ref(SubmissionCollection),
    datafile=get_collection_ref(DataFileCollection),
    annotation=get_collection_ref(AnnotationReference),
    group=get_collection_ref(GroupCollection),
    repository=get_collection_ref(RepositoryCollection),
    cgcore=get_collection_ref(CGCoreCollection),
    textannotation=get_collection_ref(TextAnnotationCollection),
    metadata_template=get_collection_ref(MetadataTemplateCollection))


class ProfileInfo:
Exemplo n.º 20
0
 def __init__(self, profile_id=None):
     self.DataFileCollection = get_collection_ref(DataFileCollection)
     self.profile_id = profile_id
Exemplo n.º 21
0
__author__ = 'felixshaw'

import bson.objectid as o
from django.urls import reverse

from dal.base_resource import Resource
from dal.mongo_util import get_collection_ref
from web.apps.web_copo.schemas.utils import data_utils
from web.apps.web_copo.vocab.status_vocab import STATUS_CODES

Schemas = get_collection_ref("Schemas")
Collections = get_collection_ref("CollectionHeads")
Profiles = get_collection_ref("Profiles")


class Collection_Head(Resource):
    # method to create a skelton collection object
    def PUT(self):
        return Collections.insert({})

    def update(self, collection_head_id, doc):
        Collections.update(
            {
                '_id': collection_head_id
            },
            {
                '$set': doc
            }
        )

    def GET(self, id):
Exemplo n.º 22
0
 def __init__(self, profile_id=None):
     self.PersonCollection = get_collection_ref(PersonCollection)
     self.schema = DataSchemas("COPO").get_ui_template_as_obj().copo.person.fields
     self.profile_id = profile_id
Exemplo n.º 23
0

__author__ = 'felixshaw'

import bson.objectid as o
from django_tools.middlewares import ThreadLocal
from django.core.urlresolvers import reverse

from web.apps.web_copo.vocab.status_vocab import STATUS_CODES
from web.apps.web_copo.schemas.utils.data_formats import DataFormats
import web.apps.web_copo.schemas.utils.data_utils as d_utils
from dal.mongo_util import get_collection_ref
from dal.base_resource import Resource

Schemas = get_collection_ref("Schemas")
Collections = get_collection_ref("CollectionHeads")
Profiles = get_collection_ref("Profiles")


class Collection_Head(Resource):
    # method to create a skelton collection object
    def PUT(self):
        return Collections.insert({})

    def update(self, collection_head_id, doc):
        Collections.update(
            {
                '_id': collection_head_id
            },
            {
                '$set': doc
Exemplo n.º 24
0
 def __init__(self, profile_id=None):
     self.RemoteFileCollection = get_collection_ref(RemoteFileCollection)
     self.profile_id = profile_id
Exemplo n.º 25
0
 def __init__(self):
     self.OAuthToken = get_collection_ref('OAuthToken')
Exemplo n.º 26
0
__author__ = 'etuka'

from django.core.management.base import BaseCommand

import os
import glob
import json
import pandas as pd
import xml.etree.ElementTree as ET
from dal.copo_base_da import DataSchemas
from dal.mongo_util import get_collection_ref
from web.apps.web_copo.lookup.resolver import RESOLVER
import web.apps.web_copo.schemas.utils.data_utils as d_utils
from web.apps.web_copo.schemas.utils.data_formats import DataFormats

Schemas = get_collection_ref("Schemas")
Lookups = get_collection_ref("Lookups")

drop_downs_pth = RESOLVER['copo_drop_downs']


class Command(BaseCommand):
    help = 'Generate UI schemas and dropdown lookups'

    def handle(self, *args, **options):
        # generate ui schemas
        self.generate_ui_schemas()

        self.convert_crp_list()

        self.generate_lookup_datasource()
Exemplo n.º 27
0
 def __init__(self):
     super(CopoGroup, self).__init__(None, "group")
     self.Group = get_collection_ref(GroupCollection)