def setUp(self):
from dal.mongo_util import get_collection_ref
# empty all collections in test_db
get_collection_ref('Profiles').drop()
get_collection_ref('FigshareCollections').drop()
get_collection_ref('OrcidCollections').drop()
get_collection_ref('EnaCollections').drop()
get_collection_ref('CollectionHeads').drop()
def __init__(self, profile_id=None, source_id=None):
self.SampleCollection = get_collection_ref(SampleCollection)
self.schema = DataSchemas("COPO").get_ui_template_as_obj().copo.sample.fields
#self.schema_characeteristics = DataSchemas("COPO").get_ui_template_as_obj().copo.sample.characteristics
#self.schema_factors = DataSchemas("COPO").get_ui_template_as_obj().copo.sample.factors
self.profile_id = profile_id
self.source_id = source_id
def upload_to_dropbox(request):
return_structure = {}
# set up an Aspera collection handle
AsperaCollection = get_collection_ref("AsperaCollections")
transfer_token = ""
task = request.POST['task']
if task == "initiate_transfer": # initiate the transfer process
# get the target datafile and obtain the file reference
study_id = request.POST["study_id"]
ena_collection_id = request.POST["ena_collection_id"]
data_file_id = request.POST["data_file_id"]
data_file = EnaCollection().get_study_datafile(study_id, ena_collection_id, data_file_id)
chunked_upload = ChunkedUpload.objects.get(id=int(data_file["fileId"]))
# set a new document in the aspera collection,
# thus obtaining a transfer token to orchestrate the transfer process
db_template = d_utils.json_to_pytype(lkup.DB_TEMPLATES['ASPERA_COLLECTION'])
transfer_token = AsperaCollection.insert(db_template)
path_to_file = os.path.join(MEDIA_ROOT, chunked_upload.file.name)
# update some initial fields
# assume transfer_status is 'transferring' initially to allow the progress monitor to kick-start
AsperaCollection.update({"_id": transfer_token},
{"$set": {"transfer_commenced": str(datetime.now()),
"file_path": path_to_file,
"transfer_status": "transferring",
"pct_completed": 0}
})
# instantiate an aspera transfer process
process = Thread(target=AsperaTransfer, args=(transfer_token,))
process.start()
return_structure['initiate_data'] = {"transfer_token": str(transfer_token)}
elif task == "transfer_progress":
tokens = ast.literal_eval(request.POST["tokens"])
progress_list = []
for key, value in tokens.items():
doc = AsperaCollection.find_one({"_id": ObjectId(key)})
if doc:
progress = {"transfer_token": key, "pct_completed": doc["pct_completed"],
"transfer_status": doc["transfer_status"]}
progress_list.append(progress)
return_structure['progress_data'] = progress_list
return_structure['exit_status'] = 'success'
out = jsonpickle.encode(return_structure)
return HttpResponse(out, content_type='json')
def __init__(self, profile_id=None):
self.profile_id = profile_id
self.PubCollection = get_collection_ref(PubCollection)
self.PersonCollection = get_collection_ref(PersonCollection)
self.DataCollection = get_collection_ref(DataCollection)
self.SourceCollection = get_collection_ref(SourceCollection)
self.SampleCollection = get_collection_ref(SampleCollection)
self.SubmissionCollection = get_collection_ref(SubmissionCollection)
def do_aspera_transfer(self):
# reference to the database collection
AsperaCollection = get_collection_ref("AsperaCollections")
# test start
for i in range(21):
if i > 0:
time.sleep(1)
print("executing token: " + str(self.transfer_token) + " " + str(i * 2))
AsperaCollection.update(
{"_id": self.transfer_token},
{"$set": {"transfer_status": "transferring",
"pct_completed": str(i * 5)
}
})
print("finished for: " + str(self.transfer_token))
AsperaCollection.update(
{"_id": self.transfer_token},
{"$set": {"transfer_status": "completed"
}
})
def __init__(self, profile_id=None):
self.SourceCollection = get_collection_ref(SourceCollection)
self.schema = DataSchemas("COPO").get_ui_template_as_obj().copo.source.fields
#self.schema_chars = DataSchemas("COPO").get_ui_template_as_obj().copo.source.characteristics.fields
self.profile_id = profile_id
from datetime import date
import difflib
import string
import uuid
import ast
import re
import random
import web.apps.web_copo.schemas.utils.data_utils as d_utils
from chunked_upload.models import ChunkedUpload
from dal.mongo_util import get_collection_ref
from dal.base_resource import Resource
from dal import ObjectId
from dal.copo_base_da import DataSchemas
EnaCollections = get_collection_ref("EnaCollections")
class EnaCollection(Resource):
def GET(self, id):
doc = EnaCollections.find_one({"_id": ObjectId(id)})
if not doc:
pass
return doc
def PUT(self, doc):
return EnaCollections.insert(doc)
def add_ena_study(self, ena_collection_id, study_type_list):
# get study template from the ENA db template
study_template = d_utils.get_db_template("ENA")['studies'][0]
def __init__(self):
self.OAuthToken = get_collection_ref('OAuthToken')
def __init__(self):
super(Repository, self).__init__(None, "repository")
self.Repository = get_collection_ref(RepositoryCollection)
def __init__(self, profile_id=None):
self.DescriptionCollection = get_collection_ref(DescriptionCollection)
self.profile_id = profile_id
from web.apps.web_copo.schemas.utils import data_utils
from web.apps.web_copo.schemas.utils.data_utils import DecoupleFormSubmission
PubCollection = 'PublicationCollection'
PersonCollection = 'PersonCollection'
DataCollection = 'DataCollection'
SampleCollection = 'SampleCollection'
SubmissionCollection = 'SubmissionCollection'
SourceCollection = 'SourceCollection'
DataFileCollection = 'DataFileCollection'
RemoteFileCollection = 'RemoteFileCollection'
DescriptionCollection = 'DescriptionCollection'
ProfileCollection = 'Profiles'
AnnotationReference = 'AnnotationCollection'
handle_dict = dict(publication=get_collection_ref(PubCollection),
person=get_collection_ref(PersonCollection),
sample=get_collection_ref(SampleCollection),
source=get_collection_ref(SourceCollection),
profile=get_collection_ref(ProfileCollection),
submission=get_collection_ref(SubmissionCollection),
datafile=get_collection_ref(DataFileCollection),
annotation=get_collection_ref(AnnotationReference)
)
class ProfileInfo:
def __init__(self, profile_id=None):
self.profile_id = profile_id
def get_counts(self):
__author__ = '[email protected] - 24/06/15' import json from allauth.socialaccount.models import SocialAccount from dal.base_resource import Resource from dal.mongo_util import get_collection_ref ORCID = get_collection_ref("OrcidCollections") class Orcid(Resource): def store_orcid_profile(self, profile_data, user): social_account = SocialAccount.objects.get(user_id=user) profile_data = social_account.extra_data profile_data = json.dumps(profile_data).replace('-', '_') orcid_dict = {'user': user, 'op': json.loads(profile_data)} ORCID.update({'user': user}, orcid_dict, True) def get_orcid_profile(self, user): u_id = user.id orc = ORCID.find_one({'user': u_id}) if (orc is not None): return orc else: return ''
import os
import pandas as pd
from bson import ObjectId
from dal import cursor_to_list
from dal.copo_da import Sample, Source, CGCore
from dal.mongo_util import get_collection_ref
from web.apps.web_copo.lookup.resolver import RESOLVER
import web.apps.web_copo.schemas.utils.data_utils as d_utils
"""
class is a service for the resolution of search terms to local objects in COPO.
Each resolver should provide a mechanism for:
1. resolving a search term to valid objects
2. resolving accessions (i.e., id, values, etc.) to obtain matching or corresponding objects
"""
Lookups = get_collection_ref("Lookups")
class COPOLookup:
def __init__(self, **kwargs):
self.param_dict = kwargs
self.search_term = self.param_dict.get("search_term", str()).lower()
self.accession = self.param_dict.get("accession", dict())
self.data_source = self.param_dict.get("data_source", str())
self.profile_id = self.param_dict.get("profile_id", str())
self.referenced_field = self.param_dict.get("referenced_field", str())
self.drop_downs_pth = RESOLVER['copo_drop_downs']
def broker_component_search(self):
dispatcher = {
'agrovoclabels': self.get_lookup_type,
def __init__(self, profile_id=None):
self.FigshareSubmission = get_collection_ref(FigshareSubmissionName)
self.PartialSubmission = get_collection_ref(PartialSubmissionName)
self.FigshareTokens = get_collection_ref(FigshareTokenCollection)
__author__ = '[email protected] - 24/06/15' import json from allauth.socialaccount.models import SocialAccount from dal.base_resource import Resource from dal.mongo_util import get_collection_ref ORCID = get_collection_ref("OrcidCollections") class Orcid(Resource): def store_orcid_profile(self, profile_data, user): user_id = user.id social_account = SocialAccount.objects.get(user_id=user_id) profile_data = social_account.extra_data profile_data = json.dumps(profile_data).replace('-', '_') orcid_dict = {'user': user_id, 'op': json.loads(profile_data)} ORCID.update({'user': user_id}, orcid_dict, True) def get_orcid_profile(self, user): u_id = user.id orc = ORCID.find_one({'user': u_id})
def __init__(self, profile_id=None):
self.FigshareSubmission = get_collection_ref(FigshareSubmissionName)
self.PartialSubmission = get_collection_ref(PartialSubmissionName)
self.FigshareTokens = get_collection_ref(FigshareTokenCollection)
def __init__(self, profile_id=None):
self.DescriptionCollection = get_collection_ref(DescriptionCollection)
self.profile_id = profile_id
self.component = str()
def __init__(self, profile_id=None):
self.RemoteFileCollection = get_collection_ref(RemoteFileCollection)
self.profile_id = profile_id
SubmissionCollection = 'SubmissionCollection'
SourceCollection = 'SourceCollection'
DataFileCollection = 'DataFileCollection'
RemoteFileCollection = 'RemoteFileCollection'
DescriptionCollection = 'DescriptionCollection'
ProfileCollection = 'Profiles'
AnnotationReference = 'AnnotationCollection'
GroupCollection = 'GroupCollection'
RepositoryCollection = 'RepositoryCollection'
CGCoreCollection = 'CGCoreCollection'
TextAnnotationCollection = 'TextAnnotationCollection'
SubmissionQueueCollection = 'SubmissionQueueCollection'
MetadataTemplateCollection = 'MetadataTemplateCollection'
handle_dict = dict(
publication=get_collection_ref(PubCollection),
person=get_collection_ref(PersonCollection),
sample=get_collection_ref(SampleCollection),
source=get_collection_ref(SourceCollection),
profile=get_collection_ref(ProfileCollection),
submission=get_collection_ref(SubmissionCollection),
datafile=get_collection_ref(DataFileCollection),
annotation=get_collection_ref(AnnotationReference),
group=get_collection_ref(GroupCollection),
repository=get_collection_ref(RepositoryCollection),
cgcore=get_collection_ref(CGCoreCollection),
textannotation=get_collection_ref(TextAnnotationCollection),
metadata_template=get_collection_ref(MetadataTemplateCollection))
class ProfileInfo:
def __init__(self, profile_id=None):
self.DataFileCollection = get_collection_ref(DataFileCollection)
self.profile_id = profile_id
__author__ = 'felixshaw'
import bson.objectid as o
from django.urls import reverse
from dal.base_resource import Resource
from dal.mongo_util import get_collection_ref
from web.apps.web_copo.schemas.utils import data_utils
from web.apps.web_copo.vocab.status_vocab import STATUS_CODES
Schemas = get_collection_ref("Schemas")
Collections = get_collection_ref("CollectionHeads")
Profiles = get_collection_ref("Profiles")
class Collection_Head(Resource):
# method to create a skelton collection object
def PUT(self):
return Collections.insert({})
def update(self, collection_head_id, doc):
Collections.update(
{
'_id': collection_head_id
},
{
'$set': doc
}
)
def GET(self, id):
def __init__(self, profile_id=None):
self.PersonCollection = get_collection_ref(PersonCollection)
self.schema = DataSchemas("COPO").get_ui_template_as_obj().copo.person.fields
self.profile_id = profile_id
__author__ = 'felixshaw'
import bson.objectid as o
from django_tools.middlewares import ThreadLocal
from django.core.urlresolvers import reverse
from web.apps.web_copo.vocab.status_vocab import STATUS_CODES
from web.apps.web_copo.schemas.utils.data_formats import DataFormats
import web.apps.web_copo.schemas.utils.data_utils as d_utils
from dal.mongo_util import get_collection_ref
from dal.base_resource import Resource
Schemas = get_collection_ref("Schemas")
Collections = get_collection_ref("CollectionHeads")
Profiles = get_collection_ref("Profiles")
class Collection_Head(Resource):
# method to create a skelton collection object
def PUT(self):
return Collections.insert({})
def update(self, collection_head_id, doc):
Collections.update(
{
'_id': collection_head_id
},
{
'$set': doc
def __init__(self, profile_id=None):
self.RemoteFileCollection = get_collection_ref(RemoteFileCollection)
self.profile_id = profile_id
def __init__(self):
self.OAuthToken = get_collection_ref('OAuthToken')
__author__ = 'etuka'
from django.core.management.base import BaseCommand
import os
import glob
import json
import pandas as pd
import xml.etree.ElementTree as ET
from dal.copo_base_da import DataSchemas
from dal.mongo_util import get_collection_ref
from web.apps.web_copo.lookup.resolver import RESOLVER
import web.apps.web_copo.schemas.utils.data_utils as d_utils
from web.apps.web_copo.schemas.utils.data_formats import DataFormats
Schemas = get_collection_ref("Schemas")
Lookups = get_collection_ref("Lookups")
drop_downs_pth = RESOLVER['copo_drop_downs']
class Command(BaseCommand):
help = 'Generate UI schemas and dropdown lookups'
def handle(self, *args, **options):
# generate ui schemas
self.generate_ui_schemas()
self.convert_crp_list()
self.generate_lookup_datasource()
def __init__(self):
super(CopoGroup, self).__init__(None, "group")
self.Group = get_collection_ref(GroupCollection)
