def get_download_url(oh_member): files = api.exchange_oauth2_member( access_token=oh_member.get_access_token())['data'] for f in files: if f['basename'] == 'oura-data.json': return {'url': f['download_url'], 'created': f['created']} return None
def dashboard(request): oh_member = request.user.oh_member context = { 'oh_member': oh_member, } try: oh_member_data = api.exchange_oauth2_member( oh_member.get_access_token()) except: messages.error(request, "You need to re-authenticate with Open Humans") logout(request) return redirect("/") requested_sources = { 'direct-sharing-128': '23andMe Upload', 'direct-sharing-129': 'AncestryDNA Upload', 'direct-sharing-120': 'FamilyTreeDNA integration', 'direct-sharing-40': 'Gencove', 'direct-sharing-131': 'Genome/Exome Upload', 'direct-sharing-139': 'Harvard Personal Genome Project', 'direct-sharing-55': 'openSNP' } matching_sources = {} found_source_ids = [] for data_source in oh_member_data['data']: if (data_source['source'] in requested_sources and 'vcf' in data_source['basename'] and 'metadata' not in data_source['basename']): matching_sources[data_source['basename']] = { 'project': requested_sources[data_source['source']], 'id': data_source['id'], 'source_id': data_source['source'] } found_source_ids.append(data_source['source']) for source_id, source_name in requested_sources.items(): if source_id not in found_source_ids: matching_sources[source_id] = { 'project': source_name, 'id': None, 'source_id': source_id } # check position in queue active_sorted = ImputerMember.objects.filter(active=True).order_by('id') queue_position = None for index, active in enumerate(active_sorted): if int(oh_member.oh_id) == int(active.oh_id): queue_position = index context = { 'base_url': request.build_absolute_uri("/").rstrip('/'), 'section': 'dashboard', 'all_datasources': requested_sources, 'matching_sources': matching_sources, 'queue_position': queue_position } return render(request, 'main/dashboard.html', context=context)
def get_vcf(data_source_id, oh_id, calculate_variant_length=True): """Download member .vcf.""" oh_member = OpenHumansMember.objects.get(oh_id=oh_id) imputer_record = ImputerMember.objects.get(oh_id=oh_id, active=True) imputer_record.step = 'get_vcf' imputer_record.save() logger.info('Downloading vcf for member {}'.format(oh_id)) user_details = api.exchange_oauth2_member(oh_member.get_access_token()) for data_source in user_details['data']: if str(data_source['id']) == str(data_source_id): data_file_url = data_source['download_url'] imputer_record.data_source_id = data_source['id'] imputer_record.save() datafile = requests.get(data_file_url) os.makedirs('{}/{}'.format(DATA_DIR, oh_id), exist_ok=True) with open('{}/{}/member.{}.vcf'.format(DATA_DIR, oh_id, oh_id), 'wb') as handle: try: try: try: textobj = bz2.decompress(datafile.content) handle.write(textobj) except OSError: textobj = gzip.decompress(datafile.content) handle.write(textobj) except OSError: for block in datafile.iter_content(1024): handle.write(block) except: logger.critical('your data source file is malformated') time.sleep(5) # download takes a few seconds # set CHROMOSOMES variable appropriately by checking which were submitted for imputation. if calculate_variant_length is True: with open('{}/{}/member.{}.vcf'.format(DATA_DIR, oh_id, oh_id)) as vcf: member_chroms = set() longest_variant = 0 for line in vcf: if not line.startswith('#'): member_chroms.add(str(line.split('\t')[0]).replace("chr", "")) reflen = len(str(line.split('\t')[3])) altlen = len(str(line.split('\t')[4])) if reflen + altlen > longest_variant: longest_variant = reflen + altlen # store the variant length, but only if less than 1000 if longest_variant <= 990: imputer_record.variant_length = longest_variant + 10 imputer_record.save() elif calculate_variant_length is False: with open('{}/{}/member.{}.vcf'.format(DATA_DIR, oh_id, oh_id)) as vcf: member_chroms = set() for line in vcf: if not line.startswith('#'): member_chroms.add(str(line.split('\t')[0]).replace("chr", "")) global CHROMOSOMES default_chroms = set(CHROMOSOMES) CHROMOSOMES = default_chroms.intersection(member_chroms) if "X" in member_chroms or "chrX" in member_chroms: CHROMOSOMES.add('23') CHROMOSOMES = sorted(CHROMOSOMES, key=int)
def clean_uploaded_file(self, access_token, file_id): member = api.exchange_oauth2_member(access_token) for dfile in member['data']: if dfile['id'] == file_id: print(dfile) process_target(dfile, access_token, member, dfile['metadata']) pass
def get_vcf(data_source_id, oh_id): """Download member .vcf.""" oh_member = OpenHumansMember.objects.get(oh_id=oh_id) imputer_record = ImputerMember.objects.get(oh_id=oh_id, active=True) imputer_record.step = 'get_vcf' imputer_record.save() logger.info('Downloading vcf for member {}'.format(oh_id)) user_details = api.exchange_oauth2_member(oh_member.get_access_token()) for data_source in user_details['data']: if str(data_source['id']) == str(data_source_id): data_file_url = data_source['download_url'] datafile = requests.get(data_file_url) os.makedirs('{}/{}'.format(DATA_DIR, oh_id), exist_ok=True) with open('{}/{}/member.{}.vcf'.format(DATA_DIR, oh_id, oh_id), 'wb') as handle: try: textobj = bz2.decompress(datafile.content) handle.write(textobj) except OSError: textobj = gzip.decompress(datafile.content) handle.write(textobj) except OSError: for block in datafile.iter_content(1024): handle.write(block) except: logger.critical('your data source file is malformated') time.sleep(5) # download takes a few seconds
def clean_uploaded_file(self, access_token, file_id, taxonomy): member = api.exchange_oauth2_member(access_token, base_url=OH_BASE_URL) for dfile in member['data']: if dfile['id'] == file_id: process_file(dfile, access_token, member, dfile['metadata'], taxonomy) pass
def handle(self, *args, **options): users = OpenHumansMember.objects.all() for user in users: member = api.exchange_oauth2_member(user.get_access_token()) for dfile in member['data']: if 'GoogleFit' in dfile['metadata']['tags']: print(dfile)
def get_last_id(oh_access_token): member = api.exchange_oauth2_member( oh_access_token, all_files=True) twitter_files = {} tweet_id = None like_id = None for dfile in member['data']: if 'Twitter' in dfile['metadata']['tags']: twitter_files[dfile['basename']] = dfile if twitter_files: filenames = list(twitter_files.keys()) filenames.sort() last_file = twitter_files[filenames[-1]] tf_in = tempfile.NamedTemporaryFile(suffix='.json') tf_in.write(requests.get(last_file['download_url']).content) tf_in.flush() twitter_data = json.load(open(tf_in.name)) print("fetched last ID from OH") print(twitter_data['tweets'][0]['id_str']) print('---') if len(twitter_data['tweets']): tweet_id = twitter_data['tweets'][0]['id_str'] if len(twitter_data['likes']): like_id = twitter_data['tweets'][0]['id_str'] return tweet_id, like_id
def bundle(handler, model): """Create a compressed tarball containing the notebook document. Parameters ---------- handler : tornado.web.RequestHandler Handler that serviced the bundle request model : dict Notebook model from the configured ContentManager """ redirect_url = os.getenv("JH_BUNDLE_REDIRECT", "http://127.0.0.1:5000/shared") try: access_token = os.getenv('OH_ACCESS_TOKEN') ohmember = api.exchange_oauth2_member(access_token) project_member_id = ohmember['project_member_id'] notebook_filename = model['name'] api.delete_file(access_token, project_member_id, file_basename=notebook_filename) print('deleted old_file') notebook_content = nbformat.writes(model['content']).encode('utf-8') upload_notebook(notebook_content, notebook_filename, access_token, project_member_id) handler.redirect(redirect_url) except: print('whopsy, something went wrong') handler.finish(("Your upload failed. " "Please restart your notebook server " "and try again."))
def get_existing_data(garmin_user_id): oh_user = get_oh_user_from_garmin_id(garmin_user_id) member = api.exchange_oauth2_member(oh_user.get_access_token()) for dfile in member['data']: if 'Garmin' in dfile['metadata']['tags']: download_url = dfile['download_url'] return download_to_json(download_url), dfile['id'] return {'dailies': []}, None
def dashboard(request): oh_member = request.user.oh_member context = { 'oh_member': oh_member, } try: oh_member_data = api.exchange_oauth2_member( oh_member.get_access_token()) except: messages.error(request, "You need to re-authenticate with Open Humans") logout(request) return redirect("/") requested_sources = { 'direct-sharing-128': '23andMe Upload', 'direct-sharing-129': 'AncestryDNA Upload', 'direct-sharing-120': 'FamilyTreeDNA integration', 'direct-sharing-40': 'Gencove', 'direct-sharing-131': 'Genome/Exome Upload', 'direct-sharing-139': 'Harvard Personal Genome Project', 'direct-sharing-55': 'openSNP' } # check position in queue active_sorted = ImputerMember.objects.filter(active=True).order_by('id') queue_position = None for index, active in enumerate(active_sorted): if oh_member.oh_id == active.oh_id: queue_position = index matching_sources = {} for data_source in oh_member_data['data']: matching_source = requested_sources.get(data_source['source'], ) vcf_in_name = 'vcf' in data_source['basename'] metadata_in_name = 'metadata' in data_source['basename'] if matching_source is not None and vcf_in_name is True and metadata_in_name is False: # check if the data source id has been imputed imputed = ImputerMember.objects.filter( active=False, oh_id=oh_member.oh_id, data_source_id=data_source['id']) matching_sources[data_source['id']] = { 'project': requested_sources[data_source['source']], 'source_id': data_source['source'], 'basename': data_source['basename'], 'already_imputed': True if len(imputed) > 0 else False } context = { 'base_url': request.build_absolute_uri("/").rstrip('/'), 'section': 'dashboard', 'all_datasources': requested_sources, 'matching_sources': matching_sources, 'queue_position': queue_position } return render(request, 'main/dashboard.html', context=context)
def get_latest_googlefit_file_url(oh_access_token): member = api.exchange_oauth2_member(oh_access_token) latest_month = GOOGLEFIT_DEFAULT_START_DATE.strftime("%Y-%m") download_url = None for dfile in member['data']: if 'GoogleFit' in dfile['metadata']['tags']: if dfile['metadata'].get('month', '') >= latest_month: latest_month = dfile['metadata']['month'] download_url = dfile['download_url'] return download_url
def get_download_url(oh_member): try: files = api.exchange_oauth2_member( access_token=oh_member.get_access_token())['data'] for f in files: if f['basename'] == 'spotify-listening-archive.json': return {'url': f['download_url'], 'created': f['created']} return None except: return 'token-broken'
def iterate_member_files(self, ohmember): client_info = ProjectConfiguration.objects.get(id=1).client_info ohmember_data = api.exchange_oauth2_member( ohmember.get_access_token(**client_info)) files = ohmember_data['data'] for f in files: fname = f['basename'] if not fname.endswith('.zip') and not fname.endswith('.json'): api.delete_file(ohmember.access_token, ohmember.oh_id, file_id=f['id'])
def get_existing_data(oh_access_token): member = api.exchange_oauth2_member(oh_access_token) for dfile in member['data']: if 'demo' in dfile['metadata']['tags']: # get file here and read the json into memory tf_in = tempfile.NamedTemporaryFile(suffix='.json') tf_in.write(requests.get(dfile['download_url']).content) tf_in.flush() demo_data = json.load(open(tf_in.name)) return demo_data return []
def get_latest_github_file_url(oh_access_token): member = api.exchange_oauth2_member(oh_access_token) download_url = None last_updated_at = None for dfile in member['data']: if 'Github' in dfile['metadata']['tags']: if last_updated_at is None or dfile['metadata'].get( 'updated_at', '') >= last_updated_at: last_updated_at = dfile['metadata']['updated_at'] download_url = dfile['download_url'] return download_url
def get_nokia_file(oh_member): try: oh_access_token = oh_member.get_access_token( client_id=settings.OPENHUMANS_CLIENT_ID, client_secret=settings.OPENHUMANS_CLIENT_SECRET) user_object = api.exchange_oauth2_member(oh_access_token) for dfile in user_object['data']: if 'nokia' in dfile['metadata']['tags']: return dfile['download_url'] return '' except: return 'error'
def get_existing_rescuetime(oh_access_token): member = api.exchange_oauth2_member(oh_access_token) for dfile in member['data']: if 'Rescuetime' in dfile['metadata']['tags']: # get file here and read the json into memory tf_in = tempfile.NamedTemporaryFile(suffix='.json') tf_in.write(requests.get(dfile['download_url']).content) tf_in.flush() rescuetime_data = json.load(open(tf_in.name)) if 'rows' in rescuetime_data.keys(): if len(rescuetime_data['rows']) > 0: return rescuetime_data return {}
def get_runkeeper_file(oh_member): try: oh_access_token = oh_member.get_access_token( client_id=settings.OPENHUMANS_CLIENT_ID, client_secret=settings.OPENHUMANS_CLIENT_SECRET) user_object = api.exchange_oauth2_member(oh_access_token) files = {} for dfile in user_object['data']: if 'Runkeeper' in dfile['metadata']['tags']: files[dfile['basename']] = dfile['download_url'] return files except: return 'error'
def get_latest_googlefit_file_updated_dt(oh_access_token): member = api.exchange_oauth2_member(oh_access_token) latest_month = GOOGLEFIT_DEFAULT_START_DATE.strftime("%Y-%m") last_updated = None for dfile in member['data']: if 'GoogleFit' in dfile['metadata']['tags']: if dfile['metadata'].get('month', '') >= latest_month: latest_month = dfile['metadata']['month'] last_updated = dfile['metadata']['updated_at'] if last_updated: return datetime.strptime(last_updated, "%Y-%m-%d %H:%M:%S.%f") else: return None
def oh_code_to_member(code): """ Exchange code for token, use this to create and return OpenHumansMember. If a matching OpenHumansMember exists, update and return it. """ if settings.OPENHUMANS_CLIENT_SECRET and \ settings.OPENHUMANS_CLIENT_ID and code: data = { 'grant_type': 'authorization_code', 'redirect_uri': '{}/complete'.format(settings.OPENHUMANS_APP_BASE_URL), 'code': code, } req = requests.post('{}/oauth2/token/'.format( settings.OPENHUMANS_OH_BASE_URL), data=data, auth=requests.auth.HTTPBasicAuth( settings.OPENHUMANS_CLIENT_ID, settings.OPENHUMANS_CLIENT_SECRET)) data = req.json() if 'access_token' in data: oh_memberdata = api.exchange_oauth2_member(data['access_token']) oh_id = oh_memberdata['project_member_id'] oh_username = oh_memberdata['username'] try: oh_member = OpenHumansMember.objects.get(oh_id=oh_id) logger.debug('Member {} re-authorized.'.format(oh_id)) oh_member.access_token = data['access_token'] oh_member.refresh_token = data['refresh_token'] oh_member.token_expires = OpenHumansMember.get_expiration( data['expires_in']) except OpenHumansMember.DoesNotExist: oh_member = OpenHumansMember.create( oh_id=oh_id, oh_username=oh_username, access_token=data['access_token'], refresh_token=data['refresh_token'], expires_in=data['expires_in']) logger.debug('Member {} created.'.format(oh_id)) oh_member.save() return oh_member elif 'error' in req.json(): logger.debug('Error in token exchange: {}'.format(req.json())) else: logger.warning('Neither token nor error info in OH response!') else: logger.error('OH_CLIENT_SECRET or code are unavailable') return None
def iterate_member_files(self, ohmember): client_info = ProjectConfiguration.objects.get(id=1).client_info ohmember_data = api.exchange_oauth2_member( ohmember.get_access_token(**client_info)) files = ohmember_data['data'] metadata_files = self.get_metadata_list(files) for f in files: fname = f['basename'] if fname.endswith('.gz'): fname = fname.replace('.gz', '') elif fname.endswith('bz2'): fname = fname.replace('.bz2', '') if fname + '.vcf.metadata.json' not in metadata_files: clean_uploaded_file.delay(ohmember.access_token, f['id'])
def get_existing_fitbit(oh_access_token): print("entered get_existing_fitbit") member = api.exchange_oauth2_member(oh_access_token) for dfile in member['data']: if 'Fitbit' in dfile['metadata']['tags']: print("got inside fitbit if") # get file here and read the json into memory tf_in = tempfile.NamedTemporaryFile(suffix='.json') tf_in.write(requests.get(dfile['download_url']).content) tf_in.flush() fitbit_data = json.load(open(tf_in.name)) print("fetched data from OH") print(fitbit_data) return fitbit_data return []
def merge_with_existing_and_upload(oh_user, summaries, file_name): access_token = oh_user.get_access_token() oh_user_data = api.exchange_oauth2_member(access_token) existing_file = find_existing_data_file(oh_user_data, file_name) if existing_file: download_url = existing_file['download_url'] old_summaries = json.loads(requests.get(download_url).content) summaries = merge_summaries(summaries, old_summaries) else: summaries = merge_summaries(summaries, []) # Remove duplicates existing_file_id = existing_file['id'] if existing_file else None _LOGGER.info(f"Uploading {len(summaries)} summaries to file {file_name} for user {oh_user.oh_id}") upload_summaries(oh_user, summaries, file_name, existing_file_id) return summaries
def get_twitter_files(oh_member): try: oh_access_token = oh_member.get_access_token( client_id=settings.OPENHUMANS_CLIENT_ID, client_secret=settings.OPENHUMANS_CLIENT_SECRET) user_object = api.exchange_oauth2_member(oh_access_token, all_files=True) files = [] for dfile in user_object['data']: if 'Twitter' in dfile['metadata']['tags']: files.append(dfile) if files: files.sort(key=lambda x: x['basename'], reverse=True) return files except: return 'error'
def get_jawbone_files(oh_member): try: files = [] oh_access_token = oh_member.get_access_token( client_id=settings.OPENHUMANS_CLIENT_ID, client_secret=settings.OPENHUMANS_CLIENT_SECRET) user_object = api.exchange_oauth2_member(oh_access_token) for dfile in user_object['data']: if 'Jawbone' in dfile['metadata']['tags']: files.append({ 'url': dfile['download_url'], 'name': dfile['basename'] }) return files except: return 'error'
def get_existing_nokia(oh_access_token): print("Entering get_existing_nokia function...") member = api.exchange_oauth2_member(oh_access_token) for dfile in member['data']: if 'nokiahealthdata' in dfile['metadata']['tags']: print("Found file with tag...") # get file here and read the json into memory tf_in = tempfile.NamedTemporaryFile(suffix='.json') tf_in.write(requests.get(dfile['download_url']).content) tf_in.flush() nokia_data = json.load(open(tf_in.name)) # print("getting existing data:") # print(nokia_data) # print(type(nokia_data)) # for key in nokia_data: # print(key) return nokia_data print('no existing data with nokiahealthdata tag') return {}
def write_new_tweets(oh_member, twitter_api, month, new_data): existing_files = api.exchange_oauth2_member( oh_member.get_access_token(), all_files=True) old_data = None file_id = None for dfile in existing_files['data']: if dfile['basename'] == 'twitter-data-{}.json'.format(month): old_data = requests.get(dfile['download_url']).json() file_id = dfile['id'] break if old_data: old_data['tweets'] = new_data['tweets'] + old_data['tweets'] old_data['likes'] = new_data['likes'] + old_data['likes'] else: old_data = {'tweets': new_data['tweets'], 'likes': new_data['likes'], 'followers': [], 'following': []} if month == str(datetime.datetime.today())[:7]: me = twitter_api.me() old_data['followers'].append( {'timestamp': str(datetime.datetime.today()), 'value': me.followers_count}) old_data['following'].append( {'timestamp': str(datetime.datetime.today()), 'value': me.friends_count}) with tempfile.TemporaryFile() as f: js = json.dumps(old_data) js = str.encode(js) f.write(js) f.flush() f.seek(0) api.upload_stream( f, "twitter-data-{}.json".format(month), metadata={ "description": "Twitter Data", "tags": ["Twitter"] }, access_token=oh_member.get_access_token()) if file_id: api.delete_file( oh_member.get_access_token(), project_member_id=oh_member.oh_id, file_id=file_id)
def dashboard(request): oh_member = request.user.oh_member context = { 'oh_member': oh_member, } try: oh_member_data = api.exchange_oauth2_member( oh_member.get_access_token()) except: messages.error(request, "You need to re-authenticate with Open Humans") logout(request) return redirect("/") all_available_notebooks = get_notebook_files(oh_member_data) existing_notebooks = SharedNotebook.objects.filter(oh_member=oh_member) context['notebook_files'] = all_available_notebooks context['existing_notebooks'] = existing_notebooks context['JH_URL'] = settings.JUPYTERHUB_BASE_URL context['base_url'] = request.build_absolute_uri("/").rstrip('/') context['section'] = 'dashboard' return render(request, 'main/dashboard.html', context=context)
def add_notebook(request, notebook_id): oh_member = request.user.oh_member try: oh_member_data = api.exchange_oauth2_member( oh_member.get_access_token()) except: messages.error(request, "You need to re-authenticate with Open Humans") logout(request) return redirect("/") notebook_name, notebook_url = get_notebook_oh(oh_member_data, notebook_id) if request.method == 'POST': add_notebook_helper(request, notebook_url, notebook_name, oh_member) return redirect('/dashboard') else: if len( SharedNotebook.objects.filter( oh_member=oh_member, notebook_name=notebook_name)) > 0: existing_notebook = SharedNotebook.objects.get( oh_member=oh_member, notebook_name=notebook_name) context = { 'description': existing_notebook.description, 'tags': existing_notebook.get_tags(), 'data_sources': existing_notebook.get_data_sources(), 'name': notebook_name, 'notebook_id': str(notebook_id), 'edit': True } else: notebook_content = download_notebook_oh(notebook_url) suggested_sources = suggest_data_sources(notebook_content) context = { 'description': '', 'name': notebook_name, 'notebook_id': str(notebook_id), 'tags': '', 'data_sources': suggested_sources } return render(request, 'main/add_notebook.html', context=context)