def get_all_albums(credentials: Credentials) -> Dict[id, Album]: a_session = AuthorizedSession(credentials) ub = Builder('photoslibrary.googleapis.com', 'v1') url = ub.build('/albums') # dump all eg: params = {'pageSize': 50} wb = {} context = RequestContext('get_all_albums') while True: req = a_session.get(url, params=params) assert req.status_code == 200 res = req.json() if 'albums' not in res: break dup = 0 for _item in res['albums']: item = Album.from_dict(_item) if item.id in wb: dup += 1 wb[item.id] = item context.add(req.elapsed.total_seconds(), len(res['albums']), dup) if 'nextPageToken' not in res: break params['pageToken'] = res['nextPageToken'] context.stat() return wb
def getObject(verticalType, objectId): headers = { 'Accept': 'application/json', 'Content-Type': 'application/json; charset=UTF-8' } credentials = makeOauthCredential() response = None # Define get() REST call of target vertical uri = 'https://walletobjects.googleapis.com/walletobjects/v1' postfix = 'Object' path = createPath(verticalType, postfix, objectId) # There is no Google API for Passes Client Library for Python. # Authorize a http client with credential generated from Google API client library. ## see https://google-auth.readthedocs.io/en/latest/user-guide.html#making-authenticated-requests authed_session = AuthorizedSession(credentials) # make the GET request to make an get(); this returns a response object # other methods require different http methods; for example, get() requires authed_Session.get(...) # check the reference API to make the right REST call ## https://developers.google.com/pay/passes/reference/v1/ ## https://google-auth.readthedocs.io/en/latest/user-guide.html#making-authenticated-requests response = authed_session.get( uri + path # REST API endpoint , headers=headers # Header; optional ) return response
def run_query(baseurl, query, credentials=None): url = '{baseurl}&tq={query}'.format(baseurl=baseurl, query=parse.quote(query, safe='/()')) headers = {'X-DataSource-Auth': 'true'} if credentials: session = AuthorizedSession(credentials) else: session = Session() r = session.get(url, headers=headers) if r.encoding is None: r.encoding = 'utf-8' # raise any error messages if r.status_code != 200: raise ProgrammingError(r.text) if r.text.startswith(LEADING): result = json.loads(r.text[len(LEADING):]) else: result = r.json() return result
def evaluate(self): credentials, project = google.auth.default(scopes=['https://www.googleapis.com/auth/cloud-platform']) authed_session = AuthorizedSession(credentials) basename="https://automl.googleapis.com/v1beta1/" cmd = basename + self.operation_name response=authed_session.get(cmd) result=json.loads(response.content) self.ctx.log("Operation name: {}".format(result["name"])) if (("done" in result.keys()) and result["done"]): self.ctx.log("Model training complete.") self.model_name = result["response"]["name"] self.ctx.log("Model full name: {}".format(self.model_name)) self.ctx.config['google'].yaml['model_name'] = self.model_name self.ctx.config['google'].write() response = self.client.list_model_evaluations(self.model_name) self.ctx.log("List of model evaluations:") for evaluation in response: self.ctx.log("Model evaluation name: {}".format(evaluation.name)) self.ctx.log("Model evaluation id: {}".format(evaluation.name.split("/")[-1])) self.ctx.log("Model evaluation example count: {}".format( evaluation.evaluated_example_count)) self.ctx.log("Model evaluation time: {} seconds".format(evaluation.create_time.seconds)) self.ctx.log("Full model evaluation: {}".format(inspect.getmembers(evaluation) )) self.ctx.log("\n") else: self.ctx.log("Model still training...")
def authenticate_user(host, debug=False): """ Begins authentication flow, getting Google auth, opening web browser if needed, getting username and state from ARENA server. host: The hostname of the ARENA webserver. debug: True to skip SSL verify for localhost tests. Returns: Username from arena-account, or None. """ global debug_toggle global _id_token debug_toggle = debug print("Signing in to the ARENA...") creds = None browser = None try: browser = webbrowser.get() except (webbrowser.Error) as err: print("Console-only login. {0}".format(err)) # store the user's access and refresh tokens if os.path.exists(_user_gauth_path): print("Using cached Google authentication.") with open(_user_gauth_path, 'rb') as token: creds = pickle.load(token) session = AuthorizedSession(creds) # if no credentials available, let the user log in. if not creds or not creds.valid: if creds and creds.expired and creds.refresh_token: print("Requesting refreshed Google authentication.") creds.refresh(Request()) session = AuthorizedSession(creds) else: print("Requesting new Google authentication.") gauth_json = _get_gauthid(host) flow = InstalledAppFlow.from_client_config(json.loads(gauth_json), _scopes) if browser: # TODO: select best client port to avoid likely conflicts creds = flow.run_local_server(port=8989) else: creds = flow.run_console() session = flow.authorized_session() with open(_user_gauth_path, 'wb') as token: # save the credentials for the next run pickle.dump(creds, token) os.chmod(_user_gauth_path, 0o600) # set user-only perms. username = None _id_token = creds.id_token user_info = _get_user_state(host, _id_token) _user_info = json.loads(user_info) if 'authenticated' in _user_info and 'username' in _user_info: username = _user_info["username"] profile_info = session.get( 'https://www.googleapis.com/userinfo/v2/me').json() if profile_info: print(f'Authenticated Google account: {profile_info["email"]}') return username
def DownloadDiscoveryDocument(api_name, version, path=_DEFAULT_DISCOVERY_PATH, label=None): """Downloads a discovery document for the given api_name and version. This utility assumes that the API for which a discovery document is being retrieved is publicly accessible. However, you may access whitelisted resources for a public API if you are added to its whitelist and specify the associated label. Args: api_name: a str indicating the name of the API for which a discovery document is to be downloaded. version: a str indicating the version number of the API. path: a str indicating the path to which you want to save the downloaded discovery document. label: a str indicating a label to be applied to the discovery service request. This is not applicable when downloading the discovery document of a legacy API. For non-legacy APIs, this may be used as a means of programmatically retrieving a copy of a discovery document containing whitelisted content. Raises: ValueError: If either the specified API name and version can't be found by the discovery service, or if downloading the discovery document fails. """ credentials = _GetCredentials() auth_session = AuthorizedSession(credentials) discovery_service = build('discovery', 'v1') discovery_rest_url = None discovery_response = discovery_service.apis().list(name=api_name).execute() if 'items' in discovery_response: for api in discovery_response['items']: if api['version'] == version: discovery_rest_url = api['discoveryRestUrl'] break if discovery_rest_url: if label: # Apply the label query parameter if it exists. path_params = '&labels=%s' % label discovery_rest_url += path_params discovery_response = auth_session.get(discovery_rest_url) if discovery_response.status_code == 200: with open(path, 'wb') as handler: handler.write(discovery_response.text) else: raise ValueError( 'Unable to retrieve discovery document for api name "%s" ' 'and version "%s" via discovery URL: %s' % (api_name, version, discovery_rest_url)) else: raise ValueError('API with name "%s" and version "%s" was not found.' % (api_name, version))
class SheriffConfigClient(object): """Wrapping of sheriff-config HTTP API.""" def __init__(self): """Make the Cloud Endpoints request from this handler.""" credentials, _ = google.auth.default( scopes=['https://www.googleapis.com/auth/userinfo.email']) jwt_credentials = jwt.Credentials.from_signing_credentials( credentials, 'sheriff-config-dot-chromeperf.appspot.com') self._session = AuthorizedSession(jwt_credentials) @staticmethod def _ParseSubscription(revision, subscription): return Subscription( revision=revision, name=subscription.name, rotation_url=subscription.rotation_url, notification_email=subscription.notification_email, bug_labels=list(subscription.bug_labels), bug_components=list(subscription.bug_components), bug_cc_emails=list(subscription.bug_cc_emails), visibility=subscription.visibility, ) def Match(self, path): response = self._session.post( 'https://sheriff-config-dot-chromeperf.appspot.com/subscriptions/match', json={'path': path}) if response.status_code == 404: # If no subscription matched return [], None if not response.ok: return None, '%r\n%s' % (response, response.text) match_resp = json_format.Parse(response.text, sheriff_config_pb2.MatchResponse()) return [ self._ParseSubscription(s.revision, s.subscription) for s in match_resp.subscriptions ], None def List(self): response = self._session.post( 'https://sheriff-config-dot-chromeperf.appspot.com/subscriptions/list', json={'identity_email': GetEmail()}) if not response.ok: return None, '%r\n%s' % (response, response.text) list_resp = json_format.Parse(response.text, sheriff_config_pb2.ListResponse()) return [ self._ParseSubscription(s.revision, s.subscription) for s in list_resp.subscriptions ], None def Update(self): response = self._session.get( 'https://sheriff-config-dot-chromeperf.appspot.com/configs/update') if response.ok: return True, None return False, '%r\n%s' % (response, response.text)
def instances_list(request, service, version): """ API endpoint that lists the versions of a service """ credentials = compute_engine.Credentials() authed_session = AuthorizedSession(credentials) res = authed_session.get('https://appengine.googleapis.com/v1/apps/{}/services/{}/versions/{}/instances'.format(os.environ.get("GOOGLE_CLOUD_PROJECT"), service, version)) return Response(res.json())
def service_details(request, service): """ API endpoint that lists the services for the current application """ credentials = compute_engine.Credentials() authed_session = AuthorizedSession(credentials) res = authed_session.get('https://appengine.googleapis.com/v1/apps/{}/services/{}'.format(os.environ.get("GOOGLE_CLOUD_PROJECT"), service)) return Response(res.json())
def application_details(request): """ API endpoint that gets information about the running application """ credentials = compute_engine.Credentials() authed_session = AuthorizedSession(credentials) res = authed_session.get('https://appengine.googleapis.com/v1/apps/{}'.format(os.environ.get("GOOGLE_CLOUD_PROJECT"))) return Response(res.json())
def get_file(credentials, tsv_download): credentials = \ service_account.Credentials.from_service_account_info(credentials) scoped_credentials = credentials.with_scopes( ['https://www.googleapis.com/auth/drive.readonly']) session = AuthorizedSession(scoped_credentials) response = session.get(tsv_download) return response
def get_credentials(project_name=None, credentials_file=None): """Get the service account credentials for the given project""" if not project_name and not credentials_file: raise GCEProjectCredentialsException( 'Either project name or credentials file path must be given') if credentials_file and not os.path.exists(credentials_file): raise GCEProjectCredentialsException( 'Provided credentials file "%s" not found' % credentials_file) if not credentials_file: credentials_file = os.path.expanduser('~/.config/gce/%s.json' % project_name) if not os.path.exists(credentials_file): raise GCEProjectCredentialsException('"%s" credentials not found' % credentials_file) try: credentials = service_account.Credentials.from_service_account_file( credentials_file) except Exception as error: raise GCEProjectCredentialsException( 'Could not extract credentials from "{credentials_file}": ' '{error}'.format(credentials_file=credentials_file, error=error)) try: # https://developers.google.com/identity/protocols/oauth2/scopes#google-sign-in scoped_credentials = credentials.with_scopes(['profile']) authed_session = AuthorizedSession(scoped_credentials) authed_session.get('https://www.googleapis.com/oauth2/v2/userinfo') except RefreshError: raise GCEProjectCredentialsException( 'The provided credentials are invalid or expired: ' '{creds_file}'.format(creds_file=credentials_file)) except Exception as error: raise GCEProjectCredentialsException( 'GCP authentication failed: {error}'.format(error=error)) return credentials
def get(self): """Make the Cloud Endpoints request from this handler.""" credentials, _ = google.auth.default( scopes=['https://www.googleapis.com/auth/userinfo.email']) jwt_credentials = jwt.Credentials.from_signing_credentials( credentials, 'sheriff-config-dot-chromeperf.appspot.com') authed_session = AuthorizedSession(jwt_credentials) response = authed_session.get( 'https://sheriff-config-dot-chromeperf.appspot.com/configs/update') if response.status_code != 200: return webapp2.Response('FAILED: %r\n%s' % (response, response.text)) return webapp2.Response('OK')
def _authorize_with_google(token): credentials = Credentials(token) authed_session = AuthorizedSession(credentials) response = authed_session.get( 'https://www.googleapis.com/oauth2/v1/userinfo?alt=json') raw_data = json.loads(response.text) user_data = { 'name': raw_data['given_name'], 'surname': raw_data['family_name'], 'email': raw_data['email'], 'avatar': raw_data['picture'] } return user_data
class Configuration: def __init__(self): self.database_base_url = "https://one-star-reviews-1553143808077.firebaseio.com" self.auth_json = "/Users/victorious/OneStarReview/files/firebase_auth.json" self.scopes = [ "https://www.googleapis.com/auth/userinfo.email", "https://www.googleapis.com/auth/firebase.database" ] self.credentials = service_account.Credentials.from_service_account_file( self.auth_json, scopes=self.scopes) # Create auth Session self.authed_session = AuthorizedSession(self.credentials) self.response = self.authed_session.get(self.database_base_url) self.request = google.auth.transport.requests.Request() # Set Access Token for subsequent access to Firebase DB self.credentials.refresh(self.request) self.access_token = self.credentials.token # Database values self.version = "" self.id = "/" def get_request(self, version): self.version = version url_params = ".json?access_token=" + self.access_token response = requests.get(self.database_base_url + self.version + url_params) return json.loads(response.text) def post_request(self): json_object = {"key": "value"} url_params = ".json?access_token=" + self.access_token response = requests.post((self.database_base_url + self.version + self.id + str(id) + url_params), json.dumps(json_object)) return json.loads(response.text) def put_request(self, version, update_object): self.version = version # THIS WILL OVERRIDE ENTIRE JSON IN DB! url_params = ".json?access_token=" + self.access_token # print(self.database_base_url + self.version + url_params) response = requests.put( (self.database_base_url + self.version + url_params), json.dumps(update_object)) return json.loads(response.text)
def testDataServerDataServing(): # Get the url of the service. service_url = os.environ.get('SERVICE_URL').strip('"') print('SERVICE_URL={}'.format(service_url)) # Get service account credentials to make request to private URL creds = service_account.IDTokenCredentials.from_service_account_file( os.environ.get('PATH_TO_SA_CREDS'), target_audience=service_url) authed_session = AuthorizedSession(creds) resp = authed_session.get(service_url) assert resp.ok assert b'Running data server.' in resp.content
class SheriffConfigClient(object): """Wrapping of sheriff-config HTTP API.""" def __init__(self): """Make the Cloud Endpoints request from this handler.""" credentials, _ = google.auth.default( scopes=['https://www.googleapis.com/auth/userinfo.email']) jwt_credentials = jwt.Credentials.from_signing_credentials( credentials, 'sheriff-config-dot-chromeperf.appspot.com') self._session = AuthorizedSession(jwt_credentials) @staticmethod def _SubscriptionToSheriff(subscription): sheriff = Sheriff( key=ndb.Key('Sheriff', subscription.name), internal_only=(subscription.visibility != sheriff_pb2.Subscription.PUBLIC), # Sheriff model only support glob patterns patterns=[p.glob for p in subscription.patterns if p.glob], labels=(list(subscription.bug_labels) + [ 'Component-' + c.replace('>', '-') for c in subscription.bug_components ]), ) if subscription.rotation_url: sheriff.url = subscription.rotation_url if subscription.notification_email: sheriff.email = subscription.notification_email return sheriff def Match(self, path): response = self._session.post( 'https://sheriff-config-dot-chromeperf.appspot.com/subscriptions/match', json={'path': path}) if not response.ok: return None, '%r\n%s' % (response, response.text) match = json_format.Parse(response.text, sheriff_config_pb2.MatchResponse()) return [ self._SubscriptionToSheriff(s.subscription) for s in match.subscriptions ], None def Update(self): response = self._session.get( 'https://sheriff-config-dot-chromeperf.appspot.com/configs/update') if response.ok: return True, None return False, '%r\n%s' % (response, response.text)
def _update_token(self, request): """Updates credentials with a new access_token representing the downscoped credentials. Args: request (google.auth.transport.requests.Request): Request object to use for refreshing credentials. """ # Refresh our source credentials. self._source_credentials.refresh(request) request = google.auth.transport.requests.Request() ac = AnonymousCredentials() authed_session = AuthorizedSession(credentials=ac) body = { "grant_type": 'urn:ietf:params:oauth:grant-type:token-exchange', "subject_token_type": 'urn:ietf:params:oauth:token-type:access_token', "requested_token_type": 'urn:ietf:params:oauth:token-type:access_token', "subject_token": self._source_credentials.token, "options": json.dumps(self._downscoped_options) } resp = authed_session.post(_STS_ENDPOINT, data=body) if resp.status_code != http_client.OK: raise exceptions.RefreshError(_REFRESH_ERROR) data = resp.json() self.token = data['access_token'] if 'expires_in' in data: self.expiry = datetime.now() + \ timedelta(seconds=int(data['expires_in'])) else: authed_session = AuthorizedSession(credentials=ac) payload = {'access_token': self._source_credentials.token} token_response = authed_session.get(_TOKEN_INFO_ENDPOINT, params=payload) if token_response.status_code != http_client.OK: raise exceptions.RefreshError(_TOKEN_INFO_ERROR) tokeninfo_data = token_response.json() self.expiry = datetime.now() + \ timedelta(seconds=int(tokeninfo_data['expires_in']))
def messages(request): q = request.POST['q'] # Load credential user = request.session['user'] credentials = google_credentials[user] authed_session = AuthorizedSession(credentials) # Call Gmail API parameter = {'q': q} response = authed_session.get( 'https://www.googleapis.com/gmail/v1/users/me/messages', params=parameter) messages = json.loads(response.text)['messages'] context = {'messages': messages, 'q': q} return render(request, 'gmail/messages.html', context)
def download_with_order( request: Request, order_id: str) -> Union[Response, StreamingHttpResponse]: # Validate that the order exists and is a valid UUIDv4 # We also check if the grant query argument exists if not uuid4_is_valid(order_id): return Response(status=404) grant = request.query_params.get("grant", None) if grant is None: return Response( { "error": "MISSING_ARGS", "msg": "'grant' is required as a query argument" }, 400) order = get_object_or_404(Order.objects.all(), pk=order_id) # Validate that the grant is valid signer = TimestampSigner(settings.SECRET_KEY) try: order_pk = signer.unsign(grant, max_age=settings.DOWNLOAD_EXPIRY_TIME) order_pk = order_pk.decode() except SignatureExpired or BadSignature or BadTimeSignature: return Response({ "error": "BAD_SIGNATURE", "msg": "Invalid Signature" }, 400) if not order_pk == order_id: return Response(status=404) download_url = order.product.download.file.name download_name = order.product.download.file.file.name # Authenticate to Cloud Storage and prepare a streaming request, # for the users file download credentials = service_account.Credentials.from_service_account_file( settings.GCP_KEYFILE_PATH / "CLOUD_STORAGE_OPERATOR.json", scopes=["https://www.googleapis.com/auth/devstorage.read_only"]) sess = AuthorizedSession(credentials) req = sess.get(download_url, stream=True) res = StreamingHttpResponse(streaming_content=req) # Adjust the headers for the download and send the response res["Content-Disposition"] = f"attachement; filename={download_name}" res["Cache-Control"] = "no-store" patch_cache_control(res, max_age=0, public=True) return res
def download_file(): credentials = Credentials.from_service_account_file(config.CREDENTIALS_FILE).with_scopes([config.SCOPE]) session = AuthorizedSession(credentials) response = session.get(config.DOCUMENT_URL) raw_text = response.content.decode(response.encoding) doc_body = ''.join(re.findall(r'"s":"(.*?)"', raw_text)).strip() doc_body = doc_body.encode().decode('unicode_escape') doc_content = "".join(filter(lambda char: char in PRINTABLE_CHARS, doc_body)) if not doc_content: raise ConnectionError(raw_text) time_str = datetime.now().strftime(config.DATE_FORMAT) new_file_name = config.TMP_DIR / f'{time_str}.txt' with open(new_file_name, 'w+') as file: file.write(doc_content) return new_file_name
def _download_image(session: AuthorizedSession, queue, size): while True: m = queue.get() if m.get("mimeType", "").startswith("image/"): outfile = f"thumbs/{m['id']}.{size}" if not os.path.exists(outfile) or os.stat(outfile).st_size == 0: try: with open(outfile, "wb") as out: r = session.get(m["baseUrl"] + f"=w{size}-h{size}-c") r.raise_for_status() out.write(r.content) except HTTPError as e: print(e) os.unlink(outfile) else: print(f"Skipping {m} since {m.get('mimeType','')} is not 'image/'") queue.task_done()
def download_instance(dicom_web_url, dicom_store_id, study_uid, series_uid, instance_uid, credentials): """Downloads a DICOM instance and saves it under the current folder.""" instance_url = posixpath.join(dicom_web_url, 'studies', study_uid, 'series', series_uid, 'instances', instance_uid) authed_session = AuthorizedSession(credentials) response = authed_session.get( instance_url, headers={'Accept': 'application/dicom; transfer-syntax=*'}) file_path = posixpath.join(dicom_store_id, study_uid, series_uid, instance_uid) filename = '%s.dcm' % file_path if not os.path.exists(filename): os.makedirs(os.path.dirname(filename)) with open(filename, 'wb') as f: f.write(response.content)
def get_match_image(): content = request.get_json() error_or_none = check_key_in_json(content, ['name', 'key', 'endpoint']) if error_or_none: return error_or_none image_full_name = content['name'] endpoint = content['endpoint'] if not endpoint.startswith('https://'): return _Error('Invalid api endpoint') product_id = image_full_name.split('/')[5] url = os.path.join(endpoint, image_full_name) (json_key, success) = validate_json_key(content['key']) if not success: return json_key try: credentials = service_account.Credentials.from_service_account_info( json_key) scoped_credentials = credentials.with_scopes(_DEFAULT_SCOPES) authed_session = AuthorizedSession(scoped_credentials) response = authed_session.get(url=url).json() except Exception as e: return _Error('%r: %r' % (image_full_name, e)) if 'uri' not in response: return _Error('Image not found: %r' % (response)) try: gcs_client = storage.Client(project=json_key['project_id'], credentials=credentials) bucket_name, path = parse_gcs_uri(response['uri']) blob = gcs_client.bucket(bucket_name).blob(path) except Exception as e: return _Error(str(e)) res = json.dumps({ 'success': True, 'image_url': blob.generate_signed_url(_DEFAULT_URL_EXPIRATION), 'label': product_id, }) return res
def get_gcals(): '''Get Google Calendar API event Responces''' creds = service_account.Credentials.from_service_account_file( CREDENTIAL_FILE, scopes=SCOPES).with_subject(SUBJECT) authed_session = AuthorizedSession(creds) time_min = datetime.utcnow() + relativedelta(months=-RANGE_OFFSET_MONTHS) time_max = datetime.utcnow() + relativedelta(months=+RANGE_OFFSET_MONTHS) response = authed_session.get( API_ENDPOINT.format(calendar_id=CALENDAR_ID, time_min=time_min.strftime(APIQUERY_DT_FMT), time_max=time_max.strftime(APIQUERY_DT_FMT), max_results=APIQUERY_MAX_RESULTS)).json() timezone = response.get('timeZone') events = response.get('items', []) if events: return generate_ics(events, timezone) else: return None
def main(token_file: str, credential_file: str) -> None: with open(token_file, "r") as token: d = json.load(token) creds = Credentials(token=None, **d, id_token=None) # creds = creds.with_quota_project("") s = AuthorizedSession(creds) # res = s.get("https://www.googleapis.com/discovery/v1/apis/sheets/v4/rest") # print(res, json.dumps(res.json(), indent=2)) sheet_id = SAMPLE_SPREADSHEET_ID range_name = SAMPLE_RANGE_NAME res = s.get( f"https://sheets.googleapis.com/v4/spreadsheets/{sheet_id}/values/{range_name}?alt=json", hooks={ "response": lambda *args, **kwargs: print(args, kwargs, file=sys.stderr) }, ) print(json.dumps(res.json(), indent=2, ensure_ascii=False))
def is_api_enabled(session: AuthorizedSession, project_id: str, api: str) -> bool: """Checks if Cloud API is enabled for given project. Args: session: The authorised session. project_id: GCP project id. api: The API to be checked. Returns: True: If the API is enabled. False: If the API is not enabled. """ get_service_url = '{}/{}/services/{}'.format(_SERVICE_URL, project_id, api) response = session.get(get_service_url) service = json.loads(response.content) if service['state'] == 'ENABLED': return True return False
def runTests(): # Get the url of the service. service_url = os.environ.get('SERVICE_URL').strip('"') print('SERVICE_URL={}'.format(service_url)) # Get service account credentials to make request to private URL creds = service_account.IDTokenCredentials.from_service_account_file( os.environ.get('PATH_TO_SA_CREDS'), target_audience=service_url) authed_session = AuthorizedSession(creds) resp = authed_session.get(service_url) if not resp.ok: print('Request failed with code {}'.format(resp.status_code)) exit(1) if b'Running data server.' not in resp.content: print('Unexpected response. Expected: "Running data server."' 'Received: {}'.format(resp.content)) exit(1) print('Test completed with no errors')
def get_dupes_info(session: AuthorizedSession, dupes: List[Set[str]]) -> List[List[Dict[str, Any]]]: out = [] for dupe in dupes: cur = [] for mid in dupe: info = session.get( f"https://photoslibrary.googleapis.com/v1/mediaItems/{mid}" ).json() if info.get("error", {}).get("code") == 404: # Delete all versions of that image, in any size for filename in os.listdir("thumbs/"): if filename.startswith(mid): print(f"Deleting {filename}") os.unlink(f"thumbs/{filename}") else: cur.append(info) if len(cur) > 1: out.append(cur) return out
def download_all_instances(dicom_store_id, credentials): """Downloads all DICOM instances in the specified DICOM store.""" # Get a list of all instances. dicom_web_url = posixpath.join(CHC_API_URL, 'projects', PROJECT_ID, 'locations', REGION, 'datasets', DATASET_ID, 'dicomStores', dicom_store_id, 'dicomWeb') qido_url = posixpath.join(dicom_web_url, 'instances') authed_session = AuthorizedSession(credentials) response = authed_session.get(qido_url, params={'limit': '15000'}) if response.status_code != 200: print(response.text) return content = response.json() # DICOM Tag numbers study_instance_uid_tag = '0020000D' series_instance_uid_tag = '0020000E' sop_instance_uid_tag = '00080018' value_key = 'Value' with futures.ThreadPoolExecutor() as executor: future_to_study_uid = {} for instance in content: study_uid = instance[study_instance_uid_tag][value_key][0] series_uid = instance[series_instance_uid_tag][value_key][0] instance_uid = instance[sop_instance_uid_tag][value_key][0] if SLEEP: time.sleep(SLEEP) future = executor.submit(download_instance, dicom_web_url, dicom_store_id, study_uid, series_uid, instance_uid, credentials) future_to_study_uid[future] = study_uid processed_count = 0 for future in futures.as_completed(future_to_study_uid): try: future.result() processed_count += 1 if not processed_count % 100 or processed_count == len( content): print('Processed instance %d out of %d' % (processed_count, len(content))) except Exception as e: print('Failed to download a study. UID: %s \n exception: %s' % (future_to_study_uid[future], e))
def DownloadDiscoveryDocument(api_name, version, path=_DEFAULT_DISCOVERY_PATH, label=None): """Downloads a discovery document for the given api_name and version. This utility assumes that the API for which a discovery document is being retrieved is publicly accessible. However, you may access whitelisted resources for a public API if you are added to its whitelist and specify the associated label. Args: api_name: a str indicating the name of the API for which a discovery document is to be downloaded. version: a str indicating the version number of the API. path: a str indicating the path to which you want to save the downloaded discovery document. label: a str indicating a label to be applied to the discovery service request. This is not applicable when downloading the discovery document of a legacy API. For non-legacy APIs, this may be used as a means of programmatically retrieving a copy of a discovery document containing whitelisted content. Raises: ValueError: if labels are specified for a legacy API, which is incompatible with labels. """ credentials = _GetCredentials() auth_session = AuthorizedSession(credentials) discovery_service = build('discovery', 'v1') discovery_rest_url = None discovery_response = discovery_service.apis().list( name=api_name).execute() if 'items' in discovery_response: for api in discovery_response['items']: if api['version'] == version: discovery_rest_url = api['discoveryRestUrl'] break if discovery_rest_url: is_legacy = _IsLegacy(discovery_rest_url) else: raise ValueError('API with name "%s" and version "%s" was not found.' % (api_name, version)) if all((is_legacy, label)): raise ValueError('The discovery URL associated with the api_name "%s" and ' 'version "%s" is for a legacy API. These are not ' 'compatible with labels.' % (api_name, version)) if label: # Apply the label query parameter if it exists. path_params = '&labels=%s' % label discovery_rest_url += path_params discovery_response = auth_session.get(discovery_rest_url) if discovery_response.status_code == 200: with open(path, 'wb') as handler: handler.write(discovery_response.text) else: raise ValueError('Unable to retrieve discovery document for api name "%s" ' 'and version "%s" via discovery URL: %s' % (api_name, version, discovery_rest_url))