def removeAllFiles(self):
data = Util.parseToken(Util.loadToken(self.userId, self.port))
logger.debug("remove files in service {}".format(self.servicename))
found = False
if self.fileStorage:
# todo: implements me
found = True
if self.metadata:
req = requests.delete(
f"{self.portaddress}/metadata/project/{self.getProjectId()}/files",
json=data,
verify=(os.environ.get("VERIFY_SSL", "True") == "True"),
)
if req.status_code < 300:
found = True
if found:
self.reload()
return found
def check_api_key(*args, **kwargs):
g.zenodo = None
try:
req = request.get_json(force=True, cache=True)
except:
req = request.form.to_dict()
try:
service, userId, apiKey = Util.parseUserId(req.get("userId"))
except:
apiKey = Util.loadToken(req.get("userId"), "port-openscienceframework").access_token
logger.debug("req data: {}".format(req))
if apiKey is None:
logger.error("apiKey or userId not found.")
abort(401)
logger.debug("found apiKey")
g.osf = OSF(
token=apiKey,
address=os.getenv(
"OPENSCIENCEFRAMEWORK_API_ADDRESS", "https://api.test.osf.io/v2"
),
)
return api_method(*args, **kwargs)
def check_api_key(*args, **kwargs):
g.zenodo = None
try:
req = request.get_json(force=True)
except Exception as e:
logger.error(e, exc_info=True)
req = request.form.to_dict()
logger.debug("got request data: {}".format(req))
try:
service, userId, apiKey = Util.parseUserId(req.get("userId"))
except Exception as e:
logger.error(e, exc_info=True)
apiKey = Util.loadToken(
req.get("userId"), "port-zenodo").access_token
if apiKey is None:
logger.error("apiKey or userId not found.")
abort(401)
logger.debug("found apiKey")
g.zenodo = Zenodo(apiKey, address=current_app.zenodo_address)
return api_method(*args, **kwargs)
def reload(self):
if self.fileStorage:
data = {"filepath": self.getFilepath(), "userId": self.userId}
req = requests.get(f"{self.portaddress}/storage/folder",
json=data,
verify=(os.environ.get("VERIFY_SSL",
"True") == "True"))
if req.status_code >= 300:
# for convenience
data.update(
Util.parseToken(Util.loadToken(self.userId, self.port)))
req = requests.get(f"{self.portaddress}/storage/folder",
json=data,
verify=(os.environ.get(
"VERIFY_SSL", "True") == "True"))
if req.status_code >= 300:
return False
json = req.json()
logger.debug("reload fileStorage in Service got: {}".format(json))
self.files = json.get("files")
if self.metadata:
# TODO: metadata ports can also response with files
self.reloadInformations()
def updateMetadataForResearch(self, researchId: int, updateMetadata: dict):
"""
This method changes the metadata in all available ports to the given metadata values in given dict for specified researchId.
"""
# get all ports registered to researchId
allMetadata = []
# get all ports registered to researchId
logger.debug("start update for research method")
research = Research(testing=self.testing, researchId=researchId)
ports = research.getPortsWithProjectId()
logger.debug("research ports: {}".format(ports))
# FIXME: parallize me
for (port, projectId) in ports:
if projectId is None:
continue
portname = port["port"]
if not portname.startswith("port-"):
portname = "port-{}".format(portname)
logger.debug("work on port {}".format(port))
data = Util.parseToken(Util.loadToken(research.userId, portname))
data["metadata"] = updateMetadata
metadata = self.updateMetadataForResearchFromPort(
portname, projectId, data)
d = {"port": portname, "metadata": metadata}
allMetadata.append(d)
return allMetadata
def bootstrap(name="MicroService", testing=False, *args, **kwargs):
list_openapi = Util.load_oai("central-service_token-storage.yml")
app = App(name, *args, **kwargs)
opts = {
"use_in_memory_on_failure": (os.getenv("use_inmemory_as_fallover",
"False") == "True")
}
if testing:
app.app.config.update({"TESTING": True})
opts = {"use_in_memory_on_failure": True}
CommonUtil.monkeypatch(app=app.app)
ServerUtil.storage = Storage(**opts)
for oai in list_openapi:
app.add_api(
oai,
resolver=MultipleResourceResolver(
"api", collection_endpoint_name="index"),
validate_responses=True,
)
return app
def bootstrap(name="MicroService", testing=False, *args, **kwargs):
list_openapi = Util.load_oai("central-service_research-manager.yml")
app = App(name, *args, **kwargs)
RDSUtil.monkeypatch("getDict", app=app.app)
opts = {
"use_in_memory_on_failure": (os.getenv("use_inmemory_as_fallover",
"False") == "True")
}
if testing:
app.app.config.update({"TESTING": True})
opts = {"use_in_memory_on_failure": True}
Singleton.ProjectService = ProjectService(**opts)
for oai in list_openapi:
app.add_api(
oai,
resolver=MultipleResourceResolver(
"api", collection_endpoint_name="index"),
validate_responses=True,
)
return app
def removeFile(self, file_id):
found = False
file = self.files[file_id]
data = Util.parseToken(Util.loadToken(self.userId, self.port))
if self.fileStorage:
data["filepath"] = "{}/{}".format(self.getFilepath(), file)
req = requests.delete(
f"{self.portaddress}/storage/file",
json=data,
verify=(os.environ.get("VERIFY_SSL", "True") == "True"),
)
if req.status_code < 300:
found = True
if self.metadata:
req = requests.delete(
f"{self.portaddress}/metadata/project/{self.getProjectId()}/files/{file_id}",
json=data,
verify=(os.environ.get("VERIFY_SSL", "True") == "True"),
)
if req.status_code < 300:
found = True
if found:
del self.files[file_id]
return True
return False
def bootstrap(name="MicroService", *args, **kwargs):
import os
from connexion_plus import App, MultipleResourceResolver, Util
from lib.TokenService import TokenService
list_openapi = Util.load_oai("use-case_port.yml")
if "testing" in kwargs:
ServerUtil.tokenService = TokenService(kwargs["testing"])
del kwargs["testing"]
else:
ServerUtil.tokenService = TokenService()
app = App(name, *args, **kwargs)
CommonUtil.monkeypatch(app=app.app)
for oai in list_openapi:
app.add_api(
oai,
resolver=MultipleResourceResolver(
"api", collection_endpoint_name="index"),
validate_responses=True,
)
return app
def triggerPassiveMode(self, folder, servicename):
"""Trigger passive upload for given folder
Args:
folder (str): Set the folder.
servicename (str): Given port, where files should be taken from.
Returns:
bool: Return True, if the trigger was successfully, otherwise False.
"""
data = {
"folder": folder,
"service": servicename,
"username": self.userId
}
data.update(Util.parseToken(Util.loadToken(self.userId, self.port)))
logger.debug("start passive mode with data {} in service {}".format(
data, self.getJSON()))
if self.metadata:
response_to = requests.post(
f"{self.portaddress}/metadata/project/{self.getProjectId()}/files",
data=data,
verify=(os.environ.get("VERIFY_SSL", "True") == "True"),
)
if response_to.status_code >= 300:
logger.error(response_to.json())
return False
pass
return True
def addFile(self, filename, fileContent):
"""Adds given file with filename to this service.
Args:
filename (str): Set the filename of this file.
fileContent (io.BytesIO): Set the content of this file.
Returns:
bool: Return True, if the file was uploaded successfully, otherwise False.
"""
data = Util.parseToken(Util.loadToken(self.userId, self.port))
files = {"file": (filename, fileContent.getvalue())}
data["filename"] = filename
logger.debug("add file {} with data {} in service {}".format(
files, data, self.getJSON()))
if self.metadata:
response_to = requests.post(
f"{self.portaddress}/metadata/project/{self.getProjectId()}/files",
files=files,
data=data,
verify=(os.environ.get("VERIFY_SSL", "True") == "True"),
)
if response_to.status_code >= 300:
logger.error(response_to.json())
return False
if self.fileStorage:
# TODO: fileStorage can also add files
return False
return True
def post(project_id):
# trigger upload on datasafe
req = request.get_json(force=True, silent=True, cache=True)
if req is None:
req = request.form.to_dict()
logger.debug("got request body: {}", req)
try:
service, userId, password = Util.parseUserId(req["userId"])
if service != "port-datasafe":
logger.debug("got wrong service token")
raise ValueError
except ValueError:
token = Util.loadToken(req["userId"], "port-datasafe")
userId = token.user.username
password = token.access_token
owncloud_token = Util.loadToken(req["username"], "port-owncloud")
data = Util.parseToken(owncloud_token)
data.update({
"filepath": "{}/ro-crate-metadata.json".format(req["folder"])
})
logger.debug("send data: {}".format(data))
metadata = json.loads(
BytesIO(
requests.get(
"http://circle1-{}/storage/file".format("port-owncloud"),
json=data,
verify=(os.environ.get("VERIFY_SSL", "True") == "True"),
).content
)
.read()
.decode("UTF-8")
)
logger.debug("got metadata: {}".format(metadata))
doc = ROParser(metadata)
logger.debug("parsed metadata: {}".format(doc))
datasafe = Datasafe(
userId,
owncloud_token.access_token,
doc.getElement(doc.rootIdentifier, expand=True, clean=True),
req["folder"],
os.getenv("DATASAFE_PUBLICKEY"),
os.getenv("DATASAFE_PRIVATEKEY")
)
logger.debug("Trigger file upload")
success = datasafe.triggerUploadForProject()
logger.debug(f"Finished trigger, result was: {success}")
return jsonify({"success": success}), 200 if success else 500
def from_dict(cls, tokenDict: dict):
"""
Returns a token object from a dict.
"""
from RDS import Util
return cls(
Util.getUserObject(tokenDict["user"]),
Util.getServiceObject(tokenDict["service"]),
tokenDict["access_token"],
)
def test_init_objects(self):
self.assertEqual(Util.getServiceObject(json.dumps(self.oauthservice1)),
self.oauthservice1)
svc1 = LoginService("MusterService", ["fileStorage"])
self.assertEqual(Util.getServiceObject(json.dumps(svc1)), svc1)
self.assertNotEqual(
Util.getServiceObject(json.dumps(svc1)).__class__,
self.oauthservice1.__class__)
self.assertEqual(Util.getUserObject(json.dumps(self.user1)),
self.user1)
self.assertEqual(Util.getTokenObject(json.dumps(self.token1)),
self.token1)
def getMetadataForResearch(
self,
userId: str = None,
researchIndex: int = None,
researchId: int = None,
metadataFields=None,
):
"""
This method returns the metadata from all available ports for specified researchId.
"""
allMetadata = []
logger.debug("start get metadata method for research")
research = Research(
testing=self.testing,
userId=userId,
researchIndex=researchIndex,
researchId=researchId,
)
ports = research.getPortsWithProjectId()
logger.debug(f"got ports {ports}")
# FIXME: parallize me
for port, projectId in ports:
# beware, that projectId could also be a string or sth else
if projectId is None:
continue
portname = port["port"]
if not portname.startswith("port-"):
portname = "port-{}".format(portname)
token = Util.loadToken(research.userId, portname)
data = Util.parseToken(token)
data["metadata"] = metadataFields
logger.debug(f"work on port {port} with apiKey {token}")
port = port["port"]
metadata = self.getMetadataForProjectFromPort(
port,
projectId,
apiKeyMetadata=data,
)
d = {"port": port, "metadata": metadata}
allMetadata.append(d)
return allMetadata
def index():
json = request.json
try:
service, userId, apiKey = Util.parseUserId(json.get("userId"))
except:
apiKey = Util.loadToken(json.get("userId"),
"port-owncloud").access_token
filepath = json.get("filepath")
logger.debug(f"userid {userId}")
files = OwncloudUser(userId, apiKey).getFolder(filepath)
return jsonify({"files": files})
def setUp(self):
Util.monkeypatch()
self.empty_storage = Storage(**get_opts())
self.user1 = User("Max Mustermann")
self.user2 = User("Mimi Mimikri")
self.service1 = LoginService(servicename="MusterService",
implements=["metadata"])
self.service2 = LoginService(servicename="FahrService",
implements=["metadata"])
self.oauthservice1 = OAuth2Service(
servicename="BetonService",
implements=["metadata"],
authorize_url="http://localhost/oauth/authorize",
refresh_url="http://localhost/oauth/token",
client_id="MNO",
client_secret="UVW",
)
self.oauthservice2 = OAuth2Service(
servicename="FlugService",
implements=["metadata"],
authorize_url="http://localhost21/oauth/authorize",
refresh_url="http://localhost21/oauth/token",
client_id="XCA",
client_secret="BCXY",
)
self.empty_storage.addService(self.service1)
self.empty_storage.addService(self.oauthservice1)
self.empty_storage.addService(self.oauthservice2)
self.token1 = Token(self.user1, self.service1, "ABC")
self.token_like_token1 = Token(self.user1, self.service1, "DEF")
self.token2 = Token(self.user1, self.oauthservice1, "XYZ")
self.token3 = Token(self.user2, self.service2, "XASD")
self.token4 = Token(self.user2, self.service1, "IOAJSD")
self.oauthtoken1 = OAuth2Token(self.user1, self.oauthservice1,
"ABC", "X_ABC")
self.oauthtoken_like_token1 = OAuth2Token(self.user1,
self.oauthservice1,
"ABC", "X_DEF")
self.oauthtoken2 = OAuth2Token(self.user1, self.oauthservice1,
"XYZ", "X_XYZ")
self.oauthtoken3 = OAuth2Token(self.user1, self.oauthservice2,
"XYZ", "X_XYZ")
def bootstrap(name="MicroService", *args, **kwargs):
list_openapi = Util.load_oai("use-case_metadata.yml")
app = App(name, *args, **kwargs)
RDSUtil.monkeypatch(app=app)
for oai in list_openapi:
app.add_api(
oai,
resolver=MultipleResourceResolver(
"api", collection_endpoint_name="index"),
validate_responses=True,
)
return app
def refreshService(self, service: Union[str, BaseService]) -> bool:
try:
servicename = service.servicename
except:
servicename = service
response = requests.get(
f"{self.address}/service/{servicename}",
verify=(os.environ.get("VERIFY_SSL", "True") == "True"),
)
if response.status_code != 200:
try:
raise ServiceNotFoundError(service)
except:
raise ServiceNotFoundError(
BaseService(servicename=servicename,
implements=["metadata"]))
svc = Util.getServiceObject(response.json())
if not svc in self._services:
self._services.append(svc)
return svc
def getTokenForServiceFromUser(self, service: BaseService,
user: User) -> Token:
"""
Returns the token from type Token (struct: servicename: str, access_token: str) for given service from given user.
Raise ServiceNotExistsError, if no token for service was found.
"""
response = requests.get(
f"{self.address}/user/{user.username}/token/{service.servicename}",
verify=(os.environ.get("VERIFY_SSL", "True") == "True"),
)
data = response.json()
while type(data) is not dict:
data = json.loads(data)
if response.status_code != 200:
if "error" in data:
if data["error"] == "TokenNotExistsError":
raise TokenNotFoundError(Token(user, service, "NOT_USED"))
if data["error"] == "UserNotExistsError":
raise UserNotFoundError(user)
if data["error"] == "ServiceNotExistsError":
raise ServiceNotFoundError(service)
raise Exception(data)
token = Util.getTokenObject(data)
if isinstance(token.service, OAuth2Service):
token.service._client_secret = ""
if isinstance(token, OAuth2Token):
token._refresh_token = ""
return token
def createProjectForUserInService(self, user: User,
service: BaseService) -> int:
"""
Create a project in service, which the token is for.
Returns the id for the new created project. If something went wrong, raise an ProjectNotCreated
"""
token = self.getTokenForServiceFromUser(service, user)
data = Util.parseToken(token)
port = get_port_string(service.servicename)
if self.address.startswith("http://localhost"):
port = self.address
req = requests.post(
"{}/metadata/project".format(port),
json=data,
verify=(os.environ.get("VERIFY_SSL", "True") == "True"),
)
if req.status_code < 300:
project = req.json()
return project.get("projectId"), project
raise ProjectNotCreatedError(service)
def getAllServicesForUser(self, user: User) -> list:
"""
Returns a `list` for all services which the user has registered a token for.
"""
response = requests.get(
f"{self.address}/user/{user.username}/token",
verify=(os.environ.get("VERIFY_SSL", "True") == "True"),
)
data = response.json()
# TODO: adjust to oai spec
services = []
try:
for index, l in enumerate(data["list"]):
token = Util.getTokenObject(l)
services.append({
"id":
index,
"servicename":
token.servicename,
"access_token":
token.access_token,
"projects":
self.getProjectsForToken(token),
"implements":
token._service.implements,
"informations":
self.getInformations(token.service)
})
except Exception as e:
logger.error(e)
raise UserNotFoundError(user)
return services
def __init__(self, rc=None, use_in_memory_on_failure=True):
# format: {user: [<type project>]}
try:
from redis_pubsub_dict import RedisDict
import redis_pubsub_dict
import functools
import json
redis_pubsub_dict.dumps = lambda x: json.dumps(x)
redis_pubsub_dict.loads = lambda x: Util.try_function_on_dict(
[Project.fromJSON])(x)
redis_pubsub_dict.RedisDict.to_json = lambda x: dict(x.items())
redis_pubsub_dict.RedisDict.__eq__ = (
lambda x, other: dict(x.items()) == other)
redis_pubsub_dict.RedisDict.keys = keys
# runs in RDS ecosystem
if rc is None:
logger.debug("No redis client was given. Create one.")
startup_nodes = [{
"host": os.getenv("REDIS_HOST", "localhost"),
"port": os.getenv("REDIS_PORT", "6379"),
}]
try:
logger.debug("first try cluster")
from rediscluster import RedisCluster
rc = RedisCluster(
startup_nodes=startup_nodes,
decode_responses=True,
)
except Exception as e:
logger.error(e)
logger.debug(
"Cluster has an error, try standardalone redis")
from redis import Redis
rc = Redis(
**(startup_nodes[0]),
db=0,
decode_responses=True,
)
rc.info() # provoke an error message
logger.debug("set redis backed dict")
self.projects = RedisDict(rc, "researchmanager_projects")
except Exception as e:
logger.error(e)
logger.info("no redis found.")
if not use_in_memory_on_failure:
logger.info("exit...")
import sys
sys.exit()
logger.info("use in-memory")
self.projects = {}
def post(user_id):
logger.debug(f"get token string: {request.json}.")
from RDS import Token
token = Util.getTokenObject(request.json)
logger.debug(f"parsed token: {token}.")
code = 200
try:
user = utility.storage.getUser(user_id)
except UserNotExistsError as e:
user = User(user_id)
try:
utility.storage.addTokenToUser(token, user)
except UserHasTokenAlreadyError as e:
abort(409, description=str(e))
except UserNotExistsError as e:
# only force adding, if user not exists, otherwise it also overwrites existing tokens.
utility.storage.addTokenToUser(token, user, Force=True)
code = 201
return jsonify({"success": True}), code
def reloadInformations(self):
"""Updates all metadata informations from port.
"""
json = requests.get("{}/service/{}".format(
os.getenv("USE_CASE_SERVICE_PORT_SERVICE",
"{}/port-service".format(self.portaddress)),
self.port)).json()
logger.debug("reload metadata informations: got {}".format(json))
svc = Util.getServiceObject(json["informations"])
self.useZipForFolder = svc.fileTransferArchive == FileTransferArchive.zip
self.fileTransferMode = svc.fileTransferMode
self.fileTransferArchive = svc.fileTransferArchive
if isinstance(svc, OAuth2Service):
self.loginMode = 0
self.credentials = svc.to_dict().get("credentials", {})
else:
self.loginMode = 1
logger.debug("got svc: {}, loginmode: {}".format(
svc.to_dict(), self.loginMode))
def getFile(self, file_id):
from io import BytesIO
file = self.files[file_id]
if self.fileStorage:
# this condition is for ports, which does not comply to the doc for urls
path = "{}/{}".format(self.getFilepath(), file)
if str(file).startswith(self.getFilepath()):
path = file
data = {
"userId": self.userId,
"filepath": path,
}
logger.debug("request data {}".format(data))
response_to = requests.get(
f"{self.portaddress}/storage/file",
json=data,
verify=(os.environ.get("VERIFY_SSL", "True") == "True"),
)
if response_to.status_code >= 300:
data.update(
Util.parseToken(Util.loadToken(self.userId, self.port)))
logger.debug("request data {}".format(data))
response_to = requests.get(
f"{self.portaddress}/storage/file",
json=data,
verify=(os.environ.get("VERIFY_SSL", "True") == "True"),
)
cnt = response_to.content
logger.debug("got content size: {}".format(len(cnt)))
return BytesIO(cnt)
if self.metadata:
# TODO: metadata can respond with files too.
pass
return BytesIO(b"")
def load_service_with_tokens(jsonStr):
d = json.loads(jsonStr)
user = User.from_json(json.dumps(d["data"]))
tokens = []
for t in d["tokens"]:
tokens.append(
Util.try_function_on_dict([OAuth2Token.from_json,
Token.from_json])(json.dumps(t)))
return {"data": user, "tokens": tokens}
def get(self, endpoint):
"""
For convenience in this test suite.
"""
data_result = []
data = self.client.get(endpoint).json
for d in data["list"]:
data_result.append(Util.initialize_object_from_json(json.dumps(d)))
return data_result
def test_initialize_object(self):
self.assertEqual(
Util.initialize_object_from_json(json.dumps(self.token1)),
self.token1)
self.assertEqual(
Util.initialize_object_from_json(json.dumps(self.oauthtoken1)),
self.oauthtoken1,
)
self.assertEqual(
Util.initialize_object_from_json(json.dumps(self.service1)),
self.service1)
self.assertEqual(
Util.initialize_object_from_json(json.dumps(self.oauthservice1)),
self.oauthservice1,
)
self.assertEqual(
Util.initialize_object_from_json(json.dumps(self.user1)),
self.user1)
def post():
user = None
try:
user = Util.getUserObject(request.json)
except:
abort(400, description=f"Request not give a valid user object: {request.json}")
utility.storage.addUser(user)
data = {"success": True}
return jsonify(data)