def __init__(self, env):
log.info('GETTING CONFIG FOR: {}'.format(env))
if env == "dev":
log.info('SETTING UP CONFIG FOR: {}'.format(env))
self.env = "dev"
self.last_updated = None
blackfynn_host = "https://api.pennsieve.net"
blackfynn_api_token = os.environ.get("PENNSIEVE_API_TOKEN")
blackfynn_api_secret = os.environ.get("PENNSIEVE_API_SECRET")
self.bf = Pennsieve(api_token=blackfynn_api_token,
api_secret=blackfynn_api_secret,
host=blackfynn_host)
elif env == "prod":
log.info('SETTING UP CONFIG FOR: {}'.format(env))
self.env = "prod"
self.last_updated = None
blackfynn_host = "https://api.pennsieve.io"
blackfynn_api_token = os.environ.get("PENNSIEVE_API_TOKEN")
blackfynn_api_secret = os.environ.get("PENNSIEVE_API_SECRET")
self.bf = Pennsieve(api_token=blackfynn_api_token,
api_secret=blackfynn_api_secret,
host=blackfynn_host)
else:
raise (Exception('Incorrect input argument'))
def generate_data(
organization_id: int,
input_file: str,
environment: str,
jumpbox: Optional[str] = "non-prod",
dry_run: bool = True,
):
assert input_file is not None and os.path.exists(input_file)
log.info(f"Loading input: {input_file}")
try:
from pennsieve import Pennsieve
except: # noqa: E722
raise ValueError("Run `pip install pennsieve` first")
settings = SSMParameters(environment)
bf = Pennsieve("Pennsieve_PROD")
ds_name = f"Test Dataset GENERATED-{''.join([random.choice(string.ascii_uppercase) for _ in range(10)])}"
ds = bf.create_dataset(ds_name, "Test dataset (generated)")
log.info(f'Creating dataset "{ds_name}" ({ds.id}, {ds.int_id}')
with with_databases(settings, jumpbox) as (neo4j_db, _):
partitioned_db = PartitionedDatabase(
neo4j_db,
organization_id=OrganizationId(organization_id),
dataset_id=DatasetId(ds.int_id),
user_id=UserNodeId("dataset-generate-migration"),
dataset_node_id=DatasetNodeId(ds.id),
)
generator.load(partitioned_db, input_file)
def get_test_client(profile=None,
api_token=None,
api_secret=None,
**overrides):
""" Utility function to get a Pennsieve client object """
ps = Pennsieve(profile=profile,
api_token=api_token,
api_secret=api_secret,
**overrides)
assert ps.context is not None
orgs = ps.organizations()
assert len(orgs) > 0
# explicitly set context to Blackfyn org
assert ps.context is not None
return ps
class BFWorker(object):
def __init__(self, id):
self.bf = Pennsieve(api_token=Config.PENNSIEVE_API_TOKEN,
api_secret=Config.PENNSIEVE_API_SECRET)
def getCollectionAndMetaFromPackageId(self, packageId):
pkg = self.bf.get(packageId)
if type(pkg) is pennsieve.DataPackage:
colId = pkg.parent
col = self.bf.get(colId)
items = col.items
for item in items:
if packageId == item.id:
return [colId, item.name]
return None
def getURLFromCollectionIdAndFileName(self, collectionId, fileName):
col = self.bf.get(collectionId)
if type(col) is pennsieve.Collection:
items = col.items
for item in items:
if fileName == item.name:
pkg = item
try:
bfFile = pkg.files[0]
url = bfFile.url
return url
except:
return None
return None
def getUrlfromPackageId(self, packageId):
pId = packageId
if ('N:' not in packageId):
pId = 'N:' + packageId
pk = self.bf.get(pId)
return pk.files[0].url
def getImagefromPackageId(self, packageId):
pId = packageId
if ('N:' not in packageId):
pId = 'N:' + packageId
pk = self.bf.get(pId)
# resp = requests.get(pk.files[0].url)
return pk.files[0].url if pk is not None else ''
def test_client_global_headers():
global_headers = {
"X-Custom-Header1": "Custom Value",
"X-Custom-Header2": "Custom Value2",
}
ps = Pennsieve(headers=global_headers)
assert ps.settings.headers == global_headers
for header, header_value in global_headers.items():
assert header in ps._api._session.headers
assert ps._api._session.headers[header] == header_value
def connect_to_pennsieve():
global ps
try:
ps = Pennsieve(api_token=Config.PENNSIEVE_API_TOKEN,
api_secret=Config.PENNSIEVE_API_SECRET,
env_override=False,
host=Config.PENNSIEVE_API_HOST)
except requests.exceptions.HTTPError as err:
logging.error("Unable to connect to Pennsieve host")
logging.error(err)
except PSUnauthorizedException as err:
logging.error("Unable to authorise with Pennsieve Api")
logging.error(err)
except Exception as err:
logging.error("Unknown Error")
logging.error(err)
def main():
args = docopt(__doc__, version=pennsieve.__version__)
# Test for these two commands first as they
# do not require a Pennsieve client or reading the settings file
if args["help"]:
print(__doc__.strip("\n"))
return
elif args["version"]:
print(pennsieve.__version__)
return
settings = Settings(args["--profile"])
if not os.path.exists(settings.config_file):
setup_assistant(settings)
elif args["create"]:
create_profile(settings, args["<name>"])
elif args["show"]:
show_profile(settings, args["<name>"])
elif args["delete"]:
delete_profile(settings, args["<name>"], args["--force"])
elif args["list"]:
list_profiles(settings, args["--contents"])
elif args["set-default"]:
set_default(settings, args["<name>"])
elif args["unset-default"]:
unset_default(settings, args["--force"])
elif args["set"]:
set_key(
settings, args["<key>"], args["<value>"], args["--profile"], args["--force"]
)
elif args["unset"]:
unset_key(settings, args["<key>"], args["--profile"], args["--force"])
elif args["keys"]:
list_keys(settings, args["--profile"])
elif args["status"]:
ps = Pennsieve(args["--profile"])
show_status(ps)
else:
invalid_usage()
with io.open(settings.config_file, "w") as configfile:
settings.config.write(configfile)
def _create_pennsieve_client(api_key: str, api_secret: str) -> Pennsieve:
logger.debug("creating new client for key/secret [%s/%s]", api_key,
api_secret)
return Pennsieve(api_token=api_key, api_secret=api_secret)
def __init__(self, id):
self.bf = Pennsieve(api_token=Config.PENNSIEVE_API_TOKEN,
api_secret=Config.PENNSIEVE_API_SECRET)
def test_exception_raise():
ps = Pennsieve()
with pytest.raises(Exception) as excinfo:
ps._api._call("get", "/datasets/plop")
assert "plop not found" in str(excinfo.value)
# CREATED: 09/06/2017 16:54:33 EDT
# REVISION: 2021-04-22
#===============================================================================
import getopt
import sys
from datetime import datetime
from pennsieve import Pennsieve
from pennsieve.models import BaseCollection
from pennsieve.models import Collection
from termcolor import colored
__version__ = "0.6.2"
bf = Pennsieve() # use 'default' profile
# extensions unknown to Pennsieve
extensions = ['ome.tiff', 'fastq.gz', 'bigWig', 'bw', 'metadata']
def flush_print(message="", with_time=False):
if with_time:
message = f"{datetime.now()}: {message}"
print(message, flush=True)
def get_datasets():
"""
Return list of tuples with short and long dataset names and
# REQUIREMENTS: python2, pennsieve python library, pennsieve key # UPDATES: 180215: Added -f option # AUTHOR: Pete Schmitt (discovery), [email protected] # COMPANY: University of Pennsylvania # VERSION: 0.2.0 # CREATED: Wed Feb 14 13:27:32 EST 2018 # REVISION: Thu Feb 15 11:49:51 EST 2018 #=============================================================================== from pennsieve import Pennsieve from pennsieve.models import BaseCollection from pennsieve.models import Collection import sys import getopt import os bf = Pennsieve() # use 'default' profile ############################################################################### def syntax(): SYNTAX = "\nbfmove -d <dataset> \n" SYNTAX += " --all (loop on all HPAP datasets)\n" SYNTAX += " -f <file containing datasets>\n" SYNTAX += " -S <source path>\n" SYNTAX += " -D <destination path> (MUST be directory)\n\n" SYNTAX += " -h (help)\n" SYNTAX += " -l (list datasets)\n\n" SYNTAX += "Note: -d, -f and --all are mutually exlusive\n" return SYNTAX ############################################################################### def printf(format, *args): """ works just like the C/C++ printf function """ sys.stdout.write(format % args)
element._check_exists()
if indent != 0:
printf("Creating new Collection:%s%s\n", " " * (indent + 4),
element.name)
for item in element.items:
if isinstance(item, BaseCollection):
c = Collection(item.name)
newdset.add(c)
create_duplicate(item, c, indent=indent + 4)
###############################################################################
# program starts HERE
bf = Pennsieve() # use 'default' profile
if len(sys.argv) < 2:
printf("%s\n", syntax())
sys.exit()
argv = sys.argv[1:]
try:
opts, args = getopt.getopt(argv, "hld:n:")
except getopt.GetoptError:
printf("%s\n", syntax())
sys.exit()
dsets, dsdict = get_datasets()
for opt, arg in opts:
def _create_pennsieve_client(api_key: str, api_secret: str) -> Pennsieve:
return Pennsieve(api_token=api_key, api_secret=api_secret)
for d in ds:
if 'HPAP-' in d.name:
tmp = d.name.split()
dsdict[str(tmp[0])] = str(d.name)
dsets.append((str(tmp[0]), str(d.name), d))
else:
dsdict[str(d.name)] = str(d.name)
dsets.append((str(d.name), str(d.name), d))
dsets.sort()
return dsets, dsdict
###############################################################################
# program starts HERE
bf = Pennsieve() # use 'default' profile
ALL = False
FILE = False
DATASET = False
FORCE = False
if len(sys.argv) < 2:
printf("%s\n", syntax())
sys.exit()
argv = sys.argv[1:]
#################
# resolve options
#################
try:
opts, args = getopt.getopt(argv, "hlp:d:f:", ['force', 'all'])
except getopt.GetoptError:
###############################################################################
def extension_remover(file_name):
for ext in extensions:
if "." + ext in file_name:
return file_name.replace("." + ext, "")
if file_name.count(".") == 1:
return file_name.rsplit(".", 1)[0]
return file_name
###############################################################################
# program starts HERE
bf = Pennsieve() # use 'default' profile
outdir = './'
NODATA = MIRROR = EXCEPT = DATASET = CATEGORY = False
REFRESH = True # default value
QUICKSYNC = False
HPAPURL = 'https://hpap.pmacs.upenn.edu/services/refreshDirectories'
if len(sys.argv) < 2:
printf("%s\n", syntax())
sys.exit()
argv = sys.argv[1:]
try:
opts, args = getopt.getopt(argv, "hlp:d:c:e:q",
['mirror', 'nodata', 'norefresh', 'refresh'])
if not find(d, locpaths, CASE):
printf("%s\n",d)
printf("\n")
printf("Data found in %s that are NOT in %s:\n", 'LOCAL', dset.name)
for c in locpaths:
if not find(c, dspaths, CASE):
printf("%s\n",c)
###############################################################################
# program starts HERE
ALL = False
FILE = False
CASE = True
PATH = None
bf = Pennsieve() # use 'default' profile
if len(sys.argv) < 2:
printf("%s\n", syntax())
sys.exit()
argv = sys.argv[1:]
try:
opts, args = getopt.getopt(argv, "hilp:d:c:",
['all', 'data'])
except getopt.GetoptError:
printf("%s\n", syntax())
sys.exit()
dsets, dsdict = get_datasets()