def pullArtifact(artifact, version, user=None, token=None):
dpl = artifact.deploy
file = "{filename}_v{version}.{ext}".format(
filename=artifact.name,
version=version,
ext=artifact.file.split(".")[1])
url = "{url}/{repo}/{path}/{file}".format(url=dpl.url,
repo=dpl.repo,
path=dpl.path,
file=file)
if (ArtifactoryPath("{url}".format(url=url)).exists()):
if (user == None):
user = input("Please provide a valid Artifactory user: "******"Please provide a valid Artifactory token: ")
print("Pulling {file} from {url}".format(file=file, url=url))
path = ArtifactoryPath(url, auth=(user, token))
with path.open() as fd:
with open(file, "wb") as out:
out.write(fd.read())
else:
print("Artifact Not Found: {url}".format(url=url))
def pushArtifact(artifact, version, user=None, token=None, force=False):
dpl = artifact.deploy
file = "{filename}_v{version}.{ext}".format(
filename=artifact.name,
version=version,
ext=artifact.file.split(".")[1])
url = "{url}/{repo}/{path}/{file}".format(url=dpl.url,
repo=dpl.repo,
path=dpl.path,
file=file)
if (force == False and ArtifactoryPath("{url}".format(url=url)).exists()):
print(
"This artifact version has already been pushed. Please bump the version before pushing (skelebot bump) or force push (-f)."
)
sys.exit(1)
if (user == None):
user = input("Please provide a valid Artifactory user: "******"Please provide a valid Artifactory token: ")
print("Deploying {file} to {url}".format(file=file, url=url))
path = ArtifactoryPath(url, auth=(user, token))
os.rename(artifact.file, file)
try:
path.deploy_file(file)
except:
os.rename(file, artifact.file)
raise
os.rename(file, artifact.file)
def test_listdir(self):
a = self.cls()
# Directory
path = ArtifactoryPath(
"http://artifactory.local/artifactory/libs-release-local")
constructed_url = (
"http://artifactory.local/artifactory/api/storage/libs-release-local"
)
responses.add(
responses.GET,
constructed_url,
status=200,
json=self.dir_stat,
)
children = a.listdir(path)
self.assertEqual(children, [".index", "com"])
# Regular File
path = ArtifactoryPath(
"http://artifactory.local/artifactory/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz"
)
constructed_url = (
"http://artifactory.local/artifactory/api/storage/libs-release-local"
)
responses.add(
responses.GET,
constructed_url,
status=200,
json=self.file_stat,
)
self.assertRaises(OSError, a.listdir, path)
def upload():
path = ArtifactoryPath(
"http://artifactory.calormen.net:8040/artifactory/generic-local/test",
auth=('admin', 'f22-demo'))
path.deploy_file("test.txt")
path = ArtifactoryPath(
"http://artifactory.calormen.net:8040/artifactory/generic-local/test/test_results",
auth=('admin', 'f22-demo'))
path.deploy_file("test-reports/results.xml")
path.deploy_file("test-reports/results_download.xml")
def get_latest_image(self,url):
auth = str(
base64.b64encode(
bytes('%s:%s' % (cicd_user,cicd_pw ), 'utf-8')
),
'ascii'
).strip()
headers = {'Authorization': 'Basic ' + auth}
''' FIND THE LATEST FILE NAME'''
print(url)
req = urllib.request.Request(url, headers=headers)
response = urllib.request.urlopen(req)
html = response.read()
soup = BeautifulSoup(html, features="html.parser")
last_link = soup.find_all('a', href=True)[-1]
latest_file=last_link['href']
filepath = local_dir
os.chdir(filepath)
#file_url = url + latest_file
''' Download the binary file from Jfrog'''
path = ArtifactoryPath(url,auth=(cicd_user, cicd_pw))
path.touch()
for file in path:
print('File =', file)
path = ArtifactoryPath(file, auth=(cicd_user, cicd_pw))
print("file to be downloaded :" ,latest_file)
print("File Path:",file)
with path.open() as des:
with open(latest_file, "wb") as out:
out.write(des.read())
des.close()
print("Extract the tar.gz file and upgrade the AP ")
housing_tgz = tarfile.open(latest_file)
housing_tgz.extractall()
housing_tgz.close()
return "pass"
print("Extract the tar file, and copying the file to Linksys AP directory")
#with open("/Users/syamadevi/Desktop/syama/ea8300/ap_sysupgrade_output.log", "a") as output:
# subprocess.call("scp /Users/syamadevi/Desktop/syama/ea8300/openwrt-ipq40xx-generic-linksys_ea8300-squashfs-sysupgrade.bin [email protected]:/tmp/openwrt-ipq40xx-generic-linksys_ea8300-squashfs-sysupgrade.bin",shell=True, stdout=output,
# stderr=output)
print('SSH to Linksys and upgrade the file')
'''
def __CreateArtifactoryPath(self, path=None):
uri = self.__uri
if not (path is None):
uri += '/' + path
if self.__apiKey is None:
self.__RequestCredentials()
path = ArtifactoryPath(uri,
auth=(self.__username, self.__password),
verify=self.__verifyCertificate)
else:
path = ArtifactoryPath(uri,
apikey=self.__apiKey,
verify=self.__verifyCertificate)
return path
def setUp(self):
self.arti = ArtifactoryPath("http://b.com/artifactory")
self.users_request_url = f"{self.arti.drive}/api/security/users"
self.users = [
{
"name": "user_1",
"uri": "http://b.com/artifactory/api/security/users/user_1",
"realm": "internal",
},
{
"name": "user_2",
"uri": "http://b.com/artifactory/api/security/users/user_2",
"realm": "internal",
},
]
self.user_1 = {"name": "user_1", "email": "*****@*****.**"}
self.user_2 = {"name": "user_2", "email": "*****@*****.**"}
self.groups_request_url = f"{self.arti.drive}/api/security/groups"
self.groups = [
{
"name": "group_1",
"uri": "http://b.com/artifactory/api/security/groups/group_1",
},
{
"name": "group_2",
"uri": "http://b.com/artifactory/api/security/groups/group_2",
},
]
self.group_1 = {
"name": "group_1",
"realm": "internal",
}
self.group_2 = {
"name": "group_2",
"realm": "internal",
}
self.projects_request_url = (
f"{self.arti.drive.rstrip('/artifactory')}/access/api/v1/projects")
self.projects = [
{
"project_key": "project_key_1",
"description": "description_1",
},
{
"project_key": "project_key_2",
"description": "description_2",
},
]
self.project_1 = {
"project_key": "project_key_1",
"description": "description_1",
"admin_privileges": {},
}
self.project_2 = {
"project_key": "project_key_2",
"description": "description_2",
"admin_privileges": {},
}
def artifactory_aql(artifactory_url, username, password, kerberos,
aql_query_dict, verify):
"""
Send AQL to Artifactory and get list of Artifacts
:param artifactory_url:
:param username:
:param password:
:param kerberos: Boolean if kerberos authentication should be used
:param verify: Boolean if SSL certificate should be checked
:param aql_query_dict:
:param max_depth_print:
:param human_readable:
:param all:
:return:
"""
if kerberos:
auth = HTTPKerberosAuth(mutual_authentication=DISABLED,
sanitize_mutual_error_response=False)
else:
if not password:
raise ValueError(
"argument 'password' needs to be set for basic authentication")
auth = (username, password)
aql = ArtifactoryPath(artifactory_url, auth=auth, verify=verify)
logging.debug("AQL query: items.find({})".format(aql_query_dict))
artifacts = aql.aql('items.find', aql_query_dict)
logging.debug('Artifacts count: {}'.format(len(artifacts)))
artifacts_size = sum([x['size'] for x in artifacts])
logging.debug('Summary size: {}'.format(size(artifacts_size)))
return artifacts
def update_updi_devices():
"""
Update all UPDI devices
"""
packs = []
PACKS_OF_INTEREST = [
'Microchip.ATmega_DFP', 'Microchip.ATtiny_DFP', 'Microchip.AVR-'
]
# Search entire path
from artifactory import ArtifactoryPath
path = ArtifactoryPath("{}/{}/{}".format(MICROCHIP_ARTIFACTORY_SERVER_URL,
REPO, ORG))
for a in path:
name = str(a)
packname = name.split('/')[-1]
# Filter:
if name.endswith("_DFP") and not name.endswith(
"ENG_DFP") and packname.startswith(tuple(PACKS_OF_INTEREST)):
packs.append(packname)
for pack in packs:
print("Fetching latest pack for: {}".format(pack))
temp_dir = tempfile.TemporaryDirectory()
pack_info = fetch_latest_device_atpack(pack, temp_dir.name)
devices = os.listdir(temp_dir.name.replace("\\", "/") + "/atdf")
print("Parsing {} devices".format(len(devices)))
for device in devices:
add_new_updi_device(
temp_dir.name.replace("\\", "/") + "/atdf/" + device,
pack_info)
print("Success.")
def get_raw_manifest_list(self):
"""Return the docker manifest list in json format
Raises:
ManifestListNotFound: [description]
Returns:
DICT: manifest.list.json content
"""
listpath = '/'.join([
self.artifactory_base,
self._get_artifactory_repo(
), # We have to massage the repo for artifactory
self.image.get_image_name(),
self.image.get_tag(),
"list.manifest.json"
])
list_path = ArtifactoryPath(listpath,
auth=(self.artifactory_user,
self.artifactory_key))
try:
f = list_path.open()
except FileNotFoundError as e:
raise ManifestListNotFound(e)
except RuntimeError as e:
raise ManifestListNotFound(e)
return json.loads(f.read().decode('utf-8'))
def path(url: str, ) -> ArtifactoryPath:
r"""Authenticate at Artifactory and get path object.
You can set your username and API key in the console:
.. code-block:: bash
$ export ARTIFACTORY_USERNAME=...
$ export ARTIFACTORY_API_KEY=...
If they are not specified,
they are read from :file:`~/.artifactory_python.cfg`
by matching ``url`` against the available entries
to pick the matching Artifactory server.
Args:
url: URL to path on Artifactory
Returns:
Artifactory path object similar to pathlib.Path
Example:
>>> artifactory_path = path(
... 'https://audeering.jfrog.io/artifactory/data-public/emodb/'
... )
>>> for content in artifactory_path:
... print(os.path.basename(str(content)))
...
db
media
meta
"""
username, apikey = authentification(url)
return ArtifactoryPath(url, auth=(username, apikey))
def connect(self):
self.client = ArtifactoryPath(
os.path.join(self.address, 'artifactory', self.repo),
auth=(self.compass.user, self.compass.pswd),
verify=self.compass.check_certificate
)
def load(self, file_path: Pathlike) -> Estimator:
"""
Loads a pickled estimator from given filepath and returns the estimator
Parameters
----------
file_path: Pathlike
Path to load the estimator relative to ArtifactoryStorage
Example
-------
We can load a saved pickled estimator from disk directly from Artifactory:
storage = ArtifactoryStorage('http://artifactory.com', 'path/to/repo')
my_estimator = storage.load('estimatorfile')
We now have a trained estimator loaded.
Returns
-------
Object
estimator unpickled object
"""
if ArtifactoryPath(file_path).is_file():
artifactory_path = file_path
else:
artifactory_path = self.artifactory_path / file_path
with artifactory_path.open() as f:
by = BytesIO()
by.write(f.read())
by.seek(0)
return joblib.load(by)
def test_unlink_raises_on_404(self):
"""
Test that folder/file unlink raises exception if we checked that file
exsists and we still get 404. This is a result of permission issue
"""
path = ArtifactoryPath(
"http://artifactory.local/artifactory/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz"
)
constructed_url = (
"http://artifactory.local/artifactory"
"/api/storage"
"/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz")
responses.add(
responses.GET,
constructed_url,
status=200,
json=self.file_stat,
)
responses.add(
responses.DELETE,
str(path),
status=404,
)
with self.assertRaises(ArtifactoryException) as context:
path.unlink()
self.assertTrue(
"insufficient Artifactory privileges" in str(context.exception))
def _deploy_helper(filename, module_name, get_module, get_today_fn, hash_check=True, auth=None):
"""Deploys a file to the Artifactory BEL namespace cache
:param str filename: The physical path
:param str module_name: The name of the module to deploy to
:param tuple[str] auth: A pair of (str username, str password) to give to the auth keyword of the constructor of
:class:`artifactory.ArtifactoryPath`. Defaults to the result of :func:`get_arty_auth`.
:return: The resource path, if it was deployed successfully, else none.
:rtype: Optional[str]
"""
path = ArtifactoryPath(
get_module(module_name),
auth=get_arty_auth() if auth is None else auth
)
path.mkdir(exist_ok=True)
if hash_check:
deployed_semantic_hashes = {
get_bel_resource_hash(subpath.as_posix())
for subpath in path
}
semantic_hash = get_bel_resource_hash(filename)
if semantic_hash in deployed_semantic_hashes:
return # Don't deploy if it's already uploaded
target = path / get_today_fn(module_name)
target.deploy_file(filename)
log.info('deployed %s', module_name)
return target.as_posix()
def test_unlink(self):
"""
Test that folder/file unlink works
"""
path = ArtifactoryPath(
"http://artifactory.local/artifactory/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz"
)
constructed_url = (
"http://artifactory.local/artifactory"
"/api/storage"
"/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz")
responses.add(
responses.GET,
constructed_url,
status=200,
json=self.file_stat,
)
responses.add(
responses.DELETE,
str(path),
status=200,
)
path.unlink()
def test_stat_no_sha256(self):
"""
Test file stats. No sha256 checksum is available.
Check that stat() works on instance itself
:return:
"""
# Regular File
path = ArtifactoryPath(
"http://artifactory.local/artifactory/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz"
)
constructed_url = (
"http://artifactory.local/artifactory"
"/api/storage"
"/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz")
file_stat = {
"repo": "ext-release-local",
"path": "/org/company/tool/1.0/tool-1.0.tar.gz",
"created": "2014-02-24T21:20:59.999+04:00",
"createdBy": "someuser",
"lastModified": "2014-02-24T21:20:36.000+04:00",
"modifiedBy": "anotheruser",
"lastUpdated": "2014-02-24T21:20:36.000+04:00",
"downloadUri":
"http://artifactory.local/artifactory/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz",
"mimeType": "application/octet-stream",
"size": "26776462",
"checksums": {
"sha1": "fc6c9e8ba6eaca4fa97868ac900570282133c095",
"md5": "2af7d54a09e9c36d704cb3a2de28aff3",
},
"originalChecksums": {
"sha1": "fc6c9e8ba6eaca4fa97868ac900570282133c095",
"md5": "2af7d54a09e9c36d704cb3a2de28aff3",
},
"uri": constructed_url,
}
responses.add(
responses.GET,
constructed_url,
status=200,
json=file_stat,
)
stats = path.stat()
self.assertEqual(
stats.ctime,
dateutil.parser.parse("2014-02-24T21:20:59.999+04:00"))
self.assertEqual(
stats.mtime,
dateutil.parser.parse("2014-02-24T21:20:36.000+04:00"))
self.assertEqual(stats.created_by, "someuser")
self.assertEqual(stats.modified_by, "anotheruser")
self.assertEqual(stats.mime_type, "application/octet-stream")
self.assertEqual(stats.size, 26776462)
self.assertEqual(stats.sha1,
"fc6c9e8ba6eaca4fa97868ac900570282133c095")
self.assertEqual(stats.sha256, None)
def get_latest_version_age(artifact_url):
path = ArtifactoryPath(artifact_url, apikey=api_key)
properties = ArtifactoryPath.stat(path)
modification_date = properties.mtime
# removing timezone
modification_date = modification_date.replace(tzinfo=None)
aritfact_age = today - modification_date
return aritfact_age.days
def _get_path_helper(module_name, getter):
"""Helps get the Artifactory path for a certain module
:param str module_name: The name of the module
:param types.FunctionType getter: The function that gets the modules from the Artifactory repository
:rtype: artifactory.ArtifactoryPath
"""
return ArtifactoryPath(getter(module_name))
def download():
path = ArtifactoryPath(
"http://artifactory.calormen.net:8040/artifactory/generic-local/test/numbers.txt",
auth=('admin', 'f22-demo'))
with path.open() as fd:
with open("old.txt", "wb") as out:
out.write(fd.read())
def generate_url(artifactory_url: str,
repo: str,
apikey=None,
auth=None) -> "ArtifactoryPath":
if not artifactory_url.endswith("artifactory"):
artifactory_url = f"{artifactory_url}/artifactory"
return ArtifactoryPath(f"{artifactory_url}/{repo}",
apikey=apikey,
auth=auth)
def search_packages_dohq(self, url, dictionary={}, **kwargs):
"""
This method parses the remote artifactory repository structure
and chooses those files that are having the extension .py or .whl.
Uses dohq-artifactory API.
:param url (str): remote artifactory repository address
:param dictionary (dict): dict where will be passed the result
:**kwargs may contain 'login', 'password'
:return dict formatted like {'module link', 'module name'}
"""
if 'login' in kwargs and 'password' in kwargs:
path = ArtifactoryPath(url,
auth=(kwargs['login'], kwargs['password']))
else:
path = ArtifactoryPath(url, auth=(self.login, self.password))
for p in path.glob('**/*.*'):
link = str(p)
if link.endswith('.py') or link.endswith('.whl'):
dictionary.update({link: link.split('/')[-1]})
return dictionary
def delete_old_artifacts(list_of_artifacts, chart_name):
"""Extract list of artifacts for this chart and only keep the last NUMBER_OF_CHARTS versions uploaded"""
artifacts = list(filter(lambda x: chart_name == x[0], list_of_artifacts))
if len(artifacts) < NUMBER_OF_CHARTS:
return
artifacts.sort(key=lambda date: time.strptime(date[1], '%d-%b-%Y %H:%M'))
for artifact in artifacts[:-(NUMBER_OF_CHARTS + 1)]:
artifact_path = ArtifactoryPath(
full_artifactory_url + artifact[2],
auth=credentials,
)
if artifact_path.exists():
print("Deleting artifact " + artifact[2])
artifact_path.unlink()
def _get_raw_image_digest(self):
manifestpath = '/'.join([
self.artifactory_base,
self._get_artifactory_repo(), # We have to massage the repo for artifactory
self.image.get_image_name(),
self.image.get_tag(),
"manifest.json"
])
manifest_path = ArtifactoryPath(manifestpath, auth=(self.artifactory_user, self.artifactory_key))
try:
return ArtifactoryPath.stat(manifest_path).sha256
except FileNotFoundError as e:
raise ManifestNotFound(e)
def getArtifact(type, version, artifact, dest):
target = os.path.join(dest, '%s-%s.jar' % (artifact, version))
if not os.path.exists(target):
print('Downloading artifact %s-%s.jar' % (artifact, version))
baseurl = '%s/%s/%s' % (ARTIFACTORY_URL, type.repositoryId,
type.groupId.replace('.', '/'))
url = '%s/%s/%s/%s-%s.jar' % (baseurl, artifact, version, artifact,
version)
path = ArtifactoryPath(
url,
auth=(os.environ.get('CORDA_ARTIFACTORY_USERNAME'),
os.environ.get('CORDA_ARTIFACTORY_PASSWORD')))
with path.open() as fd:
with open(target, 'wb') as out:
out.write(fd.read())
return target
def _create_archive_obj(self):
"""
Create archive object for tests.
During archive creation we call stats() to check if it is_dir(), thus, mock response
:return:
"""
ArtifactoryPath = self.cls
folder = ArtifactoryPath("http://b/artifactory/reponame/folder")
constructed_url = "http://b/artifactory/api/storage/reponame/folder"
responses.add(
responses.GET,
constructed_url,
status=200,
json=self.dir_stat,
)
archive_obj = folder.archive(check_sum=True)
return archive_obj
def integration_artifactory_path_repo(artifactory):
"""
Create repo if not exist and remove all files from this repo
:param artifactory:
:return:
"""
# Create repo if not exist
name = 'integration-artifactory-path-repo'
repo_ = artifactory.find_repository_local(name)
if repo_ is None:
repo_ = RepositoryLocal(artifactory=artifactory, name=name)
repo_.create()
# Remove all file from repo
repo_path = ArtifactoryPath(str(artifactory) + "/" + name, auth=artifactory.auth)
for path_ in repo_path.glob('*'):
path_.unlink()
yield repo_
def _collect_docker_size(self, new_result):
docker_repos = list(set(x['repo'] for x in new_result))
if docker_repos:
aql = ArtifactoryPath(self.artifactory_server,
session=self.artifactory_session)
args = [
'items.find', {
"$or": [{
"repo": repo
} for repo in docker_repos]
}
]
artifacts_list = aql.aql(*args)
for artifact in new_result:
artifact['size'] = sum([
docker_layer['size'] for docker_layer in artifacts_list
if docker_layer['path'] == '{}/{}'.format(
artifact['path'], artifact['name'])
])
def _mock_properties_response():
"""
Function to mock responses on HTTP requests
:return: ArtifactoryPath instance object
"""
# Regular File
path = ArtifactoryPath(
"http://artifactory.local/artifactory/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz"
)
constructed_url = (
"http://artifactory.local/artifactory"
"/api/storage/"
"ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz")
reference_props = {
"test": ["test_property"],
"removethis": ["removethis_property"],
"time": ["2018-01-16 12:17:44.135143"],
}
responses.add(
responses.GET,
constructed_url,
status=204,
json={
"properties": reference_props,
"uri": constructed_url,
},
)
responses.add(
responses.DELETE,
constructed_url,
status=204,
body="",
)
responses.add(
responses.PUT,
constructed_url,
status=204,
body="",
)
return path
def test_unlink_raises_not_found(self):
"""
Test that folder/file unlink raises OSError if file does not exist
"""
path = ArtifactoryPath(
"http://artifactory.local/artifactory/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz"
)
constructed_url = (
"http://artifactory.local/artifactory"
"/api/storage"
"/ext-release-local/org/company/tool/1.0/tool-1.0.tar.gz")
responses.add(
responses.GET,
constructed_url,
status=404,
body="Unable to find item",
)
with self.assertRaises(OSError) as context:
path.unlink()
self.assertTrue(
"No such file or directory" in context.exception.strerror)