def artifactory_aql(artifactory_url, username, password, kerberos,
aql_query_dict, verify):
"""
Send AQL to Artifactory and get list of Artifacts
:param artifactory_url:
:param username:
:param password:
:param kerberos: Boolean if kerberos authentication should be used
:param verify: Boolean if SSL certificate should be checked
:param aql_query_dict:
:param max_depth_print:
:param human_readable:
:param all:
:return:
"""
if kerberos:
auth = HTTPKerberosAuth(mutual_authentication=DISABLED,
sanitize_mutual_error_response=False)
else:
if not password:
raise ValueError(
"argument 'password' needs to be set for basic authentication")
auth = (username, password)
aql = ArtifactoryPath(artifactory_url, auth=auth, verify=verify)
logging.debug("AQL query: items.find({})".format(aql_query_dict))
artifacts = aql.aql('items.find', aql_query_dict)
logging.debug('Artifacts count: {}'.format(len(artifacts)))
artifacts_size = sum([x['size'] for x in artifacts])
logging.debug('Summary size: {}'.format(size(artifacts_size)))
return artifacts
def _collect_docker_size(self, new_result):
docker_repos = list(set(x['repo'] for x in new_result))
if docker_repos:
aql = ArtifactoryPath(self.artifactory_server,
session=self.artifactory_session)
args = [
'items.find', {
"$or": [{
"repo": repo
} for repo in docker_repos]
}
]
artifacts_list = aql.aql(*args)
for artifact in new_result:
artifact['size'] = sum([
docker_layer['size'] for docker_layer in artifacts_list
if docker_layer['path'] == '{}/{}'.format(
artifact['path'], artifact['name'])
])
def _collect_docker_size(self, new_result):
docker_repos = list(set(x['repo'] for x in new_result))
if docker_repos:
aql = ArtifactoryPath(self.artifactory_server,
session=self.artifactory_session)
args = [
'items.find', {
"$or": [{
"repo": repo
} for repo in docker_repos]
}
]
artifacts_list = aql.aql(*args)
images_dict = defaultdict(int)
for docker_layer in artifacts_list:
images_dict[docker_layer['path']] += docker_layer['size']
for artifact in new_result:
image = f"{artifact['path']}/{artifact['name']}"
artifact['size'] = images_dict[image]
def artifactory_aql(artifactory_url, username, password, aql_query_dict):
"""
Send AQL to Artifactory and get list of Artifacts
:param artifactory_url:
:param username:
:param password:
:param aql_query_dict:
:param max_depth_print:
:param human_readable:
:param all:
:return:
"""
aql = ArtifactoryPath(artifactory_url,
auth=(username, password),
verify=False)
logging.debug("AQL query: items.find({})".format(aql_query_dict))
artifacts = aql.aql('items.find', aql_query_dict)
logging.debug('Artifacts count: {}'.format(len(artifacts)))
artifacts_size = sum([x['size'] for x in artifacts])
logging.debug('Summary size: {}'.format(size(artifacts_size)))
return artifacts
## to validate the retention period variable
if int(retention_period) < 90:
print("retention is under 90 days so exiting")
sys.exit()
else:
print()
repo_name = repo_name.split(',')
print("Delete is for artifactory url=" + artifactory_url)
aql = ArtifactoryPath("{}/artifactory/".format(artifactory_url),
auth=('repluser',
'AP49A5SMDpZuQb7e9g7Tn5c45fbUfJkZMzmUSM'))
# multiple repo list
for r in repo_name:
build_numbers = []
artifacts = aql.aql("items.find", {
"type": "folder",
"repo": "{}".format(r)
})
build_info = []
for a in range(len(artifacts)):
if (artifacts[a]["name"] != "."):
build_info.append(int(artifacts[a]["name"]))
build_max_no = max(build_info)
for b in range(len(artifacts)):
if (artifacts[b]["name"] != "."
and build_max_no == int(artifacts[b]["name"])):
created_date = artifacts[b]["created"]
zulu_time, string, seconds = created_date.rpartition('-')
zulu_time = zulu_time + 'Z'
utc_dt = datetime.strptime(zulu_time, '%Y-%m-%dT%H:%M:%S.%fZ')
#last_build_epoch = (utc_dt - datetime(1970, 1, 1)).total_seconds()
retention_time = utc_dt - timedelta(
#!/usr/local/bin/python3.5
import requests
import json
from artifactory import ArtifactoryPath
aql = ArtifactoryPath("http://artifact.corp.continuum.net:8081/artifactory/",
auth=('repluser',
'AP49A5SMDpZuQb7e9g7Tn5c45fbUfJkZMzmUSM'))
repo_name = 'dt-dev_its-portal-net'
build_numbers = []
artifacts = aql.aql("items.find", {
"type": "any",
"repo": "{0}".format(repo_name),
"modified": {
"$before": "8mo"
}
})
with open('data.json', 'w') as outfile:
json.dump(artifacts, outfile)
for element in range(len(artifacts)):
if (artifacts[element]["type"] == "folder"
and artifacts[element]["name"] != "."):
build_numbers.append(int(artifacts[element]["name"]))
build_numbers.sort(key=int)
for i in build_numbers:
print(i)
# response = requests.delete('http://artifact.corp.continuum.net:8081/artifactory/api/build/{}?buildNumbers={}&artifacts=1'.format(repo_name,i),auth=('repluser', 'AP49A5SMDpZuQb7e9g7Tn5c45fbUfJkZMzmUSM'))
response = requests.delete(
'http://artifact.corp.continuum.net:8081/artifactory/{}/{}'.format(
repo_name, i),
auth=('repluser', 'AP49A5SMDpZuQb7e9g7Tn5c45fbUfJkZMzmUSM'))
print(response)
#repo_name = ['dt-dev_its-portal-net','dt_dev_yash']
repo_name = os.environ['repo_name']
#retention_period = '8mo'
retention_period = os.environ['retention_period']
# dry run to list the delete info
dry_run = os.environ['dry_run']
ret_time = "2018-08-03"
aql = ArtifactoryPath("{}/artifactory/".format(artifactory_url),
auth=('repluser',
'AP49A5SMDpZuQb7e9g7Tn5c45fbUfJkZMzmUSM'))
artifacts = aql.aql(
"builds.find", {
"name": {
"$match": "dev_its-portal-net"
},
"created": {
"$lt": "{}".format(ret_time)
},
"promotion.status": {
"$ne": "Released"
}
}, ".include", ["promotion.status"])
#"promotion.status": {"$ne":"released"}})
print(artifacts)
'''
## to validate the retention period variable
if int(retention_period) < 90:
print("retention is under 90 days so exiting")
sys.exit()
else:
print()
repo_name = repo_name.split(',')
from pprint import pprint
#aql = ArtifactoryPath("http://35.193.65.132/artifactory", apikey="AKCp5ekmesKJi3ccYGVL5EAxGTtwvQyHwKrkQ9jH3D4KFdL91BLERDphTXjSAPJMp2x3zwXTC")# path to artifactory, NO repo
#print ("The script has the name %s" % (sys.argv[0])")
# Count the arguments
#arguments=len(sys.argv) - 1
#print ("The script is called with %i arguments" % (arguments))
#art_url=sys.argv[1]
#apikey=sys.argv[2]
art_url="http://jfrog.local/artifactory"
key="AKCp5ekmesKJi3ccYGVL5EAxGTtwvQyHwKrkQ9jH3D4KFdL91BLERDphTXjSAPJMp2x3zwXTC"
aql = ArtifactoryPath("http://35.193.65.132/artifactory", apikey="AKCp5ekmesKJi3ccYGVL5EAxGTtwvQyHwKrkQ9jH3D4KFdL91BLERDphTXjSAPJMp2x3zwXTC")# path to artifactory, NO repo
#aql = ArtifactoryPath(art_url."/artifactory", apikey=key)# path to artifactory, NO repo
artifacts = aql.aql("items.find()", ".include", ["name","stat.downloads"])
# The following steps will filter the artifacts list and give a list of only jars
jarlistall=[]
for artifactdetails in artifacts:
for key, value in artifactdetails.items():
count = 0
if "jar" in value:
jarlistall.append(artifactdetails)
#print (jarlistall)
total_jars=len(jarlistall)
print("total_jars=", total_jars)
#The following steps will provide a list of jar names and stats
jardownloadlist=[]
for jardetails in jarlistall:
},
"name": {
"$match": "*." + artType
},
"stat.downloads": {
"$gt": "0"
}
}]
}
]
args1 = [".sort", {"$desc": ["name"]}]
args2 = [".limit(" + limits + ")"]
finalargs = args + args1 + args2
#This is just to check whether Query is forming correctly or not.
aql_query_text = create_aql_text(*finalargs)
print("Generated AQL query --> " + aql_query_text)
# artifacts_list contains raw data (list of dict)
# Send query:
# items.find({"$and": [{"repo": {"$eq": "repo"}}, {"$or": [{"path": {"$match": "*path1"}}, {"path": {"$match": "*path2"}}]}]})
artifacts_list = aql.aql(*finalargs)
result = json.dumps(artifacts_list)
print("Most Popupar Download Based on Search Criteria--->" + result)
"type": "file"
},
# {"created_by":"abanzon"},
# {"size":{"$gt":"0"}}
# {"created": {"$gt":"2020-05-05"}}
]
},
#sort({"$desc": ["size","name"]}),
#limit(100)
]
try:
#artifacts_list = aql.aql(*args)
artifacts_list = aql.aql(*args, ".include", [
"stat.*", "release_artifact.*", "dependency.*", "artifact.*",
"archive.*", "property.*"
])
artifact_pathlib = map(aql.from_aql, artifacts_list)
artifact_pathlib_list = list(map(aql.from_aql, artifacts_list))
except:
print("Invalid value, please check arguments.")
sys.exit()
pathlist = []
for mylist in artifacts_list:
checkpath = str(mylist['repo'] + "/" + mylist['path'])
if checkpath not in pathlist:
pathlist.append(checkpath)
## to validate the retention period variable
if "mo" in retention_period or "w" in retention_period:
print()
else:
print("Exiting due to variable declaration issue")
sys.exit()
repo_name = repo_name.split(',')
aql = ArtifactoryPath("{}/artifactory/".format(artifactory_url),
auth=('repluser',
'AP49A5SMDpZuQb7e9g7Tn5c45fbUfJkZMzmUSM'))
# multiple repo list
for r in repo_name:
build_numbers = []
artifacts = aql.aql("items.find", {
"type": "folder",
"repo": "{}".format(r)
})
build_info = []
for a in range(len(artifacts)):
if (artifacts[a]["name"] != "."):
build_info.append(int(artifacts[a]["name"]))
build_max_no = max(build_info)
for b in range(len(artifacts)):
if (artifacts[b]["name"] != "."
and build_max_no == int(artifacts[b]["name"])):
created_date = artifacts[b]["created"]
zulu_time, string, seconds = created_date.rpartition('-')
zulu_time = zulu_time + 'Z'
utc_dt = datetime.strptime(zulu_time, '%Y-%m-%dT%H:%M:%S.%fZ')
#last_build_epoch = (utc_dt - datetime(1970, 1, 1)).total_seconds()
retention_time = utc_dt - timedelta(14)
try:
options = parser.parse_args()
except:
sys.exit(0)
repo = options.repo
path = options.path
days = options.time
delete = options.delete
verbose = options.verbose
timesetting = options.timesetting
aql = ArtifactoryPath(art_url, apikey=art_apikey)
print("Searching....")
args = aql_search.package_search(repo, path, days, timesetting)
artifacts_list = aql.aql(*args)
if options.show:
print(len(artifacts_list))
exit()
total_file_count = 0
if delete:
log_name = log_path + "/log-deleting-" + datetime.now().strftime(
"%Y%m%d-%H%M%S") + ".log"
else:
log_name = log_path + "/log-dry_run-" + datetime.now().strftime(
"%Y%m%d-%H%M%S") + ".log"
log_file = open(log_name, 'w')
log_msg = '-'
print("Processing list....")