class PivNetUpdater:
def __init__(self):
self.url = "https://network.pivotal.io"
self.headers = {"content-type": "application/json"}
self.database = Database()
def update_db(self):
self.database.clear_all_tables()
reader = codecs.getreader("utf-8")
conf = "conf.toml"
if not os.path.exists(conf):
return "no valid confi.toml with key"
with open(conf) as conffile:
config = toml.loads(conffile.read())
api_key = config.get("api_key")
if not api_key or len(api_key) <= 0:
return "no valid confi.toml with key"
products = self.getProducts()
for product in products:
product_id = product.get("id")
slug = product.get("slug")
pname = product.get("name")
p = Product(id=product_id, slug=slug, name=pname)
self.database.session.add(p)
# print('id=%s,slug=%s,name=%s'%(product_id,slug,pname))
print("Found %s" % slug)
releases = self.getReleases(slug)
if releases:
for release in releases:
rid = release.get("id")
version = release.get("version")
r = Release(id=rid, version=version, product_slug=p.slug)
self.database.session.add(r)
print("Found %s,%s" % (slug, version))
# print('id=%s,version=%s,slud=%s'%(rid,version,p.id))
files = self.getProductFiles(product_id, rid)
if files:
for file in files:
links = file.get("_links")
url = links.get("download").get("href")
name = file.get("aws_object_key").split("/")[-1]
f = ProductFile(id=file.get("id"), release_id=r.id, filename=name, download_url=url)
# print('id=%s,releasid=%s,filename=%s,url=%s'%(f.id,r.id,name,url))
self.database.session.add(f)
print("Found %s,%s,%s" % (slug, version, name))
self.database.commit()
self.database.commit()
print("Local Pivotal Network db has been updated.")
def getProducts(self):
url = self.url + "/api/v2/products/"
r = requests.get(url, headers=self.headers, proxies=proxies)
data = json.loads(r.content.decode("utf-8"))
return data.get("products")
def getReleases(self, slug):
url = self.url + "/api/v2/products/" + slug + "/releases"
r = requests.get(url, headers=self.headers, proxies=proxies)
data = json.loads(r.content.decode("utf-8"))
releases = data.get("releases")
# print(releases)
return releases
def getProductFiles(self, product_id, release_id):
url = self.url + "/api/v2/products/" + str(product_id) + "/releases/" + str(release_id) + "/product_files"
r = requests.get(url, headers=self.headers, proxies=proxies)
data = json.loads(r.content.decode("utf-8"))
return data.get("product_files")
class PivNetUpdater:
def __init__(self, api_key):
self.token = api_key
self.secure_url = end_point
self.secure_headers = {
'content-type': 'application/json',
'Accept': 'application/json',
'Authorization': 'Token token=' + self.token
}
self.database = Database()
# print('token=%s, secure_url=%s, secure_headers=%s' %
# (self.token, self.secure_url, self.secure_headers))
def update_db(self):
self.database.clear_all_tables()
products = self.getProducts()
for product in products:
product_id = product.get('id')
slug = product.get('slug')
pname = product.get('name')
p_file_groups = product.get('_links').get('file_groups').get(
'href')
p_product_files = product.get('_links').get('product_files').get(
'href')
p = Product(id=product_id,
slug=slug,
name=pname,
file_groups_url=p_file_groups,
product_files_url=p_product_files)
self.database.session.add(p)
self.database.commit()
# print(
# 'Found product %s,%s,%s,%s,%s' %
# (product_id,
# slug,
# pname,
# p_file_groups,
# p_product_files))
# if p_file_groups:
# p_groups = self.getFileGroups(p_file_groups)
# if p_groups:
# for p_group in p_groups:
# print(
# json.dumps(
# p_group,
# sort_keys=True,
# indent=4))
# if p_product_files:
# p_files = self.getProductFiles(p_product_files)
# if p_files:
# for p_file in p_files:
# print(
# json.dumps(
# p_file,
# sort_keys=True,
# indent=4))
releases = self.getReleases(slug)
if releases:
for release in releases:
rid = release.get('id')
version = release.get('version')
r_file_groups = release.get('_links').get(
'file_groups').get('href')
r_product_files = release.get('_links').get(
'product_files').get('href')
r = Release(id=rid,
version=version,
product_slug=p.slug,
file_groups_url=r_file_groups,
product_files_url=r_product_files)
self.database.session.add(r)
self.database.commit()
# print(
# 'Found release %s,%s,%s,%s,%s' %
# (rid, version, p.slug, r_file_groups, r_product_files))
if r_file_groups:
groups = self.getFileGroups(r_file_groups)
if groups:
for group in groups:
self.addFiles(group.get('product_files'),
product_id, rid)
self.addFiles(self.getProductFiles(r_product_files),
product_id, rid)
print("Local Pivotal Network db has been updated.")
def addFiles(self, files, product_id, rid):
if files:
for file in files:
try:
file_id = file.get('id')
file_detail = self.getProductFile(product_id, rid, file_id)
url = file_detail.get('_links').get('download').get('href')
name = file_detail.get('aws_object_key').split('/')[-1]
md5 = file_detail.get('md5').lower()
released_at = file_detail.get('released_at')
f = ProductFile(id=file_id,
release_id=rid,
filename=name,
download_url=url,
md5=md5,
release_date=released_at)
# print(
# 'file id=%s,release id=%s,filename=%s,url=%s,md5=%s,date=%s' %
# (f.id, r.id, name, url, md5, released_at))
self.database.session.add(f)
self.database.commit()
except exc.IntegrityError:
self.database.session.rollback()
print('Duplicate: %s' % (file_detail))
except:
print('addFile (%s, %s, %s) exception: %s' %
(product_id, rid, file_id, sys.exc_info()[0]))
# print(
# json.dumps(
# file,
# sort_keys=True,
# indent=4))
# print(
# json.dumps(
# file_detail,
# sort_keys=True,
# indent=4))
def getProducts(self):
url = self.secure_url + "/api/v2/products/"
for i in range(0, 3):
try:
r = requests.get(url,
headers=self.secure_headers,
proxies=proxies)
data = json.loads(r.content.decode('utf-8'))
# print('getProducts %s' % (url))
# print(json.dumps(data, sort_keys=True, indent=4))
products = data.get('products')
return products
except requests.exceptions.RequestException as e:
print('getProducts (i=%s) %s e=%s' % (i, url, e))
print('getProducts giving up after %s tries' % (i))
def getReleases(self, slug):
url = self.secure_url + "/api/v2/products/" + slug + "/releases"
for i in range(0, 3):
try:
r = requests.get(url,
headers=self.secure_headers,
proxies=proxies)
data = json.loads(r.content.decode('utf-8'))
# print('getReleases %s' % (url))
# print(json.dumps(data, sort_keys=True, indent=4))
releases = data.get('releases')
return releases
except requests.exceptions.RequestException as e:
print('getReleases (i=%s) %s e=%s' % (i, url, e))
print('getReleases giving up after %s tries' % (i))
def getFileGroups(self, url):
for i in range(0, 3):
try:
r = requests.get(url,
headers=self.secure_headers,
proxies=proxies)
data = json.loads(r.content.decode('utf-8'))
# print('getFileGroups %s' % (url))
# print(json.dumps(data, sort_keys=True, indent=4))
file_groups = data.get('file_groups')
return file_groups
except requests.exceptions.RequestException as e:
print('getFileGroups (i=%s) %s e=%s' % (i, url, e))
print('getFileGroups giving up after %s tries' % (i))
def getProductFiles(self, url):
for i in range(0, 3):
try:
r = requests.get(url,
headers=self.secure_headers,
proxies=proxies)
data = json.loads(r.content.decode('utf-8'))
# print('getProductFiles %s' % (url))
# print(json.dumps(data, sort_keys=True, indent=4))
product_files = data.get('product_files')
return product_files
except requests.exceptions.RequestException as e:
print('getProductFiles (i=%s) %s e=%s' % (i, url, e))
print('getProductFiles giving up after %s tries' % (i))
def getProductFile(self, product_id, release_id, file_id):
url = self.secure_url + '/api/v2/products/' + str(product_id) \
+ '/releases/' + str(release_id) + '/product_files/' \
+ str(file_id)
# print(url)
for i in range(0, 3):
try:
r = requests.get(url,
headers=self.secure_headers,
proxies=proxies)
data = json.loads(r.content.decode('utf-8'))
# print('getProductFile %s' % (url))
# print(json.dumps(data, sort_keys=True, indent=4))
product_file = data.get('product_file')
return product_file
except requests.exceptions.RequestException as e:
print('getProductFile (i=%s) %s e=%s' % (i, url, e))
print('getProductFile giving up after %s tries' % (i))
