def build_releases(package_id: str, releases: dict) -> List[Release]:
db_releases = []
for k in releases.keys():
all_releases_for_version = releases.get(k)
if not all_releases_for_version:
continue
v = all_releases_for_version[-1]
r = Release()
r.package_id = package_id
r.major_ver, r.minor_ver, r.build_ver = make_version_num(k)
r.created_date = parse(v.get('upload_time'))
r.comment = v.get('comment_text')
r.url = v.get('url')
r.size = int(v.get('size', 0))
db_releases.append(r)
return db_releases
def test_getters(self):
release1 = Release({
'tasks': [0, 2, 1, 2, 1],
'name': 'release-260',
'links': ['https://youtu.be/dQw4w9WgXcQ'],
'started': 1158781609,
'branch': 'master',
'release_type': 'daily',
'state': 4,
'last_modified': 341231047,
'last_active_task': 'task4_ID'
})
self.assertEqual(release1.name, 'release-260')
self.assertEqual(release1.tasks, [0, 2, 1, 2, 1])
self.assertEqual(release1.links, ['https://youtu.be/dQw4w9WgXcQ'])
self.assertEqual(release1.started, datetime.fromtimestamp(1158781609))
self.assertEqual(release1.branch, 'master')
self.assertEqual(release1.release_type, 'daily')
self.assertEqual(release1.state, 4)
self.assertEqual(release1.last_modified,
datetime.fromtimestamp(341231047))
self.assertEqual(release1.last_active_task, 'task4_ID')
def read_releases(release_data, airflow_db):
"""Reads in releases in the SQL tuple format, transforms them into objects.
Args:
release_data: the raw release data from SQL (tuple)
airflow_db: the airflow database connection object
Returns:
release_objects
"""
release_named_tuple = namedtuple('Row', ['dag_id', 'execution_date'])
release_objects = {} # resources.py expects a dict of objects
for item in release_data: # iterate through each release in release_data
item = release_named_tuple(*item)
release = Release() # initialize the release object
execution_date = item.execution_date
started = int(time.mktime(execution_date.timetuple()))
task_ids, most_recent_task, state = get_task_info(
item.dag_id, started, airflow_db)
xcom_dict, green_sha = get_xcom(started, item.dag_id, airflow_db)
release.release_id = item.dag_id + '@' + str(item.execution_date)
release.tasks = task_ids
release.started = int(started)
release.labels = [item.dag_id]
release.state = state
if green_sha is None:
release.links = [{'name': 'no links available', 'url': None}]
else:
release.links = construct_repo_links(green_sha) # TODO(dommarques) these need to be implemented into airflow first, or we make our own way to get the links pylint: disable=line-too-long
if xcom_dict is None:
continue
else:
release.name = xcom_dict[xcomKeys.VERSION]
try:
release.branch = xcom_dict[xcomKeys.BRANCH]
release.download_link = construct_download_link(
xcom_dict[xcomKeys.GCS_BUILD_PATH])
except KeyError:
_, release.branch = parse_dag_id(item.dag_id)
try:
# it seems that some xcom dicts do not have this value for some reason?
release.release_type = xcom_dict[xcomKeys.RELEASE_TYPE]
except KeyError:
# old releases don't have release types, so this is a fallback
release.release_type, _ = parse_dag_id(item.dag_id)
if most_recent_task:
# allows for continuation even if tasks have not begun/been generated yet
last_modified = most_recent_task.last_modified
release.last_modified = int(time.mktime(last_modified.timetuple()))
release.last_active_task = most_recent_task.task_name
else:
release.last_modified = int(time.mktime(
execution_date.timetuple()))
release.last_active_task = ''
release_objects[release.release_id] = release
logging.info('Release ' + str(release.release_id) + ' has been read')
return release_objects
def get_latest_release_for_package(package_name: str) -> Optional[Release]:
return Release("1.2.0", datetime.datetime.now())
def test_setters(self):
release2 = Release()
release2.name = 'new release'
self.assertEqual(release2.name, 'new release')
with self.assertRaises(ValueError):
release2.name = 0
release2.tasks = []
self.assertEqual(release2.tasks, [])
with self.assertRaises(ValueError):
release2.tasks = 0
with self.assertRaises(ValueError):
release2.tasks = ['task1', 'task2']
release2.links = []
self.assertEqual(release2.links, [])
with self.assertRaises(ValueError):
release2.links = 0
with self.assertRaises(ValueError):
release2.links = [0, 1]
release2.started = 0
self.assertEqual(release2.started, datetime.fromtimestamp(0))
with self.assertRaises(ValueError):
release2.started = 'started'
release2.branch = '0.8'
self.assertEqual(release2.branch, '0.8')
with self.assertRaises(ValueError):
release2.branch = 0
release2.release_type = 'monthly'
self.assertEqual(release2.release_type, 'monthly')
with self.assertRaises(ValueError):
release2.release_type = []
release2.state = 0
self.assertEqual(release2.state, 0)
with self.assertRaises(ValueError):
release2.state = 'state'
release2.last_modified = 0
self.assertEqual(release2.last_modified, datetime.fromtimestamp(0))
with self.assertRaises(ValueError):
release2.last_modified = 'last_modified'
release2.last_active_task = 'last_active_task'
self.assertEqual(release2.last_active_task, 'last_active_task')
with self.assertRaises(ValueError):
release2.last_active_task = 0