def check_bundle(): dist_dir = os.path.join(repo_path, 'dist') if os.path.isdir(dist_dir) and len(os.listdir(dist_dir)) == 1: logger.info('Bundle has been built successfully') return os.path.join(dist_dir, os.listdir(dist_dir)[0]) else: raise BuildProcessError('Bundle file was not generated correctly')
def xo_file_exists(assets): for asset in assets: if '.xo' in asset['name']: logger.info('Attached xo file has been found.') return asset logger.info('No attached xo file has been found.') return None
def get_sugar_details(activity, repo_path): logger.info('Applying heuristic to determine min sugar supported version.') def is_gtk3(): GTK3_IMPORT_TYPES = { 'sugar3': 3, 'from gi.repository import Gtk': 3, 'sugar.': 2, 'import pygtk': 2, 'pygtk.require': 2 } setup_py_path = os.path.join(repo_path, 'setup.py') all_files = os.listdir(repo_path) try_paths = [setup_py_path] + all_files for path in try_paths: if os.path.isfile(path): with open(path) as f: text = f.read() for sign in GTK3_IMPORT_TYPES: if sign in text: version = GTK3_IMPORT_TYPES[sign] return version == 3 # Fallback to assuming GTK3 return True def is_web(): if 'exec' in activity: return activity['exec'] == 'sugar-activity-web' return False # Fallback def has_old_toolbars(): OLD_TOOLBAR_SIGNS = ['activity.ActivityToolbox', 'gtk.Toolbar'] for path in os.listdir(repo_path): if os.path.isfile(path): with open(path) as f: text = f.read() for sign in OLD_TOOLBAR_SIGNS: if sign in text: return True return False def determine_min_sugar_version(is_gtk3, is_web, has_old_toolbars): min_sugar_version = '0.100' if is_web else ('0.96' if is_gtk3 else ( '0.86' if not has_old_toolbars else '0.82')) return min_sugar_version sugar = {} sugar['is_gtk3'] = is_gtk3() sugar['is_web'] = is_web() sugar['has_old_toolbars'] = has_old_toolbars() sugar['min_sugar_version'] = determine_min_sugar_version( sugar['is_gtk3'], sugar['is_web'], sugar['has_old_toolbars']) return sugar
def download_attached_xo(xo): # save on blocks of 1024 logger.info("Downloading {} file...".format(xo['name'])) response = requests.get(xo['browser_download_url'], stream=True) tmp_bundle_path = os.path.join(app.config['TEMP_BUNDLE_DIR'], xo['name']) with open(tmp_bundle_path, "wb") as fh: for block in response.iter_content(chunk_size=1024): fh.write(block) return tmp_bundle_path
def store_bundle(tmp_bundle_path): try: shutil.copy2(tmp_bundle_path, app.config['BUILD_BUNDLE_DIR']) stored_bundle = os.path.join(app.config['BUILD_BUNDLE_DIR'], os.path.basename(tmp_bundle_path)) os.chmod(stored_bundle, 0o644) except IOError as e: raise ReleaseError('Bundle copying has failed: %s', e) logger.info('Bundle succesfully stored at %s', stored_bundle)
def verify_xo(xo_archive): logger.info('Searching for activity.info inside xo file.') valid = any('activity.info' in filename for filename in xo_archive.namelist()) if not valid: raise ReleaseError('activity.info not found in xo file.') else: logger.info('activity.info file has been found in xo file.') # TODO: are we going to store this locally and/or in remote server? bundle_name = os.path.basename(tmp_bundle_path) bundle_path = get_bundle_path(bundle_name) if os.path.exists(bundle_path) and os.path.isfile(bundle_path): raise ReleaseError('Bundle {} already exist.'.format(bundle_name))
def store_bundle(tmp_bundle_path, bundle_id): # Create bundle_id folder if it doesn't exist bundle_id_path = os.path.join(app.config['BUILD_BUNDLE_DIR'], bundle_id) if not os.path.exists(bundle_id_path): os.makedirs(bundle_id_path, exist_ok=True) try: shutil.copy2(tmp_bundle_path, bundle_id_path) stored_bundle = os.path.join(bundle_id_path, os.path.basename(tmp_bundle_path)) os.chmod(stored_bundle, 0o644) except IOError as e: raise ReleaseError('Bundle copying has failed: %s', e) logger.info('Bundle succesfully stored at %s', stored_bundle)
def release_process(self, gh_json): try: handle_release(gh_json) except (ReleaseError, BuildProcessError) as error: logger.exception("Error in activity release process!") tag_commit = find_tag_commit( gh_json['repository']['full_name'], gh_json['release']['tag_name']) comment_on_commit( tag_commit, "Build Failed :x: Details: {}".format(error)) else: logger.info('Activity release process has finished successfully!') tag_commit = find_tag_commit( gh_json['repository']['full_name'], gh_json['release']['tag_name']) comment_on_commit(tag_commit, "Build Passed :white_check_mark:")
def invoke_bundle_build(repo_path): def check_bundle(): dist_dir = os.path.join(repo_path, 'dist') if os.path.isdir(dist_dir) and len(os.listdir(dist_dir)) == 1: logger.info('Bundle has been built successfully') return os.path.join(dist_dir, os.listdir(dist_dir)[0]) else: raise BuildProcessError('Bundle file was not generated correctly') logger.info('Building bundle.') volume = repo_path + ':/activity' docker_image = app.config['BUILD_DOCKER_IMAGE'] docker_cmd = ['docker', 'run', '--rm', '-v', volume, docker_image] logger.info('Running docker command: "%s"', ' '.join(docker_cmd)) if sp.call(docker_cmd) != 0: raise BuildProcessError('Docker building process has failed') bundle_path = check_bundle() return bundle_path
def clone_repo(url, tag, repo_path): target_dir = app.config['BUILD_CLONE_REPO'] if not os.path.isdir(target_dir): raise BuildProcessError('Directory %s does not exist' % target_dir) if os.path.isdir(repo_path): logger.info('Removing existing cloned repo %s', repo_path) try: shutil.rmtree(repo_path) except IOError as e: raise BuildProcessError( 'Can\'t remove existing repo {}. Exception: {}'.format( repo_path, e)) cmd = [ 'git', '-c', 'advice.detachedHead=false', '-C', target_dir, 'clone', '-b', tag, '--depth', '1', url ] logger.info('Cloning repo %s', url) if sp.call(cmd) != 0: raise BuildProcessError('[%s] command has failed' % ' '.join(cmd))
def get_activity_metadata(repo_path): def metadata_file_exists(): activity_file = os.path.join(repo_path, 'activity/activity.info') if not os.path.isfile(activity_file): raise ReleaseError('Activity file %s does not exist' % activity_file) return activity_file def parse_metadata_file(activity_file): parser = configparser.ConfigParser() if len(parser.read(activity_file)) == 0: raise ReleaseError('Error parsing metadata file') try: attributes = dict(parser.items('Activity')) except configparser.NoSectionError as e: raise ReleaseError( 'Error parsing metadata file. Exception message: %s' % e) return attributes def validate_mandatory_attributes(attributes): MANDATORY_ATTRIBUTES = [ 'name', 'bundle_id', 'license', 'icon', 'activity_version' ] for attr in MANDATORY_ATTRIBUTES: if attr not in attributes: raise ReleaseError('%s field missing in activity metadata' % attr) return True logger.info('Getting activity metadata from activity.info file.') activity_file = metadata_file_exists() attributes = parse_metadata_file(activity_file) validate_mandatory_attributes(attributes) return attributes
def get_translations(repo_path): logger.info('Getting translations.') po_files_location = os.path.join(repo_path, 'po/') translations = {} matched_files = glob.glob(os.path.join(po_files_location, '*.po')) if len(matched_files) == 0: # If no po files are found just continue working return translations po_files = list(map(polib.pofile, matched_files)) language_codes = list(map(get_language_code, matched_files)) # Initialize the dictionary for language_code in language_codes: translations[language_code] = {} for po_file, language_code in zip(po_files, language_codes): for entry in po_file.translated_entries(): translations[language_code][entry.msgid] = entry.msgstr return translations
def handle_release(gh_json): repo_url = gh_json['repository']['clone_url'] repo_name = gh_json['repository']['name'] release = gh_json['release'] tag = release['tag_name'] tag_commit = gh.find_tag_commit(gh_json['repository']['full_name'], tag) xo_asset = None bundle_name = None # TODO: Extract message to constants file gh.comment_on_commit(tag_commit, "Build has started :hourglass_flowing_sand:") if 'assets' in release and len(release['assets']) != 0: xo_asset = xo_file_exists(release['assets']) if xo_asset: logger.info('[bundle-release] No bundle building process needed.') tmp_bundle_path = download_attached_xo(xo_asset) repo_path = verify_and_extract_xo(tmp_bundle_path) bundle_name = xo_asset['name'] else: logger.info('[sourcecode-release] Building bundle from source code.') repo_path = os.path.join(app.config['BUILD_CLONE_REPO'], repo_name) clone_repo(repo_url, tag, repo_path) tmp_bundle_path = invoke_bundle_build(repo_path) bundle_name = os.path.basename(tmp_bundle_path) metadata = get_activity_metadata(repo_path) compare_version_in_bundlename_and_metadata(tmp_bundle_path, metadata) translations = i18n.get_translations(repo_path) if translations: metadata['i18n_name'] = i18n.translate_field(metadata['name'], translations) metadata['i18n_summary'] = i18n.translate_field( metadata.get('summary', ''), translations) # name and summary fields might have empty values or missing transl. if not metadata['i18n_name']: metadata['i18n_name'] = {'en': metadata['name']} if not metadata['i18n_summary']: metadata['i18n_summary'] = {'en': metadata.get('summary', '')} else: metadata['i18n_name'] = {'en': metadata['name']} metadata['i18n_summary'] = {'en': metadata.get('summary', '')} metadata['repository'] = repo_url metadata['developers'] = gh.get_developers( gh_json['repository']['full_name']) metadata['icon_bin'] = img.get_icon(repo_path, metadata['icon']) try: screenshots = img.get_screenshots(repo_path, metadata['bundle_id']) except ScreenshotDoesNotExist as e: screenshots = {} logger.info(e) finally: metadata['screenshots'] = screenshots metadata['sugar'] = get_sugar_details(metadata, repo_path) metadata['bundle_name'] = bundle_name metadata['release'] = {} metadata['release']['notes'] = gh.render_markdown( gh_json['release']['body']) metadata['release']['time'] = datetime.datetime.strptime( gh_json['release']['published_at'], '%Y-%m-%dT%H:%M:%SZ') logger.info('Inserting activity into db.') activity_service.insert_activity(metadata) logger.info('Saving bundle.') store_bundle(tmp_bundle_path, metadata['bundle_id']) logger.info('Cleaning up.') clean_up(tmp_bundle_path, repo_path)