def authenticate(self, user, password): if user is None: # The user is already in DB, just need the password prev_user = self._localdb.get_username(self._remote.url) if prev_user is None: raise ConanException("User for remote '%s' is not defined" % self._remote.name) else: user = prev_user # Create a connection to the org organization_url, _ = self._get_organization_url_and_feed(self._remote) credentials = BasicAuthentication(user, password) connection = Connection(base_url=organization_url, creds=credentials) # Authenticate the connection try: connection.authenticate() except Exception as ex: raise AuthenticationException(ex) # Generate token token_raw_bytes = password.encode() token_bytes = base64.b64encode(token_raw_bytes) # Store result in DB remote_name, prev_user, user = update_localdb( self._local_db, user, token_bytes.decode(), self._sxor(user, password), self._remote) return remote_name, prev_user, user
class NpmClientV1Methods(object): def __init__(self, remote, local_db, output, config, artifacts_properties=None): # Set to instance self._local_db = local_db self._remote = remote self._output = output self._connection = None self._artifacts_properties = artifacts_properties self._revisions_enabled = config.revisions_enabled self._config = config # Create download cache for conan-npm packages self._npm_cache = os.path.join(Path.home(), '.conan-npm') if not os.path.exists(self._npm_cache): os.mkdir(self._npm_cache) def _download_recipe_npm(self, ref): npm_name = ref.name + '-recipe' files = self._download_npm(npm_name, ref.version, ref.revision) return files def _download_package_npm(self, pref): npm_ref = pref.ref npm_name = npm_ref.name + '-' + pref.id files = self._download_npm(npm_name, npm_ref.version, npm_ref.revision) return files def _download_npm(self, npm_name, version, revision): """ Downloads the npm package from azure feed """ # Get npm package version revision = revision if revision else "0" npm_version = self._get_npm_version(version, revision) # Build npm cache path npm_package_path = os.path.join(self._npm_cache, 'packages', npm_name, npm_version) npm_package_file = os.path.join(npm_package_path, npm_name + '.tgz') # Check if already downloaded if not os.path.isfile(npm_package_file): self.check_credentials() _, feed_id = self._get_organization_url_and_feed(self._remote) # Find correct feed_project feed_project = None feed_client = self._connection.clients_v6_0.get_feed_client() feeds = feed_client.get_feeds() for feed in feeds: if feed.name == feed_id: feed_project = feed.project if feed_project: feed_project = feed.project.id break # Check for npm package hits feed_packages = feed_client.get_packages( feed_id, project=feed_project, package_name_query=npm_name, include_all_versions=True) if not feed_packages: raise NotFoundException( ("No packages found matching pattern '%s'" % npm_name)) # Find correct version npm_package = None npm_package_version = None for feed_package in feed_packages: for feed_package_version in feed_package.versions: # Check of versions matches if feed_package_version.normalized_version == npm_version: npm_package = feed_package npm_package_version = feed_package_version.normalized_version break # Alternative fall back to latest version if feed_package_version.is_latest and revision == "0": npm_package = feed_package npm_package_version = feed_package_version.normalized_version break if npm_package and npm_package_version: break if not npm_package_version: raise NotFoundException( ("No packages found matching version '%s'" % npm_version)) # Update paths and file names npm_package_path = os.path.join(self._npm_cache, 'packages', npm_package.name, npm_package_version) npm_package_file = os.path.join(npm_package_path, npm_package.name + '.tgz') # Check again file exists if not os.path.isfile(npm_package_file): # Download package npm_client = self._connection.clients_v6_0.get_npm_client() npm_download_generator = npm_client.get_content_unscoped_package( feed_id, npm_package.name, npm_package_version, project=feed_project) if self._output and not self._output.is_terminal: self._output.writeln( "Downloading npm package '%s/%s'..." % (npm_package.name, npm_package_version)) os.makedirs(npm_package_path, exist_ok=True) with open(npm_package_file, 'wb') as file: for chunk in npm_download_generator: file.write(chunk) # Extract downloaded file npm_package_content_path = os.path.join(npm_package_path, "package") if not os.path.exists(npm_package_content_path): untargz(npm_package_file, npm_package_path) # Get all files from cache found_files = [ fn for fn in glob.glob(npm_package_content_path + os.path.sep + "**", recursive=True) if not os.path.basename(fn).startswith('package.json') and os.path.isfile(fn) ] # Convert to dict files = {} for found_file in found_files: files[os.path.basename(found_file)] = found_file return files def _get_file_contents(self, files, filters): contents = {} for filename, filepath in files.items(): if filename in filters: with open(filepath, 'rb') as handle: file_content = handle.read() contents[filename] = file_content return contents def _copy_files_to_folder(self, src_files, dest_folder): copied_files = {} for src_filename, src_filepath in src_files.items(): # Copy file to dest_folder dest_filepath = os.path.join(dest_folder, src_filename) os.makedirs(os.path.dirname(dest_filepath), exist_ok=True) shutil.copyfile(src_filepath, dest_filepath) # Add copied file to dict copied_files[src_filename] = dest_filepath return copied_files def get_recipe_manifest(self, ref): """Gets a FileTreeManifest from conans""" # Obtain the files from npm package npm_package_files = self._download_recipe_npm(ref) contents = self._get_file_contents(npm_package_files, [CONAN_MANIFEST]) # Unroll generator and decode (plain text) contents = {key: decode_text(value) for key, value in contents.items()} return FileTreeManifest.loads(contents[CONAN_MANIFEST]) def get_package_manifest(self, pref): """Gets a FileTreeManifest from a package""" pref = pref.copy_with_revs(None, None) # Obtain the files from npm package npm_package_files = self._download_package_npm(pref) contents = self._get_file_contents(npm_package_files, [CONAN_MANIFEST]) # Unroll generator and decode (plain text) contents = {key: decode_text(value) for key, value in contents.items()} return FileTreeManifest.loads(contents[CONAN_MANIFEST]) def get_package_info(self, pref, headers): """Gets a ConanInfo file from a package""" pref = pref.copy_with_revs(None, None) # Obtain the files from npm package npm_package_files = self._download_package_npm(pref) if CONANINFO not in npm_package_files: raise NotFoundException("Package %s doesn't have the %s file!" % (pref, CONANINFO)) # Get the info (in memory) contents = self._get_file_contents(npm_package_files, [CONANINFO]) # Unroll generator and decode (plain text) contents = { key: decode_text(value) for key, value in dict(contents).items() } return ConanInfo.loads(contents[CONANINFO]) def get_recipe(self, ref, dest_folder): npm_package_files = self._download_recipe_npm(ref) npm_package_files.pop(EXPORT_SOURCES_TGZ_NAME, None) check_compressed_files(EXPORT_TGZ_NAME, npm_package_files) # Copy files from cache to dest_folder recipe_files = self._copy_files_to_folder(npm_package_files, dest_folder) return recipe_files def get_recipe_snapshot(self, ref): try: # Get the digest and calculate md5 of package files npm_package_files = self._download_recipe_npm(ref) snapshot = {} for src_filename, src_filepath in npm_package_files.items(): snapshot[src_filename] = md5sum(src_filepath) except NotFoundException: snapshot = [] return snapshot def get_recipe_sources(self, ref, dest_folder): npm_package_files = self._download_recipe_npm(ref) check_compressed_files(EXPORT_SOURCES_TGZ_NAME, npm_package_files) if EXPORT_SOURCES_TGZ_NAME not in npm_package_files: return None npm_package_files = { EXPORT_SOURCES_TGZ_NAME: npm_package_files[EXPORT_SOURCES_TGZ_NAME] } # Copy files from cache to dest_folder recipe_sources_files = self._copy_files_to_folder( npm_package_files, dest_folder) return recipe_sources_files def get_package(self, pref, dest_folder): npm_package_files = self._download_package_npm(pref) check_compressed_files(PACKAGE_TGZ_NAME, npm_package_files) # Copy files from cache to dest_folder package_files = self._copy_files_to_folder(npm_package_files, dest_folder) return package_files def get_package_snapshot(self, pref): try: # Get the digest and calculate md5 of package files npm_package_files = self._download_package_npm(pref) snapshot = {} for src_filename, src_filepath in npm_package_files.items(): snapshot[src_filename] = md5sum(src_filepath) except NotFoundException: snapshot = [] return snapshot def _get_path(self, npm_package_files, path): files = npm_package_files.keys() def is_dir(the_path): if the_path == ".": return True for _the_file in files: if the_path == _the_file: return False elif _the_file.startswith(the_path): return True raise NotFoundException("The specified path doesn't exist") if is_dir(path): ret = [] for the_file in files: if path == "." or the_file.startswith(path): tmp = the_file[len(path) - 1:].split("/", 1)[0] if tmp not in ret: ret.append(tmp) return sorted(ret) else: contents = self._get_file_contents(npm_package_files, [path]) content = contents[path] return decode_text(content) def get_recipe_path(self, ref, path): """Gets a file content or a directory list""" npm_package_files = self._download_recipe_npm(ref) return self._get_path(npm_package_files, path) def get_package_path(self, pref, path): """Gets a file content or a directory list""" npm_package_files = self._download_package_npm(pref) return self._get_path(npm_package_files, path) def upload_recipe(self, ref, files_to_upload, deleted, retry, retry_wait): npm_name = ref.name + '-recipe' self._upload_as_npm(npm_name, ref.version, ref.revision, files_to_upload, deleted, retry, retry_wait) def upload_package(self, pref, files_to_upload, deleted, retry, retry_wait): npm_ref = pref.ref npm_name = npm_ref.name + '-' + pref.id self._upload_as_npm(npm_name, npm_ref.version, npm_ref.revision, files_to_upload, deleted, retry, retry_wait) def _upload_as_npm(self, npm_name, version, revision, files_to_upload, deleted, retry, retry_wait): self.check_credentials() user, token, refresh_token = self._local_db.get_login(self._remote.url) # Get npm package version npm_version = self._get_npm_version(version, revision) with tempfile.TemporaryDirectory() as tmp_dir_name: remote_url = self._remote.url remote_url_no_https = remote_url.replace('https:', '') remote_url_no_https_and_registry = remote_url_no_https.replace( 'registry/', '') # Create .npmrc file npmrc = os.path.join(os.path.abspath(tmp_dir_name), '.npmrc') npmrc_file = open(npmrc, "w", encoding='utf8') npmrc_file.write(('registry = %s\n' % self._remote.url)) npmrc_file.write('always-auth = true\n') npmrc_file.write('; begin auth token\n') npmrc_file.write( ('%s:username=%s\n' % (remote_url_no_https, user))) npmrc_file.write( ('%s:_password=%s\n' % (remote_url_no_https, token))) npmrc_file.write(( '%s:email=npm requires email to be set but doesn\'t use the value\n' % remote_url_no_https)) npmrc_file.write(('%s:username=%s\n' % (remote_url_no_https_and_registry, user))) npmrc_file.write(('%s:_password=%s\n' % (remote_url_no_https_and_registry, token))) npmrc_file.write(( '%s:email=npm requires email to be set but doesn\'t use the value\n' % remote_url_no_https_and_registry)) npmrc_file.write('; end auth token\n') npmrc_file.close() # Create package.json package_json_data = { "name": npm_name, "description": npm_name + " package created by conan.io", "version": npm_version, } package_json = os.path.join(tmp_dir_name, 'package.json') package_json_file = open(package_json, "w", encoding='utf8') package_json_file.write( json.dumps(package_json_data, sort_keys=True, ensure_ascii=False)) package_json_file.close() # Copy needed files for filename, filepath in files_to_upload.items(): shutil.copyfile(filepath, os.path.join(tmp_dir_name, filename)) # Run NPM publish subprocess.run("npm publish", shell=True, cwd=tmp_dir_name) def _get_npm_version(self, version, revision): # Convert to valid npm version npm_version_build_number = None npm_version_tokens = version.split(".") if len(npm_version_tokens) > 3: npm_version_build_number = npm_version_tokens.pop() # Build npm version npm_version = ".".join(npm_version_tokens) # Check version is semantic 2.0 compatible match = re.match( "^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)" "(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)" "(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))" "?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$", npm_version) if not match: npm_version = "1.0.0-conan-" + npm_version if revision and revision != "0": npm_version += "-" + revision if npm_version_build_number: npm_version += "." + npm_version_build_number else: if npm_version_build_number: npm_version += "-" + npm_version_build_number return npm_version def _get_organization_url_and_feed(self, remote): # Extract data from given url remote_url = remote.url remote_url_parts = remote_url.split('/') if len(remote_url_parts) < 6: ConanException( "Cannot parse Organization or Package from given url") if remote_url_parts[2].find('dev.azure.com') == -1: ConanException("Cannot handle platform not equal to dev.azure.com") # Find npm keyword in remote url npm_pos = 0 npm_index = 0 for remote_url_part in remote_url_parts: if 'npm' == remote_url_part: npm_pos = npm_index break npm_index += 1 # Check npm keyword was found if npm_pos == 0: ConanException( "Cannot handle packages with given type not equal to npm") organization = remote_url_parts[3] organization_url_len = remote_url.find(organization) + len( organization) return remote_url[:organization_url_len], remote_url_parts[npm_pos - 1] def authenticate(self, user, password): if user is None: # The user is already in DB, just need the password prev_user = self._localdb.get_username(self._remote.url) if prev_user is None: raise ConanException("User for remote '%s' is not defined" % self._remote.name) else: user = prev_user # Create a connection to the org organization_url, _ = self._get_organization_url_and_feed(self._remote) credentials = BasicAuthentication(user, password) connection = Connection(base_url=organization_url, creds=credentials) # Authenticate the connection try: connection.authenticate() except Exception as ex: raise AuthenticationException(ex) # Generate token token_raw_bytes = password.encode() token_bytes = base64.b64encode(token_raw_bytes) # Store result in DB remote_name, prev_user, user = update_localdb( self._local_db, user, token_bytes.decode(), self._sxor(user, password), self._remote) return remote_name, prev_user, user def check_credentials(self): organization_url, _ = self._get_organization_url_and_feed(self._remote) user, token, refresh_token = self._local_db.get_login(self._remote.url) if not user or not refresh_token: raise AuthenticationException('Username or password not valid.') # Decrypt password password = self._sxor(user, refresh_token) # Create a connection to the org credentials = BasicAuthentication(user, password) self._connection = Connection(base_url=organization_url, creds=credentials) # Authenticate the connection try: self._connection.authenticate() except Exception as ex: raise AuthenticationException(ex) def search(self, pattern=None, ignorecase=True): self.check_credentials() _, feed_id = self._get_organization_url_and_feed(self._remote) # Find correct feed_project feed_project = None feed_client = self._connection.clients_v6_0.get_feed_client() feeds = feed_client.get_feeds() for feed in feeds: if feed.name == feed_id: feed_project = feed.project if feed_project: feed_project = feed.project.id break # Check for npm package hits search_results = [] pattern, _ = self._split_pair(pattern, "/") or (pattern, None) feed_packages = feed_client.get_packages(feed_id, project=feed_project, package_name_query=pattern, include_all_versions=True) for feed_package in feed_packages: if '-recipe' in feed_package.normalized_name: conan_package_name = feed_package.normalized_name.replace( '-recipe', '') for feed_package_version in feed_package.versions: npm_package_version = feed_package_version.normalized_version # Remove non semantic version extension form npm version npm_package_version = npm_package_version.replace( '1.0.0-conan-', '') version, revision_build = self._split_pair(npm_package_version, '-') \ or (npm_package_version, None) revision, build = self._split_pair(revision_build, '.') \ or (None, revision_build) conan_package = conan_package_name + '/' + version conan_package = conan_package + ( '.' + build) if build else conan_package conan_package = conan_package + '@_/_' conan_package = conan_package + ( '#' + revision) if revision else conan_package search_results.append(conan_package) return [ ConanFileReference.loads(reference) for reference in search_results ] def search_packages(self, reference, query): self.check_credentials() raise RuntimeError('Not implemented, yet') def remove_recipe(self, ref): self.check_credentials() raise RuntimeError('Not implemented, yet') def remove_packages(self, ref, package_ids=None): self.check_credentials() raise RuntimeError('Not implemented, yet') def server_capabilities(self): raise RuntimeError('Not implemented, yet') def get_recipe_revisions(self, ref): raise NoRestV2Available("The remote doesn't support revisions") def get_package_revisions(self, pref): raise NoRestV2Available("The remote doesn't support revisions") def get_latest_recipe_revision(self, ref): raise NoRestV2Available("The remote doesn't support revisions") def get_latest_package_revision(self, pref, headers): raise NoRestV2Available("The remote doesn't support revisions") def _sxor(self, s1, s2): """ XOR two byte strings """ zip_list = zip(s1, cycle(s2)) if len(s1) > len(s2) else zip( cycle(s1), s2) return ''.join(chr(ord(a) ^ ord(b)) for a, b in zip_list) def _split_pair(self, pair, split_char): if not pair or pair == split_char: return None, None if split_char not in pair: return None words = pair.split(split_char) if len(words) != 2: raise ConanException( "The reference has too many '{}'".format(split_char)) else: return words