def dowload_artifact(self, project_id, job_id, store): # project_id = '13539397' # job_id = '261939258' is_cache = self.config.get('cache') save_dir = os.path.dirname(store) if is_cache: cache_files = db.collection('artifacts').find({ 'project_id': project_id, 'job_id': job_id, 'app_type': 'gitlab' }) cache_files = list(cache_files) if cache_files: for record in cache_files: file_id = record.get('file_id') if not file_id: continue filename = os.path.join(save_dir, record['filename']) with open(filename, 'wb') as stream: db.fs_bucket().download_to_stream(file_id, stream) extract(filename, save_dir) os.unlink(filename) return True project = self.projects.get(project_id) # pipeline = project.jobs.get(job_id) jobs = project.jobs job = jobs.get(job_id) if job.status != 'success': raise Exception('gitlab job status must be success, %s got'.format(job.status)) with open(store, 'wb') as f: job.artifacts(streamed=True, action=f.write) if is_cache: with open(store, mode='rb') as fd: name = os.path.basename(store) file_id = db.save_file(filename=name, fileobj=fd) store_info = { 'app_type': 'jenkins', 'file_id': str(file_id), 'project_id': project_id, 'job_id': job_id, 'filename': name, 'created_at': time.time() } db.collection('artifacts').insert_one(store_info) with zipfile.ZipFile(store, 'r') as zip_ref: zip_ref.extractall(os.path.dirname(store)) os.unlink(store) return True
def build_book_from_history(self, build_id): history = db.collection('build_history').find_one( {'_id': ObjectId(build_id)}) task_id = history.get('task_id') file_id = history.get('file_id') job_info = history.get('job_info') book = Book.find_one({'_id': ObjectId(job_info.get('book_id'))}) bookspace = self.get_book_space(book.get('name')) bookspace = os.path.join(bookspace, md5(str(task_id))) self.mkdir(bookspace) save_file = NamedTemporaryFile(delete=False, suffix='.zip').name with open(save_file, 'wb') as fd: db.fs_bucket().download_to_stream(ObjectId(file_id), fd) extract(save_file, bookspace) os.unlink(save_file) return bookspace
def load_book_from_db(self, name, roles=None): wk = Workspace() workspace = wk.workspace wk.check_workspace() books = db.collection('playbook').find({ 'book_name': name }).sort('seq_no', pymongo.ASCENDING) for item in books: if roles: folder = item.get('name') if folder and folder not in roles: continue filename = workspace + item['path'] if item['is_dir']: if os.path.isdir(filename): continue else: os.mkdir(filename, 0o600) else: if os.path.isfile(filename): file_hash = self.file_md5(filename) if item.get('md5') and item['md5'] == file_hash: continue dirname = os.path.dirname(filename) if not os.path.exists(dirname): os.makedirs(dirname) if item['is_edit']: db.collection('playbook').update_one( {'_id': item['_id']}, {'$set': { 'md5': md5(item['content'].encode('utf8')) }}) with open(filename, 'w') as stream: stream.write(item['content']) else: with open(filename, 'wb') as stream: db.fs_bucket().download_to_stream( item['file_id'], stream) return books
def setup(): wk = Workspace() workspace = wk.workspace check_workspace() if not os.path.exists(workspace): os.makedirs(workspace, 0o755) books = db.collection('playbook').find().sort('path', pymongo.ASCENDING) # books = collection_array(books) start = time.time() for item in books: filename = workspace + item['path'] if item['is_dir']: if os.path.isdir(filename): continue else: os.makedirs(filename, 0o755) else: if os.path.isfile(filename): file_hash = file_md5(filename) if item.get('md5') and item['md5'] == file_hash: continue dirname = os.path.dirname(filename) if not os.path.exists(dirname): os.makedirs(dirname) if item['is_edit']: db.collection('playbook').update_one({'_id': item['_id']}, {'$set': {'md5': md5(item['content'])}}) with open(filename, 'w+') as stream: stream.write(item['content']) else: with open(filename, 'wb') as stream: db.fs_bucket().download_to_stream(item['file_id'], stream) end = time.time() return jsonify({ "message": "ok", "code": 0, 'runtime': end - start })
def load_book_from_db(self, name, roles=None, build_id=False): book = Book.find_one({'name': name}) if not book: return False files = Model.build_model('playbook').find({'book_id': str(book['_id'])})\ .sort([('is_edit', pymongo.ASCENDING), ('path', pymongo.ASCENDING)]) files = list(files) if not files: return False if build_id: bookspace = os.path.join(self.book, md5(str(build_id))) else: bookspace = self.get_book_space(name) def parse_register(record): register = record.get('register') if not register: return record c_ids = map(lambda i: ObjectId(i), register) cfg_records = Configuration.find({'_id': {'$in': list(c_ids)}}) if not cfg_records: return record try: variables = {} content = yaml.safe_load(record.get('content', '')) if not content: return record vault = Vault({'vault_pass': config.vault.get('secret')}) for cfg in cfg_records: config_vars = cfg.get('variables') if not config_vars: continue for k, v in config_vars.items(): key = '_'.join( ['ECLOGUE', 'CONFIG', cfg.get('name', ''), k]) is_encrypt = Vault.is_encrypted(v) value = v if is_encrypt: value = vault.decrypt_string(value) variables[key] = value content = dict(content) content.update(variables) record['content'] = yaml.safe_dump(content) except Exception as e: print(e) return record self.check_workspace(path=self._check_make(bookspace)) for item in files: item = parse_register(item) if roles and item.get('folder'): folder = item.get('folder') if folder and folder not in roles: continue filename = bookspace + item.get('path') if item['is_dir']: if os.path.isdir(filename): continue self.mkdir(filename) else: if os.path.isfile(filename): file_hash = file_md5(filename) if item.get('md5') and item['md5'] == file_hash: continue dirname = os.path.dirname(filename) if not os.path.exists(dirname): os.makedirs(dirname) if item['is_edit']: Model.build_model('playbooks').update_one( {'_id': item['_id']}, {'$set': { 'md5': md5(item['content']) }}) with open(filename, 'w') as stream: stream.write(item['content']) else: with open(filename, 'wb') as stream: db.fs_bucket().download_to_stream( item['file_id'], stream) return bookspace
def save_artifacts(self, save_dir, job_name, build_id=None, strict_validation=False, artifact_name=None): is_cache = self.config.get('cache') self.logger.info('use cached:{}'.format(is_cache)) if is_cache: cache_files = db.collection('artifacts').find({ 'job_name': job_name, 'build_id': build_id, 'app_type': 'jenkins' }) cache_files = list(cache_files) if cache_files: for record in cache_files: file_id = record.get('file_id') if not file_id: continue msg = 'load file from cached, save_dir:{}, filename:{}'.format( save_dir, record['filename']) self.logger.info(msg) filename = os.path.join(save_dir, record['filename']) with open(filename, 'wb') as stream: db.fs_bucket().download_to_stream(file_id, stream) extract(filename, save_dir) os.unlink(filename) return True artifacts = self.get_artifacts(job_name, build_id) store_files = [] for artifact in artifacts: if artifact_name and artifact.filename != artifact_name: continue file_path = artifact.save_to_dir(save_dir, strict_validation) msg = 'download artifacts from {}, save_dir:{}, filename:{}' msg = msg.format(self.config.get('base_url'), save_dir, artifact.filename) self.logger.info(msg) extract(file_path, save_dir) store_files.append({ 'filename': artifact.filename, 'path': file_path }) for file in store_files: filename = file['filename'] path = file['path'] if is_cache: with open(path, mode='rb') as stream: file_id = db.save_file(filename=filename, fileobj=stream) store_info = { 'app_type': 'jenkins', 'file_id': file_id, 'job_name': job_name, 'build_id': build_id, 'filename': os.path.basename(filename), 'created_at': time.time() } db.collection('artifacts').insert_one(store_info) os.unlink(path)