def get_dtable_export_content(username, repo_id, dtable_uuid, asset_dir_id, config): """ 1. prepare file content at /tmp/dtable-io/<dtable_id>/dtable_asset/... 2. make zip file 3. return zip file's content """ dtable_io_logger.info('Start prepare /tmp/dtable-io/{}/zip_file.zip for export DTable.'.format(dtable_uuid)) tmp_file_path = os.path.join('/tmp/dtable-io', dtable_uuid, 'dtable_asset/') # used to store asset files and json from file_server tmp_zip_path = os.path.join('/tmp/dtable-io', dtable_uuid, 'zip_file') + '.zip' # zip path of zipped xxx.dtable try: db_session = init_db_session_class(config)() except Exception as e: db_session = None dtable_io_logger.error('create db session failed. ERROR: {}'.format(e)) dtable_io_logger.info('Clear tmp dirs and files before prepare.') clear_tmp_files_and_dirs(tmp_file_path, tmp_zip_path) os.makedirs(tmp_file_path, exist_ok=True) # import here to avoid circular dependency # 1. create 'content.json' from 'xxx.dtable' dtable_io_logger.info('Create content.json file.') try: prepare_dtable_json_from_memory(dtable_uuid, username) except Exception as e: dtable_io_logger.error('prepare dtable json failed. ERROR: {}'.format(e)) # 2. get asset file folder, asset could be empty if asset_dir_id: dtable_io_logger.info('Create asset dir.') try: prepare_asset_file_folder(username, repo_id, dtable_uuid, asset_dir_id) except Exception as e: dtable_io_logger.error('create asset folder failed. ERROR: {}'.format(e)) # 3. copy forms try: copy_src_forms_to_json(dtable_uuid, tmp_file_path, db_session) except Exception as e: dtable_io_logger.error('copy forms failed. ERROR: {}'.format(e)) """ /tmp/dtable-io/<dtable_uuid>/dtable_asset/ |- asset/ |- content.json we zip /tmp/dtable-io/<dtable_uuid>/dtable_asset/ to /tmp/dtable-io/<dtable_id>/zip_file.zip and download it notice than make_archive will auto add .zip suffix to /tmp/dtable-io/<dtable_id>/zip_file """ dtable_io_logger.info('Make zip file for download...') try: shutil.make_archive('/tmp/dtable-io/' + dtable_uuid + '/zip_file', "zip", root_dir=tmp_file_path) except Exception as e: dtable_io_logger.error('make zip failed. ERROR: {}'.format(e)) dtable_io_logger.info('Create /tmp/dtable-io/{}/zip_file.zip success!'.format(dtable_uuid))
def post_dtable_import_files(username, repo_id, workspace_id, dtable_uuid, dtable_file_name, config): """ post files at /tmp/<dtable_uuid>/dtable_zip_extracted/ to file server unzip django uploaded tmp file is suppose to be done in dtable-web api. """ dtable_io_logger.info('Start import DTable: {}.'.format(dtable_uuid)) try: db_session = init_db_session_class(config)() except Exception as e: db_session = None dtable_io_logger.error('create db session failed. ERROR: {}'.format(e)) dtable_io_logger.info('Prepare dtable json file and post it at file server.') try: post_dtable_json(username, repo_id, workspace_id, dtable_uuid, dtable_file_name) except Exception as e: dtable_io_logger.error('post dtable json failed. ERROR: {}'.format(e)) dtable_io_logger.info('Post asset files in tmp path to file server.') try: post_asset_files(repo_id, dtable_uuid, username) except Exception as e: dtable_io_logger.error('post asset files faield. ERROR: {}'.format(e)) dtable_io_logger.info('create forms from src dtable.') try: create_forms_from_src_dtable(workspace_id, dtable_uuid, db_session) except Exception as e: dtable_io_logger.error('create forms failed. ERROR: {}'.format(e)) # remove extracted tmp file dtable_io_logger.info('Remove extracted tmp file.') try: shutil.rmtree(os.path.join('/tmp/dtable-io', dtable_uuid)) except Exception as e: dtable_io_logger.error('rm extracted tmp file failed. ERROR: {}'.format(e)) dtable_io_logger.info('Import DTable: {} success!'.format(dtable_uuid))
def __init__(self, config): self._enabled = True self._logfile = None self._parse_config(config) self._prepare_logfile() self._db_session_class = init_db_session_class(config)
def __init__(self, config): Thread.__init__(self) self._finished = Event() self._db_session_class = init_db_session_class(config) self._redis_client = RedisClient(config)
def __init__(self, config): self._db_session_class = scoped_session(init_db_session_class(config)) self._redis_client = RedisClient(config) self._subscriber = self._redis_client.get_subscriber('table-events') self.post_queue = Queue()