def create_space(space_desc: SpaceDesc, sess: Session):
def exec(stmt):
response(sess, stmt)
exec(space_desc.drop_stmt())
exec(space_desc.create_stmt())
time.sleep(3)
exec(space_desc.use_stmt())
def create_space(space_desc: SpaceDesc, sess: Session):
def exec(stmt):
response(sess, stmt)
stmts = [
space_desc.drop_stmt(),
space_desc.create_stmt(),
space_desc.use_stmt(),
]
exec(";".join(stmts))
def load_csv_data_once(
tmp_path_factory,
pytestconfig,
worker_id,
conn_pool: ConnectionPool,
space: str,
):
root_tmp_dir = tmp_path_factory.getbasetemp().parent
fn = root_tmp_dir / f"csv-data-{space}"
is_file = True
with FileLock(str(fn) + ".lock"):
if not fn.is_file():
data_dir = os.path.join(CURR_PATH, "data", space)
user = pytestconfig.getoption("user")
password = pytestconfig.getoption("password")
sess = conn_pool.get_session(user, password)
space_desc = load_csv_data(pytestconfig, sess, data_dir)
sess.release()
fn.write_text(json.dumps(space_desc.__dict__))
is_file = False
else:
space_desc = SpaceDesc.from_json(json.loads(fn.read_text()))
if is_file:
logging.info(f"session-{worker_id} need not to load {space} csv data")
yield space_desc
else:
logging.info(f"session-{worker_id} load {space} csv data")
yield space_desc
os.remove(str(fn))
def new_space(request, session, graph_spaces):
name = "EmptyGraph_" + space_generator()
space_desc = SpaceDesc(
name=name,
partition_num=9,
replica_factor=1,
vid_type="FIXED_STRING(30)",
charset="utf8",
collate="utf8_bin",
)
create_space(space_desc, session)
graph_spaces["space_desc"] = space_desc
graph_spaces["drop_space"] = True
def new_space(options, session, graph_spaces):
lines = csv.reader(io.StringIO(options), delimiter="|")
opts = {line[1].strip(): line[2].strip() for line in lines}
name = "EmptyGraph_" + space_generator()
space_desc = SpaceDesc(
name=name,
partition_num=int(opts.get("partition_num", 7)),
replica_factor=int(opts.get("replica_factor", 1)),
vid_type=opts.get("vid_type", "FIXED_STRING(30)"),
charset=opts.get("charset", "utf8"),
collate=opts.get("collate", "utf8_bin"),
)
create_space(space_desc, session)
graph_spaces["space_desc"] = space_desc
graph_spaces["drop_space"] = True
def load_csv_data(
sess: Session,
data_dir: str,
space_name: str = "",
):
"""
Before loading CSV data files, you must create and select a graph
space. The `config.yaml' file only create schema about tags and
edges, not include space.
"""
config_path = os.path.join(data_dir, 'config.yaml')
with open(config_path, 'r') as f:
config = yaml.full_load(f)
space = config.get('space', None)
assert space is not None
if not space_name:
space_name = space.get('name', "A" + space_generator())
space_desc = SpaceDesc(
name=space_name,
vid_type=space.get('vidType', 'FIXED_STRING(32)'),
partition_num=space.get('partitionNum', 7),
replica_factor=space.get('replicaFactor', 1),
charset=space.get('charset', 'utf8'),
collate=space.get('collate', 'utf8_bin'),
)
create_space(space_desc, sess)
schemas = config['schema']
for line in schemas.splitlines():
resp_ok(sess, line.strip(), True)
# wait heartbeat_interval_secs + 1 seconds for schema synchronization
time.sleep(2)
if config["files"] is not None:
for fd in config["files"]:
_load_data_from_file(sess, data_dir, fd)
return space_desc
def load_csv_data(
pytestconfig,
sess: Session,
data_dir: str,
space_name: str = "",
):
"""
Before loading CSV data files, you must create and select a graph
space. The `config.yaml' file only create schema about tags and
edges, not include space.
"""
config_path = os.path.join(data_dir, 'config.yaml')
with open(config_path, 'r') as f:
config = yaml.full_load(f)
space = config.get('space', None)
assert space is not None
if not space_name:
space_name = space.get('name', "A" + space_generator())
space_desc = SpaceDesc(
name=space_name,
vid_type=space.get('vidType', 'FIXED_STRING(32)'),
partition_num=space.get('partitionNum', 7),
replica_factor=space.get('replicaFactor', 1),
charset=space.get('charset', 'utf8'),
collate=space.get('collate', 'utf8_bin'),
)
create_space(space_desc, sess)
schemas = config['schema']
stmts = ' '.join(map(lambda x: x.strip(), schemas.splitlines()))
response(sess, stmts)
time.sleep(3)
for fd in config["files"]:
_load_data_from_file(sess, data_dir, fd)
return space_desc
def create_space(space_desc: SpaceDesc, sess: Session):
resp_ok(sess, space_desc.drop_stmt(), True)
resp_ok(sess, space_desc.create_stmt(), True)
resp_ok(sess, space_desc.use_stmt(), True)
def load_csv_data_once(space: str):
with open(SPACE_TMP_PATH, "r") as f:
for sp in json.loads(f.readline()):
if sp.get("name", None) == space:
return SpaceDesc.from_json(sp)
raise ValueError(f"Invalid space name: {space}")
