def insert_properties(properties: Set[str], session: scoped_session) -> Optional[Any]: """Insert all the properties as defined in the APIDocumentation into DB.""" prop_list = [ BaseProperty(name=prop) for prop in properties if not session.query(exists().where( BaseProperty.name == prop)).scalar() ] session.add_all(prop_list) session.commit() return None # if __name__ == "__main__": # Session = sessionmaker(bind=engine) # session = Session() # # doc = doc_gen("test", "test") # # Extract all classes with supportedProperty from both # classes = get_classes(doc.generate()) # # # Extract all properties from both # # import pdb; pdb.set_trace() # properties = get_all_properties(classes) # # Add all the classes # insert_classes(classes, session) # print("Classes inserted successfully") # # Add all the properties # insert_properties(properties, session) # print("Properties inserted successfully")
def test_broadcast_move( fx_server: WSGIServer, fx_session: scoped_session, fx_other_server: WSGIServer, fx_other_session: Session, fx_user: User, fx_novice_status: typing.Mapping[str, str], ): now = datetime.datetime.utcnow() node = Node(url=fx_server.url, last_connected_at=now) node2 = Node(url=fx_other_server.url, last_connected_at=datetime.datetime.utcnow()) move = fx_user.create_novice(fx_novice_status) fx_session.add_all([node, node2, move]) fx_session.commit() assert not fx_other_session.query(Move).get(move.id) serialized = move.serialize( use_bencode=False, include_signature=True, include_id=True, ) multicast(serialized=serialized, broadcast=broadcast_move) assert fx_other_session.query(Move).get(move.id) assert node.last_connected_at > now
def delete_object(query_info: Dict[str, str], session: scoped_session, collection: bool = False) -> None: """ Delete the object from the database :param query_info: Dict containing the id and @type of object that has to retrieved :param session: sqlalchemy session :param collection: True if the type_ is of a collection, False for any other class """ type_ = query_info["@type"] id_ = query_info["id_"] database_class = get_database_class(type_) if collection: try: objects = session.query(database_class).filter_by( collection_id=id_).delete() except NoResultFound: raise InstanceNotFound(type_=type_, id_=id_) try: session.commit() except InvalidRequestError: session.rollback() return id_ else: try: object_ = (session.query(database_class).filter( database_class.id == id_).one()) except NoResultFound: raise InstanceNotFound(type_=type_, id_=id_) session.delete(object_) try: session.commit() except InvalidRequestError: session.rollback()
def test_move_broadcast(fx_session: scoped_session, fx_user: User, fx_novice_status: typing.Mapping[str, str]): move = fx_user.create_novice(fx_novice_status) fx_session.add(move) fx_session.commit() with unittest.mock.patch('nekoyume.tasks.multicast') as m: move_broadcast(move.id, 'http://localhost:5000', 'http://localhost:5001', session=fx_session) serialized = move.serialize( use_bencode=False, include_signature=True, include_id=True, ) assert m.called args = m.call_args[1] assert serialized == args['serialized'] assert isinstance(args['sent_node'], Node) assert args['sent_node'].url == 'http://localhost:5000' assert isinstance(args['my_node'], Node) assert args['my_node'].url == 'http://localhost:5001' broadcast = args['broadcast'] assert isinstance(broadcast, typing.Callable) assert broadcast.__name__ == 'broadcast_move'
def delete_multiple(id_: List[int], type_: str, session: scoped_session) -> None: """ To delete multiple rows in a single request :param id_: list of ids for objects to be deleted\ :param type_: type of object to be deleted :param session: sqlalchemy scoped session Raises: ClassNotFound: If `type_` does not represent a valid/defined RDFClass. InstanceNotFound: If any instance with type 'type_' and any id in 'id_' list does not exist. """ id_ = id_.split(',') try: rdf_class = session.query(RDFClass).filter( RDFClass.name == type_).one() except NoResultFound: raise ClassNotFound(type_=type_) instances = list() data_III = list() data_IAC = list() data_IIT = list() for index in id_: try: instance = session.query(Instance).filter( Instance.id == index and type_ == rdf_class.id).one() instances.append(instance) except NoResultFound: raise InstanceNotFound(type_=rdf_class.name, id_=index) data_IIT += session.query(triples).filter( triples.GraphIIT.subject == index).all() data_IAC += session.query(triples).filter( triples.GraphIAC.subject == index).all() data_III += session.query(triples).filter( triples.GraphIII.subject == index).all() data = data_III + data_IIT + data_IAC for item in data: session.delete(item) for data in data_IIT: terminal = session.query(Terminal).filter( Terminal.id == data.object_).one() session.delete(terminal) for data in data_III: III_instance = session.query(Instance).filter( Instance.id == data.object_).one() III_instance_type = session.query(RDFClass).filter( RDFClass.id == III_instance.type_).one() # Get the III object type_ delete(III_instance.id, III_instance_type.name, session=session) for instance in instances: session.delete(instance) session.commit()
def test_broadcast_node_same_url(fx_session: scoped_session): url = 'http://test.neko' now = datetime.datetime.utcnow() node = Node(url=url, last_connected_at=now) fx_session.add(node) fx_session.commit() with Mocker() as m: broadcast_node(serialized={'url': url}, sent_node=node) assert not m.called assert node.last_connected_at == now
def insert_modification_record(method: str, resource_url: str, session: scoped_session) -> int: """ Insert a modification record into the database. :param method: HTTP method type of related operation. :param resource_url: URL of resource modified. :param session: sqlalchemy session. :return: ID of new modification record. """ modification = Modification(method=method, resource_url=resource_url) session.add(modification) session.commit() return modification.job_id
def test_broadcast_my_node(fx_session: scoped_session): url = 'http://test.neko' now = datetime.datetime.utcnow() node = Node(url=url, last_connected_at=now) fx_session.add(node) fx_session.commit() with Mocker() as m: m.post('http://test.neko/nodes', json={'result': 'success'}) broadcast_node({'url': url}, my_node=node) assert node.last_connected_at > now # check request.json value assert m.request_history[0].json() == { 'url': 'http://test.neko', 'sent_node': 'http://test.neko' }
def delete(id_: str, type_: str, session: scoped_session) -> None: """Delete an Instance and all its relations from DB given id [DELETE]. :param id_: id of object to be deleted :param type_: type of object to be deleted :param session: sqlalchemy scoped session Raises: ClassNotFound: If `type_` does not represent a valid/defined RDFClass. InstanceNotFound: If no instace of type `type_` with id `id_` exists. """ try: rdf_class = session.query(RDFClass).filter( RDFClass.name == type_).one() except NoResultFound: raise ClassNotFound(type_=type_) try: instance = session.query(Instance).filter( Instance.id == id_ and type_ == rdf_class.id).one() except NoResultFound: raise InstanceNotFound(type_=rdf_class.name, id_=id_) data_IIT = session.query(triples).filter( triples.GraphIIT.subject == id_).all() data_IAC = session.query(triples).filter( triples.GraphIAC.subject == id_).all() data_III = session.query(triples).filter( triples.GraphIII.subject == id_).all() data = data_III + data_IIT + data_IAC for item in data: session.delete(item) for data in data_IIT: terminal = session.query(Terminal).filter( Terminal.id == data.object_).one() session.delete(terminal) for data in data_III: III_instance = session.query(Instance).filter( Instance.id == data.object_).one() III_instance_type = session.query(RDFClass).filter( RDFClass.id == III_instance.type_).one() # Get the III object type_ delete(III_instance.id, III_instance_type.name, session=session) session.delete(instance) session.commit()
def broadcast_move_failed(fx_session: scoped_session, fx_user: User, fx_novice_status: typing.Mapping[str, str], error): now = datetime.datetime.utcnow() move = fx_user.create_novice(fx_novice_status) node = Node(url='http://test.neko', last_connected_at=now) fx_session.add_all([node, move]) fx_session.commit() with Mocker() as m: serialized = move.serialize( use_bencode=False, include_signature=True, include_id=True, ) m.post('http://test.neko', exc=error) broadcast_move(serialized=serialized) assert node.last_connected_at == now
def broadcast_node_failed(fx_session: scoped_session, fx_other_session: Session, error): now = datetime.datetime.utcnow() node = Node(url='http://test.neko', last_connected_at=now) node2 = Node(url='http://other.neko', last_connected_at=datetime.datetime.utcnow()) fx_session.add(node) fx_session.commit() fx_other_session.add(node2) fx_other_session.commit() assert not fx_session.query(Node).filter(Node.url == node2.url).first() with Mocker() as m: m.post('http://test.neko', exc=error) broadcast_node(serialized={'url': fx_other_server.url}) assert not fx_session.query(Node).filter(Node.url == node2.url).first() assert node.last_connected_at == now
def test_broadcast_move_same_url(fx_session: scoped_session, fx_user: User, fx_novice_status: typing.Mapping[str, str]): url = 'http://test.neko' now = datetime.datetime.utcnow() node = Node(url=url, last_connected_at=now) move = fx_user.create_novice(fx_novice_status) fx_session.add_all([node, move]) fx_session.commit() with Mocker() as m: serialized = move.serialize( use_bencode=False, include_signature=True, include_id=True, ) broadcast_move(serialized=serialized, sent_node=node) assert not m.called assert node.last_connected_at == now
def test_broadcast_node( fx_server: WSGIServer, fx_session: scoped_session, fx_other_server: WSGIServer, fx_other_session: Session, ): now = datetime.datetime.utcnow() node = Node(url=fx_server.url, last_connected_at=now) node2 = Node(url=fx_other_server.url, last_connected_at=datetime.datetime.utcnow()) fx_session.add(node) fx_session.commit() fx_other_session.add(node2) fx_other_session.commit() assert not fx_session.query(Node).filter(Node.url == node2.url).first() broadcast_node(serialized={'url': fx_other_server.url}) assert fx_session.query(Node).filter(Node.url == node2.url).first() assert node.last_connected_at > now
def test_post_block_return_block_id(fx_test_client: FlaskClient, fx_user: User, fx_session: scoped_session): block = Block.create(fx_user, []) fx_session.add(block) fx_session.commit() block2 = Block.create(fx_user, []) des = block2.serialize(use_bencode=False, include_suffix=True, include_moves=True, include_hash=True) des['id'] = 3 resp = fx_test_client.post('/blocks', data=json.dumps(des), content_type='application/json') assert resp.status_code == 403 data = json.loads(resp.get_data()) assert data['result'] == 'failed' assert data['message'] == "new block isn't our next block." assert data['block_id'] == 2
def insert_classes(classes: List[Dict[str, Any]], session: scoped_session) -> Optional[Any]: """Insert all the classes as defined in the APIDocumentation into DB.""" # print(session.query(exists().where(RDFClass.name == "Datastream")).scalar()) class_list = [ RDFClass(name=class_["label"].strip('.')) for class_ in classes if "label" in class_ and not session.query(exists().where( RDFClass.name == class_["label"].strip('.'))).scalar() ] class_list = class_list + [ RDFClass(name=class_["title"].strip('.')) for class_ in classes if "title" in class_ and not session.query(exists().where( RDFClass.name == class_["title"].strip('.'))).scalar() ] # print(class_list) session.add_all(class_list) session.commit() return None
def test_block_broadcast(fx_session: scoped_session, fx_user: User): block = Block.create(fx_user, []) fx_session.add(block) fx_session.commit() with unittest.mock.patch('nekoyume.tasks.broadcast_block') as m: block_broadcast(block.id, 'http://localhost:5000', 'http://localhost:5001', session=fx_session) serialized = block.serialize(use_bencode=False, include_suffix=True, include_moves=True, include_hash=True) assert m.called args = m.call_args[1] assert serialized == args['serialized'] assert isinstance(args['sent_node'], Node) assert args['sent_node'].url == 'http://localhost:5000' assert isinstance(args['my_node'], Node) assert args['my_node'].url == 'http://localhost:5001'
def test_broadcast_move_my_node(fx_session: scoped_session, fx_user: User, fx_novice_status: typing.Mapping[str, str]): url = 'http://test.neko' now = datetime.datetime.utcnow() node = Node(url=url, last_connected_at=now) move = fx_user.create_novice(fx_novice_status) fx_session.add_all([node, move]) fx_session.commit() with Mocker() as m: m.post('http://test.neko/moves', json={'result': 'success'}) expected = serialized = move.serialize( use_bencode=False, include_signature=True, include_id=True, ) broadcast_move(serialized=serialized, my_node=node) expected['sent_node'] = 'http://test.neko' assert node.last_connected_at > now # check request.json value assert m.request_history[0].json() == expected
def delete(id_: int, type_: str, session: scoped_session) -> None: """Delete an Instance and all its relations from DB given id [DELETE].""" try: rdf_class = session.query(RDFClass).filter( RDFClass.name == type_).one() except NoResultFound: print(type_) raise ClassNotFound(type_=type_) try: instance = session.query(Instance).filter( Instance.id == id_ and type_ == rdf_class.id).one() except NoResultFound: raise InstanceNotFound(type_=rdf_class.name, id_=id_) data_IIT = session.query(triples).filter( triples.GraphIIT.subject == id_).all() data_IAC = session.query(triples).filter( triples.GraphIAC.subject == id_).all() data_III = session.query(triples).filter( triples.GraphIII.subject == id_).all() data = data_III + data_IIT + data_IAC for item in data: session.delete(item) for data in data_IIT: terminal = session.query(Terminal).filter( Terminal.id == data.object_).one() session.delete(terminal) for data in data_III: III_instance = session.query(Instance).filter( Instance.id == data.object_).one() III_instance_type = session.query(RDFClass).filter( RDFClass.id == III_instance.type_).one() # Get the III object type_ delete(III_instance.id, III_instance_type.name, session=session) session.delete(instance) session.commit()
async def fix_directories( posts: list[api_table], subscription: user_types, database_session: scoped_session, api_type: str, ): new_directories = [] authed = subscription.get_authed() api = authed.api site_settings = api.get_site_settings() async def fix_directories2( post: api_table, media_db: list[template_media_table], all_files: list[Path] ): delete_rows = [] final_api_type = ( os.path.join("Archived", api_type) if post.archived else api_type ) post_id = post.post_id media_db = [x for x in media_db if x.post_id == post_id] for media in media_db: media_id = media.media_id if media.link: url_path = urlparse.urlparse(media.link).path url_path = Path(url_path) else: url_path = Path(media.filename) new_filename = url_path.name original_filename, ext = (url_path.stem, url_path.suffix) ext = ext.replace(".", "") file_directory_format = site_settings.file_directory_format filename_format = site_settings.filename_format date_format = site_settings.date_format text_length = site_settings.text_length download_path = subscription.directory_manager.root_download_directory option = {} option["site_name"] = api.site_name option["post_id"] = post_id option["media_id"] = media_id option["profile_username"] = authed.username option["model_username"] = subscription.username option["api_type"] = final_api_type option["media_type"] = media.media_type option["filename"] = original_filename option["ext"] = ext option["text"] = post.text option["postedAt"] = media.created_at option["price"] = post.price option["date_format"] = date_format option["text_length"] = text_length option["directory"] = download_path option["preview"] = media.preview option["archived"] = post.archived prepared_format = prepare_reformat(option) file_directory = await prepared_format.reformat_2(file_directory_format) prepared_format.directory = file_directory old_filepath = "" if media.linked: filename_format = filename_format.with_name(f"linked_{filename_format}") new_filepath = await prepared_format.reformat_2(filename_format) old_filepaths = [ x for x in all_files if original_filename in x.name and x.parts != new_filepath.parts ] if not old_filepaths: old_filepaths = [x for x in all_files if str(media_id) in x.name] print if not media.linked: old_filepaths: list[Path] = [ x for x in old_filepaths if "linked_" not in x.parts ] if old_filepaths: old_filepath = old_filepaths[0] # a = randint(0,1) # await asyncio.sleep(a) if old_filepath and old_filepath != new_filepath: moved = None while not moved: try: if old_filepath.exists(): _old_filename, old_ext = (url_path.stem, url_path.suffix) if ".part" == old_ext: old_filepath.unlink() continue if media.size: media.downloaded = True found_dupes = [ x for x in media_db if x.filename == new_filename and x.id != media.id ] delete_rows.extend(found_dupes) os.makedirs(os.path.dirname(new_filepath), exist_ok=True) if media.linked: if os.path.dirname(old_filepath) == os.path.dirname( new_filepath ): moved = shutil.move(old_filepath, new_filepath) else: moved = shutil.copy(old_filepath, new_filepath) else: moved = shutil.move(old_filepath, new_filepath) else: break except OSError as e: print(traceback.format_exc()) print print if os.path.exists(new_filepath): if media.size: media.downloaded = True if prepared_format.text: pass media.directory = file_directory.as_posix() media.filename = os.path.basename(new_filepath) new_directories.append(os.path.dirname(new_filepath)) return delete_rows base_directory = subscription.directory_manager.user.find_legacy_directory( "download", api_type ) temp_files: list[Path] = await subscription.directory_manager.walk(base_directory) result = database_session.query(user_database.media_table) media_db = result.all() pool = api.pool # tasks = pool.starmap(fix_directories2, product(posts, [media_db])) tasks = [ asyncio.ensure_future(fix_directories2(post, media_db, temp_files)) for post in posts ] settings = {"colour": "MAGENTA", "disable": False} delete_rows = await tqdm.gather(*tasks, **settings) delete_rows = list(chain(*delete_rows)) for delete_row in delete_rows: database_session.query(user_database.media_table).filter( user_database.media_table.id == delete_row.id ).delete() database_session.commit() new_directories = list(set(new_directories)) return posts, new_directories
def insert(object_: Dict[str, Any], session: scoped_session, id_: Optional[int] = None) -> int: """Insert an object to database [POST] and returns the inserted object.""" rdf_class = None instance = None # Check for class in the begging try: rdf_class = session.query(RDFClass).filter( RDFClass.name == object_["@type"]).one() except NoResultFound: raise ClassNotFound(type_=object_["@type"]) if id_ is not None: if session.query(exists().where(Instance.id == id_)).scalar(): raise InstanceExists(type_=rdf_class.name, id_=id_) else: instance = Instance(id=id_, type_=rdf_class.id) else: instance = Instance(type_=rdf_class.id) session.add(instance) session.flush() for prop_name in object_: if prop_name not in ["@type", "@context"]: try: property_ = session.query(properties).filter( properties.name == prop_name).one() except NoResultFound: # Adds new Property session.close() raise PropertyNotFound(type_=prop_name) # For insertion in III if type(object_[prop_name]) == dict: instance_id = insert(object_[prop_name], session=session) instance_object = session.query(Instance).filter( Instance.id == instance_id).one() if property_.type_ == "PROPERTY" or property_.type_ == "INSTANCE": property_.type_ = "INSTANCE" session.add(property_) triple = GraphIII(subject=instance.id, predicate=property_.id, object_=instance_object.id) session.add(triple) else: session.close() raise NotInstanceProperty(type_=prop_name) # For insertion in IAC elif session.query(exists().where( RDFClass.name == str(object_[prop_name]))).scalar(): if property_.type_ == "PROPERTY" or property_.type_ == "ABSTRACT": property_.type_ = "ABSTRACT" session.add(property_) class_ = session.query(RDFClass).filter( RDFClass.name == object_[prop_name]).one() triple = GraphIAC(subject=instance.id, predicate=property_.id, object_=class_.id) session.add(triple) else: session.close() raise NotAbstractProperty(type_=prop_name) # For insertion in IIT else: terminal = Terminal(value=object_[prop_name]) session.add(terminal) session.flush() # Assigns ID without committing if property_.type_ == "PROPERTY" or property_.type_ == "INSTANCE": property_.type_ = "INSTANCE" session.add(property_) triple = GraphIIT(subject=instance.id, predicate=property_.id, object_=terminal.id) # Add things directly to session, if anything fails whole transaction is aborted session.add(triple) else: session.close() raise NotInstanceProperty(type_=prop_name) session.commit() return instance.id
def insert(object_: Dict[str, Any], session: scoped_session, id_: Optional[str] = None) -> str: """Insert an object to database [POST] and returns the inserted object. :param object_: object to be inserted :param session: sqlalchemy scoped session :param id_: id of the object to be inserted (optional param) :return: ID of object inserted Raises: ClassNotFound: If `object_["@type"] is not a valid/defined RDFClass. InstanceExists: If an Instance `id_` already exists. PropertyNotFound: If any property name of `object_` other than `@type` or `@context` is not a valid/defined property. NotInstanceProperty: If any property of `object_` is a dictionary but not an Instance property NotAbstractProperty: If any property of `object_` is a valid/defined RDFClass but is not a dictionary neither an Abstract Property """ rdf_class = None instance = None # Check for class in the begging try: rdf_class = session.query(RDFClass).filter( RDFClass.name == object_["@type"]).one() except NoResultFound: raise ClassNotFound(type_=object_["@type"]) if id_ is not None and session.query(exists().where(Instance.id == id_)).scalar(): raise InstanceExists(type_=rdf_class.name, id_=id_) elif id_ is not None: instance = Instance(id=id_, type_=rdf_class.id) else: instance = Instance(type_=rdf_class.id) session.add(instance) session.flush() for prop_name in object_: if prop_name not in ["@type", "@context"]: try: property_ = session.query(properties).filter( properties.name == prop_name).one() except NoResultFound: # Adds new Property session.close() raise PropertyNotFound(type_=prop_name) # For insertion in III if isinstance(object_[prop_name], dict): instance_id = insert(object_[prop_name], session=session) instance_object = session.query(Instance).filter( Instance.id == instance_id).one() if property_.type_ == "PROPERTY" or property_.type_ == "INSTANCE": property_.type_ = "INSTANCE" session.add(property_) triple = GraphIII( subject=instance.id, predicate=property_.id, object_=instance_object.id) session.add(triple) else: session.close() raise NotInstanceProperty(type_=prop_name) # For insertion in IAC elif session.query(exists().where(RDFClass.name == str(object_[prop_name]))).scalar() \ and property_.type_ == "PROPERTY" or property_.type_ == "ABSTRACT": property_.type_ = "ABSTRACT" session.add(property_) class_ = session.query(RDFClass).filter( RDFClass.name == object_[prop_name]).one() triple = GraphIAC( subject=instance.id, predicate=property_.id, object_=class_.id) session.add(triple) elif session.query(exists().where(RDFClass.name == str(object_[prop_name]))).scalar(): session.close() raise NotAbstractProperty(type_=prop_name) # For insertion in IIT else: terminal = Terminal(value=object_[prop_name]) session.add(terminal) session.flush() # Assigns ID without committing if property_.type_ == "PROPERTY" or property_.type_ == "INSTANCE": property_.type_ = "INSTANCE" session.add(property_) triple = GraphIIT( subject=instance.id, predicate=property_.id, object_=terminal.id) # Add things directly to session, if anything fails whole # transaction is aborted session.add(triple) else: session.close() raise NotInstanceProperty(type_=prop_name) session.commit() return instance.id
def insert_multiple(objects_: List[Dict[str, Any]], session: scoped_session, id_: Optional[str] = "") -> List[int]: """ Adds a list of object with given ids to the database :param objects_: List of dict's to be added to the database :param session: scoped session from getSession in utils :param id_: optional parameter containing the ids of objects that have to be inserted :return: Ids that have been inserted """ # instance list to store instances instance_list = list() triples_list = list() properties_list = list() instances = list() id_list = id_.split(',') instance_id_list = list() # the number of objects would be the same as number of instances for index in range(len(objects_)): try: rdf_class = session.query(RDFClass).filter( RDFClass.name == objects_[index]["@type"]).one() except NoResultFound: raise ClassNotFound(type_=objects_[index]["@type"]) if index in range(len(id_list)) and id_list[index] != "": if session.query( exists().where( Instance.id == id_list[index])).scalar(): print(session.query( exists().where( Instance.id == id_list[index]))) # TODO handle where intance already exists , if instance is # fetched later anyways remove this raise InstanceExists(type_=rdf_class.name, id_=id_list[index]) else: instance = Instance(id=id_list[index], type_=rdf_class.id) instances.append(instance) else: instance = Instance(type_=rdf_class.id) instances.append(instance) session.add_all(instances) session.flush() for i in range(len(instances)): instance_id_list.append(instances[i].id) for index in range(len(objects_)): for prop_name in objects_[index]: if prop_name not in ["@type", "@context"]: try: property_ = session.query(properties).filter( properties.name == prop_name).one() except NoResultFound: # Adds new Property session.close() raise PropertyNotFound(type_=prop_name) # For insertion in III if isinstance(objects_[index][prop_name], dict): instance_id = insert( objects_[index][prop_name], session=session) instance_object = session.query(Instance).filter( Instance.id == instance_id).one() if property_.type_ == "PROPERTY" or property_.type_ == "INSTANCE": property_.type_ = "INSTANCE" properties_list.append(property_) triple = GraphIII( subject=instances[index].id, predicate=property_.id, object_=instance_object.id) triples_list.append(triple) else: session.close() raise NotInstanceProperty(type_=prop_name) # For insertion in IAC elif session.query( exists().where(RDFClass.name == str(objects_[index][prop_name]))).scalar(): if property_.type_ == "PROPERTY" or property_.type_ == "ABSTRACT": property_.type_ = "ABSTRACT" properties_list.append(property_) class_ = session.query(RDFClass).filter( RDFClass.name == objects_[index][prop_name]).one() triple = GraphIAC( subject=instances[index].id, predicate=property_.id, object_=class_.id) triples_list.append(triple) else: session.close() raise NotAbstractProperty(type_=prop_name) # For insertion in IIT else: terminal = Terminal(value=objects_[index][prop_name]) session.add(terminal) session.flush() # Assigns ID without committing if property_.type_ == "PROPERTY" or property_.type_ == "INSTANCE": property_.type_ = "INSTANCE" properties_list.append(property_) triple = GraphIIT( subject=instances[index].id, predicate=property_.id, object_=terminal.id) # Add things directly to session, if anything fails # whole transaction is aborted triples_list.append(triple) else: session.close() raise NotInstanceProperty(type_=prop_name) session.bulk_save_objects(properties_list) session.bulk_save_objects(triples_list) session.commit() return instance_id_list
def fix_directories(api,posts, all_files, database_session: scoped_session, folder, site_name, parent_type, api_type, username, base_directory, json_settings): new_directories = [] def fix_directories(post: api_table, media_db: list[media_table]): delete_rows = [] final_type = "" if parent_type: final_type = f"{api_type}{os.path.sep}{parent_type}" final_type = final_type if final_type else api_type post_id = post.post_id media_db = [x for x in media_db if x.post_id == post_id] for media in media_db: media_id = media.media_id if media.link: path = urlparse.urlparse(media.link).path else: path: str = media.filename new_filename = os.path.basename(path) original_filename, ext = os.path.splitext(new_filename) ext = ext.replace(".", "") file_directory_format = json_settings["file_directory_format"] filename_format = json_settings["filename_format"] date_format = json_settings["date_format"] text_length = json_settings["text_length"] download_path = base_directory today = datetime.today() today = today.strftime("%d-%m-%Y %H:%M:%S") option = {} option["site_name"] = site_name option["post_id"] = post_id option["media_id"] = media_id option["username"] = username option["api_type"] = final_type if parent_type else api_type option["media_type"] = media.media_type option["filename"] = original_filename option["ext"] = ext option["text"] = post.text option["postedAt"] = media.created_at option["price"] = post.price option["date_format"] = date_format option["text_length"] = text_length option["directory"] = download_path option["preview"] = media.preview prepared_format = prepare_reformat(option) file_directory = main_helper.reformat( prepared_format, file_directory_format) prepared_format.directory = file_directory old_filepath = "" if media.linked: filename_format = f"linked_{filename_format}" old_filepaths = [ x for x in all_files if original_filename in os.path.basename(x)] if not old_filepaths: old_filepaths = [ x for x in all_files if str(media_id) in os.path.basename(x)] print if not media.linked: old_filepaths = [x for x in old_filepaths if "linked_" not in x] if old_filepaths: old_filepath = old_filepaths[0] new_filepath = main_helper.reformat( prepared_format, filename_format) if old_filepath and old_filepath != new_filepath: if os.path.exists(new_filepath): os.remove(new_filepath) moved = None while not moved: try: if os.path.exists(old_filepath): if media.size: media.downloaded = True found_dupes = [ x for x in media_db if x.filename == new_filename and x.id != media.id] delete_rows.extend(found_dupes) os.makedirs(os.path.dirname( new_filepath), exist_ok=True) if media.linked: if os.path.dirname(old_filepath) == os.path.dirname(new_filepath): moved = shutil.move(old_filepath, new_filepath) else: moved = shutil.copy(old_filepath, new_filepath) else: moved = shutil.move(old_filepath, new_filepath) else: break except OSError as e: print(traceback.format_exc()) print print if os.path.exists(new_filepath): if media.size: media.downloaded = True if prepared_format.text: pass media.directory = file_directory media.filename = os.path.basename(new_filepath) new_directories.append(os.path.dirname(new_filepath)) return delete_rows result = database_session.query(folder.media_table) media_db = result.all() pool = api.pool delete_rows = pool.starmap(fix_directories, product( posts, [media_db])) delete_rows = list(chain(*delete_rows)) for delete_row in delete_rows: database_session.query(folder.media_table).filter( folder.media_table.id == delete_row.id).delete() database_session.commit() new_directories = list(set(new_directories)) return posts, new_directories
def insert_object(object_: Dict[str, Any], session: scoped_session, collection: bool = False) -> str: """ Insert the object in the database :param object_: Dict containing object properties :param session: sqlalchemy session :return: The ID of the inserted object """ type_ = get_type(object_) database_class = get_database_class(type_) id_ = object_.get("id", None) if collection: # if type_ is of a collection class members = object_['members'] collection_id = id_ if id_ else str(uuid.uuid4()) for member in members: # add all the members of that collection inserted_object = database_class( members=member['id_'], collection_id=collection_id, member_type=member['@type'], ) try: session.add(inserted_object) session.commit() except InvalidRequestError: session.rollback() return collection_id else: # when type_ is of a non-collection class if (id_ is not None and session.query( exists().where(database_class.id == id_)).scalar()): raise InstanceExists(type_, id_) foreign_keys = database_class.__table__.foreign_keys for fk in foreign_keys: # the name of the column through which this foreign key relationship # is being established fk_column = fk.info["column_name"] try: fk_object = object_[fk_column] except KeyError as e: wrong_property = e.args[0] raise PropertyNotGiven(type_=wrong_property) # insert the foreign key object fk_object_id = insert_object(fk_object, session) # put the id of the foreign instance in this table's column object_[fk_column] = fk_object_id try: # remove the @type from object before using the object to make a # instance of it using sqlalchemy class object_.pop("@type") inserted_object = database_class(**object_) except TypeError as e: # extract the wrong property name from TypeError object wrong_property = e.args[0].split("'")[1] raise PropertyNotFound(type_=wrong_property) try: session.add(inserted_object) session.commit() except InvalidRequestError: session.rollback() return inserted_object.id
async def fix_directories( posts, api: onlyfans.start, subscription: create_user, all_files, database_session: scoped_session, folder, site_name, api_type, base_directory, json_settings, ): new_directories = [] async def fix_directories2(post: api_table, media_db: list[template_media_table]): delete_rows = [] final_api_type = (os.path.join("Archived", api_type) if post.archived else api_type) post_id = post.post_id media_db = [x for x in media_db if x.post_id == post_id] for media in media_db: media_id = media.media_id if media.link: path = urlparse.urlparse(media.link).path else: path: str = media.filename new_filename = os.path.basename(path) original_filename, ext = os.path.splitext(new_filename) ext = ext.replace(".", "") file_directory_format = json_settings["file_directory_format"] filename_format = json_settings["filename_format"] date_format = json_settings["date_format"] text_length = json_settings["text_length"] download_path = base_directory option = {} option["site_name"] = site_name option["post_id"] = post_id option["media_id"] = media_id option["profile_username"] = subscription.subscriber.username option["model_username"] = subscription.username option["api_type"] = final_api_type option["media_type"] = media.media_type option["filename"] = original_filename option["ext"] = ext option["text"] = post.text option["postedAt"] = media.created_at option["price"] = post.price option["date_format"] = date_format option["text_length"] = text_length option["directory"] = download_path option["preview"] = media.preview option["archived"] = post.archived prepared_format = prepare_reformat(option) file_directory = await main_helper.reformat( prepared_format, file_directory_format) prepared_format.directory = file_directory old_filepath = "" if media.linked: filename_format = f"linked_{filename_format}" old_filepaths = [ x for x in all_files if original_filename in os.path.basename(x) ] if not old_filepaths: old_filepaths = [ x for x in all_files if str(media_id) in os.path.basename(x) ] print if not media.linked: old_filepaths = [ x for x in old_filepaths if "linked_" not in x ] if old_filepaths: old_filepath = old_filepaths[0] # a = randint(0,1) # await asyncio.sleep(a) new_filepath = await main_helper.reformat(prepared_format, filename_format) if old_filepath and old_filepath != new_filepath: if os.path.exists(new_filepath): os.remove(new_filepath) moved = None while not moved: try: if os.path.exists(old_filepath): if media.size: media.downloaded = True found_dupes = [ x for x in media_db if x.filename == new_filename and x.id != media.id ] delete_rows.extend(found_dupes) os.makedirs(os.path.dirname(new_filepath), exist_ok=True) if media.linked: if os.path.dirname( old_filepath) == os.path.dirname( new_filepath): moved = shutil.move( old_filepath, new_filepath) else: moved = shutil.copy( old_filepath, new_filepath) else: moved = shutil.move(old_filepath, new_filepath) else: break except OSError as e: print(traceback.format_exc()) print print if os.path.exists(new_filepath): if media.size: media.downloaded = True if prepared_format.text: pass media.directory = file_directory media.filename = os.path.basename(new_filepath) new_directories.append(os.path.dirname(new_filepath)) return delete_rows result = database_session.query(folder.media_table) media_db = result.all() pool = api.pool # tasks = pool.starmap(fix_directories2, product(posts, [media_db])) tasks = [ asyncio.ensure_future(fix_directories2(post, media_db)) for post in posts ] settings = {"colour": "MAGENTA", "disable": False} delete_rows = await tqdm.gather(*tasks, **settings) delete_rows = list(chain(*delete_rows)) for delete_row in delete_rows: database_session.query(folder.media_table).filter( folder.media_table.id == delete_row.id).delete() database_session.commit() new_directories = list(set(new_directories)) return posts, new_directories