def setUp(self):
database_factory = DatabaseFactory(
settings.SQL_HOST, settings.SQL_PORT, settings.SQL_USER,
settings.SQL_PASSWD, settings.SQL_DB, settings.SQL_COLLECTION_NAME)
self._cache_db = database_factory.get_database(CollectionTypes.cache)
self._history_db = database_factory.get_database(
CollectionTypes.history)
self._item_db = database_factory.get_database(CollectionTypes.item)
self._page_url = "https://sa.opensooq.com/ar/search/30002057/استراحة-سديم-للايجار-اليومي-والشهري-والسنوي-حي-الأمانة-شمال-الرياض"
self._page_from = WebsiteTypes.opensooq.value
self._ads_id = 70
self._ads_item = Ad.get_default(
section_item=SectionItem.get_default(),
ads_title="اسكاليد موديل 2016 بسعر جي",
city_id='44',
ads_contact="123454321",
ads_body="test mysql db",
image_link=
"https://img1cdn.haraj.com.sa/userfiles30/2015-07-18/55aa1ba3366cd.jpeg,https://img1cdn.haraj.com.sa/userfiles30/2015-07-18/55aa1bab49a3b.jpeg,https://img1cdn.haraj.com.sa/userfiles30/2015-07-18/55aa1bb0b6ca6.jpeg",
His_announcement_id=60,
type_ads_or=1,
_close_ads=0)
self._member_id = 60
self._memberName = "djzhang"
self.opensooq_phone_id = 24
def setUp(self):
database_factory = DatabaseFactory(settings.SQL_HOST, settings.SQL_PORT,
settings.SQL_USER, settings.SQL_PASSWD,
settings.SQL_DB, settings.SQL_COLLECTION_NAME)
self._cache_db = database_factory.get_database(CollectionTypes.cache)
self._history_db = database_factory.get_database(CollectionTypes.history)
self._item_db = database_factory.get_database(CollectionTypes.item)
self._page_url = "https://sa.opensooq.com/ar/search/30002057/استراحة-سديم-للايجار-اليومي-والشهري-والسنوي-حي-الأمانة-شمال-الرياض"
self._page_from = WebsiteTypes.opensooq.value
self._ads_id = 70
self._ads_item = Ad.get_default(
section_item=SectionItem.get_default(),
ads_title="اسكاليد موديل 2016 بسعر جي",
city_id='44',
ads_contact="123454321",
ads_body="test mysql db",
image_link="https://img1cdn.haraj.com.sa/userfiles30/2015-07-18/55aa1ba3366cd.jpeg,https://img1cdn.haraj.com.sa/userfiles30/2015-07-18/55aa1bab49a3b.jpeg,https://img1cdn.haraj.com.sa/userfiles30/2015-07-18/55aa1bb0b6ca6.jpeg",
His_announcement_id=60,
type_ads_or=1, _close_ads=0
)
self._member_id = 60
self._memberName = "djzhang"
self.opensooq_phone_id = 24
def setUp(self):
database_factory = DatabaseFactory(settings.SQL_HOST, settings.SQL_PORT,
settings.SQL_USER, settings.SQL_PASSWD,
settings.SQL_DB, settings.SQL_COLLECTION_NAME)
self._item_db = database_factory.get_database(CollectionTypes.item)
self.section_data = SectionData()
self.expect = self.section_data.expect
self.section_mgr = HarajsSection(self.section_data.sections, self._item_db)
def __init__(self, name=None, **kwargs):
from cwharaj.database_factory import DatabaseFactory, CollectionTypes
database_factory = DatabaseFactory(kwargs['host'], kwargs['port'],
kwargs['user'], kwargs['passwd'],
kwargs['db'], kwargs['collection_name'])
self._cache_db = database_factory.get_database(CollectionTypes.cache)
self._history_db = database_factory.get_database(CollectionTypes.history)
self._item_db = database_factory.get_database(CollectionTypes.item)
from cwharaj.parser.opensooq_parser import OpensooqParse
self.opensooq_parse = OpensooqParse()
super(OpensooqDebugCommentDateSpider, self).__init__(name, **kwargs)
def __init__(self, name=None, **kwargs):
from cwharaj.database_factory import DatabaseFactory, CollectionTypes
database_factory = DatabaseFactory(kwargs['host'], kwargs['port'],
kwargs['user'], kwargs['passwd'],
kwargs['db'],
kwargs['collection_name'])
self._cache_db = database_factory.get_database(CollectionTypes.cache)
self._history_db = database_factory.get_database(
CollectionTypes.history)
self._item_db = database_factory.get_database(CollectionTypes.item)
from cwharaj.parser.opensooq_parser import OpensooqParse
self.opensooq_parse = OpensooqParse()
super(OpensooqDebugWatchSpider, self).__init__(name, **kwargs)
def __init__(self, name=None, **kwargs):
from cwharaj.database_factory import DatabaseFactory, CollectionTypes
database_factory = DatabaseFactory(kwargs['host'], kwargs['port'],
kwargs['user'], kwargs['passwd'],
kwargs['db'], kwargs['collection_name'])
self._cache_db = database_factory.get_database(CollectionTypes.cache)
self._history_db = database_factory.get_database(CollectionTypes.history)
self._item_db = database_factory.get_database(CollectionTypes.item)
from cwharaj.spiders.dispatch.spider_watch_dispatch import SpiderWatchDispatch
self.watch_dispatch = SpiderWatchDispatch()
# Dynamic the domains and start url.
self.allowed_domains = self.watch_dispatch.get_allowed_domains()
self.start_urls = self.watch_dispatch.get_pagination_websites()
super(HarajsSpiderWatch, self).__init__(name, **kwargs)
def __init__(self, name=None, **kwargs):
from cwharaj.database_factory import DatabaseFactory, CollectionTypes
database_factory = DatabaseFactory(kwargs['host'], kwargs['port'],
kwargs['user'], kwargs['passwd'],
kwargs['db'],
kwargs['collection_name'])
self._cache_db = database_factory.get_database(CollectionTypes.cache)
self._history_db = database_factory.get_database(
CollectionTypes.history)
self._item_db = database_factory.get_database(CollectionTypes.item)
from cwharaj.spiders.dispatch.spider_watch_dispatch import SpiderWatchDispatch
self.watch_dispatch = SpiderWatchDispatch()
# Dynamic the domains and start url.
self.allowed_domains = self.watch_dispatch.get_allowed_domains()
self.start_urls = self.watch_dispatch.get_pagination_websites()
super(HarajsSpiderWatch, self).__init__(name, **kwargs)
def __init__(self, name=None, **kwargs):
self.allowed_domains = [websites_allowed_domains.get(self.url_from)]
if is_pagination:
self.start_urls = [WebsiteTypes.get_pagination_url(self.url_from)]
else:
self.start_urls = self.details_urls
from cwharaj.database_factory import DatabaseFactory, CollectionTypes
database_factory = DatabaseFactory(kwargs['host'], kwargs['port'],
kwargs['user'], kwargs['passwd'],
kwargs['db'], kwargs['collection_name'])
self._cache_db = database_factory.get_database(CollectionTypes.cache)
self._history_db = database_factory.get_database(CollectionTypes.history)
self._item_db = database_factory.get_database(CollectionTypes.item)
from cwharaj.parser.opensooq_parser import OpensooqParse
self._parser = OpensooqParse()
super(OpensooqDebugSpider, self).__init__(name, **kwargs)
def __init__(self, name=None, **kwargs):
from cwharaj.database_factory import DatabaseFactory, CollectionTypes
database_factory = DatabaseFactory(kwargs['host'], kwargs['port'],
kwargs['user'], kwargs['passwd'],
kwargs['db'], kwargs['collection_name'])
self._cache_db = database_factory.get_database(CollectionTypes.cache)
self._history_db = database_factory.get_database(CollectionTypes.history)
self._item_db = database_factory.get_database(CollectionTypes.item)
from cwharaj.spiders.dispatch.spider_dispatch import SpiderDispatch
self.spider_dispatch = SpiderDispatch()
# Dynamic the domains
self.allowed_domains = self.spider_dispatch.get_allowed_domains()
# Get the start urls from the cache database
row = self.get_row_from_cache("", "")
if row:
self.start_urls = [row['url']]
super(HarajsSpider, self).__init__(name, **kwargs)
def __init__(self, name=None, **kwargs):
self.allowed_domains = [websites_allowed_domains.get(self.url_from)]
if is_pagination:
self.start_urls = [WebsiteTypes.get_pagination_url(self.url_from)]
else:
self.start_urls = self.details_urls
from cwharaj.database_factory import DatabaseFactory, CollectionTypes
database_factory = DatabaseFactory(kwargs['host'], kwargs['port'],
kwargs['user'], kwargs['passwd'],
kwargs['db'],
kwargs['collection_name'])
self._cache_db = database_factory.get_database(CollectionTypes.cache)
self._history_db = database_factory.get_database(
CollectionTypes.history)
self._item_db = database_factory.get_database(CollectionTypes.item)
from cwharaj.parser.mstaml_parser import MstamlParse
self._parser = MstamlParse()
super(MstamlDebugWatchSpider, self).__init__(name, **kwargs)
def __init__(self, name=None, **kwargs):
from cwharaj.database_factory import DatabaseFactory, CollectionTypes
database_factory = DatabaseFactory(kwargs['host'], kwargs['port'],
kwargs['user'], kwargs['passwd'],
kwargs['db'],
kwargs['collection_name'])
self._cache_db = database_factory.get_database(CollectionTypes.cache)
self._history_db = database_factory.get_database(
CollectionTypes.history)
self._item_db = database_factory.get_database(CollectionTypes.item)
from cwharaj.spiders.dispatch.spider_dispatch import SpiderDispatch
self.spider_dispatch = SpiderDispatch()
# Dynamic the domains
self.allowed_domains = self.spider_dispatch.get_allowed_domains()
# Get the start urls from the cache database
row = self.get_row_from_cache("", "")
if row:
self.start_urls = [row['url']]
super(HarajsSpider, self).__init__(name, **kwargs)