def __init__(self, login_or_token=None, password=None, jwt=None, base_url=DEFAULT_BASE_URL, timeout=DEFAULT_TIMEOUT, client_id=None, client_secret=None, user_agent='PyGithub/Python', per_page=DEFAULT_PER_PAGE, api_preview=False, verify=True, retry=None):
"""
:param login_or_token: string
:param password: string
:param base_url: string
:param timeout: integer
:param client_id: string
:param client_secret: string
:param user_agent: string
:param per_page: int
:param verify: boolean or string
:param retry: int or urllib3.util.retry.Retry object
"""
assert login_or_token is None or isinstance(login_or_token, (str, unicode)), login_or_token
assert password is None or isinstance(password, (str, unicode)), password
assert jwt is None or isinstance(jwt, (str, unicode)), jwt
assert isinstance(base_url, (str, unicode)), base_url
assert isinstance(timeout, (int, long)), timeout
assert client_id is None or isinstance(client_id, (str, unicode)), client_id
assert client_secret is None or isinstance(client_secret, (str, unicode)), client_secret
assert user_agent is None or isinstance(user_agent, (str, unicode)), user_agent
assert isinstance(api_preview, (bool))
assert retry is None or isinstance(retry, (int)) or isinstance(retry, (urllib3.util.Retry))
self.__requester = Requester(login_or_token, password, jwt, base_url, timeout, client_id, client_secret, user_agent, per_page, api_preview, verify, retry)
def __init__(self):
self.logger = logging.getLogger()
self.logger.setLevel(logging.INFO)
self.cash = None
self.date = None
self.holdings = {}
self.r = Requester()
def __init__(self,
login_or_token=None,
password=None,
base_url=DEFAULT_BASE_URL,
timeout=DEFAULT_TIMEOUT):
self.__requester = Requester(login_or_token, password, base_url,
timeout)
def __init__(self,
login_or_token=None,
password=None,
base_url=DEFAULT_BASE_URL,
timeout=DEFAULT_TIMEOUT,
client_id=None,
client_secret=None,
user_agent='PyGithub/Python',
per_page=DEFAULT_PER_PAGE):
"""
:param login_or_token: string
:param password: string
:param base_url: string
:param timeout: integer
:param client_id: string
:param client_secret: string
:param user_agent: string
:param per_page: int
"""
assert login_or_token is None or isinstance(
login_or_token, (str, unicode)), login_or_token
assert password is None or isinstance(password,
(str, unicode)), password
assert isinstance(base_url, (str, unicode)), base_url
assert isinstance(timeout, (int, long)), timeout
assert client_id is None or isinstance(client_id,
(str, unicode)), client_id
assert client_secret is None or isinstance(
client_secret, (str, unicode)), client_secret
assert user_agent is None or isinstance(user_agent,
(str, unicode)), user_agent
self.__requester = Requester(login_or_token, password, base_url,
timeout, client_id, client_secret,
user_agent, per_page)
def run_requester_threads(thread_number):
for i in range(thread_number):
thread = Requester(i, wordpress_url, plugins_directory,
sleep_between_req_in_milis, proxies,
basic_auth_user, basic_auth_password)
thread.start()
threads.append(thread)
Printer.p(NAME, 'Requester threads started')
def __init__(self,
login_or_token=None,
password=None,
base_url=DEFAULT_BASE_URL,
timeout=DEFAULT_TIMEOUT,
client_id=None,
client_secret=None,
user_agent=None):
self.__requester = Requester(login_or_token, password, base_url,
timeout, client_id, client_secret,
user_agent)
def __init__(self, dictionary, logger, perm_dict):
# email, password, name, server_id, server_name
threading.Thread.__init__(self)
self.email = dictionary["email"]
self.password = dictionary["password"]
self.server = {
"id": dictionary["server_id"],
"status": None,
"name": dictionary["server_name"],
"last_login": None,
"url": {
"region": None,
"map": None,
"main": None
}
}
self.data = {
"player": {
"-1": {
"nick": dictionary["name"]
}
},
# player with index -1 is used when player_id is None so we can have name for logs before
# logging in
"player_rank": {},
"alliance": {},
"habitat": {},
"habitat_unit": [],
"report": {},
"transit": [],
"diplomacy": {
"red": [],
"green": [],
"blue": [],
"orange": []
}
}
self.player_id = "-1"
self.requester = Requester(self)
self.tasks = []
self.scheduled_tasks = {}
self.active = True
self.last_time_idle = False
self.logger = logger
self.perm_dict = perm_dict
self.add_startup_tasks()
print(str(self))
def __init__(self) -> None:
# get all the api keys
load_dotenv()
self.rapidAPIKey = os.getenv('RAPID_API_KEY')
# dictionaries containing api urls and host urls from rapid api
self.apiURLs = {
# each key maps to a list with the first idex containing the api url and
# the second containing the host url
"edamam": [os.getenv('EDAMAM_URL'),
os.getenv('EDAMAM_HOST_URL')],
# ... More to add
}
self.requester = Requester()
def main():
if len(sys.argv) > 1:
config_path = sys.argv[1]
else:
config_path = './configs/config_default.txt'
if not Path(config_path).is_file():
logging.error("Could not find config file!")
sys.exit(1) # exiting with error code
# load config
config = configparser.ConfigParser()
config.read(config_path)
log_dir = config['PATHS']['log_dir']
log_file_name = config['PATHS']['log_file_name']
# check if config dir is present
if not Path(log_dir).is_dir():
logging.error("Logging directory is not present!")
sys.exit(1) # exiting with error code
file_handler = TimedRotatingFileHandler(os.path.join(
os.path.dirname(__file__), log_dir, log_file_name),
when='midnight',
interval=1)
console_handler = logging.StreamHandler()
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
handlers=[file_handler, console_handler])
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("apscheduler.scheduler").setLevel(logging.WARNING)
logging.getLogger("apscheduler.executors.default").setLevel(
logging.WARNING)
logging.getLogger("chardet.charsetprober").setLevel(logging.WARNING)
logger.info("=======Starting=Crawler=========")
# store config preferences in variables
article_download_pattern = ([
(int(config['ARTICLE_DOWNLOAD_PATTERN']['number']),
int(config['ARTICLE_DOWNLOAD_PATTERN']['delay'])),
]) # [(application number, period in seconds) ... ]
number_download_worker = int(config['CRAWLING']['number_download_worker'])
website_request_timeout = int(
config['REQUESTS']['website_request_timeout'])
rss_feed_crawl_period = int(config['CRAWLING']['rss_feed_crawl_period'])
rss_feed_request_timeout = int(
config['REQUESTS']['rss_feed_request_timeout'])
warmup_iterations = int(config['CRAWLING']['warmup_iterations'])
throttle_velocity = float(config['CRAWLING']['throttle_velocity'])
max_offset = int(config['CRAWLING']['max_offset'])
downloads_path = config['PATHS']['downloads']
crawled_rss_articles_path = config['PATHS']['rss_articles']
feed_path = config['PATHS']['feeds_list']
requests_path = config['PATHS']['requests']
# partly validating the config
if not Path(feed_path).is_file():
logging.error("Could not find RSS feeds list file!")
sys.exit(1) # exiting with error code
parent_dir = os.path.dirname(requests_path)
if not Path(parent_dir).is_dir():
logging.error("Could not find requests directory!")
sys.exit(1) # exiting with error code
writer = Writer()
writer.start()
throttle = Throttle(request_velocity=throttle_velocity)
rss_requester = Requester(tag="RSS Requester",
path=requests_path,
throttle=throttle)
website_requester = Requester(tag="Website Requester",
path=requests_path,
throttle=throttle)
scheduler = Scheduler(patterns=article_download_pattern)
crawler = Crawler(requester=rss_requester,
scheduler=scheduler,
feed_path=feed_path,
crawled_rss_articles_path=crawled_rss_articles_path,
rss_feed_crawl_period=rss_feed_crawl_period,
rss_feed_request_timeout=rss_feed_request_timeout,
warmup_iterations=warmup_iterations,
max_offset=max_offset)
crawler.start()
for i in range(number_download_worker):
logger.info("Starting download worker #%d", i)
DownloadWorker(requester=website_requester,
timeout=website_request_timeout,
path=downloads_path).start()
while True:
time.sleep(60)
logger.debug("Number of threads running: %d", threading.active_count())
process = psutil.Process(os.getpid())
ram_usage = process.memory_full_info()
# percent = absolute/mem.total
logger.info("RAM usage: %s%%, %s", process.memory_percent(),
ram_usage)
def test_get_stock(self):
r = Requester()
print(r.get_stock("INX"))
def __init__(self, api_key=None):
self.base_requester = Requester(api_key)
def run():
from Requester import Requester
for i in range(10000):
Requester('http://www.hello.com', raw_request)
def __init__(self, login_or_token=None, password=None):
self.__requester = Requester(login_or_token, password)
def __init__(self, url):
self.__requester = Requester(url)
LINKERFINAL = Graphline(LINKRESOLVE=LinkResolver(bitlyusername,
bitlyapikey),
LINKREQUESTER=HTTPGetter(proxy, "BBC R&D Grabber",
10),
linkages={
("self", "inbox"): ("LINKRESOLVE", "inbox"),
("LINKRESOLVE", "outbox"): ("self", "outbox"),
("LINKRESOLVE", "urlrequests"):
("LINKREQUESTER", "inbox"),
("LINKREQUESTER", "outbox"):
("LINKRESOLVE", "responses")
}).activate()
system = Graphline(
CURRENTPROG=WhatsOn(proxy),
REQUESTER=Requester(
"all", dbuser, dbpass
), # Can set this for specific channels to limit Twitter requests whilst doing dev
FIREHOSE=TwitterStream(
username, password, proxy, True, 40
), # Twitter API sends blank lines every 30 secs so timeout of 40 should be fine
SEARCH=PeopleSearch(consumerkeypair, keypair, proxy),
COLLECTOR=DataCollector(dbuser, dbpass),
RAWCOLLECTOR=RawDataCollector(dbuser, dbpass),
HTTPGETTER=HTTPGetter(proxy, "BBC R&D Grabber", 10),
HTTPGETTERRDF=HTTPGetter(proxy, "BBC R&D Grabber", 10),
TWOWAY=TwoWaySplitter(),
ANALYSIS=LiveAnalysis(dbuser, dbpass),
NLTKANALYSIS=LiveAnalysisNLTK(dbuser, dbpass),
TWEETCLEANER=Pipeline(
LINKER, RetweetFixer(), RetweetCorrector(dbuser, dbpass),
TweetCleaner(['user_mentions', 'urls', 'hashtags'])),
