def user_agent(): """ HTTP User-Agent header string derived from version, used by various HTTP requests sent to learningequality.org for stats """ from requests.utils import default_user_agent return "ka-lite/%s " % VERSION + default_user_agent()
def fetch_bse_star_master_data(): """Download BSE STARMF master data file""" logger.info("BSE Master data not provided. Downloading.") session = requests.Session() session.headers.update({ "User-Agent": default_user_agent("folioman-python-requests"), }) response = session.get(settings.BSE_STARMF_SCHEME_MASTER_URL, timeout=30) page = fromstring(response.content) form_data = { x.get("name"): x.get("value") for x in page.xpath( './/form[@id="frmOrdConfirm"]//input[@type="hidden"]') } form_data.update({ "ddlTypeOption": "SCHEMEMASTERPHYSICAL", "btnText": "Export to Text" }) response = session.post(settings.BSE_STARMF_SCHEME_MASTER_URL, data=form_data, timeout=600) if response.status_code != 200: raise ValueError("Invalid response from BSE. Cannot continue...") logger.info("BSE Master data downloaded.") return response.text
def get_api_result(path, params=None, data=None): result = None r = None try: headers = { "User-Agent": "PlatformIO/%s %s" % (__version__, default_user_agent()) } if data: r = post(__apiurl__ + path, params=params, data=data, headers=headers) else: r = get(__apiurl__ + path, params=params, headers=headers) result = r.json() r.raise_for_status() except HTTPError as e: if result and "errors" in result: raise APIRequestError(result['errors'][0]['title']) else: raise APIRequestError(e) except ConnectionError: raise APIRequestError( "Could not connect to PlatformIO Registry Service") except ValueError: raise APIRequestError("Invalid response: %s" % r.text) finally: if r: r.close() return result
def get_api_result(path, params=None, data=None): result = None r = None try: headers = {"User-Agent": "PlatformIO/%s %s" % ( __version__, default_user_agent())} if data: r = post(__apiurl__ + path, params=params, data=data, headers=headers) else: r = get(__apiurl__ + path, params=params, headers=headers) result = r.json() r.raise_for_status() except HTTPError as e: if result and "errors" in result: raise APIRequestError(result['errors'][0]['title']) else: raise APIRequestError(e) except ConnectionError: raise APIRequestError( "Could not connect to PlatformIO Registry Service") except ValueError: raise APIRequestError("Invalid response: %s" % r.text) finally: if r: r.close() return result
def test_basic(self): gdn = SpaceGDN() self.assertEqual(gdn.endpoint, pyspacegdn.DEFAULT_ENDPOINT) self.assertTrue( gdn.user_agent.endswith('{}/{} {}'.format(pyspacegdn.__title__, pyspacegdn.__version__, default_user_agent())))
def get_manifest(): try: return PackageManager._cached_manifest except AttributeError: headers = {"User-Agent": "PlatformIO/%s %s" % ( __version__, default_user_agent())} PackageManager._cached_manifest = get(__pkgmanifesturl__, headers=headers).json() return PackageManager._cached_manifest
def __init__(self, config={}): self.precision = {} if self.precision is None else self.precision self.limits = {} if self.limits is None else self.limits self.exceptions = {} if self.exceptions is None else self.exceptions self.headers = {} if self.headers is None else self.headers self.balance = {} if self.balance is None else self.balance self.orderbooks = {} if self.orderbooks is None else self.orderbooks self.orders = {} if self.orders is None else self.orders self.trades = {} if self.trades is None else self.trades self.currencies = {} if self.currencies is None else self.currencies self.options = {} if self.options is None else self.options # Python does not allow to define properties in run-time with setattr # version = '.'.join(map(str, sys.version_info[:3])) # self.userAgent = { # 'User-Agent': 'ccxt/' + __version__ + ' (+https://github.com/ccxt/ccxt) Python/' + version # } self.userAgent = default_user_agent() settings = self.deep_extend(self.describe(), config) for key in settings: if hasattr(self, key) and isinstance(getattr(self, key), dict): setattr(self, key, self.deep_extend(getattr(self, key), settings[key])) else: setattr(self, key, settings[key]) if self.api: self.define_rest_api(self.api, 'request') if self.markets: self.set_markets(self.markets) # format camel case for attr in dir(self): if attr[0] != '_' and attr[-1] != '_' and '_' in attr: conv = attr.split('_') camel_case = conv[0] + ''.join(i[0].upper() + i[1:] for i in conv[1:]) setattr(self, camel_case, getattr(self, attr)) self.tokenBucket = self.extend( { 'refillRate': 1.0 / self.rateLimit, 'delay': 1.0, 'capacity': 1.0, 'defaultCost': 1.0, 'maxCapacity': 1000, }, getattr(self, 'tokenBucket') if hasattr(self, 'tokenBucket') else {}) self.session = self.session if self.session else Session() self.logger = self.logger if self.logger else logging.getLogger( __name__)
def _create_user_agent(self): """ Create the user agent and return it as a string. """ user_agent = '{}/{} {}'.format(pyspacegdn.__title__, pyspacegdn.__version__, default_user_agent()) if self.client_name: user_agent = '{}/{} {}'.format(self.client_name, self.client_version, user_agent) return user_agent
def default_headers(): return CaseInsensitiveDict({ 'User-Agent': ' '.join(['python-boxview/1.0', default_user_agent()]), 'Accept': '*/*', 'Accept-Encoding': ', '.join(('gzip', 'deflate', 'compress')), })
def get_manifest(): try: return PackageManager._cached_manifest except AttributeError: headers = { "User-Agent": "PlatformIO/%s %s" % (__version__, default_user_agent()) } PackageManager._cached_manifest = get(__pkgmanifesturl__, headers=headers).json() return PackageManager._cached_manifest
def __init__(self, config={}): self.userAgent = default_user_agent() self.session = self.session if self.session else Session() self.logger = self.logger if self.logger else logging.getLogger(__name__) self.headers = {} if self.headers is None else self.headers settings = self.deep_extend(self.describe(), config) for key in settings: if hasattr(self, key) and isinstance(getattr(self, key), dict): setattr(self, key, self.deep_extend(getattr(self, key), settings[key])) else: setattr(self, key, settings[key])
def __init__(self): # supply intent and contact info in user-agent website = 'https://github.com/tjsantos/practiceipa' user_agent_add = '( wiktionary pronunciations download; {} )'.format( website) self.headers = { 'User-Agent': default_user_agent() + ' ' + user_agent_add } self.last_request_time = time.time() self.delay = self.default_delay
def __init__(self, config={}): self.precision = {} if self.precision is None else self.precision self.limits = {} if self.limits is None else self.limits self.exceptions = {} if self.exceptions is None else self.exceptions self.headers = {} if self.headers is None else self.headers self.balance = {} if self.balance is None else self.balance self.orderbooks = {} if self.orderbooks is None else self.orderbooks self.orders = {} if self.orders is None else self.orders self.trades = {} if self.trades is None else self.trades self.currencies = {} if self.currencies is None else self.currencies self.options = {} if self.options is None else self.options # Python does not allow to define properties in run-time with setattr self.decimalToPrecision = self.decimal_to_precision = decimal_to_precision # version = '.'.join(map(str, sys.version_info[:3])) # self.userAgent = { # 'User-Agent': 'ccxt/' + __version__ + ' (+https://github.com/ccxt/ccxt) Python/' + version # } self.userAgent = default_user_agent() settings = self.deep_extend(self.describe(), config) for key in settings: if hasattr(self, key) and isinstance(getattr(self, key), dict): setattr(self, key, self.deep_extend(getattr(self, key), settings[key])) else: setattr(self, key, settings[key]) if self.api: self.define_rest_api(self.api, 'request') if self.markets: self.set_markets(self.markets) # format camel case for attr in dir(self): if attr[0] != '_'and attr[-1] != '_' and '_' in attr: conv = attr.split('_') camel_case = conv[0] + ''.join(i[0].upper() + i[1:] for i in conv[1:]) setattr(self, camel_case, getattr(self, attr)) self.tokenBucket = self.extend({ 'refillRate': 1.0 / self.rateLimit, 'delay': 1.0, 'capacity': 1.0, 'defaultCost': 1.0, }, getattr(self, 'tokenBucket') if hasattr(self, 'tokenBucket') else {}) self.session = self.session if self.session else Session() self.logger = self.logger if self.logger else logging.getLogger(__name__)
def __init__(self, endpoint_url, oauth_token, debug=False): self.endpoint_url = endpoint_url self.session = AuthPreservingRedirectSession(oauth_token) build_url = "" if os.getenv( "BUILD_URL") is None else " BUILD_URL={}".format( os.getenv("BUILD_URL")) self.session.headers['User-Agent'] = '{} fallout-cli/{}{}'.\ format(default_user_agent(), VERSION_STRING, build_url) if debug: HTTPConnection.debuglevel = 1 logging.basicConfig(level=logging.DEBUG) requests_log = logging.getLogger("requests.packages.urllib3") requests_log.setLevel(logging.DEBUG) requests_log.propagate = True
def test_extra_headers(self): from requests.utils import default_user_agent user_agent = "slumber %s" % default_user_agent( ) # cannot import version from setup.py extra_headers = { "User-Agent": user_agent, "Cookie": "session=1234567890", } client = slumber.API(base_url="http://httpbin.org/", append_slash=False, extra_headers=extra_headers) resp = client.headers.get() self.assertDictContainsSubset(extra_headers, resp["headers"])
def __init__(self): super(CFSession, self).__init__() self.headers = OrderedDict([ ('User-Agent', default_user_agent()), ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' ), ('Accept-Language', 'en-US,en;q=0.5'), ('Accept-Encoding', 'gzip, deflate'), ('Connection', 'keep-alive'), ('Pragma', 'no-cache'), ('Cache-Control', 'no-cache'), ('Upgrade-Insecure-Requests', '1'), ('DNT', '1'), ]) self.debug = os.environ.get("CF_DEBUG", False)
def create_concurrent_request(url_list, headers_list=[], json_body_list=[], method_list=[], auth="", params_list=[]): default_headers = { 'User-Agent': default_user_agent(), 'Accept-Encoding': ', '.join(('gzip', 'deflate')), 'Accept': '*/*', 'Connection': 'keep-alive', } # monkey patch for fixing proxy if len(headers_list) != 0: default_headers = headers_list[0] while len(headers_list) != len(url_list) and len(headers_list) < len(url_list): headers_list.append(default_headers) while len(params_list) != len(url_list) and len(params_list) < len(url_list): params_list.append(None) while len(json_body_list) != len(url_list) and len(json_body_list) < len(url_list): default_json = {} json_body_list.append(default_json) while len(method_list) != len(url_list) and len(method_list) < len(url_list): default_method = 'GET' method_list.append(default_method) def send_concurrent_request(url, headers, json_body, method, param): response = None if method.upper() in ['POST', 'PUT', 'DELETE']: response = requests.request(method.upper(), url=url, json=json_body, headers=headers) if method.upper() == 'GET': response = requests.request(method.upper(), url=url, json=json_body, headers=headers, auth=auth, params=param) if response: try: return response.json() except ValueError: return response.text return None with futures.ThreadPoolExecutor(max_workers=5) as executor: res = executor.map(send_concurrent_request, url_list, headers_list, json_body_list, method_list, params_list) return list(res)
def get_jobs(circle_token, project_slug): r = requests.get( "https://circleci.com/api/v2/insights/{0}/workflows/build/jobs".format( project_slug ), headers={ "Circle-Token": circle_token, "User-Agent": "{0} {1}".format("tmhall99/buildstats", default_user_agent()), }, ) r.raise_for_status() jobs = r.json() if jobs["next_page_token"] is not None: print("Too many jobs") return return jobs
def __init__(self, config={}): # version = '.'.join(map(str, sys.version_info[:3])) # self.userAgent = { # 'User-Agent': 'ccxt/' + __version__ + ' (+https://github.com/ccxt/ccxt) Python/' + version # } self.userAgent = default_user_agent() settings = self.deep_extend(self.describe(), config) for key in settings: if hasattr(self, key) and isinstance(getattr(self, key), dict): setattr(self, key, self.deep_extend(getattr(self, key), settings[key])) else: setattr(self, key, settings[key]) if self.api: self.define_rest_api(self.api, 'request') if self.markets: self.set_markets(self.markets) # format camel case for attr in dir(self): if attr[0] != '_' and attr[-1] != '_' and '_' in attr: conv = attr.split('_') camel_case = conv[0] + ''.join(i[0].upper() + i[1:] for i in conv[1:]) setattr(self, camel_case, getattr(self, attr)) self.tokenBucket = self.extend( { 'refillRate': 1.0 / self.rateLimit, 'delay': 1.0, 'capacity': 1.0, 'defaultCost': 1.0, 'maxCapacity': 1000, }, getattr(self, 'tokenBucket') if hasattr(self, 'tokenBucket') else {}) self.session = self.session if self.session else Session()
def __init__(self, config={}): # version = '.'.join(map(str, sys.version_info[:3])) # self.userAgent = { # 'User-Agent': 'ccxt/' + __version__ + ' (+https://github.com/ccxt/ccxt) Python/' + version # } self.userAgent = default_user_agent() settings = self.deep_extend(self.describe(), config) for key in settings: if hasattr(self, key) and isinstance(getattr(self, key), dict): setattr(self, key, self.deep_extend(getattr(self, key), settings[key])) else: setattr(self, key, settings[key]) if self.api: self.define_rest_api(self.api, 'request') if self.markets: self.set_markets(self.markets) # format camel case for attr in dir(self): if attr[0] != '_'and attr[-1] != '_' and '_' in attr: conv = attr.split('_') camel_case = conv[0] + ''.join(i[0].upper() + i[1:] for i in conv[1:]) setattr(self, camel_case, getattr(self, attr)) self.tokenBucket = self.extend({ 'refillRate': 1.0 / self.rateLimit, 'delay': 1.0, 'capacity': 1.0, 'defaultCost': 1.0, 'maxCapacity': 1000, }, getattr(self, 'tokenBucket') if hasattr(self, 'tokenBucket') else {}) self.session = self.session if self.session else Session()
def handle(self, **options): if not settings.MEDIA_ROOT: raise ImproperlyConfigured('Please set MEDIA_ROOT in your settings file') remote_url = options['remote_url'] try: val = URLValidator() val(remote_url) except ValidationError: raise CommandError('Please enter a valid URL') exercise_api = "{0}/api/v2/exercise/?limit=999&status=2" image_api = "{0}/api/v2/exerciseimage/?exercise={1}" thumbnail_api = "{0}/api/v2/exerciseimage/{1}/thumbnails/" headers = {'User-agent': default_user_agent('wger/{} + requests'.format(get_version()))} # Get all exercises result = requests.get(exercise_api.format(remote_url), headers=headers).json() for exercise_json in result['results']: exercise_name = exercise_json['name'].encode('utf-8') exercise_uuid = exercise_json['uuid'] exercise_id = exercise_json['id'] self.stdout.write('') self.stdout.write(u"*** Processing {0} (ID: {1}, UUID: {2})".format(exercise_name, exercise_id, exercise_uuid)) try: exercise = Exercise.objects.get(uuid=exercise_uuid) except Exercise.DoesNotExist: self.stdout.write(' Remote exercise not found in local DB, skipping...') continue # Get all images images = requests.get(image_api.format(remote_url, exercise_id), headers=headers).json() if images['count']: for image_json in images['results']: image_id = image_json['id'] result = requests.get(thumbnail_api.format(remote_url, image_id), headers=headers).json() image_name = os.path.basename(result['original']) self.stdout.write(' Fetching image {0} - {1}'.format(image_id, image_name)) try: image = ExerciseImage.objects.get(pk=image_id) self.stdout.write(' --> Image already present locally, skipping...') continue except ExerciseImage.DoesNotExist: self.stdout.write(' --> Image not found in local DB, creating now...') image = ExerciseImage() image.pk = image_id # Save the downloaded image, see link for details # http://stackoverflow.com/questions/1308386/programmatically-saving-image-to- retrieved_image = requests.get(result['original'], headers=headers) img_temp = NamedTemporaryFile(delete=True) img_temp.write(retrieved_image.content) img_temp.flush() image.exercise = exercise image.is_main = image_json['is_main'] image.status = image_json['status'] image.image.save( os.path.basename(image_name), File(img_temp), ) image.save() else: self.stdout.write(' No images for this exercise, nothing to do')
def _user_agent(self): """Create User-Agent header.""" return 'minFraud-API/%s %s' % (__version__, default_user_agent())
def determine_user_agent(user_agent): if not user_agent or user_agent == default_user_agent(): return FAKE_USER_AGENT return user_agent
def _user_agent(self): return 'GeoIP2 Python Client v%s (%s)' % (geoip2.__version__, default_user_agent())
def default(cls): cls.useragents = [default_user_agent()]
def __init__(self, app=App(), token=Token()): super(ClientBase, self).__init__() self.app = app self.token = token self._session.headers['User-Agent'] = default_user_agent( '%s/%s requests' % (__title__, __version__))
def __init__(self, client_id=None, client_secret=None, api_base_url=DEFAULT_API_URL, api_ssl_verify=CA_BUNDLE_PATH, site_base_url=DEFAULT_SITE_URL, site_ssl_verify=CA_BUNDLE_PATH): """Initialize an Iconfinder API client. Note that if :param:`client_id` is provided, :param:`client_secret` must also be provided and vice versa. :param client_id: Optional client ID. Default ``None``. :param client_secret: Optional client secret. Default ``None``. :param api_base_url: API base URL. Default ``https://api.iconfinder.com/v2``. :param api_ssl_verify: API SSL verification. Refer to the `Requests documentation <http://docs.python-requests.org/>`_ for further details. Defaults to the included CA bundle for the Iconfinder wildcard SSL certificate. :param site_url: Site base URL. Default ``https://www.iconfinder.com``. :param site_ssl_verify: Site SSL verification. Refer to the `Requests documentation <http://docs.python-requests.org/>`_ for further details. Defaults to the included CA bundle for the Iconfinder wildcard SSL certificate. """ # Validate client ID and secret. if (client_id is None) != (client_secret is None): raise ValueError('client_id and client_secret must both be ' 'provided if one is provided') if client_id is not None and not isinstance(client_id, string_types): raise TypeError('client_id must be a string value') if client_secret is not None and \ not isinstance(client_secret, string_types): raise TypeError('client_secret must be a string value') self._client_id = client_id or None self._client_secret = client_secret or None # Set up URLs etc. self._api_base_url = api_base_url.rstrip('/') self._api_ssl_verify = api_ssl_verify self._site_base_url = site_base_url.rstrip('/') self._site_ssl_verify = site_ssl_verify # Set up sessions. from . import __version__ user_agent = 'pyiconfinder/%s %s' % (__version__, default_user_agent()) self._api_session = requests.Session() self._api_session.verify = api_ssl_verify self._api_session.headers['User-Agent'] = user_agent self._site_session = requests.Session() self._site_session.verify = site_ssl_verify self._site_session.headers['User-Agent'] = user_agent # Set up model class proxies. self.Author = ModelClassProxy(Author, self) self.Category = ModelClassProxy(Category, self) self.IconSet = ModelClassProxy(IconSet, self) self.License = ModelClassProxy(License, self) self.Style = ModelClassProxy(Style, self)
def __init__( self, expire_after: ExpirationTime = None, cache_file: FileOrPath = CACHE_FILE, per_second: int = REQUESTS_PER_SECOND, per_minute: int = REQUESTS_PER_MINUTE, per_day: float = REQUESTS_PER_DAY, burst: int = REQUEST_BURST_RATE, bucket_class: Type[AbstractBucket] = SQLiteBucket, retries: int = REQUEST_RETRIES, backoff_factor: float = RETRY_BACKOFF, timeout: int = REQUEST_TIMEOUT, user_agent: str = None, **kwargs, ): """Get a Session object, optionally with custom settings for caching and rate-limiting. Args: expire_after: How long to keep cached API requests; for advanced options, see `requests-cache: Expiration <https://requests-cache.readthedocs.io/en/latest/user_guide/expiration.html>`_ cache_file: Cache file path to use; defaults to the system default cache directory per_second: Max requests per second per_minute: Max requests per minute per_day: Max requests per day burst: Max number of consecutive requests allowed before applying per-second rate-limiting bucket_class: Rate-limiting backend to use. Defaults to a persistent SQLite database. retries: Maximum number of times to retry a failed request backoff_factor: Factor for increasing delays between retries timeout: Maximum number of seconds to wait for a response from the server user_agent: Additional User-Agent info to pass to API requests kwargs: Additional keyword arguments for :py:class:`~requests_cache.session.CachedSession` and/or :py:class:`~requests_ratelimiter.requests_ratelimiter.LimiterSession` """ # If not overridden, use Cache-Control when possible, and some default expiration times if not expire_after: kwargs.setdefault('cache_control', True) kwargs.setdefault('urls_expire_after', CACHE_EXPIRATION) self.timeout = timeout super().__init__( # type: ignore # false positive # Cache settings cache_name=cache_file, backend='sqlite', expire_after=expire_after, ignored_parameters=['Authorization', 'access_token'], old_data_on_error=True, # Rate limit settings bucket_class=bucket_class, bucket_kwargs={'path': RATELIMIT_FILE}, per_second=per_second, per_minute=per_minute, per_day=per_day, per_host=True, burst=burst, max_delay=MAX_DELAY, **kwargs, ) # Set default headers self.headers['Accept'] = 'application/json' user_agent_details = [ default_user_agent(), f'pyinaturalist/{pyinaturalist.__version__}', user_agent or '', ] self.headers['User-Agent'] = ' '.join(user_agent_details).strip() # Mount an adapter to apply retry settings retry = Retry(total=retries, backoff_factor=backoff_factor) adapter = HTTPAdapter(max_retries=retry) self.mount('https://', adapter)
def user_agent_header(): return u'%s/%s %s' % (metadata.package, metadata.version, default_user_agent())
def handle(self, **options): remote_url = options['remote_url'] try: val = URLValidator() val(remote_url) except ValidationError: raise CommandError('Please enter a valid URL') headers = { 'User-agent': default_user_agent('wger/{} + requests'.format(get_version())) } # # Categories # self.stdout.write('*** Synchronizing categories...') result = requests.get(CATEGORY_API.format(remote_url), headers=headers).json() for category_data in result['results']: category_id = category_data['id'] category_name = category_data['name'] try: category = ExerciseCategory.objects.get(pk=category_id) category.name = category_name category.save() except ExerciseCategory.DoesNotExist: self.stdout.write( self.style.WARNING(f'Saving new category {category_name}')) category = ExerciseCategory(id=category_id, name=category_name) category.save() self.stdout.write(self.style.SUCCESS('done!\n')) # # Muscles # self.stdout.write('*** Synchronizing muscles...') result = requests.get(MUSCLE_API.format(remote_url), headers=headers).json() for muscle_data in result['results']: muscle_id = muscle_data['id'] muscle_name = muscle_data['name'] muscle_is_front = muscle_data['is_front'] muscle_url_main = muscle_data['image_url_main'] muscle_url_secondary = muscle_data['image_url_secondary'] try: muscle = Muscle.objects.get(pk=muscle_id) muscle.name = muscle_name muscle_is_front = muscle_is_front muscle.save() except Muscle.DoesNotExist: muscle = Muscle(id=muscle_id, name=muscle_name, is_front=muscle_is_front) muscle.save() self.stdout.write( self.style.WARNING( f'Saved new muscle {muscle_name}. ' f'Save the corresponding images manually')) self.stdout.write(self.style.WARNING(muscle_url_main)) self.stdout.write(self.style.WARNING(muscle_url_secondary)) self.stdout.write(self.style.SUCCESS('done!\n')) # # Equipment # self.stdout.write('*** Synchronizing equipment...') result = requests.get(EQUIPMENT_API.format(remote_url), headers=headers).json() for equipment_data in result['results']: equipment_id = equipment_data['id'] equipment_name = equipment_data['name'] try: equipment = Equipment.objects.get(pk=equipment_id) equipment.name = equipment_name equipment.save() except Equipment.DoesNotExist: self.stdout.write(f'Saved new equipment {equipment_name}') equipment = Equipment(id=equipment_id, name=equipment_name) equipment.save() self.stdout.write(self.style.SUCCESS('done!\n')) # # Exercises # self.stdout.write('*** Synchronizing exercises...') page = 1 all_exercise_processed = False result = requests.get(EXERCISE_API.format(remote_url), headers=headers).json() while not all_exercise_processed: for data in result['results']: exercise_uuid = data['uuid'] exercise_name = data['name'] exercise_description = data['description'] equipment = [ Equipment.objects.get(pk=i['id']) for i in data['equipment'] ] muscles = [ Muscle.objects.get(pk=i['id']) for i in data['muscles'] ] muscles_sec = [ Muscle.objects.get(pk=i['id']) for i in data['muscles_secondary'] ] try: exercise = Exercise.objects.get(uuid=exercise_uuid) exercise.name = exercise_name exercise.description = exercise_description # Note: this should not happen and is an unnecessary workaround # https://github.com/wger-project/wger/issues/840 if not exercise.exercise_base: warning = f'Exercise {exercise.uuid} has no base, this should not happen!' \ f'Skipping...\n' self.stdout.write(self.style.WARNING(warning)) continue exercise.exercise_base.category_id = data['category']['id'] exercise.exercise_base.muscles.set(muscles) exercise.exercise_base.muscles_secondary.set(muscles_sec) exercise.exercise_base.equipment.set(equipment) exercise.exercise_base.save() exercise.save() except Exercise.DoesNotExist: self.stdout.write(f'Saved new exercise {exercise_name}') exercise = Exercise( uuid=exercise_uuid, name=exercise_name, description=exercise_description, language_id=data['language']['id'], license_id=data['license']['id'], license_author=data['license_author'], ) base = ExerciseBase() base.category_id = data['category']['id'] base.save() base.muscles.set(muscles) base.muscles_secondary.set(muscles_sec) base.equipment.set(equipment) base.save() exercise.save() if result['next']: page += 1 result = requests.get(result['next'], headers=headers).json() else: all_exercise_processed = True self.stdout.write(self.style.SUCCESS('done!\n'))
def _generate_user_agent(name='t1-python'): return '{name}/{version} {ua}'.format(name=name, version=__version__, ua=default_user_agent())
def useragentify(session): """declare a specific user-agent for commit --blog""" user_agent = default_user_agent('commit --blog') session.headers.update({'User-Agent': user_agent})
def user_agent_header(): return u'{0}/{1} {2}'.format(metadata.package, metadata.version, default_user_agent())
def handle(self, **options): if not settings.MEDIA_ROOT: raise ImproperlyConfigured('Please set MEDIA_ROOT in your settings file') remote_url = options['remote_url'] try: val = URLValidator() val(remote_url) except ValidationError: raise CommandError('Please enter a valid URL') headers = {'User-agent': default_user_agent('wger/{} + requests'.format(get_version()))} # Get all images page = 1 all_images_processed = False result = requests.get(IMAGE_API.format(remote_url), headers=headers).json() self.stdout.write('*** Processing images ***') while not all_images_processed: self.stdout.write('') self.stdout.write(f'*** Page {page}') self.stdout.write('') page += 1 if result['next']: result = requests.get(result['next'], headers=headers).json() else: all_images_processed = True for image_data in result['results']: image_uuid = image_data['uuid'] self.stdout.write(f'Processing image {image_uuid}') try: exercise_base = ExerciseBase.objects.get(id=image_data['exercise']) except ExerciseBase.DoesNotExist: self.stdout.write(' Remote exercise base not found in local DB, skipping...') continue try: image = ExerciseImage.objects.get(uuid=image_uuid) self.stdout.write(' Image already present locally, skipping...') continue except ExerciseImage.DoesNotExist: self.stdout.write(' Image not found in local DB, creating now...') image = ExerciseImage() image.uuid = image_uuid # Save the downloaded image # http://stackoverflow.com/questions/1308386/programmatically-saving-image-to- retrieved_image = requests.get(image_data['image'], headers=headers) # Temporary files on windows don't support the delete attribute if os.name == 'nt': img_temp = NamedTemporaryFile() else: img_temp = NamedTemporaryFile(delete=True) img_temp.write(retrieved_image.content) img_temp.flush() image.exercise = exercise_base image.is_main = image_data['is_main'] image.status = image_data['status'] image.image.save( os.path.basename(os.path.basename(image_data['image'])), File(img_temp), ) image.save() self.stdout.write(self.style.SUCCESS(' successfully saved'))
def handle(self, **options): if not settings.MEDIA_ROOT: raise ImproperlyConfigured( 'Please set MEDIA_ROOT in your settings file') remote_url = options['remote_url'] try: val = URLValidator() val(remote_url) except ValidationError: raise CommandError('Please enter a valid URL') headers = { 'User-agent': default_user_agent('wger/{} + requests'.format(get_version())) } # Get all exercises result = requests.get(EXERCISE_API.format(remote_url), headers=headers).json() for exercise_json in result['results']: exercise_name = exercise_json['name'] exercise_uuid = exercise_json['uuid'] exercise_id = exercise_json['id'] self.stdout.write('') self.stdout.write( f"*** {exercise_name} (ID: {exercise_id}, UUID: {exercise_uuid})" ) try: exercise = Exercise.objects.get(uuid=exercise_uuid) except Exercise.DoesNotExist: self.stdout.write( ' Remote exercise not found in local DB, skipping...') continue # Get all images images = requests.get(IMAGE_API.format(remote_url, exercise_id), headers=headers).json() if images['count']: for image_json in images['results']: image_id = image_json['id'] image_uuid = image_json['uuid'] result = requests.get(THUMBNAIL_API.format( remote_url, image_id), headers=headers).json() image_name = os.path.basename(result['original']) self.stdout.write(' Fetching image {0} - {1}'.format( image_id, image_name)) try: image = ExerciseImage.objects.get(uuid=image_uuid) self.stdout.write( ' --> Image already present locally, skipping...' ) continue except ExerciseImage.DoesNotExist: self.stdout.write( ' --> Image not found in local DB, creating now...' ) image = ExerciseImage() image.uuid = image_uuid # Save the downloaded image # http://stackoverflow.com/questions/1308386/programmatically-saving-image-to- retrieved_image = requests.get(remote_url + result['original'], headers=headers) # Temporary files on windows don't support the delete attribute if os.name == 'nt': img_temp = NamedTemporaryFile() else: img_temp = NamedTemporaryFile(delete=True) img_temp.write(retrieved_image.content) img_temp.flush() image.exercise = exercise.exercise_base image.is_main = image_json['is_main'] image.status = image_json['status'] image.image.save( os.path.basename(image_name), File(img_temp), ) image.save() else: self.stdout.write( ' No images for this exercise, nothing to do')
def set_user_agent(self, extra=None): version = get_distribution('steelscript').version ua = '%s SteelScript/%s' % (default_user_agent(), version) if extra: ua = '%s %s' % (ua, extra) self.conn.headers['User-Agent'] = ua
def get_user_agent(self): if not hasattr(self, "_user_agent"): self._user_agent = default_user_agent() return self._user_agent
def __init__(self, app=App(), token=Token()): super(ClientBase, self).__init__() self.app = app self.token = token self._session.headers['User-Agent'] = default_user_agent('%s/%s requests' % (__title__, __version__))
def handle(self, **options): if not settings.MEDIA_ROOT: raise ImproperlyConfigured( 'Please set MEDIA_ROOT in your settings file') remote_url = options['remote_url'] try: val = URLValidator() val(remote_url) except ValidationError: raise CommandError('Please enter a valid URL') exercise_api = "{0}/api/v2/exercise/?limit=999" image_api = "{0}/api/v2/exerciseimage/?exercise={1}" thumbnail_api = "{0}/api/v2/exerciseimage/{1}/thumbnails/" headers = { 'User-agent': default_user_agent('wger/{} + requests'.format(get_version())) } # Get all exercises result = requests.get( exercise_api.format(remote_url), headers=headers).json() for exercise_json in result['results']: exercise_name = exercise_json['name'].encode('utf-8') exercise_uuid = exercise_json['uuid'] exercise_id = exercise_json['id'] self.stdout.write('') self.stdout.write( u"*** Processing {0} (ID: {1}, UUID: {2})".format( exercise_name, exercise_id, exercise_uuid)) try: exercise = Exercise.objects.get(uuid=exercise_uuid) except Exercise.DoesNotExist: self.stdout.write( ' Remote exercise not found in local DB, skipping...') continue # Get all images images = requests.get( image_api.format(remote_url, exercise_id), headers=headers).json() if images['count']: for image_json in images['results']: image_id = image_json['id'] result = requests.get( thumbnail_api.format(remote_url, image_id), headers=headers).json() image_name = os.path.basename(result['original']) self.stdout.write(' Fetching image {0} - {1}'.format( image_id, image_name)) try: image = ExerciseImage.objects.get(pk=image_id) self.stdout.write( ' --> Image already present locally, skipping...' ) continue except ExerciseImage.DoesNotExist: self.stdout.write( ' --> Image not found in local DB, creating now...' ) image = ExerciseImage() image.pk = image_id # Save the downloaded image, see link for details # http://stackoverflow.com/questions/1308386/programmatically-saving-image-to- retrieved_image = requests.get( result['original'], headers=headers) img_temp = NamedTemporaryFile(delete=True) img_temp.write(retrieved_image.content) img_temp.flush() image.exercise = exercise image.is_main = image_json['is_main'] image.status = image_json['status'] image.image.save( os.path.basename(image_name), File(img_temp), ) image.save() else: self.stdout.write( ' No images for this exercise, nothing to do')