def user_agent():
"""
HTTP User-Agent header string derived from version, used by various HTTP
requests sent to learningequality.org for stats
"""
from requests.utils import default_user_agent
return "ka-lite/%s " % VERSION + default_user_agent()
def fetch_bse_star_master_data():
"""Download BSE STARMF master data file"""
logger.info("BSE Master data not provided. Downloading.")
session = requests.Session()
session.headers.update({
"User-Agent":
default_user_agent("folioman-python-requests"),
})
response = session.get(settings.BSE_STARMF_SCHEME_MASTER_URL, timeout=30)
page = fromstring(response.content)
form_data = {
x.get("name"): x.get("value")
for x in page.xpath(
'.//form[@id="frmOrdConfirm"]//input[@type="hidden"]')
}
form_data.update({
"ddlTypeOption": "SCHEMEMASTERPHYSICAL",
"btnText": "Export to Text"
})
response = session.post(settings.BSE_STARMF_SCHEME_MASTER_URL,
data=form_data,
timeout=600)
if response.status_code != 200:
raise ValueError("Invalid response from BSE. Cannot continue...")
logger.info("BSE Master data downloaded.")
return response.text
def get_api_result(path, params=None, data=None):
result = None
r = None
try:
headers = {
"User-Agent":
"PlatformIO/%s %s" % (__version__, default_user_agent())
}
if data:
r = post(__apiurl__ + path,
params=params,
data=data,
headers=headers)
else:
r = get(__apiurl__ + path, params=params, headers=headers)
result = r.json()
r.raise_for_status()
except HTTPError as e:
if result and "errors" in result:
raise APIRequestError(result['errors'][0]['title'])
else:
raise APIRequestError(e)
except ConnectionError:
raise APIRequestError(
"Could not connect to PlatformIO Registry Service")
except ValueError:
raise APIRequestError("Invalid response: %s" % r.text)
finally:
if r:
r.close()
return result
def user_agent():
"""
HTTP User-Agent header string derived from version, used by various HTTP
requests sent to learningequality.org for stats
"""
from requests.utils import default_user_agent
return "ka-lite/%s " % VERSION + default_user_agent()
def get_api_result(path, params=None, data=None):
result = None
r = None
try:
headers = {"User-Agent": "PlatformIO/%s %s" % (
__version__, default_user_agent())}
if data:
r = post(__apiurl__ + path, params=params, data=data,
headers=headers)
else:
r = get(__apiurl__ + path, params=params, headers=headers)
result = r.json()
r.raise_for_status()
except HTTPError as e:
if result and "errors" in result:
raise APIRequestError(result['errors'][0]['title'])
else:
raise APIRequestError(e)
except ConnectionError:
raise APIRequestError(
"Could not connect to PlatformIO Registry Service")
except ValueError:
raise APIRequestError("Invalid response: %s" % r.text)
finally:
if r:
r.close()
return result
def test_basic(self):
gdn = SpaceGDN()
self.assertEqual(gdn.endpoint, pyspacegdn.DEFAULT_ENDPOINT)
self.assertTrue(
gdn.user_agent.endswith('{}/{} {}'.format(pyspacegdn.__title__,
pyspacegdn.__version__,
default_user_agent())))
def get_manifest():
try:
return PackageManager._cached_manifest
except AttributeError:
headers = {"User-Agent": "PlatformIO/%s %s" % (
__version__, default_user_agent())}
PackageManager._cached_manifest = get(__pkgmanifesturl__,
headers=headers).json()
return PackageManager._cached_manifest
def __init__(self, config={}):
self.precision = {} if self.precision is None else self.precision
self.limits = {} if self.limits is None else self.limits
self.exceptions = {} if self.exceptions is None else self.exceptions
self.headers = {} if self.headers is None else self.headers
self.balance = {} if self.balance is None else self.balance
self.orderbooks = {} if self.orderbooks is None else self.orderbooks
self.orders = {} if self.orders is None else self.orders
self.trades = {} if self.trades is None else self.trades
self.currencies = {} if self.currencies is None else self.currencies
self.options = {} if self.options is None else self.options # Python does not allow to define properties in run-time with setattr
# version = '.'.join(map(str, sys.version_info[:3]))
# self.userAgent = {
# 'User-Agent': 'ccxt/' + __version__ + ' (+https://github.com/ccxt/ccxt) Python/' + version
# }
self.userAgent = default_user_agent()
settings = self.deep_extend(self.describe(), config)
for key in settings:
if hasattr(self, key) and isinstance(getattr(self, key), dict):
setattr(self, key,
self.deep_extend(getattr(self, key), settings[key]))
else:
setattr(self, key, settings[key])
if self.api:
self.define_rest_api(self.api, 'request')
if self.markets:
self.set_markets(self.markets)
# format camel case
for attr in dir(self):
if attr[0] != '_' and attr[-1] != '_' and '_' in attr:
conv = attr.split('_')
camel_case = conv[0] + ''.join(i[0].upper() + i[1:]
for i in conv[1:])
setattr(self, camel_case, getattr(self, attr))
self.tokenBucket = self.extend(
{
'refillRate': 1.0 / self.rateLimit,
'delay': 1.0,
'capacity': 1.0,
'defaultCost': 1.0,
'maxCapacity': 1000,
},
getattr(self, 'tokenBucket')
if hasattr(self, 'tokenBucket') else {})
self.session = self.session if self.session else Session()
self.logger = self.logger if self.logger else logging.getLogger(
__name__)
def _create_user_agent(self):
""" Create the user agent and return it as a string. """
user_agent = '{}/{} {}'.format(pyspacegdn.__title__,
pyspacegdn.__version__,
default_user_agent())
if self.client_name:
user_agent = '{}/{} {}'.format(self.client_name,
self.client_version, user_agent)
return user_agent
def default_headers():
return CaseInsensitiveDict({
'User-Agent':
' '.join(['python-boxview/1.0',
default_user_agent()]),
'Accept':
'*/*',
'Accept-Encoding':
', '.join(('gzip', 'deflate', 'compress')),
})
def get_manifest():
try:
return PackageManager._cached_manifest
except AttributeError:
headers = {
"User-Agent":
"PlatformIO/%s %s" % (__version__, default_user_agent())
}
PackageManager._cached_manifest = get(__pkgmanifesturl__,
headers=headers).json()
return PackageManager._cached_manifest
def __init__(self, config={}):
self.userAgent = default_user_agent()
self.session = self.session if self.session else Session()
self.logger = self.logger if self.logger else logging.getLogger(__name__)
self.headers = {} if self.headers is None else self.headers
settings = self.deep_extend(self.describe(), config)
for key in settings:
if hasattr(self, key) and isinstance(getattr(self, key), dict):
setattr(self, key, self.deep_extend(getattr(self, key), settings[key]))
else:
setattr(self, key, settings[key])
def __init__(self):
# supply intent and contact info in user-agent
website = 'https://github.com/tjsantos/practiceipa'
user_agent_add = '( wiktionary pronunciations download; {} )'.format(
website)
self.headers = {
'User-Agent': default_user_agent() + ' ' + user_agent_add
}
self.last_request_time = time.time()
self.delay = self.default_delay
def __init__(self, config={}):
self.precision = {} if self.precision is None else self.precision
self.limits = {} if self.limits is None else self.limits
self.exceptions = {} if self.exceptions is None else self.exceptions
self.headers = {} if self.headers is None else self.headers
self.balance = {} if self.balance is None else self.balance
self.orderbooks = {} if self.orderbooks is None else self.orderbooks
self.orders = {} if self.orders is None else self.orders
self.trades = {} if self.trades is None else self.trades
self.currencies = {} if self.currencies is None else self.currencies
self.options = {} if self.options is None else self.options # Python does not allow to define properties in run-time with setattr
self.decimalToPrecision = self.decimal_to_precision = decimal_to_precision
# version = '.'.join(map(str, sys.version_info[:3]))
# self.userAgent = {
# 'User-Agent': 'ccxt/' + __version__ + ' (+https://github.com/ccxt/ccxt) Python/' + version
# }
self.userAgent = default_user_agent()
settings = self.deep_extend(self.describe(), config)
for key in settings:
if hasattr(self, key) and isinstance(getattr(self, key), dict):
setattr(self, key, self.deep_extend(getattr(self, key), settings[key]))
else:
setattr(self, key, settings[key])
if self.api:
self.define_rest_api(self.api, 'request')
if self.markets:
self.set_markets(self.markets)
# format camel case
for attr in dir(self):
if attr[0] != '_'and attr[-1] != '_' and '_' in attr:
conv = attr.split('_')
camel_case = conv[0] + ''.join(i[0].upper() + i[1:] for i in conv[1:])
setattr(self, camel_case, getattr(self, attr))
self.tokenBucket = self.extend({
'refillRate': 1.0 / self.rateLimit,
'delay': 1.0,
'capacity': 1.0,
'defaultCost': 1.0,
}, getattr(self, 'tokenBucket') if hasattr(self, 'tokenBucket') else {})
self.session = self.session if self.session else Session()
self.logger = self.logger if self.logger else logging.getLogger(__name__)
def __init__(self, endpoint_url, oauth_token, debug=False):
self.endpoint_url = endpoint_url
self.session = AuthPreservingRedirectSession(oauth_token)
build_url = "" if os.getenv(
"BUILD_URL") is None else " BUILD_URL={}".format(
os.getenv("BUILD_URL"))
self.session.headers['User-Agent'] = '{} fallout-cli/{}{}'.\
format(default_user_agent(), VERSION_STRING, build_url)
if debug:
HTTPConnection.debuglevel = 1
logging.basicConfig(level=logging.DEBUG)
requests_log = logging.getLogger("requests.packages.urllib3")
requests_log.setLevel(logging.DEBUG)
requests_log.propagate = True
def test_extra_headers(self):
from requests.utils import default_user_agent
user_agent = "slumber %s" % default_user_agent(
) # cannot import version from setup.py
extra_headers = {
"User-Agent": user_agent,
"Cookie": "session=1234567890",
}
client = slumber.API(base_url="http://httpbin.org/",
append_slash=False,
extra_headers=extra_headers)
resp = client.headers.get()
self.assertDictContainsSubset(extra_headers, resp["headers"])
def __init__(self):
super(CFSession, self).__init__()
self.headers = OrderedDict([
('User-Agent', default_user_agent()),
('Accept',
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
),
('Accept-Language', 'en-US,en;q=0.5'),
('Accept-Encoding', 'gzip, deflate'),
('Connection', 'keep-alive'),
('Pragma', 'no-cache'),
('Cache-Control', 'no-cache'),
('Upgrade-Insecure-Requests', '1'),
('DNT', '1'),
])
self.debug = os.environ.get("CF_DEBUG", False)
def create_concurrent_request(url_list, headers_list=[], json_body_list=[], method_list=[], auth="",
params_list=[]):
default_headers = {
'User-Agent': default_user_agent(),
'Accept-Encoding': ', '.join(('gzip', 'deflate')),
'Accept': '*/*',
'Connection': 'keep-alive',
}
# monkey patch for fixing proxy
if len(headers_list) != 0:
default_headers = headers_list[0]
while len(headers_list) != len(url_list) and len(headers_list) < len(url_list):
headers_list.append(default_headers)
while len(params_list) != len(url_list) and len(params_list) < len(url_list):
params_list.append(None)
while len(json_body_list) != len(url_list) and len(json_body_list) < len(url_list):
default_json = {}
json_body_list.append(default_json)
while len(method_list) != len(url_list) and len(method_list) < len(url_list):
default_method = 'GET'
method_list.append(default_method)
def send_concurrent_request(url, headers, json_body, method, param):
response = None
if method.upper() in ['POST', 'PUT', 'DELETE']:
response = requests.request(method.upper(), url=url, json=json_body, headers=headers)
if method.upper() == 'GET':
response = requests.request(method.upper(), url=url, json=json_body, headers=headers, auth=auth,
params=param)
if response:
try:
return response.json()
except ValueError:
return response.text
return None
with futures.ThreadPoolExecutor(max_workers=5) as executor:
res = executor.map(send_concurrent_request, url_list, headers_list, json_body_list, method_list, params_list)
return list(res)
def get_jobs(circle_token, project_slug):
r = requests.get(
"https://circleci.com/api/v2/insights/{0}/workflows/build/jobs".format(
project_slug
),
headers={
"Circle-Token": circle_token,
"User-Agent": "{0} {1}".format("tmhall99/buildstats", default_user_agent()),
},
)
r.raise_for_status()
jobs = r.json()
if jobs["next_page_token"] is not None:
print("Too many jobs")
return
return jobs
def __init__(self, config={}):
# version = '.'.join(map(str, sys.version_info[:3]))
# self.userAgent = {
# 'User-Agent': 'ccxt/' + __version__ + ' (+https://github.com/ccxt/ccxt) Python/' + version
# }
self.userAgent = default_user_agent()
settings = self.deep_extend(self.describe(), config)
for key in settings:
if hasattr(self, key) and isinstance(getattr(self, key), dict):
setattr(self, key,
self.deep_extend(getattr(self, key), settings[key]))
else:
setattr(self, key, settings[key])
if self.api:
self.define_rest_api(self.api, 'request')
if self.markets:
self.set_markets(self.markets)
# format camel case
for attr in dir(self):
if attr[0] != '_' and attr[-1] != '_' and '_' in attr:
conv = attr.split('_')
camel_case = conv[0] + ''.join(i[0].upper() + i[1:]
for i in conv[1:])
setattr(self, camel_case, getattr(self, attr))
self.tokenBucket = self.extend(
{
'refillRate': 1.0 / self.rateLimit,
'delay': 1.0,
'capacity': 1.0,
'defaultCost': 1.0,
'maxCapacity': 1000,
},
getattr(self, 'tokenBucket')
if hasattr(self, 'tokenBucket') else {})
self.session = self.session if self.session else Session()
def __init__(self, config={}):
# version = '.'.join(map(str, sys.version_info[:3]))
# self.userAgent = {
# 'User-Agent': 'ccxt/' + __version__ + ' (+https://github.com/ccxt/ccxt) Python/' + version
# }
self.userAgent = default_user_agent()
settings = self.deep_extend(self.describe(), config)
for key in settings:
if hasattr(self, key) and isinstance(getattr(self, key), dict):
setattr(self, key, self.deep_extend(getattr(self, key), settings[key]))
else:
setattr(self, key, settings[key])
if self.api:
self.define_rest_api(self.api, 'request')
if self.markets:
self.set_markets(self.markets)
# format camel case
for attr in dir(self):
if attr[0] != '_'and attr[-1] != '_' and '_' in attr:
conv = attr.split('_')
camel_case = conv[0] + ''.join(i[0].upper() + i[1:] for i in conv[1:])
setattr(self, camel_case, getattr(self, attr))
self.tokenBucket = self.extend({
'refillRate': 1.0 / self.rateLimit,
'delay': 1.0,
'capacity': 1.0,
'defaultCost': 1.0,
'maxCapacity': 1000,
}, getattr(self, 'tokenBucket') if hasattr(self, 'tokenBucket') else {})
self.session = self.session if self.session else Session()
def handle(self, **options):
if not settings.MEDIA_ROOT:
raise ImproperlyConfigured('Please set MEDIA_ROOT in your settings file')
remote_url = options['remote_url']
try:
val = URLValidator()
val(remote_url)
except ValidationError:
raise CommandError('Please enter a valid URL')
exercise_api = "{0}/api/v2/exercise/?limit=999&status=2"
image_api = "{0}/api/v2/exerciseimage/?exercise={1}"
thumbnail_api = "{0}/api/v2/exerciseimage/{1}/thumbnails/"
headers = {'User-agent': default_user_agent('wger/{} + requests'.format(get_version()))}
# Get all exercises
result = requests.get(exercise_api.format(remote_url), headers=headers).json()
for exercise_json in result['results']:
exercise_name = exercise_json['name'].encode('utf-8')
exercise_uuid = exercise_json['uuid']
exercise_id = exercise_json['id']
self.stdout.write('')
self.stdout.write(u"*** Processing {0} (ID: {1}, UUID: {2})".format(exercise_name,
exercise_id,
exercise_uuid))
try:
exercise = Exercise.objects.get(uuid=exercise_uuid)
except Exercise.DoesNotExist:
self.stdout.write(' Remote exercise not found in local DB, skipping...')
continue
# Get all images
images = requests.get(image_api.format(remote_url, exercise_id), headers=headers).json()
if images['count']:
for image_json in images['results']:
image_id = image_json['id']
result = requests.get(thumbnail_api.format(remote_url, image_id),
headers=headers).json()
image_name = os.path.basename(result['original'])
self.stdout.write(' Fetching image {0} - {1}'.format(image_id, image_name))
try:
image = ExerciseImage.objects.get(pk=image_id)
self.stdout.write(' --> Image already present locally, skipping...')
continue
except ExerciseImage.DoesNotExist:
self.stdout.write(' --> Image not found in local DB, creating now...')
image = ExerciseImage()
image.pk = image_id
# Save the downloaded image, see link for details
# http://stackoverflow.com/questions/1308386/programmatically-saving-image-to-
retrieved_image = requests.get(result['original'], headers=headers)
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(retrieved_image.content)
img_temp.flush()
image.exercise = exercise
image.is_main = image_json['is_main']
image.status = image_json['status']
image.image.save(
os.path.basename(image_name),
File(img_temp),
)
image.save()
else:
self.stdout.write(' No images for this exercise, nothing to do')
def _user_agent(self):
"""Create User-Agent header."""
return 'minFraud-API/%s %s' % (__version__, default_user_agent())
def _user_agent(self):
"""Create User-Agent header."""
return 'minFraud-API/%s %s' % (__version__, default_user_agent())
def determine_user_agent(user_agent):
if not user_agent or user_agent == default_user_agent():
return FAKE_USER_AGENT
return user_agent
def _user_agent(self):
return 'GeoIP2 Python Client v%s (%s)' % (geoip2.__version__,
default_user_agent())
def default(cls): cls.useragents = [default_user_agent()]
def __init__(self, app=App(), token=Token()):
super(ClientBase, self).__init__()
self.app = app
self.token = token
self._session.headers['User-Agent'] = default_user_agent(
'%s/%s requests' % (__title__, __version__))
def __init__(self,
client_id=None,
client_secret=None,
api_base_url=DEFAULT_API_URL,
api_ssl_verify=CA_BUNDLE_PATH,
site_base_url=DEFAULT_SITE_URL,
site_ssl_verify=CA_BUNDLE_PATH):
"""Initialize an Iconfinder API client.
Note that if :param:`client_id` is provided, :param:`client_secret`
must also be provided and vice versa.
:param client_id: Optional client ID. Default ``None``.
:param client_secret: Optional client secret. Default ``None``.
:param api_base_url:
API base URL. Default ``https://api.iconfinder.com/v2``.
:param api_ssl_verify:
API SSL verification. Refer to the `Requests documentation
<http://docs.python-requests.org/>`_ for further details. Defaults
to the included CA bundle for the Iconfinder wildcard SSL
certificate.
:param site_url: Site base URL. Default ``https://www.iconfinder.com``.
:param site_ssl_verify:
Site SSL verification. Refer to the `Requests documentation
<http://docs.python-requests.org/>`_ for further details. Defaults
to the included CA bundle for the Iconfinder wildcard SSL
certificate.
"""
# Validate client ID and secret.
if (client_id is None) != (client_secret is None):
raise ValueError('client_id and client_secret must both be '
'provided if one is provided')
if client_id is not None and not isinstance(client_id, string_types):
raise TypeError('client_id must be a string value')
if client_secret is not None and \
not isinstance(client_secret, string_types):
raise TypeError('client_secret must be a string value')
self._client_id = client_id or None
self._client_secret = client_secret or None
# Set up URLs etc.
self._api_base_url = api_base_url.rstrip('/')
self._api_ssl_verify = api_ssl_verify
self._site_base_url = site_base_url.rstrip('/')
self._site_ssl_verify = site_ssl_verify
# Set up sessions.
from . import __version__
user_agent = 'pyiconfinder/%s %s' % (__version__, default_user_agent())
self._api_session = requests.Session()
self._api_session.verify = api_ssl_verify
self._api_session.headers['User-Agent'] = user_agent
self._site_session = requests.Session()
self._site_session.verify = site_ssl_verify
self._site_session.headers['User-Agent'] = user_agent
# Set up model class proxies.
self.Author = ModelClassProxy(Author, self)
self.Category = ModelClassProxy(Category, self)
self.IconSet = ModelClassProxy(IconSet, self)
self.License = ModelClassProxy(License, self)
self.Style = ModelClassProxy(Style, self)
def __init__(
self,
expire_after: ExpirationTime = None,
cache_file: FileOrPath = CACHE_FILE,
per_second: int = REQUESTS_PER_SECOND,
per_minute: int = REQUESTS_PER_MINUTE,
per_day: float = REQUESTS_PER_DAY,
burst: int = REQUEST_BURST_RATE,
bucket_class: Type[AbstractBucket] = SQLiteBucket,
retries: int = REQUEST_RETRIES,
backoff_factor: float = RETRY_BACKOFF,
timeout: int = REQUEST_TIMEOUT,
user_agent: str = None,
**kwargs,
):
"""Get a Session object, optionally with custom settings for caching and rate-limiting.
Args:
expire_after: How long to keep cached API requests; for advanced options, see
`requests-cache: Expiration <https://requests-cache.readthedocs.io/en/latest/user_guide/expiration.html>`_
cache_file: Cache file path to use; defaults to the system default cache directory
per_second: Max requests per second
per_minute: Max requests per minute
per_day: Max requests per day
burst: Max number of consecutive requests allowed before applying per-second rate-limiting
bucket_class: Rate-limiting backend to use. Defaults to a persistent SQLite database.
retries: Maximum number of times to retry a failed request
backoff_factor: Factor for increasing delays between retries
timeout: Maximum number of seconds to wait for a response from the server
user_agent: Additional User-Agent info to pass to API requests
kwargs: Additional keyword arguments for :py:class:`~requests_cache.session.CachedSession`
and/or :py:class:`~requests_ratelimiter.requests_ratelimiter.LimiterSession`
"""
# If not overridden, use Cache-Control when possible, and some default expiration times
if not expire_after:
kwargs.setdefault('cache_control', True)
kwargs.setdefault('urls_expire_after', CACHE_EXPIRATION)
self.timeout = timeout
super().__init__( # type: ignore # false positive
# Cache settings
cache_name=cache_file,
backend='sqlite',
expire_after=expire_after,
ignored_parameters=['Authorization', 'access_token'],
old_data_on_error=True,
# Rate limit settings
bucket_class=bucket_class,
bucket_kwargs={'path': RATELIMIT_FILE},
per_second=per_second,
per_minute=per_minute,
per_day=per_day,
per_host=True,
burst=burst,
max_delay=MAX_DELAY,
**kwargs,
)
# Set default headers
self.headers['Accept'] = 'application/json'
user_agent_details = [
default_user_agent(),
f'pyinaturalist/{pyinaturalist.__version__}',
user_agent or '',
]
self.headers['User-Agent'] = ' '.join(user_agent_details).strip()
# Mount an adapter to apply retry settings
retry = Retry(total=retries, backoff_factor=backoff_factor)
adapter = HTTPAdapter(max_retries=retry)
self.mount('https://', adapter)
def user_agent_header():
return u'%s/%s %s' % (metadata.package, metadata.version,
default_user_agent())
def default_headers():
return CaseInsensitiveDict({
'User-Agent': ' '.join(['python-boxview/1.0', default_user_agent()]),
'Accept': '*/*',
'Accept-Encoding': ', '.join(('gzip', 'deflate', 'compress')),
})
def handle(self, **options):
remote_url = options['remote_url']
try:
val = URLValidator()
val(remote_url)
except ValidationError:
raise CommandError('Please enter a valid URL')
headers = {
'User-agent':
default_user_agent('wger/{} + requests'.format(get_version()))
}
#
# Categories
#
self.stdout.write('*** Synchronizing categories...')
result = requests.get(CATEGORY_API.format(remote_url),
headers=headers).json()
for category_data in result['results']:
category_id = category_data['id']
category_name = category_data['name']
try:
category = ExerciseCategory.objects.get(pk=category_id)
category.name = category_name
category.save()
except ExerciseCategory.DoesNotExist:
self.stdout.write(
self.style.WARNING(f'Saving new category {category_name}'))
category = ExerciseCategory(id=category_id, name=category_name)
category.save()
self.stdout.write(self.style.SUCCESS('done!\n'))
#
# Muscles
#
self.stdout.write('*** Synchronizing muscles...')
result = requests.get(MUSCLE_API.format(remote_url),
headers=headers).json()
for muscle_data in result['results']:
muscle_id = muscle_data['id']
muscle_name = muscle_data['name']
muscle_is_front = muscle_data['is_front']
muscle_url_main = muscle_data['image_url_main']
muscle_url_secondary = muscle_data['image_url_secondary']
try:
muscle = Muscle.objects.get(pk=muscle_id)
muscle.name = muscle_name
muscle_is_front = muscle_is_front
muscle.save()
except Muscle.DoesNotExist:
muscle = Muscle(id=muscle_id,
name=muscle_name,
is_front=muscle_is_front)
muscle.save()
self.stdout.write(
self.style.WARNING(
f'Saved new muscle {muscle_name}. '
f'Save the corresponding images manually'))
self.stdout.write(self.style.WARNING(muscle_url_main))
self.stdout.write(self.style.WARNING(muscle_url_secondary))
self.stdout.write(self.style.SUCCESS('done!\n'))
#
# Equipment
#
self.stdout.write('*** Synchronizing equipment...')
result = requests.get(EQUIPMENT_API.format(remote_url),
headers=headers).json()
for equipment_data in result['results']:
equipment_id = equipment_data['id']
equipment_name = equipment_data['name']
try:
equipment = Equipment.objects.get(pk=equipment_id)
equipment.name = equipment_name
equipment.save()
except Equipment.DoesNotExist:
self.stdout.write(f'Saved new equipment {equipment_name}')
equipment = Equipment(id=equipment_id, name=equipment_name)
equipment.save()
self.stdout.write(self.style.SUCCESS('done!\n'))
#
# Exercises
#
self.stdout.write('*** Synchronizing exercises...')
page = 1
all_exercise_processed = False
result = requests.get(EXERCISE_API.format(remote_url),
headers=headers).json()
while not all_exercise_processed:
for data in result['results']:
exercise_uuid = data['uuid']
exercise_name = data['name']
exercise_description = data['description']
equipment = [
Equipment.objects.get(pk=i['id'])
for i in data['equipment']
]
muscles = [
Muscle.objects.get(pk=i['id']) for i in data['muscles']
]
muscles_sec = [
Muscle.objects.get(pk=i['id'])
for i in data['muscles_secondary']
]
try:
exercise = Exercise.objects.get(uuid=exercise_uuid)
exercise.name = exercise_name
exercise.description = exercise_description
# Note: this should not happen and is an unnecessary workaround
# https://github.com/wger-project/wger/issues/840
if not exercise.exercise_base:
warning = f'Exercise {exercise.uuid} has no base, this should not happen!' \
f'Skipping...\n'
self.stdout.write(self.style.WARNING(warning))
continue
exercise.exercise_base.category_id = data['category']['id']
exercise.exercise_base.muscles.set(muscles)
exercise.exercise_base.muscles_secondary.set(muscles_sec)
exercise.exercise_base.equipment.set(equipment)
exercise.exercise_base.save()
exercise.save()
except Exercise.DoesNotExist:
self.stdout.write(f'Saved new exercise {exercise_name}')
exercise = Exercise(
uuid=exercise_uuid,
name=exercise_name,
description=exercise_description,
language_id=data['language']['id'],
license_id=data['license']['id'],
license_author=data['license_author'],
)
base = ExerciseBase()
base.category_id = data['category']['id']
base.save()
base.muscles.set(muscles)
base.muscles_secondary.set(muscles_sec)
base.equipment.set(equipment)
base.save()
exercise.save()
if result['next']:
page += 1
result = requests.get(result['next'], headers=headers).json()
else:
all_exercise_processed = True
self.stdout.write(self.style.SUCCESS('done!\n'))
def _generate_user_agent(name='t1-python'):
return '{name}/{version} {ua}'.format(name=name,
version=__version__,
ua=default_user_agent())
def useragentify(session):
"""declare a specific user-agent for commit --blog"""
user_agent = default_user_agent('commit --blog')
session.headers.update({'User-Agent': user_agent})
def user_agent_header():
return u'{0}/{1} {2}'.format(metadata.package, metadata.version,
default_user_agent())
def handle(self, **options):
if not settings.MEDIA_ROOT:
raise ImproperlyConfigured('Please set MEDIA_ROOT in your settings file')
remote_url = options['remote_url']
try:
val = URLValidator()
val(remote_url)
except ValidationError:
raise CommandError('Please enter a valid URL')
headers = {'User-agent': default_user_agent('wger/{} + requests'.format(get_version()))}
# Get all images
page = 1
all_images_processed = False
result = requests.get(IMAGE_API.format(remote_url), headers=headers).json()
self.stdout.write('*** Processing images ***')
while not all_images_processed:
self.stdout.write('')
self.stdout.write(f'*** Page {page}')
self.stdout.write('')
page += 1
if result['next']:
result = requests.get(result['next'], headers=headers).json()
else:
all_images_processed = True
for image_data in result['results']:
image_uuid = image_data['uuid']
self.stdout.write(f'Processing image {image_uuid}')
try:
exercise_base = ExerciseBase.objects.get(id=image_data['exercise'])
except ExerciseBase.DoesNotExist:
self.stdout.write(' Remote exercise base not found in local DB, skipping...')
continue
try:
image = ExerciseImage.objects.get(uuid=image_uuid)
self.stdout.write(' Image already present locally, skipping...')
continue
except ExerciseImage.DoesNotExist:
self.stdout.write(' Image not found in local DB, creating now...')
image = ExerciseImage()
image.uuid = image_uuid
# Save the downloaded image
# http://stackoverflow.com/questions/1308386/programmatically-saving-image-to-
retrieved_image = requests.get(image_data['image'], headers=headers)
# Temporary files on windows don't support the delete attribute
if os.name == 'nt':
img_temp = NamedTemporaryFile()
else:
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(retrieved_image.content)
img_temp.flush()
image.exercise = exercise_base
image.is_main = image_data['is_main']
image.status = image_data['status']
image.image.save(
os.path.basename(os.path.basename(image_data['image'])),
File(img_temp),
)
image.save()
self.stdout.write(self.style.SUCCESS(' successfully saved'))
def useragentify(session):
"""declare a specific user-agent for commit --blog"""
user_agent = default_user_agent('commit --blog')
session.headers.update({'User-Agent': user_agent})
def _generate_user_agent(name='t1-python'):
return '{name}/{version} {ua}'.format(name=name, version=__version__,
ua=default_user_agent())
def __init__(self,
client_id=None,
client_secret=None,
api_base_url=DEFAULT_API_URL,
api_ssl_verify=CA_BUNDLE_PATH,
site_base_url=DEFAULT_SITE_URL,
site_ssl_verify=CA_BUNDLE_PATH):
"""Initialize an Iconfinder API client.
Note that if :param:`client_id` is provided, :param:`client_secret`
must also be provided and vice versa.
:param client_id: Optional client ID. Default ``None``.
:param client_secret: Optional client secret. Default ``None``.
:param api_base_url:
API base URL. Default ``https://api.iconfinder.com/v2``.
:param api_ssl_verify:
API SSL verification. Refer to the `Requests documentation
<http://docs.python-requests.org/>`_ for further details. Defaults
to the included CA bundle for the Iconfinder wildcard SSL
certificate.
:param site_url: Site base URL. Default ``https://www.iconfinder.com``.
:param site_ssl_verify:
Site SSL verification. Refer to the `Requests documentation
<http://docs.python-requests.org/>`_ for further details. Defaults
to the included CA bundle for the Iconfinder wildcard SSL
certificate.
"""
# Validate client ID and secret.
if (client_id is None) != (client_secret is None):
raise ValueError('client_id and client_secret must both be '
'provided if one is provided')
if client_id is not None and not isinstance(client_id, string_types):
raise TypeError('client_id must be a string value')
if client_secret is not None and \
not isinstance(client_secret, string_types):
raise TypeError('client_secret must be a string value')
self._client_id = client_id or None
self._client_secret = client_secret or None
# Set up URLs etc.
self._api_base_url = api_base_url.rstrip('/')
self._api_ssl_verify = api_ssl_verify
self._site_base_url = site_base_url.rstrip('/')
self._site_ssl_verify = site_ssl_verify
# Set up sessions.
from . import __version__
user_agent = 'pyiconfinder/%s %s' % (__version__, default_user_agent())
self._api_session = requests.Session()
self._api_session.verify = api_ssl_verify
self._api_session.headers['User-Agent'] = user_agent
self._site_session = requests.Session()
self._site_session.verify = site_ssl_verify
self._site_session.headers['User-Agent'] = user_agent
# Set up model class proxies.
self.Author = ModelClassProxy(Author, self)
self.Category = ModelClassProxy(Category, self)
self.IconSet = ModelClassProxy(IconSet, self)
self.License = ModelClassProxy(License, self)
self.Style = ModelClassProxy(Style, self)
def _user_agent(self):
return 'GeoIP2 Python Client v%s (%s)' % (geoip2.__version__,
default_user_agent())
def handle(self, **options):
if not settings.MEDIA_ROOT:
raise ImproperlyConfigured(
'Please set MEDIA_ROOT in your settings file')
remote_url = options['remote_url']
try:
val = URLValidator()
val(remote_url)
except ValidationError:
raise CommandError('Please enter a valid URL')
headers = {
'User-agent':
default_user_agent('wger/{} + requests'.format(get_version()))
}
# Get all exercises
result = requests.get(EXERCISE_API.format(remote_url),
headers=headers).json()
for exercise_json in result['results']:
exercise_name = exercise_json['name']
exercise_uuid = exercise_json['uuid']
exercise_id = exercise_json['id']
self.stdout.write('')
self.stdout.write(
f"*** {exercise_name} (ID: {exercise_id}, UUID: {exercise_uuid})"
)
try:
exercise = Exercise.objects.get(uuid=exercise_uuid)
except Exercise.DoesNotExist:
self.stdout.write(
' Remote exercise not found in local DB, skipping...')
continue
# Get all images
images = requests.get(IMAGE_API.format(remote_url, exercise_id),
headers=headers).json()
if images['count']:
for image_json in images['results']:
image_id = image_json['id']
image_uuid = image_json['uuid']
result = requests.get(THUMBNAIL_API.format(
remote_url, image_id),
headers=headers).json()
image_name = os.path.basename(result['original'])
self.stdout.write(' Fetching image {0} - {1}'.format(
image_id, image_name))
try:
image = ExerciseImage.objects.get(uuid=image_uuid)
self.stdout.write(
' --> Image already present locally, skipping...'
)
continue
except ExerciseImage.DoesNotExist:
self.stdout.write(
' --> Image not found in local DB, creating now...'
)
image = ExerciseImage()
image.uuid = image_uuid
# Save the downloaded image
# http://stackoverflow.com/questions/1308386/programmatically-saving-image-to-
retrieved_image = requests.get(remote_url +
result['original'],
headers=headers)
# Temporary files on windows don't support the delete attribute
if os.name == 'nt':
img_temp = NamedTemporaryFile()
else:
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(retrieved_image.content)
img_temp.flush()
image.exercise = exercise.exercise_base
image.is_main = image_json['is_main']
image.status = image_json['status']
image.image.save(
os.path.basename(image_name),
File(img_temp),
)
image.save()
else:
self.stdout.write(
' No images for this exercise, nothing to do')
def set_user_agent(self, extra=None):
version = get_distribution('steelscript').version
ua = '%s SteelScript/%s' % (default_user_agent(), version)
if extra:
ua = '%s %s' % (ua, extra)
self.conn.headers['User-Agent'] = ua
def get_user_agent(self):
if not hasattr(self, "_user_agent"):
self._user_agent = default_user_agent()
return self._user_agent
def set_user_agent(self, extra=None):
version = get_distribution('steelscript').version
ua = '%s SteelScript/%s' % (default_user_agent(), version)
if extra:
ua = '%s %s' % (ua, extra)
self.conn.headers['User-Agent'] = ua
def __init__(self, app=App(), token=Token()):
super(ClientBase, self).__init__()
self.app = app
self.token = token
self._session.headers['User-Agent'] = default_user_agent('%s/%s requests' % (__title__, __version__))
def handle(self, **options):
if not settings.MEDIA_ROOT:
raise ImproperlyConfigured(
'Please set MEDIA_ROOT in your settings file')
remote_url = options['remote_url']
try:
val = URLValidator()
val(remote_url)
except ValidationError:
raise CommandError('Please enter a valid URL')
exercise_api = "{0}/api/v2/exercise/?limit=999"
image_api = "{0}/api/v2/exerciseimage/?exercise={1}"
thumbnail_api = "{0}/api/v2/exerciseimage/{1}/thumbnails/"
headers = {
'User-agent':
default_user_agent('wger/{} + requests'.format(get_version()))
}
# Get all exercises
result = requests.get(
exercise_api.format(remote_url), headers=headers).json()
for exercise_json in result['results']:
exercise_name = exercise_json['name'].encode('utf-8')
exercise_uuid = exercise_json['uuid']
exercise_id = exercise_json['id']
self.stdout.write('')
self.stdout.write(
u"*** Processing {0} (ID: {1}, UUID: {2})".format(
exercise_name, exercise_id, exercise_uuid))
try:
exercise = Exercise.objects.get(uuid=exercise_uuid)
except Exercise.DoesNotExist:
self.stdout.write(
' Remote exercise not found in local DB, skipping...')
continue
# Get all images
images = requests.get(
image_api.format(remote_url, exercise_id),
headers=headers).json()
if images['count']:
for image_json in images['results']:
image_id = image_json['id']
result = requests.get(
thumbnail_api.format(remote_url, image_id),
headers=headers).json()
image_name = os.path.basename(result['original'])
self.stdout.write(' Fetching image {0} - {1}'.format(
image_id, image_name))
try:
image = ExerciseImage.objects.get(pk=image_id)
self.stdout.write(
' --> Image already present locally, skipping...'
)
continue
except ExerciseImage.DoesNotExist:
self.stdout.write(
' --> Image not found in local DB, creating now...'
)
image = ExerciseImage()
image.pk = image_id
# Save the downloaded image, see link for details
# http://stackoverflow.com/questions/1308386/programmatically-saving-image-to-
retrieved_image = requests.get(
result['original'], headers=headers)
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(retrieved_image.content)
img_temp.flush()
image.exercise = exercise
image.is_main = image_json['is_main']
image.status = image_json['status']
image.image.save(
os.path.basename(image_name),
File(img_temp),
)
image.save()
else:
self.stdout.write(
' No images for this exercise, nothing to do')
def determine_user_agent(user_agent):
if not user_agent or user_agent == default_user_agent():
return FAKE_USER_AGENT
return user_agent
