def get_data(self, endpoint, params):
'''helper method to get data from tmdb json API'''
if self.api_key:
params["api_key"] = self.api_key
rate_limit = None
expiration = datetime.timedelta(days=7)
else:
params["api_key"] = "ae06df54334aa653354e9a010f4b81cb"
# without personal api key = rate limiting and older info from cache
rate_limit = ("themoviedb.org",10)
expiration = datetime.timedelta(days=60)
cachestr = "tmdb.%s" % params.itervalues()
cache = self.cache.get(cachestr)
if cache:
# data obtained from cache
result = cache
else:
# no cache, grab data from API
url = u'http://api.themoviedb.org/3/%s' % endpoint
result = get_json(url, params)
# make sure that we have a plot value (if localized value fails, fallback to english)
if result and "language" in params and "overview" in result:
if not result["overview"] and params["language"] != "en":
params["language"] = "en"
result2 = get_json(url, params)
if result2 and result2.get("overview"):
result = result2
self.cache.set(url, result, expiration=expiration)
return result
def get_followers(self, user_id, raw=False):
# make request
url = 'https://api.instagram.com/v1/users/' + str(user_id) + '/followed-by'
params = {
'access_token': self.token
}
# make request with pagination
users = []
cursor = ''
counter = 0
while not (cursor is None) and counter < 100:
params['cursor'] = cursor
resp = get_json(url, params)
data = resp['data']
if raw:
print data
if resp['pagination']:
cursor = resp['pagination']['next_cursor']
else:
cursor = None
for item in data:
user = {}
keys = ['id', 'username', 'full_name']
for key in keys:
user[key] = item[key]
users.append(user)
counter += 1
return users
def get_animatedart_db(self):
'''get the full animated art database as dict with imdbid and tmdbid as key
uses 7 day cache to prevent overloading the server'''
# get all animated posters from the online json file
cache = self.cache.get("animatedartdb")
if cache:
return cache
art_db = {}
data = get_json('http://www.consiliumb.com/animatedgifs/movies.json', None)
base_url = data.get("baseURL", "")
if data and data.get('movies'):
for item in data['movies']:
for db_id in ["imdbid", "tmdbid"]:
key = item[db_id]
art_db[key] = {"posters": [], "fanarts": []}
for entry in item['entries']:
entry_new = {
"contributedby": entry["contributedBy"],
"dateadded": entry["dateAdded"],
"language": entry["language"],
"source": entry["source"],
"image": "%s/%s" % (base_url, entry["image"].replace(".gif", "_original.gif")),
"thumb": "%s/%s" % (base_url, entry["image"])}
if entry['type'] == 'poster':
art_db[key]["posters"].append(entry_new)
elif entry['type'] == 'background':
art_db[key]["fanarts"].append(entry_new)
self.cache.set("animatedartdb", art_db, expiration=timedelta(days=7))
return art_db
def update(self, data=None, etag=None):
if not data:
data, etag = get_json(self.service.url, self.service.etag)
if data:
with db.transaction():
self.update_service_data(data, etag)
self.update_mods(data["mods"])
def get_cinfo(c_list):
src_list = []
for channel in c_list:
# 频道名
cname = channel["STATION_NAME"]
code = get_code(cname)
# 如果IsSohuSource字段等于2则该频道的源来自cntv，且会自动跳转到cntv
if 1 == channel["IsSohuSource"] and code:
try:
print "getting url from %s" % code
# 频道ID
cid = channel["STATION_ID"]
t = random.uniform(1, 10)
url = api % (cid, t)
data = utils.get_json(url)
src = {}
src['code'] = code
src['hls'] = data['data']['hls']
src_list.append(src)
except Exception, e:
print e
else:
pass
finally:
def from_sirna_result(task_id):
"""Gets sh-miR result created from siRNA via task_id
Args:
task_id: Id of task which was given by RESTful API
Returns:
None
"""
data = get_json("from_sirna_result", task_id)['data']
path = "./results/sirna/{}/".format(task_id)
if data['result']:
print("Results under: {}\n".format(path))
else:
print("No sh-miR found")
for no, (points, shmir, name, mfold_id) in enumerate(data['result']):
mfold_result(mfold_id, path=path, verbose=False)
new_path = path + name
os.rename(path + mfold_id, new_path)
print("{}: name: {}, score: {}, pdf: {}\n result: {}\n".format(
no + 1, name, points, new_path, shmir
))
def find_id(self, username, raw=False):
# make request
url = 'https://api.instagram.com/v1/users/search?q=' + username + '&client_id=e467b070519f452abe8e687393081b96'
params = {
'q': username,
'access_token': self.token
}
data = get_json(url, params)['data']
# for debug
if raw:
return data
# search for username in response
res = None
for user in data:
if user['username'] == username:
res = user['id']
break
# not found handling
if not res:
print 'User not found'
return None
return res
def get_user_info(self, user_id, raw=False):
# make request
url = 'https://api.instagram.com/v1/users/' + str(user_id)
params = {
'access_token': self.token
}
try:
data = get_json(url, params)['data']
except:
print 'User not found'
return None
# for debug
if raw:
return data
# extract necessary user info from response
user = {}
keys = ['id', 'username', 'bio', 'full_name']
for key in keys:
user[key] = data[key]
user['media_count'] = data['counts']['media']
user['followed_by_count'] = data['counts']['followed_by']
user['follows_count'] = data['counts']['follows']
user['pic_url'] = data['profile_picture']
return user
def crawl(self):
# 지역구 대표
jobs = []
constant_candidates = self.constant_candidates
candidate_type = self.candidate_type
target = self.target
target_eng = self.target_eng
target_kor = self.target_kor
nth = self.nth
req_url = self.urlPath_result_list
townCode_JSON = get_json(self.urlPath_city_codes, self.urlParam_city_codes)[0]['results']
print("\x1b[1;36mWaiting to connect http://info.nec.go.kr server (%s, %d-th)...\x1b[1;m" % (target_eng, nth))
for city_index, (city_code, city_name) in list(enumerate(self.city_codes(townCode_JSON))):
req_param = self.XHTML_url_param(city_code)
job = gevent.spawn(self.parse, req_url, req_param, constant_candidates, target, target_kor, city_name, city_code, city_index, townCode_JSON)
jobs.append(job)
gevent.joinall(jobs)
every_result = [{'election_type':target_eng, 'nth':nth, 'candidate_type':candidate_type, \
'results':flatten(job.get() for job in jobs)}]
# 비례대표
if hasattr(self, 'next_crawler'):
prop_result = self.next_crawler.crawl()
every_result.extend(prop_result)
return every_result
def get_data(self, endpoint, params):
'''helper method to get data from theaudiodb json API'''
endpoint = 'http://www.theaudiodb.com/api/v1/json/%s/%s' % (self.api_key, endpoint)
data = get_json(endpoint, params)
if data:
return data
else:
return {}
def get_clist():
try:
js = utils.get_json(channels_url, pattern="par=(.*?);")
channnel_list = js["STATIONS"]
return channnel_list
except Exception, e:
print channel_error, e
exit(1)
def get_code(name):
with open("code.json", "r") as f:
js = utils.get_json(f.read())
for category in js.values():
for c in category:
if name in c:
return c[0]
def from_sirna_result(task_id):
"""Gets sh-miR result created from siRNA via task_id
Args:
task_id: Id of task which was given by RESTful API
Returns:
None
"""
data = get_json("from_sirna_result", task_id)["data"]
path = "./results/sirna/{}/".format(task_id)
if data["result"]:
print("Results in: {}/".format(path))
if not os.path.exists(path):
os.makedirs(path)
columns = [
"scaffold name",
"first sequence",
"second sequence",
"sh-miR",
"overall score",
"structure score",
"homogenity score",
"same ends score",
"pdf",
]
# parsing results into csv
with open(path + "result.csv", "w") as fp:
writer = csv.writer(fp)
writer.writerow(columns)
for no, result in enumerate(data["result"]):
mfold_result(result["pdf_reference"], path=path, verbose=False)
new_path = "{}{}/{}".format(path, result["scaffold_name"], no + 1)
if not os.path.exists(new_path):
os.makedirs(new_path)
os.rename(path + result["pdf_reference"], new_path)
parsed = [
result["scaffold_name"],
result["sequences"][0],
result["sequences"][1],
result["shmir"],
result["score"]["all"],
result["score"]["structure"],
result["score"]["homogeneity"],
result["score"]["same_ends"],
new_path,
]
writer.writerow(parsed)
print_parsed_result(no, columns, parsed)
else:
print("No sh-miR found")
def get_data(self, params):
'''helper method to get data from lastfm json API'''
params["format"] = "json"
params["api_key"] = self.api_key
data = get_json('http://ws.audioscrobbler.com/2.0/', params)
if data:
return data
else:
return {}
def add_service(self, json_url):
if ModService.select().where(ModService.url==json_url).count():
return None
data, etag = get_json(json_url)
service = ModService(url=json_url)
service.name = data["service"]["name"]
service.set_mirrors(data["service"]["filelocations"])
service.save()
L.info("Service %s added", service.name)
return service
def parse(self, url, params):
_party_list = get_json(url, params)['jsonResult']['body'] # 정당 code json을 받음.
for i in range(int(len(_party_list))):
x = _party_list[i]
x['code_thisElec'] = i+1 # 각 정당별 '기호'(순번)을 매겨줌.
if isinstance(x['CODE'], str): # if x['CODE'] is string type object...
x['CODE'] = int(x['CODE'])
print('crawled %s election #%d - 선거참여 정당, 전국(%d)...' % (self.target_eng, self.nth, len(_party_list)))
return _party_list
def parse_town(self, url, params, target, target_kor, town_code=None, town_name=None):
_sgg_list = get_json(url, params)['jsonResult']['body']
for x in _sgg_list:
if isinstance(x['CODE'], str): # if x['CODE'] is string type object...
x['CODE'] = int(x['CODE'])
_result = [dict(town_name=town_name, town_code=int(town_code), consti_list=_sgg_list)]
print('\t└└ %s, %s(%d)...' % (target_kor+' 선거구 목록', town_name, len(_sgg_list)))
return _result
def get_panoramas(self):
init = 1
end = 101
panoramio_size = 'square'
panoramio_url = "http://www.panoramio.com/map/get_panoramas.php?set=public&from=%i&to=%i&minx=%s&miny=%s&maxx=%s&maxy=%s&size=%s"
panoramio_url = panoramio_url % (init, end, self.min_x, self.min_y, self.max_x, self.max_y, panoramio_size)
json = get_json(panoramio_url)
photos = json.get('photos')
return photos
def get_data(self, params):
'''helper method to get data from omdb json API'''
base_url = 'http://www.omdbapi.com/'
params["plot"] = "short"
if self.api_key:
params["apikey"] = api_key
rate_limit = None
else:
# rate limited api key !
params["apikey"] = "ea23cea2"
rate_limit = ("omdbapi.com", 2)
params["r"] = "json"
return get_json(base_url, params, ratelimit=rate_limit)
def source_vst():
urllist = []
try:
js = utils.get_html()
info_dict = utils.get_json(js)
for item in info_dict["live"]:
url_list = item["urllist"].split("#")
for url in url_list:
urllist.append(url)
except Exception, e:
print api_error, e
exit(1)
def test_filtered_circ_policies_get(client, librarian_martigny_no_email,
circ_policy_default_martigny,
circ_policy_short_martigny,
circ_policy_temp_martigny,
librarian_sion_no_email,
circ_policy_default_sion):
"""Test circulation policies filter by organisation."""
# Martigny
login_user_via_session(client, librarian_martigny_no_email.user)
list_url = url_for('invenio_records_rest.cipo_list')
res = client.get(list_url)
assert res.status_code == 200
data = get_json(res)
assert data['hits']['total']['value'] == 3
# Sion
login_user_via_session(client, librarian_sion_no_email.user)
list_url = url_for('invenio_records_rest.cipo_list')
res = client.get(list_url)
assert res.status_code == 200
data = get_json(res)
assert data['hits']['total']['value'] == 1
def test_item_types_name_validate(client, item_type_standard_martigny):
"""Test record name validation."""
url = url_for('item_types.name_validate', name='standard')
class current_patron:
class organisation:
pid = 'org1'
with mock.patch('rero_ils.modules.item_types.views.current_patron',
current_patron):
res = client.get(url)
assert res.status_code == 200
assert get_json(res) == {'name': 'standard'}
class current_patron:
class organisation:
pid = 'does not exists'
with mock.patch('rero_ils.modules.item_types.views.current_patron',
current_patron):
res = client.get(url)
assert res.status_code == 200
assert get_json(res) == {'name': None}
def test_patrons_logged_user(client, librarian_martigny):
"""Test logged user info API."""
# No logged user (only settings are present)
res = client.get(url_for('patrons.logged_user'))
assert res.status_code == 200
data = get_json(res)
assert not data.get('metadata')
assert not data.get('patrons')
assert data.get('settings')
# logged user
login_user_via_session(client, librarian_martigny.user)
res = client.get(url_for('patrons.logged_user', resolve=1))
assert res.status_code == 200
data = get_json(res)
assert data.get('first_name')
assert data.get('last_name')
assert data.get('patrons')
assert data.get('settings')
assert 'organisation' in data.get('patrons')[0].get('libraries')[0]
assert data.get('patrons')[0].get('organisation')
class current_i18n:
class locale:
language = 'fr'
with mock.patch('rero_ils.modules.patrons.views.current_i18n',
current_i18n):
login_user_via_session(client, librarian_martigny.user)
res = client.get(url_for('patrons.logged_user'))
assert res.status_code == 200
data = get_json(res)
assert data.get('patrons')[0]['libraries'][0]['pid'] == \
librarian_martigny['libraries'][0]['$ref'].rsplit('/', 1)[-1]
assert data.get('settings').get('language') == 'fr'
def test_item_possible_actions(client, item_lib_martigny,
librarian_martigny_no_email,
patron_martigny_no_email, circulation_policies):
"""Possible action changes according to params of cipo."""
login_user_via_session(client, librarian_martigny_no_email.user)
item = item_lib_martigny
patron_pid = patron_martigny_no_email.pid
res = client.get(
url_for('api_item.item',
item_barcode=item.get('barcode'),
patron_pid=patron_pid))
assert res.status_code == 200
data = get_json(res)
actions = data.get('metadata').get('item').get('actions')
assert 'checkout' in actions
from rero_ils.modules.circ_policies.api import CircPolicy
circ_policy = CircPolicy.provide_circ_policy(item.library_pid, 'ptty1',
'itty1')
circ_policy['allow_checkout'] = False
circ_policy.update(circ_policy, dbcommit=True, reindex=True)
res = client.get(
url_for('api_item.item',
item_barcode=item.get('barcode'),
patron_pid=patron_pid))
assert res.status_code == 200
data = get_json(res)
actions = data.get('metadata').get('item').get('actions')
assert 'checkout' not in actions
circ_policy['allow_checkout'] = True
circ_policy.update(circ_policy, dbcommit=True, reindex=True)
assert circ_policy['allow_checkout']
def test_filtered_patron_types_get(client, librarian_martigny_no_email,
patron_type_children_martigny,
patron_type_adults_martigny,
librarian_sion_no_email,
patron_type_youngsters_sion,
patron_type_grown_sion):
"""Test patron types filter by organisation."""
# Martigny
login_user_via_session(client, librarian_martigny_no_email.user)
list_url = url_for('invenio_records_rest.ptty_list')
res = client.get(list_url)
assert res.status_code == 200
data = get_json(res)
assert data['hits']['total'] == 2
# Sion
login_user_via_session(client, librarian_sion_no_email.user)
list_url = url_for('invenio_records_rest.ptty_list')
res = client.get(list_url)
assert res.status_code == 200
data = get_json(res)
assert data['hits']['total'] == 2
def test_patron_types_name_validate(client, patron_type_children_martigny):
"""Test patron type name validation."""
url = url_for('patron_types.name_validate', name='children')
class current_patron:
class organisation:
pid = 'org1'
with mock.patch('rero_ils.modules.patron_types.views.current_patron',
current_patron):
res = client.get(url)
assert res.status_code == 200
assert get_json(res) == {'name': 'children'}
class current_patron:
class organisation:
pid = 'does not exists'
with mock.patch('rero_ils.modules.patron_types.views.current_patron',
current_patron):
res = client.get(url)
assert res.status_code == 200
assert get_json(res) == {'name': None}
def test_patrons_serializers(client, json_header, librarian_martigny,
librarian2_martigny):
"""Test serializers for patrons."""
login_user(client, librarian_martigny)
list_url = url_for('invenio_records_rest.ptrn_list')
response = client.get(list_url, headers=json_header)
assert response.status_code == 200
# Get the first result and check if it contains all desired keys.
data = get_json(response)
hit = data['hits']['hits'][0]
for key in ['created', 'updated', 'id', 'links', 'metadata']:
assert key in hit
assert hit[key]
def test_documents_facets(client, document, item_lib_martigny,
rero_json_header):
"""Test record retrieval."""
list_url = url_for('invenio_records_rest.doc_list', view='global')
res = client.get(list_url, headers=rero_json_header)
data = get_json(res)
aggs = data['aggregations']
# check all facets are present
for facet in [
'document_type', 'author__en', 'author__fr', 'author__de',
'author__it', 'language', 'subject', 'status'
]:
assert aggs[facet]
def get_pitch_arsenal(pitcherid, season=None):
url = API_LINK + "people/%s/stats?stats=pitchArsenal&group=pitching" % pitcherid
if season is not None:
url += "&season=%s" % season
results = utils.get_json(url)
pitches = list()
for pitch in results['stats'][0]['splits']:
p = pitch['stat']
p['code'] = pitch['stat']['type']['code']
p['description'] = pitch['stat']['type']['description']
p['percentage'] = '%.1f' % (p['percentage']*100)
p['averageSpeed'] = '%.1f' % p['averageSpeed']
pitches.append(p)
return pitches
def test_get_json(self, test_url, test_payload):
""" test get json.
"""
class MockObject(Mock):
""" MockObject class
"""
def json(self):
""" json method
"""
return test_payload
with patch('utils.requests.get') as MockClass:
MockClass.return_value = MockObject()
response = get_json(test_url)
self.assertEqual(response, test_payload)
def test_filtered_locations_get(client, librarian_martigny_no_email,
loc_public_martigny, loc_restricted_martigny,
loc_public_saxon, loc_restricted_saxon,
loc_public_fully, loc_restricted_fully,
librarian_sion_no_email, loc_public_sion,
loc_restricted_sion):
"""Test location filter by organisation."""
# Martigny
login_user_via_session(client, librarian_martigny_no_email.user)
list_url = url_for('invenio_records_rest.loc_list')
res = client.get(list_url)
assert res.status_code == 200
data = get_json(res)
assert data['hits']['total'] == 6
# Sion
login_user_via_session(client, librarian_sion_no_email.user)
list_url = url_for('invenio_records_rest.loc_list')
res = client.get(list_url)
assert res.status_code == 200
data = get_json(res)
assert data['hits']['total'] == 2
def function(self, update=None, context=None):
repo_url = 'http://api.github.com/orgs/pug-se/repos'
text = 'Os projetos da comunidade estão no '
text += f'<a href="{repo_url}">GitHub</a>\n\n'
url = 'https://api.github.com/orgs/pug-se/repos'
info_dict = get_json(url)
i = 1
for info in info_dict:
name = info['name']
description = info['description']
url = info['html_url']
text += f'{i}) <a href="{url}">{name}</a>: {description}\n'
i += 1
return text
def test_circ_policies_name_validate(client, circ_policy_default_martigny):
"""Test policy validation."""
url = url_for('circ_policies.name_validate', name='Default')
circ_policy = circ_policy_default_martigny
class current_patron:
class organisation:
pid = 'org1'
with mock.patch('rero_ils.modules.circ_policies.views.current_patron',
current_patron):
res = client.get(url)
assert res.status_code == 200
assert get_json(res) == {'name': 'Default'}
class current_patron:
class organisation:
pid = 'does not exists'
with mock.patch('rero_ils.modules.circ_policies.views.current_patron',
current_patron):
res = client.get(url)
assert res.status_code == 200
assert get_json(res) == {'name': None}
def test_acq_order_get(client, acq_order_fiction_martigny):
"""Test record retrieval."""
item_url = url_for('invenio_records_rest.acor_item', pid_value='acor1')
acq_order = deepcopy(acq_order_fiction_martigny)
res = client.get(item_url)
assert res.status_code == 200
assert res.headers['ETag'] == '"{}"'.format(acq_order.revision_id)
data = get_json(res)
assert acq_order.dumps() == data['metadata']
# Check metadata
for k in ['created', 'updated', 'metadata', 'links']:
assert k in data
# Check self links
res = client.get(to_relative_url(data['links']['self']))
assert res.status_code == 200
assert data == get_json(res)
assert acq_order.dumps() == data['metadata']
assert acq_order.get_account_statement() == \
data['metadata']['account_statement']
list_url = url_for('invenio_records_rest.acor_list', pid='acor1')
res = client.get(list_url)
assert res.status_code == 200
data = get_json(res)
metadata = data['hits']['hits'][0]['metadata']
# remove dynamically added fields
del metadata['organisation']
del metadata['order_lines']
del metadata['receipts']
assert data['hits']['hits'][0]['metadata'] == acq_order.replace_refs()
def scrap_joberty():
companies_count = get_joberty_companies_count()
company_list = []
company_count = 0
for page in range(get_joberty_page_count()):
all_companies = get_json(
"https://backend-test.joberty.rs/api/v1/user/company/search?q=&p="
+ str(page))
for company in all_companies["items"]:
company_id = company["id"]
company_details = get_json(
"https://backend-test.joberty.rs/api/v1/user/company/" +
str(company_id))
json_object = {}
json_object["id"] = company_id
json_object["name"] = str(company_details["name"])
json_object["industry"] = switch_industry(company["industryId"])
#Promenjeno 11.09, continue or pass? dodaje samo sa gradom
try:
json_object["city"] = str(company_details["city"])
except:
continue
json_object["website"] = str(company_details["website"])
json_object["employeeNumber"] = str(
company_details["employeeNumber"])
json_object["foundingYear"] = company_details["foundingYear"]
company_list.append(json_object)
company_count += 1
print(">> " + str(company_count) + "/" + str(companies_count) +
" companies added")
return company_list
def test_document_exclude_draft_records(client, document):
"""Test document exclude draft record."""
list_url = url_for('invenio_records_rest.doc_list', q='Lingliang')
res = client.get(list_url)
hits = get_json(res)['hits']
assert hits['total']['value'] == 1
data = hits['hits'][0]['metadata']
assert data['pid'] == document.get('pid')
document['_draft'] = True
document.update(document, dbcommit=True, reindex=True)
list_url = url_for('invenio_records_rest.doc_list', q='Lingliang')
res = client.get(list_url)
hits = get_json(res)['hits']
assert hits['total']['value'] == 0
document['_draft'] = False
document.update(document, dbcommit=True, reindex=True)
list_url = url_for('invenio_records_rest.doc_list', q='Lingliang')
res = client.get(list_url)
hits = get_json(res)['hits']
assert hits['total']['value'] == 1
def test_due_soon_loans(client, librarian_martigny_no_email,
patron_martigny_no_email, loc_public_martigny,
item_type_standard_martigny, item_lib_martigny,
json_header, circ_policy_short_martigny):
"""Test overdue loans."""
login_user_via_session(client, librarian_martigny_no_email.user)
item = item_lib_martigny
item_pid = item.pid
patron_pid = patron_martigny_no_email.pid
assert not item.is_loaned_to_patron(
patron_martigny_no_email.get('barcode'))
assert item.can_delete
assert item.available
from rero_ils.modules.circ_policies.api import CircPolicy
circ_policy = CircPolicy.provide_circ_policy(
item.library_pid, patron_martigny_no_email.patron_type_pid,
item.item_type_pid)
circ_policy['number_of_days_before_due_date'] = 7
circ_policy['checkout_duration'] = 3
circ_policy.update(circ_policy, dbcommit=True, reindex=True)
# checkout
res = client.post(
url_for('api_item.checkout'),
data=json.dumps(dict(item_pid=item_pid, patron_pid=patron_pid)),
content_type='application/json',
)
assert res.status_code == 200
data = get_json(res)
loan_pid = data.get('action_applied')[LoanAction.CHECKOUT].get('pid')
due_soon_loans = get_due_soon_loans()
assert due_soon_loans[0].get('pid') == loan_pid
# test due date hour
checkout_loan = Loan.get_record_by_pid(loan_pid)
end_date = ciso8601.parse_datetime(
checkout_loan.get('end_date')).astimezone()
assert end_date.minute == 59 and end_date.hour == 23
# checkin the item to put it back to it's original state
res = client.post(
url_for('api_item.checkin'),
data=json.dumps(dict(item_pid=item_pid, pid=loan_pid)),
content_type='application/json',
)
assert res.status_code == 200
def test_automatic_checkin(client, librarian_martigny_no_email, lib_martigny,
patron_martigny_no_email, loc_public_martigny,
item_lib_martigny, json_header,
item_type_standard_martigny,
circ_policy_short_martigny,
patron_type_children_martigny, lib_saxon,
loc_public_saxon, librarian_saxon_no_email):
"""Test item automatic checkin."""
login_user_via_session(client, librarian_saxon_no_email.user)
circ_policy_origin = deepcopy(circ_policy_short_martigny)
circ_policy = circ_policy_short_martigny
record, actions = item_lib_martigny.automatic_checkin()
assert actions == {'no': None}
res = client.post(
url_for('api_item.librarian_request'),
data=json.dumps(
dict(item_pid=item_lib_martigny.pid,
pickup_location_pid=loc_public_saxon.pid,
patron_pid=patron_martigny_no_email.pid)),
content_type='application/json',
)
assert res.status_code == 200
data = get_json(res)
loan_pid = data.get('action_applied')[LoanAction.REQUEST].get('pid')
res = client.post(
url_for('api_item.validate_request'),
data=json.dumps(
dict(item_pid=item_lib_martigny.pid,
pid=loan_pid,
transaction_location_pid=loc_public_martigny.pid)),
content_type='application/json',
)
assert res.status_code == 200
item = Item.get_record_by_pid(item_lib_martigny.pid)
assert item.status == ItemStatus.IN_TRANSIT
text = item_status_text(item, format='medium', locale='en')
assert text == 'not available (requested) (in_transit)'
record, actions = item.automatic_checkin()
assert 'receive' in actions
item.cancel_loan(pid=loan_pid)
assert item.status == ItemStatus.ON_SHELF
def get_quote_yahoo(symbol):
if symbol.lower() == 'futures':
return get_index_futures()
# url = "https://query1.finance.yahoo.com/v7/finance/options/%s" % symbol
# quote = utils.get_json(url)['optionChain']['result'][0]['quote']
url = "https://query1.finance.yahoo.com/v7/finance/quote?symbols=%s" % symbol
quote = utils.get_json(url)['quoteResponse']['result'][0]
output = "{shortName} ({symbol})\n".format_map(quote)
output += "```python\n"
if quote['marketState'] in ["POST", "POSTPOST"]:
if "postMarketChange" in quote:
output += "After Hours: %.02f (%.02f, %.02f%%) (%s)\n" % (
quote.get("postMarketPrice"), quote.get("postMarketChange"),
quote.get("postMarketChangePercent"),
datetime.fromtimestamp(quote.get("postMarketTime")))
output += "Market Close: %.02f (%.02f, %.02f%%) (%s)\n" % (
quote.get("regularMarketPrice"), quote.get("regularMarketChange"),
quote.get("regularMarketChangePercent"),
datetime.fromtimestamp(quote.get("regularMarketTime")))
elif quote['marketState'] in ["PRE"]:
if "preMarketPrice" in quote:
output += "PreMarket: %.02f (%.02f, %.02f%%) (%s)\n" % (
quote.get("preMarketPrice"), quote.get("preMarketChange"),
quote.get("preMarketChangePercent"),
datetime.fromtimestamp(quote.get("preMarketTime")))
output += "Market Close: %.02f (%.02f, %.02f%%) (%s)\n" % (
quote.get("regularMarketPrice"), quote.get("regularMarketChange"),
quote.get("regularMarketChangePercent"),
datetime.fromtimestamp(quote.get("regularMarketTime")))
else:
output += "Market Hours: %.02f (%.02f, %.02f%%) (%s)\n" % (
quote.get("regularMarketPrice"), quote.get("regularMarketChange"),
quote.get("regularMarketChangePercent"),
datetime.fromtimestamp(quote.get("regularMarketTime")))
output += "Day volume: %s (%s 10 day avg)\n" % (
simpleMarketCap(quote.get("regularMarketVolume")),
simpleMarketCap(quote.get("averageDailyVolume10Day")))
output += "Day range: {regularMarketDayRange}\n".format_map(quote)
output += "52w range: {fiftyTwoWeekRange}\n".format_map(quote)
output += "Market Cap: %s, P/E: %s\n" % (simpleMarketCap(
quote.get("marketCap")), "%.02f" % quote.get("trailingPE") if
"trailingPE" in quote else "N/A")
output += "```\n<https://finance.yahoo.com/quote/%s>" % (
quote.get("symbol"))
return output
def download_recent_csv():
data = get_json(f'{base_dir}/tmp/recent.json')
if data is None:
return "Nie odbyto żadnej sesji."
print(data)
rows = []
if data['mode'] != 'pilot':
# Stage two data
stage_two_row = dict.fromkeys(column_names, 'n/a')
stage_two_row["External Task Number"] = "STAGE TWO"
stage_two_row["Internal Task Number"] = "STAGE TWO"
stage_two_row["GIL Clicks"] = data["stageTwoGILClicks"]
rows.append(stage_two_row)
# Stage four tasks data
for task_data in data["taskData"]:
task_row = dict.fromkeys(column_names, '-')
task_row["External Task Number"] = task_data["externalTaskNum"]
task_row["Internal Task Number"] = task_data["internalTaskNum"]
task_row["Task Name"] = task_data["taskName"]
task_row["GIL Clicks"] = task_data["GILClicks"]
task_row["Correct"] = 1 if task_data["correct"] else 0
task_row["Card Order"] = task_data["cardOrder"]
task_row["Task Execution Time"] = task_data["executionDuration"]
task_row["Task Solution Time"] = task_data["solutionDuration"]
task_row["P Selection Times"] = task_data["selectionTimes"][0]
task_row["nP Selection Times"] = task_data["selectionTimes"][1]
task_row["Q Selection Times"] = task_data["selectionTimes"][2]
task_row["nQ Selection Times"] = task_data["selectionTimes"][3]
task_row["P Deselection Times"] = task_data["deselectionTimes"][0]
task_row["nP Deselection Times"] = task_data["deselectionTimes"][1]
task_row["Q Deselection Times"] = task_data["deselectionTimes"][2]
task_row["nQ Deselection Times"] = task_data["deselectionTimes"][3]
task_row["Final Selection"] = task_data["finalSelectionOrder"]
task_row["Final Selection Times"] = task_data[
"finalSelectionOrderedTimes"]
rows.append(task_row)
# Sending csv
return send_csv(rows, \
f'{data["nickname"]}_{data["gender"]}_{data["age"]}_mode{data["mode"]}.csv', \
column_names)
def test_documents_facets(client, document, ebook_1, ebook_2, ebook_3, ebook_4,
item_lib_martigny, rero_json_header):
"""Test record retrieval."""
list_url = url_for('invenio_records_rest.doc_list', view='global')
res = client.get(list_url, headers=rero_json_header)
data = get_json(res)
aggs = data['aggregations']
# check all facets are present
for facet in [
'document_type', 'author__en', 'author__fr', 'author__de',
'author__it', 'language', 'subject', 'status'
]:
assert aggs[facet]
# FILTERS
# person author
list_url = url_for('invenio_records_rest.doc_list',
view='global',
author__de='Peter James')
res = client.get(list_url, headers=rero_json_header)
data = get_json(res)
assert data['hits']['total'] == 2
# organisation author
list_url = url_for('invenio_records_rest.doc_list',
view='global',
author__de='Great Edition')
res = client.get(list_url, headers=rero_json_header)
data = get_json(res)
assert data['hits']['total'] == 1
# an other person author
list_url = url_for('invenio_records_rest.doc_list',
view='global',
author__de='J.K. Rowling')
res = client.get(list_url, headers=rero_json_header)
data = get_json(res)
assert data['hits']['total'] == 1
# two authors in the same document
list_url = url_for('invenio_records_rest.doc_list',
view='global',
author__de=['Great Edition', 'Peter James'])
res = client.get(list_url, headers=rero_json_header)
data = get_json(res)
assert data['hits']['total'] == 1
# two authors: each in a separate document
list_url = url_for('invenio_records_rest.doc_list',
view='global',
author__de=['J.K. Rowling', 'Peter James'])
res = client.get(list_url, headers=rero_json_header)
data = get_json(res)
assert data['hits']['total'] == 0
def test_patron_messages(client, patron_martigny):
"""Test for patron messages."""
patron_pid = patron_martigny.pid
url = url_for('api_patrons.get_messages', patron_pid=patron_pid)
res = client.get(url)
assert res.status_code == 401
login_user_via_session(client, patron_martigny.user)
url = url_for('api_patrons.get_messages', patron_pid=patron_pid)
res = client.get(url)
assert res.status_code == 200
data = get_json(res)
assert len(data) == 1
assert data[0]['type'] == 'warning'
assert data[0]['content'] == 'This person will be in vacations.\n' \
'Will be back in february.'
def check_for_null(attribute, memberUrls):
"""Provides a list of Member urls that have [attribute] is null.
param attribute to check for null value
params memberUrls List of member urls to check
return list of member urls with null [attribute] field
"""
attributeNotFound =[]
for url in memberUrls:
member_data = utils.get_json(url)
if member_data[attribute] is None:
#TODO: TBD Could grab email here if speed was an issue
attributeNotFound.append(url)
return attributeNotFound
def get(id):
output_folder = utils.make_output_folder("hathitrust", id)
metadata_url = f"https://babel.hathitrust.org/cgi/imgsrv/meta?id={id}"
metadata = utils.get_json(metadata_url)
total_pages = metadata["total_items"]
print(f"Going to download {total_pages} pages to {output_folder}")
for page in range(1, total_pages + 1):
url = f"https://babel.hathitrust.org/cgi/imgsrv/image?id={id};seq={page};width=1000000"
output_filename = utils.make_output_filename(output_folder,
page,
extension="jpg")
if os.path.exists(output_filename):
print(f"Skip downloading existing page #{page:08d}")
continue
print(f"Downloading page {page} to {output_filename}")
utils.get_binary(output_filename, url)
def profile(self) -> "Entity":
if not self.fresh and self.json_file_path.exists():
return get_json(self.json_file_path)
self.ranks
self.scores
self.stats
if self.entity_type == EntityTypeEnum["institution"]:
self.get_institution_data()
if self.entity_type != EntityTypeEnum["institution"]:
data = {k: v for k, v in self.__dict__.items()}
for attr in ["ranks", "scores", "stats"]:
data[attr] = data.pop(f"_{attr}")
str_export(self.json_file_path, EntitySchema(**data).json())
return self
def get_gif_tweet():
# clear out gif folder (to make local testing easier)
for f in os.listdir(exports_gif_dir):
if f != ".gitkeep":
os.remove(os.path.join(exports_gif_dir, f))
# generate still images with alphabetical names
# set starting date and then incremement by 7 until we hit today
vax_res_json = get_json(vax_url)
date = datetime.strptime(start_date, '%Y-%m-%d')
while date <= now:
vax_perc = get_vax_perc_by_date(vax_res_json, date, save_totals=False)
vax_colors = get_colors_dict_absolute(vax_perc, gif_colorscale, date,
"vax")
output_path = os.path.join(
os.getcwd(), "exports", "gif",
"vax-{}.png".format(date.strftime("%Y-%m-%d")))
write_svg(
gif_svg_path,
[output_path],
vax_colors,
dpi=75,
)
date += timedelta(7)
# turn them into a gif
# https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html#gif
# copy the last frame to the front so we get it as the gif preview
img_paths = sorted(glob.glob(png_input_paths))
img_paths.insert(0, img_paths[-1])
img, *imgs = [Image.open(f) for f in img_paths]
img.save(fp=gif_output_path,
format='GIF',
append_images=imgs,
save_all=True,
duration=500,
loop=0)
alt_text = '''
The animated gif shows a map of Chicago with vaccination rates for each ZIP code over time. As time passes from late January until today, the north west side pulls ahead of the rest of the city in vaccination rates.
'''
return {
"gif_path": gif_output_path,
"alt_text": alt_text,
}
def hk_freq(data_dir, hk_dir):
print("hk freq")
data = get_json_data(data_dir)
at = AutoTag()
for entry in data:
entry["text"] = clean_text(entry["text"])
if not os.path.isdir(hk_dir):
os.mkdir(hk_dir)
with open(hk_dir + "total", "w") as f:
pass
word_count = at.count_data([w for entry in data for w in entry["text"].split()], hk_dir + "total")
words = [w.encode("utf-8") for w, c in word_count if c > 40]
with open(hk_dir + "freqs.csv", "wb") as csvfile:
# data_encoded = [w.encode('utf-8') for w,c in word_count if c > 40]
w = csv.writer(csvfile, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL)
w.writerow([u"HK"] + words)
# csvfile.write(','.join([u'HK']+words) + '\n')
hkwords = {}
data_json = get_json(data_dir)
for json_entry in data_json:
if json_entry["model"] != "facebook_feeds.facebook_feed":
continue
name = json_entry["fields"]["name"]
print(name)
if not name:
continue
name = name.encode("utf-8")
word_count = at.count_data(
[w for entry in data for w in entry["text"].split() if entry["feed"] == json_entry["pk"]], hk_dir + name
)
word_dict = {w.encode("utf-8"): c for w, c in word_count}
hkwords[name] = []
for word in words:
if word not in word_dict:
hkwords[name].append(str(0))
else:
hkwords[name].append(str(word_dict[word]))
with open(hk_dir + "freqs.csv", "a") as csvfile:
writer = csv.writer(csvfile, delimiter=",")
# writer = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow([name] + hkwords[name])
with open(hk_dir + "freqs_t.csv", "a") as csvfile:
writer = csv.writer(csvfile, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL)
for name in hkwords:
writer.writerow([name] + hkwords[name])
def test_checkout_no_loan_given(client, librarian_martigny_no_email,
patron_martigny_no_email, loc_public_martigny,
item_lib_martigny, json_header,
circulation_policies):
"""Test checkout item when request loan is not given."""
login_user_via_session(client, librarian_martigny_no_email.user)
item = item_lib_martigny
patron = patron_martigny_no_email
location = loc_public_martigny
# request
res = client.post(
url_for('api_item.librarian_request'),
data=json.dumps(
dict(item_pid=item.pid,
pickup_location_pid=location.pid,
patron_pid=patron.pid)),
content_type='application/json',
)
assert res.status_code == 200
# checkout
res = client.post(
url_for('api_item.checkout'),
data=json.dumps(dict(item_pid=item.pid, patron_pid=patron.pid)),
content_type='application/json',
)
assert res.status_code == 200
data = get_json(res)
loan_pid = data.get('action_applied')[LoanAction.CHECKOUT].get('pid')
from rero_ils.modules.patrons.views import \
get_patron_from_checkout_item_pid
assert get_patron_from_checkout_item_pid(item.pid) == patron
res = client.post(
url_for('api_item.checkin'),
data=json.dumps(dict(item_pid=item.pid)),
content_type='application/json',
)
assert res.status_code == 403
res = client.post(
url_for('api_item.checkin'),
data=json.dumps(dict(item_pid=item.pid, pid=loan_pid)),
content_type='application/json',
)
assert res.status_code == 200
def hk_freq(data_dir, hk_dir):
print('hk freq')
data = get_json_data(data_dir)
at = AutoTag()
for entry in data:
entry['text'] = clean_text(entry['text'])
if not os.path.isdir(hk_dir):
os.mkdir(hk_dir)
with open(hk_dir+'total', 'w') as f:
pass
word_count = at.count_data([w for entry in data for w in entry['text'].split()],hk_dir+'total')
words = [w.encode('utf-8') for w,c in word_count if c > 40]
with open(hk_dir+'freqs.csv', 'wb') as csvfile:
# data_encoded = [w.encode('utf-8') for w,c in word_count if c > 40]
w = csv.writer(csvfile, delimiter = ',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
w.writerow([u'HK']+words)
# csvfile.write(','.join([u'HK']+words) + '\n')
hkwords = {}
data_json = get_json(data_dir)
for json_entry in data_json:
if json_entry['model'] != "facebook_feeds.facebook_feed":
continue
name = json_entry['fields']['name']
print(name)
if not name:
continue
name = name.encode('utf-8')
word_count = at.count_data([w for entry in data for w in entry['text'].split() if entry["feed"] == json_entry['pk']],hk_dir+name)
word_dict = {w.encode('utf-8'):c for w,c in word_count}
hkwords[name] = []
for word in words:
if word not in word_dict:
hkwords[name].append(str(0))
else:
hkwords[name].append(str(word_dict[word]))
with open(hk_dir+'freqs.csv', 'a') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
# writer = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow([name]+hkwords[name])
with open(hk_dir+'freqs_t.csv', 'a') as csvfile:
writer = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
for name in hkwords:
writer.writerow([name]+hkwords[name])
def login(self, mobile, password):
#response = None
result = {}
login_data = {}
if mobile:
login_data['mobile'] = mobile
if password:
login_data['password'] = password
if login_data:
response = requests.post(self.login_url, json=login_data)
else:
response = requests.post(self.login_url)
# result = response.json()
# status_code = response.status_code
# result['status_code'] = status_code
# return result
return get_json(response)
def __init__(self, filename):
"""
:param filename: str
"""
self.filename = filename
self.parsed_data = {}
self.executors = {}
self.jobs = {}
self.tasks = {}
self.block_managers = []
f = open(filename, "r")
for line in f:
json_data = get_json(line)
event_type = json_data["Event"]
if event_type == "SparkListenerLogStart":
self.do_SparkListenerLogStart(json_data)
elif event_type == "SparkListenerBlockManagerAdded":
self.do_SparkListenerBlockManagerAdded(json_data)
elif event_type == "SparkListenerEnvironmentUpdate":
self.do_SparkListenerEnvironmentUpdate(json_data)
elif event_type == "SparkListenerApplicationStart":
self.do_SparkListenerApplicationStart(json_data)
elif event_type == "SparkListenerJobStart":
self.do_SparkListenerJobStart(json_data)
elif event_type == "SparkListenerStageSubmitted":
self.do_SparkListenerStageSubmitted(json_data)
elif event_type == "SparkListenerExecutorAdded":
self.do_SparkListenerExecutorAdded(json_data)
elif event_type == "SparkListenerTaskStart":
self.do_SparkListenerTaskStart(json_data)
elif event_type == "SparkListenerTaskEnd":
self.do_SparkListenerTaskEnd(json_data)
elif event_type == "SparkListenerExecutorRemoved":
self.do_SparkListenerExecutorRemoved(json_data)
elif event_type == "SparkListenerBlockManagerRemoved":
self.do_SparkListenerBlockManagerRemoved(json_data)
elif event_type == "SparkListenerStageCompleted":
self.do_SparkListenerStageCompleted(json_data)
elif event_type == "SparkListenerJobEnd":
self.do_SparkListenerJobEnd(json_data)
elif event_type == "SparkListenerApplicationEnd":
self.do_SparkListenerApplicationEnd(json_data)
else:
pass
def test_auto_checkin_else(client, librarian_martigny_no_email,
patron_martigny_no_email, loc_public_martigny,
item_lib_martigny, json_header, lib_martigny,
loc_public_saxon):
"""Test item checkin no action."""
login_user_via_session(client, librarian_martigny_no_email.user)
res, data = postdata(
client,
'api_item.checkin',
dict(
item_pid=item_lib_martigny.pid,
transaction_library_pid=lib_martigny.pid,
transaction_user_pid=librarian_martigny_no_email.pid
)
)
assert res.status_code == 400
assert get_json(res)['status'] == 'error: No circulation action performed'
def test_item_dumps(client, item_lib_martigny, org_martigny,
librarian_martigny):
"""Test item dumps and elastic search version."""
item_dumps = Item(item_lib_martigny.dumps()).replace_refs()
assert item_dumps.get('organisation').get('pid') == org_martigny.pid
login_user_via_session(client, librarian_martigny.user)
record_url = url_for('invenio_records_rest.item_item',
pid_value=item_lib_martigny.pid)
res = client.get(record_url)
assert res.status_code == 200
item_es = Item(get_json(res).get('metadata'))
assert item_es.available
assert item_es.organisation_pid == org_martigny.pid
def get_posts(self, user_id, raw=False):
# make request
url = 'https://api.instagram.com/v1/users/' + str(user_id) + '/media/recent'
params = {
'access_token': self.token
}
try:
data = get_json(url, params)['data']
except:
print 'User not found'
return None
# for debug
if raw:
return data
# extract necessary posts data from response
posts = []
for item in data:
post = {}
post['user_id'] = item['user']['id']
if item['caption']:
post['caption'] = item['caption']['text']
else:
post['caption'] = ''
post['type'] = item['type']
post['tags'] = " ".join(item['tags']) # need to be stored in distinct db table
post['created_time'] = datetime.fromtimestamp(float(item['created_time']))
post['likes_count'] = item['likes']['count']
if item['location']:
post['loc_latitude'] = item['location']['latitude']
post['loc_longitude'] = item['location']['longitude']
else:
post['loc_latitude'] = ''
post['loc_longitude'] = ''
posts.append(post)
return posts
def from_transcript_result(task_id):
"""Gets sh-miR result created from transcript via task_id
* creates task
* wait until it's done
* gets result(s)
Args:
task_id: Id of task which was given by RESTful API
Returns:
None
"""
data = get_json("from_transcript_result", task_id)['data']
path = "./results/transcript/{}/".format(task_id)
if data['result']:
print("Results under: {}\n".format(path))
else:
print("No sh-miR found")
result_string = "{}: backbone: {}, score: {}, sequence: {}\n pdf: {}\n result: {}\n"
for no, result in enumerate(data['result']):
mfold_result(result['pdf'], path=path, verbose=False)
new_path = path + result['backbone']
if not os.path.exists(new_path):
os.makedirs(new_path)
subtask_id = result['pdf'].split("/")[-1]
shutil.move(path + subtask_id + '/', new_path)
print(result_string.format(
no + 1,
result['backbone'],
result['score'],
result['sequence'],
os.path.join(new_path, subtask_id),
result['sh_mir']
))
def get_channels(area, url):
try:
# 获取地区中所有频道的url
c_url = url % (area[0], area[0])
resp = utils.get_html(c_url)
c_match = utils.get_json(resp, json_pattern)
# c_match = re.search(json_pattern, r.text).group(1)
c_json = json.loads(c_match)
# 获取含有频道列表的html
c_html = c_json["html"]
# 频道url列表
urllist = re.findall(url_pattern, c_html)
# 频道名列表
namelist = re.findall(c_name_pattern, c_html)
# 频道字典，频道名与url对应
c_dict = {}
for i in xrange(len(namelist)):
c_dict[namelist[i]] = urllist[i]
except Exception, e:
print e
return None
def cntv_crawler(api):
print "start running cntv crawler...."
src_list = []
for channel in channel_list:
print "getting url from %s" % channel
try:
jsn = utils.get_html(api % channel)
data = utils.get_json(jsn)
src = {}
src["code"] = channel
src['auth'] = data['hls_url']['hls2']
if "cctv" in channel:
src["hls"] = data['hls_url']['hls1']
src['flv'] = data['hds_url']['hds2']
src_list.append(src)
except Exception, e:
print e
else:
pass
finally:
def city_ids(self):
list_ = get_json(self.url_city_ids_json)['body']
return map(lambda x: (x['CODE'], x['NAME']), list_)
def parse_city(self, url, params, target, target_kor, nth, city_code=None, city_name=None):
_town_list = get_json(url['town'], params['town'])['jsonResult']['body']
_townSeq_list = []
_towntoCode_dict = dict()
_codetoTown_dict = dict()
if (target=='assembly' and nth <= 16) or \
(target=='local-mp' and nth <= 3) or \
(target=='local-pp' and nth <= 3):
for x in _town_list:
x['CODE'] = int(x['CODE'])
if x['CODE'] < 10000:
x['CODE'] = x['CODE']*10 + 3
_towntoCode_dict[x['NAME']] = x['CODE']
_codetoTown_dict[x['CODE']] = x['NAME']
_townSeq_list.append(x['CODE'])
else:
for x in _town_list:
x['CODE'] = int(x['CODE'])
_towntoCode_dict[x['NAME']] = x['CODE']
_codetoTown_dict[x['CODE']] = x['NAME']
_townSeq_list.append(x['CODE'])
# _sgg_list = get_json(url['sggCity'], params['sggCity'])['jsonResult']['body']
# _sggSeq_list = []
# _sggtoCode_dict = dict()
# _codetoSgg_dict = dict()
# _codetoNumElected_dict = dict()
# for x in _sgg_list:
# x['CODE'] = int(x['CODE'])
# if not x['NAME'] in _sggtoCode_dict:
# _sggtoCode_dict[x['NAME']] = []
# _sggtoCode_dict[x['NAME']].append(x['CODE'])
# _codetoSgg_dict[x['CODE']] = x['NAME']
# _sggSeq_list.append(x['CODE'])
# if (target=='local-pp' or target=='local-mp' or target=='assembly'):
# xhtml_url = self.urlPath_elector_list
# xhtml_params = self.urlParam_elector_list
# (_codetoNumElected_dict, _sggSeq_list_neo) = self.parse_elected(xhtml_url, xhtml_params, target, city_code, copy.deepcopy(_sggtoCode_dict))
# else:
# _codetoNumElected_dict = self.constant_elected(target, nth, _sggSeq_list)
_sggSeq_list = []
_sggtoCode_dict = dict()
_codetoSgg_dict = dict()
_codetoNumElected_dict = dict()
for town_code in _townSeq_list:
params['sggTown']['townCode'] = town_code
sleep(0.3)
_sgg_list = get_json(url['sggTown'], params['sggTown'])['jsonResult']['body']
for x in _sgg_list:
print(x['NAME'])
x['CODE'] = int(x['CODE'])
if not x['NAME'] in _sggtoCode_dict:
_sggtoCode_dict[x['NAME']] = []
_sggtoCode_dict[x['NAME']].append(x['CODE'])
_codetoSgg_dict[x['CODE']] = x['NAME']
_sggSeq_list.append(x['CODE'])
if (target=='local-pp' or target=='local-mp' or target=='assembly'):
xhtml_url = self.urlPath_elector_list
xhtml_params = self.urlParam_elector_list
(_codetoNumElected_dict, _sggSeq_list_neo) = self.parse_elected(xhtml_url, xhtml_params, target, city_code, copy.deepcopy(_sggtoCode_dict))
else:
_codetoNumElected_dict = self.constant_elected(target, nth, _sggSeq_list)
sleep(1)
_PR_sggSeq_list = []
_PR_sggtoCode_dict = dict()
_PR_codetoSgg_dict = dict()
if 'PR_sgg' in params:
if (target=='assembly') and (nth==17):
_PR_sgg_list = [{'CODE':7000000, 'NAME':"비례대표"}]
_PR_sggtoCode_dict = {"비례대표": [7000000]}
_PR_codetoSgg_dict = {7000000: "비례대표"}
_PR_sggSeq_list = [7000000]
else:
for town_code in _townSeq_list:
sleep(0.2)
params['PR_sgg']['townCode'] = town_code
_PR_sgg_list = get_json(url['sggTown'], params['PR_sgg'])['jsonResult']['body']
for x in _PR_sgg_list:
print(x['NAME'])
if not x['NAME'] in _PR_sggtoCode_dict:
_PR_sggtoCode_dict[x['NAME']] = []
if not x['CODE'] in _PR_sggtoCode_dict[x['NAME']]:
_PR_sggtoCode_dict[x['NAME']].append(x['CODE'])
_PR_codetoSgg_dict[x['CODE']] = x['NAME']
if not x['CODE'] in _PR_sggSeq_list:
_PR_sggSeq_list.append(x['CODE'])
xhtml_url = self.urlPath_elector_list
xhtml_params = self.urlParam_PR_elector_list
(_PR_codetoNumElected_dict, _PR_sggSeq_list_neo) = self.parse_elected(xhtml_url, xhtml_params, target, city_code, copy.deepcopy(_PR_sggtoCode_dict))
_result = dict(city_name=city_name, city_code=int(city_code))
#_result['town_list'] = _town_list
_result['town_Seq'] = _townSeq_list
_result['town_toCode'] = _towntoCode_dict
_result['code_toTown'] = _codetoTown_dict
if len(_sggSeq_list) > 0:
#_result['consti_list'] = _sgg_list
_result['consti_Seq'] = _sggSeq_list
_result['consti_toCode'] = _sggtoCode_dict
_result['code_toConsti'] = _codetoSgg_dict
_result['code_toNumElected'] = _codetoNumElected_dict
if len(_PR_sggSeq_list) > 0:
#_result['PR_consti_list'] = _PR_sggSeq_list
_result['PR_consti_Seq'] = _PR_sggSeq_list
_result['PR_consti_toCode'] = _PR_sggtoCode_dict
_result['PR_code_toConsti'] = _PR_codetoSgg_dict
_result['PR_code_toNumElected'] = _PR_codetoNumElected_dict
print('\x1b[1;31mcrawled %s election #%d - \x1b[1;m%s' % (target, self.nth, city_name))
print('\t└ %s, %s(%d)...' % ('구시군 행정구역 목록', city_name, len(_townSeq_list)))
if len(_PR_sggtoCode_dict) > 0:
print('\t└ %s, %s(%d)...' % (target_kor+' 비례대표 선거구(자치구시군) 목록', city_name, len(_PR_sggSeq_list)))
print('\t└ %s, %s(%d)...' % (target_kor+' 선거구 목록', city_name, len(_sggSeq_list)))
return _result
def city_codes(self): # 광역자치단체 code 리스트를 json으로 받게 됨. list_ = get_json(self.urlPath_city_codes, self.urlParam_city_codes)['jsonResult']['body'] return [(x['CODE'], x['NAME']) for x in list_]
