def multiple_pattern_request(pattern_list):
pattern_list = [str(code) for code in pattern_list]
if len(pattern_list) > 500:
first_pattern_list = pattern_list[:500]
second_pattern_list = pattern_list[500:]
first_patterns_url = 'https://api.ravelry.com/patterns.json?ids={}'.format(
'+'.join(first_pattern_list))
first_patterns = requests.get(first_patterns_url,
auth=(personal_keys.username(),
personal_keys.password()))
if first_patterns.status_code is not 200:
return "first pattern batch bad"
second_patterns_url = 'https://api.ravelry.com/patterns.json?ids={}'.format(
'+'.join(second_pattern_list))
second_patterns = requests.get(second_patterns_url,
auth=(personal_keys.username(),
personal_keys.password()))
if second_patterns.status_code is not 200:
return "second pattern batch bad"
large_patterns = first_patterns.json()
large_patterns['patterns'].update(second_patterns.json()['patterns'])
return large_patterns
else:
patterns_url = 'https://api.ravelry.com/patterns.json?ids={}'.format(
'+'.join(pattern_list))
patterns = requests.get(patterns_url,
auth=(personal_keys.username(),
personal_keys.password()))
if patterns.status_code is 200:
return patterns.json()
else:
return 404
def problem_children(pattern_list):
del_list = []
batch_50_prob = []
batch_10_prob = []
pattern_list_str = [str(item) for item in pattern_list]
# below checks in batches of 50 if the patterns status code is not 200
l_of_l_patterns_50 = [
pattern_list_str[i:i + 50] for i in range(0, len(pattern_list_str), 50)
]
batch_num = 0
while batch_num < len(l_of_l_patterns_50):
# create url to request from api
patterns_url = 'https://api.ravelry.com/patterns.json?ids={}'.format(
'+'.join(l_of_l_patterns_50[batch_num]))
# make the request to the api
patterns = requests.get(patterns_url,
auth=(personal_keys.username(),
personal_keys.password()))
if patterns.status_code is not 200:
batch_50_prob.append(l_of_l_patterns_50[batch_num])
batch_num += 1
# if any of the batches come back with an error, then we go into figuring out which pattern codes are the issue
if len(batch_50_prob) > 0:
# below I break the list of 50 or more into batches of 10 so we're not individually testing 50+ pattern codes
flat_50 = [item for sublist in batch_50_prob for item in sublist]
l_of_l_patterns_10 = [
flat_50[i:i + 10] for i in range(0, len(flat_50), 10)
]
new_batch_num = 0
while new_batch_num < len(l_of_l_patterns_10):
# create url to request from api
patterns_url = 'https://api.ravelry.com/patterns.json?ids={}'.format(
'+'.join(l_of_l_patterns_10[new_batch_num]))
# make the request to the api
patterns = requests.get(patterns_url,
auth=(personal_keys.username(),
personal_keys.password()))
if patterns.status_code is not 200:
batch_10_prob.append(l_of_l_patterns_10[new_batch_num])
new_batch_num += 1
flat_10 = [item for sublist in batch_10_prob for item in sublist]
for item in flat_10:
patterns_url = 'https://api.ravelry.com/patterns.json?ids={}'.format(
'+'.join(item))
patterns = requests.get(patterns_url,
auth=(personal_keys.username(),
personal_keys.password()))
if patterns.status_code is not 200:
del_list.append(item)
return [item for item in pattern_list_str if item not in del_list]
def single_pattern_request(code):
if type(code) is not str:
code = str(code)
pattern_url = 'https://api.ravelry.com/patterns/{}.json'.format(code)
pattern = requests.get(pattern_url,
auth=(personal_keys.username(),
personal_keys.password()))
return pattern.json()['pattern']
def search(term):
search_url = 'https://api.ravelry.com/patterns/search.json?query={}'.format(
term)
search = requests.get(search_url,
auth=(personal_keys.username(),
personal_keys.password()))
s_json = search.json()
return s_json
def pattern_list_to_tf_idf_df(pattern_list):
# turn ints into strings for the request url
pattern_list = [str(item) for item in pattern_list]
# if the pattern list is less than 50, just one request is needed, otherwise, multiple requests are needed
if len(pattern_list) < 50:
# create url to request from api
patterns_url = 'https://api.ravelry.com/patterns.json?ids={}'.format(
'+'.join(pattern_list))
# make the request to the api
patterns = requests.get(patterns_url,
auth=(personal_keys.username(),
personal_keys.password()))
# create a dictionary for the attributes for each pattern where each attribute = 1
attr_list = []
for key in patterns.json()['patterns'].keys():
attr_list.append({
attr['permalink']: 1
for attr in patterns.json()['patterns'][key]
['pattern_attributes']
})
else:
# create nested list that contains lists of either 50 patterns or the remainder of length of list/50
l_of_l_patterns = [
pattern_list[i:i + 25] for i in range(0, len(pattern_list), 25)
]
batch_num = 0
attr_list = []
while batch_num < len(l_of_l_patterns):
# create url to request from api
patterns_url = 'https://api.ravelry.com/patterns.json?ids={}'.format(
'+'.join(l_of_l_patterns[batch_num]))
# make the request to the api
patterns = requests.get(patterns_url,
auth=(personal_keys.username(),
personal_keys.password()))
# create a dictionary for the attributes for each pattern where each attribute = 1
for key in patterns.json()['patterns'].keys():
attr_list.append({
attr['permalink']: 1
for attr in patterns.json()['patterns'][key]
['pattern_attributes']
})
batch_num += 1
def friend_username_list(username):
user_url = 'https://api.ravelry.com/people/{}/friends/list.json'.format(
username)
user = requests.get(user_url,
auth=(personal_keys.username(),
personal_keys.password()))
return [
user.json()['friendships'][item]['friend_username']
for item in range(0, len(user.json()['friendships']))
]
def multiple_pattern_request(pattern_list):
patterns_url = 'https://api.ravelry.com/patterns.json?ids={}'.format(
'+'.join(pattern_list))
patterns = requests.get(patterns_url,
auth=(personal_keys.username(),
personal_keys.password()))
if patterns.status_code is 200:
return patterns.json()
else:
return 404
def fav_request(username, page_size, page):
favs_url = 'https://api.ravelry.com/people/{}/favorites/list.json'.format(
username)
favs = requests.get(favs_url,
auth=(personal_keys.username(),
personal_keys.password()),
params={
'page_size': page_size,
'page': page
})
return favs.json()
def get_project_list(username):
proj_list = []
projects_url = 'https://api.ravelry.com/projects/{}/list.json'.format(
username)
projects = requests.get(projects_url,
auth=(personal_keys.username(),
personal_keys.password()))
proj_list.extend([
projects.json()['projects'][item]['pattern_id']
for item in range(0, len(projects.json()['projects']))
])
return proj_list
def proj_json(username):
projects_url = 'https://api.ravelry.com/projects/{}/list.json'.format(username)
projects = requests.get(projects_url,
auth = (personal_keys.username(),personal_keys.password()))
return projects.json()
def search_using_url(search_url):
search = requests.get(search_url,
auth=(personal_keys.username(),
personal_keys.password()))
s_json = search.json()
return s_json
def problem_children_large_scale(pattern_list):
del_list = []
batch_100_prob = []
batch_50_prob = []
batch_10_prob = []
pattern_list_str = [str(item) for item in pattern_list]
# below checks in batches of 100 if the patterns status code is not 200
l_of_l_patterns_100 = [
pattern_list_str[i:i + 100]
for i in range(0, len(pattern_list_str), 100)
]
batch_num = 0
while batch_num < len(l_of_l_patterns_100):
# create url to request from api
patterns_url = 'https://api.ravelry.com/patterns.json?ids={}'.format(
'+'.join(l_of_l_patterns_100[batch_num]))
# make the request to the api
patterns = requests.get(patterns_url,
auth=(personal_keys.username(),
personal_keys.password()))
if patterns.status_code is not 200:
batch_100_prob.append(l_of_l_patterns_100[batch_num])
batch_num += 1
# if any of the batches come back with an error, then we go into figuring out which pattern codes are the issue
if len(batch_100_prob) > 0:
# below I break the list of 50 or more into batches of 10 so we're not individually testing 50+ pattern codes
flat_100 = [item for sublist in batch_100_prob for item in sublist]
# below checks in batches of 50 if the patterns status code is not 200
l_of_l_patterns_50 = [
pattern_list_str[i:i + 50] for i in range(0, len(flat_100), 50)
]
batch_num = 0
while batch_num < len(l_of_l_patterns_50):
# create url to request from api
patterns_url = 'https://api.ravelry.com/patterns.json?ids={}'.format(
'+'.join(l_of_l_patterns_50[batch_num]))
# make the request to the api
patterns = requests.get(patterns_url,
auth=(personal_keys.username(),
personal_keys.password()))
if patterns.status_code is not 200:
batch_50_prob.append(l_of_l_patterns_50[batch_num])
batch_num += 1
# if any of the batches come back with an error, then we go into figuring out which pattern codes are the issue
if len(batch_50_prob) > 0:
# below I break the list of 50 or more into batches of 10 so we're not individually testing 50+ pattern codes
flat_50 = [item for sublist in batch_50_prob for item in sublist]
l_of_l_patterns_10 = [
flat_50[i:i + 10] for i in range(0, len(flat_50), 10)
]
new_batch_num = 0
while new_batch_num < len(l_of_l_patterns_10):
# create url to request from api
patterns_url = 'https://api.ravelry.com/patterns.json?ids={}'.format(
'+'.join(l_of_l_patterns_10[new_batch_num]))
# make the request to the api
patterns = requests.get(patterns_url,
auth=(personal_keys.username(),
personal_keys.password()))
if patterns.status_code is not 200:
batch_10_prob.append(l_of_l_patterns_10[new_batch_num])
new_batch_num += 1
flat_10 = [item for sublist in batch_10_prob for item in sublist]
for item in flat_10:
patterns_url = 'https://api.ravelry.com/patterns.json?ids={}'.format(
'+'.join(item))
patterns = requests.get(patterns_url,
auth=(personal_keys.username(),
personal_keys.password()))
if patterns.status_code is not 200:
del_list.append(item)
return [item for item in pattern_list_str if item not in del_list]
def pattern_list_to_tf_idf_df(pattern_list):
# turn ints into strings for the request url
pattern_list = [str(item) for item in pattern_list]
# if the pattern list is less than 50, just one request is needed, otherwise, multiple requests are needed
if len(pattern_list) < 50:
# create url to request from api
patterns_url = 'https://api.ravelry.com/patterns.json?ids={}'.format(
'+'.join(pattern_list))
# make the request to the api
patterns = requests.get(patterns_url,
auth=(personal_keys.username(),
personal_keys.password()))
# create a dictionary for the attributes for each pattern where each attribute = 1
attr_list = []
for key in patterns.json()['patterns'].keys():
attr_list.append({
attr['permalink']: 1
for attr in patterns.json()['patterns'][key]
['pattern_attributes']
})
else:
# create nested list that contains lists of either 50 patterns or the remainder of length of list/50
l_of_l_patterns = [
pattern_list[i:i + 25] for i in range(0, len(pattern_list), 25)
]
batch_num = 0
attr_list = []
while batch_num < len(l_of_l_patterns):
# create url to request from api
patterns_url = 'https://api.ravelry.com/patterns.json?ids={}'.format(
'+'.join(l_of_l_patterns[batch_num]))
# make the request to the api
patterns = requests.get(patterns_url,
auth=(personal_keys.username(),
personal_keys.password()))
# create a dictionary for the attributes for each pattern where each attribute = 1
for key in patterns.json()['patterns'].keys():
attr_list.append({
attr['permalink']: 1
for attr in patterns.json()['patterns'][key]
['pattern_attributes']
})
batch_num += 1
# in case pattern list is list of strings, make it into list of integers
p_l = [int(item) for item in pattern_list]
# turn list of dictionaries into a dataframe, turn all the NaN values into zero so the dataframe is ones and zeroes
df_attr = pd.DataFrame(attr_list).fillna(0)
# make a new column that is the pattern id number for each pattern
df_attr['pattern_id'] = p_l
# set the pattern id number as the index
df_attr = df_attr.set_index('pattern_id')
# instate argument for the tf/idf transformer
tfidf = TfidfTransformer()
# transform dataframe into tf/idf array
tf_array = tfidf.fit_transform(df_attr).toarray()
# turn tf/idf array into dataframe
df_tf = pd.DataFrame(tf_array, columns=df_attr.columns)
# create pattern id column
df_tf['pattern_id'] = p_l
# set pattern id column as index of tf/idf dataframe
df_tf = df_tf.set_index('pattern_id')
return df_tf