def pixivSubmissionsFromJson(bookmarks):
submissions = []
for illustration in bookmarks.illusts:
if illustration.type not in ['illust', 'manga']:
# TODO: Add more format support
logger.log("Skipping " + illustration.type)
continue
# Album
if illustration.meta_pages:
imageIndex = 0
for imagePage in illustration.meta_pages:
newSubmission = Submission()
fillPixivSubmission(illustration, newSubmission)
newSubmission.title = '{}_{}'.format(newSubmission.title,
imageIndex)
newSubmission.bodyUrl = imagePage.image_urls.original
imageIndex += 1
submissions.append(newSubmission)
# Single image
elif illustration.meta_single_page:
newSubmission = Submission()
fillPixivSubmission(illustration, newSubmission)
# The image that will be downloaded
newSubmission.bodyUrl = illustration.meta_single_page.original_image_url
submissions.append(newSubmission)
logger.log("Got {} Pixiv bookmarks".format(len(submissions)))
return submissions
def __init__(self, reddit_settings={}):
self.reddit = praw.Reddit(
client_id='Vyw-20ZFtH4msA',
client_secret='-vZkEG8s6qlRvbTcuGxmJOnpAds',
user_agent='ubuntu:arguing-agents:v1 (by /u/HolzmindenScherfede)')
if reddit_settings == {}:
print(
"Reddit: __init__: No settings given. Creating empty object.")
return
if 'mode' not in reddit_settings:
print(
"Reddit: __init__: Mode not provided. Initializing it to 'find'."
)
mode = reddit_settings['mode']
if mode == 'find':
if 'topic' in reddit_settings:
topic = reddit_settings['topic']
else:
print("Reddit: provide topic")
exit(-1003)
if 'amount' in reddit_settings:
amount = reddit_settings['amount']
else:
amount = 10
if 'sortby' in reddit_settings:
sortby = reddit_settings['sortby']
else:
sortby = 'hot'
praw_submissions = list(
self.reddit.subreddit('ChangeMyView').search(topic,
sortby))[:amount]
self.submissions = []
for praw_submission in praw_submissions:
submission = Submission(praw_submission)
self.submissions.append(submission)
elif mode == 'url':
if 'submission_urls' not in reddit_settings:
print('Reddit: provide submission_urls!')
exit(-12031)
submission_urls = reddit_settings['submission_urls']
self.submissions = []
for submission_url in submission_urls:
submission = Submission(
praw.models.Submission(self.reddit, url=submission_url))
self.submissions.append(submission)
def from_dict(self, dic):
self.submissions = []
for dic_submission in dic['submissions']:
submission = Submission()
submission.from_dict(dic_submission, self.reddit)
self.submissions.append(submission)
def evaluate(self) -> bool:
"""
The function will error out if either the submission
function raises an error or the solution and submission
outputs do not match
Thank God for stackoverflow
"""
solution = Solution()
submission = Submission()
# retrive the running functions in the Solution and Submission classes
# The names of these functions are defined in inputs.json
solution_func = getattr(solution, self.func_name)
submission_func = getattr(submission, self.func_name)
for testcase in self.inputs:
# unpack each testcase and execute
solution_res = solution_func(*testcase)
submission_res = submission_func(*testcase)
if isinstance(solution_res, Iterable):
if sorted(solution_res) != sorted(submission_res):
raise ImplementationError(
f"Error for testcase {testcase}. "
f"Expected {solution_res}, "
f"got {submission_res}")
else:
if solution_res != submission_res:
raise ImplementationError(
f"Error for testcase {testcase}. "
f"Expected {solution_res}, "
f"got {submission_res}")
def load_data(self):
"""Initialize blocchain + open submissions data from a file."""
try:
with open('blocchain-{}.bit'.format(self.node_id), mode='r') as f:
# file_content = pickle.loads(f.read())
file_content = f.readlines()
# blocchain = file_content['chain']
# open_submissions = file_content['ot']
blocchain = json.loads(file_content[0][:-1])
# We need to convert the loaded data because submissions
# should use OrderedDict
updated_blocchain = []
for bloc in blocchain:
converted_tx = [Submission(
tx['voter'],
tx['candidate'],
tx['zero'], #added to hold day zero, countdown until final vote
tx['signature'],
tx['amount']) for tx in bloc['submissions']]
updated_bloc = Bloc(
bloc['index'],
bloc['previous_hash'],
converted_tx,
bloc['proof'],
bloc['timestamp'])
updated_blocchain.append(updated_bloc)
self.chain = updated_blocchain
open_submissions = json.loads(file_content[1][:-1])
# We need to convert the loaded data because submissions
# should use OrderedDict
updated_submissions = []
for tx in open_submissions:
updated_submission = Submission(
tx['voter'],
tx['candidate'],
tx['zero'],
tx['signature'],
tx['amount'])
updated_submissions.append(updated_submission)
self.__open_submissions = updated_submissions
peer_nodes = json.loads(file_content[2])
self.__peer_nodes = set(peer_nodes)
except (IOError, IndexError):
pass
finally:
print('Cleanup!')
def add_submission_by_user(user: User, subm: str, cat: str,
lang: str) -> Submission:
submission = Submission(user=user,
value=subm,
category=cat,
language=lang,
mwe=mwe_helper.get_todays_mwe(lang))
session.add(submission)
session.commit()
return submission
def submit_handler_2(user: User, update: Update, context: CallbackContext):
todays_mwe = mwe_helper.get_todays_mwe(user.language)
submission = Submission(value=update.message.text)
context.user_data["submission"] = submission
context.user_data["state"] = "submit_example_type_1"
update.message.reply_text(
get_language_token(user.language, Tokens.SPECIAL_MEANING) %
todays_mwe.name,
parse_mode=telegram.ParseMode.MARKDOWN,
reply_markup=get_submit_category_1_keyboard_markup(
user.language, todays_mwe))
def run(pid, title, lang, code):
controller = Controller.controller
try:
submission = Submission(pid, title, lang, code)
controller.execute(submission)
return submission.sid
except Failed as e:
logging.warning(e.message)
raise
except Exception as e:
traceback.print_exc()
raise Failed('unknown error')
def handleSubmission(id, lang, code):
# Maximum time
try:
with tlim(5):
logger.info(f'[PROCESSING] {multiprocessing.current_process().name}')
# Get compiler
c = Client.__compiler.getCompiler(lang)
s = Submission(id, lang, code, c)
s.start()
except TimeoutException:
return
finally:
pass
def updateSolvedProblems(self, html):
#TODO: compile
submissionPattern = b'\t<tr class="sectiontableentry\d+">\n' \
b'\t<td align="center"><a href="[^"]*">(\d+)</a></td>\n' \
b'\t<td align="center">\d+</td>\n' \
b'\t<td align="center">\d+</td>\n' \
b'\t<td align="center">([^<]*)</td>\n' \
b'\t<td align="center">[^<]*</td>\n\t'
tablePattern = b'<div class="contentheading">Solved problems</div>\n\t<table border="0" cellspacing="0" cellpadding="4" style="width:70%" align="center">\n\t<tr class="sectiontableheader"><th align="center">Problem</th><th align="center">Ranking</th><th align="center">Submission</th><th align="center">Date</th><th align="center">Run time</th></tr>\n\t' \
b'(' + submissionPattern + b')*' \
b'\t</table>'
tableHTML = re.search(tablePattern, html).group(0)
self.submissions = list()
for match in re.finditer(submissionPattern, tableHTML):
self.submissions.append(Submission(match.group(1).decode('utf-8'), match.group(2).decode('utf-8')))
def import_old():
existing_tags = {}
for tag in Tag.query.all():
exisiting_tags[tag.name] = tag
for resource in [
'Projector', 'Microphone', 'Sound system', 'Drinking water',
'Quiet (no airwalls)'
]:
penguicontrax.db.session.add(Resource(resource))
for track in ['literature', 'tech', 'music', 'food', 'science']:
penguicontrax.db.session.add(Track(track, None))
with penguicontrax.app.open_resource('schedule2013.html', mode='r') as f:
tree = ET.fromstring(f.read())
events = tree.find('document')
for section in events:
if section.tag == 'div' and section.attrib['class'] == 'section':
name = section[0].text
tag_list = section[
1].text # Tag doesn't seem to be in the DB yet
person = section[3][0].text
# Only one presenter is supported so far
firstPerson = person.split(',')[0].split(' ')
description = section[3][0].tail
submission = Submission()
submission.email = '*****@*****.**'
submission.title = name
submission.description = description
submission.duration = 1
submission.setupTime = 0
submission.repetition = 0
submission.firstname = firstPerson[0]
submission.lastname = firstPerson[1] if len(
firstPerson) > 1 else ''
submission.followUpState = 0
submission.tags = []
for tag in tag_list.split(','):
tag = normalize_tag_name(tag)
db_tag = None
if not tag in existing_tags:
db_tag = Tag(tag)
penguicontrax.db.session.add(db_tag)
existing_tags[tag] = db_tag
else:
db_tag = existing_tags[tag]
submission.tags.append(db_tag)
penguicontrax.db.session.add(submission)
penguicontrax.db.session.commit()
def resolve(self):
"""Checks all peer nodes' blocchains and replaces the local one with
longer valid ones."""
# Initialize the winner chain with the local chain
winner_chain = self.chain
replace = False
for node in self.__peer_nodes:
url = 'http://{}/chain'.format(node)
try:
# Send a request and store the response
response = requests.get(url)
# Retrieve the JSON data as a dictionary
node_chain = response.json()
# Convert the dictionary list to a list of bloc AND
# submission objects
node_chain = [
Bloc(bloc['index'],
bloc['previous_hash'],
[
Submission(
tx['voter'],
tx['candidate'],
tx['zero'],
tx['signature'],
tx['amount']) for tx in bloc['submissions']
],
bloc['proof'],
bloc['timestamp']) for bloc in node_chain
]
node_chain_length = len(node_chain)
local_chain_length = len(winner_chain)
# Store the received chain as the current winner chain if it's
# longer AND valid
if (node_chain_length > local_chain_length and
Verification.verify_chain(node_chain)):
winner_chain = node_chain
replace = True
except requests.exceptions.ConnectionError:
continue
self.resolve_conflicts = False
# Replace the local chain with the winner chain
self.chain = winner_chain
if replace:
self.__open_submissions = []
self.save_data()
return replace
def add_submission(self,
candidate,
voter,
zero,
signature,
amount=1.0,
is_receiving=False):
""" Append a new value as well as the last blocchain value to the blocchain.
Arguments:
:voter: The person voting.
:candidate: The candidate recieving the votes.
:amount: The amount of votes sent with the submission
(default = 1.0)
"""
# submission = {
# 'voter': voter,
# 'candidate': candidate,
# 'amount': amount
# }
# if self.public_key == None:
# return False
submission = Submission(voter, candidate, zero, signature, amount)
if Verification.verify_submission(submission, self.get_balance):
self.__open_submissions.append(submission)
self.save_data()
if not is_receiving:
for node in self.__peer_nodes:
url = 'http://{}/broadcast-submission'.format(node)
try:
response = requests.post(url,
json={
'voter': voter,
'candidate': candidate,
'zero': zero,
'amount': amount,
'signature': signature
})
if (response.status_code == 400 or
response.status_code == 500):
print('Submission declined, needs resolving')
return False
except requests.exceptions.ConnectionError:
continue
return True
return False
def add_bloc(self, bloc):
"""Add a bloc which was received via broadcasting to the localb
lockchain."""
# Create a list of submission objects
submissions = [Submission(
tx['voter'],
tx['candidate'],
tx['zero'],
tx['signature'],
tx['amount']) for tx in bloc['submissions']]
# Validate the proof of work of the bloc and store the result (True
# or False) in a variable
proof_is_valid = Verification.valid_proof(
submissions[:-1], bloc['previous_hash'], bloc['proof'])
# Check if previous_hash stored in the bloc is equal to the local
# blocchain's last bloc's hash and store the result in a bloc
hashes_match = hash_bloc(self.chain[-1]) == bloc['previous_hash']
if not proof_is_valid or not hashes_match:
return False
# Create a bloc object
converted_bloc = Bloc(
bloc['index'],
bloc['previous_hash'],
submissions,
bloc['proof'],
bloc['timestamp'])
self.__chain.append(converted_bloc)
stored_submissions = self.__open_submissions[:]
# Check which open submissions were included in the received bloc
# and remove them
# This could be improved by giving each submission an ID that would
# uniquely identify it
for itx in bloc['submissions']:
for opentx in stored_submissions:
if (opentx.voter == itx['voter'] and
opentx.candidate == itx['candidate'] and
opentx.zero == itx['zero'] and
opentx.amount == itx['amount'] and
opentx.signature == itx['signature']):
try:
self.__open_submissions.remove(opentx)
except ValueError:
print('Item was already removed')
self.save_data()
return True
def criaSubmission(self, id):
urlSubmission = "https://www.urionlinejudge.com.br/judge/pt/runs/code/" + id
submissionPage = self.sess.get(urlSubmission)
submissionSoup = BeautifulSoup(submissionPage.text,
features="html.parser")
dados = submissionSoup.find("div", {
"class": "st-big-box"
}).find_all("dd")
idEnome = dados[0].text.split("-")
resultado = dados[1].text.strip()
language = dados[2].text.split("(")[0].strip()
data = dados[5].text.strip()
id = idEnome[0].strip()
nome = idEnome[1].strip()
codeLines = submissionSoup.find("pre", {"id": "code"})
code = codeLines.text
submission = Submission(id, nome, resultado, data, language, code)
return submission
def massageData(self, inflate):
try:
submissions = inflate['attached_media']
from submission import Submission
contributors = []
for s in submissions:
submission = Submission(_id=s)
print submission.emit()
contributors.append(
submission.j3m.genealogy['createdOnDevice'])
inflate['contributors'] = list(set(contributors))
except KeyError as e:
print e
pass
return inflate
def submit(self, revision, data, mode_info, log=[]):
repo = mode_info["repo"]
settings = {
"treeherder": {
'group_symbol': 'AWFY',
'group_name': 'AWFY',
'job_name': mode_info["job_name"],
'job_symbol': mode_info["job_symbol"],
"tier": mode_info["tier"],
"platform": mode_info["platform"]
}
}
logfile = create_log_item({
"repo": repo,
"revision": revision,
"settings": settings,
"perf_data": data,
"extra_log_info": log
})
loglink = "https://arewefastyet.com/data.php?file=treeherder-logs/" + logfile
retriggerlink = None
if "mode_id" in log and "machine_id" in log:
retriggerlink = "https://arewefastyet.com/retrigger/?machine_id=" + str(
log["machine_id"]) + "&mode_id=" + str(
log["mode_id"]) + "&revision=" + revision
th = Submission(repo,
revision,
treeherder_url=awfy.th_host,
treeherder_client_id=awfy.th_user,
treeherder_secret=awfy.th_secret,
settings=settings)
job = th.create_job(None)
th.submit_completed_job(job,
data,
loglink=loglink,
retriggerlink=retriggerlink)
def import_old():
existing_tags = {}
for tag in Tag.query.all():
exisiting_tags[tag.name] = tag
with penguicontrax.app.open_resource('schedule2013.html', mode='r') as f:
tree = ET.fromstring(f.read())
events = tree.find('document')
for section in events:
if section.tag == 'div' and section.attrib['class'] == 'section':
name = section[0].text
tag_list = section[1].text # Tag doesn't seem to be in the DB yet
person = section[3][0].text
# Only one presenter is supported so far
firstPerson = person.split(',')[0].split(' ')
description = section[3][0].tail
submission = Submission()
submission.email = '*****@*****.**'
submission.title = name
submission.description = description
submission.duration = 1
submission.setupTime = 0
submission.repetition = 0
submission.firstname = firstPerson[0]
submission.lastname = firstPerson[1] if len(firstPerson) > 1 else ''
submission.followUpState = 0
submission.tags = []
for tag in tag_list.split(','):
tag = tag.strip()
db_tag = None
if not tag in existing_tags:
db_tag = Tag(tag)
penguicontrax.db.session.add(db_tag)
existing_tags[tag] = db_tag
else:
db_tag = existing_tags[tag]
submission.tags.append(db_tag)
penguicontrax.db.session.add(submission)
penguicontrax.db.session.commit()
x_train = [row['UPDATE_TIME'] for row in data]
y1_train = [row['BANDWIDTH_TOTAL'] for row in data]
y2_train = [row['MAX_USER'] for row in data]
dates_predict = list(week_range(start_date_predict, end_date_predict, weekday))
model = Model()
model.fit(x_train, y1_train, y2_train)
y1_predict, y2_predict = model.predict(dates_predict)
for date, y1, y2 in zip(dates_predict, y1_predict, y2_predict):
result.add(date, hour, server, y1, y2)
pbar.close()
print()
submission = Submission()
print('Prepare submission...')
with open('data/test_id.csv', 'r') as test_file:
reader = csv.DictReader(test_file)
for row in tqdm(reader):
test_id = row['id']
update_time = datetime.strptime(row['UPDATE_TIME'], '%Y-%m-%d').date()
hour = int(row['HOUR_ID'])
server = row['SERVER_NAME']
bandwidth, max_user = result.get(update_time, hour, server)
submission.add_entry(test_id, bandwidth, max_user)
print()
print('Start write submission...')
submission.write()
from mlp import MultiLayerPerceptron
TEST_PATH = "data/test.csv"
TRAIN_PATH = "data/train.csv"
test_set = TestSet(TEST_PATH).read()
train_set = TrainSet(TRAIN_PATH).read()
x_train = train_set.drop("label", axis=1).values.astype('float32')
y_train = train_set["label"].values.astype('int32')
#preprocessing
max_value = np.max(x_train)
mean_value = np.mean(x_train)
test_set = (test_set - mean_value) / max_value
x_train = (x_train - mean_value) / max_value
#MLP
MLP_SUBMISSION = "mlp.csv"
mlp = MultiLayerPerceptron(x_train, y_train)
mlp.fit()
predictions = mlp.predict(test_set)
Submission(predictions).save(MLP_SUBMISSION)
for i in range(1, 10):
random_image = test_set[i,:]
print(predictions[i])
Pixel(random_image).display()
def main():
# Command line Interface
parser = argparse.ArgumentParser()
parser.add_argument('-d',
'--dirpath',
default=DATASETS_DIRPATH,
help="dataset directory path")
parser.add_argument('-n',
'--n_train',
default=None,
type=int,
help="number of rows to download on the train dataset")
parser.add_argument('-t',
'--n_test',
default=None,
type=int,
help="number of rows to download on the test dataset")
parser.add_argument('-e',
"--epochs",
default=3,
type=int,
help="set the number of epochs")
parser.add_argument('-b',
"--batch_size",
default=86,
type=int,
help="set batch size")
cli = parser.parse_args()
# Download and clean train dataset
train = DataSet(dirpath=cli.dirpath, filename="train.csv")
train.download(nrows=cli.n_train)
train.split_X_Y()
train.normalize()
train.reshape()
train.convert_digits_to_one_hot_vectors()
print(train)
# Split trian/validation datasets
validation = train.extract_validation(size=0.1)
print(validation)
# Download clean test dataset
test = DataSet(dirpath=cli.dirpath, filename="test.csv")
test.download(nrows=cli.n_test)
test.set_X()
test.normalize()
test.reshape()
print(test)
# Setup convolutional neural network model
model = Sequential()
model.add(
Conv2D(filters=32,
kernel_size=(5, 5),
padding='Same',
activation='relu',
input_shape=(28, 28, 1)))
model.add(
Conv2D(filters=32,
kernel_size=(5, 5),
padding='Same',
activation='relu'))
model.add(MaxPool2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(
Conv2D(filters=64,
kernel_size=(3, 3),
padding='Same',
activation='relu'))
model.add(
Conv2D(filters=64,
kernel_size=(3, 3),
padding='Same',
activation='relu'))
model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(256, activation="relu"))
model.add(Dropout(rate=0.5))
model.add(Dense(10, activation="softmax"))
# Define the optimizer
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
# Compile the model
model.compile(optimizer=optimizer,
loss="categorical_crossentropy",
metrics=["accuracy"])
# Set learning rate decay
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
# Perform synthetic data augmentation
data_generator = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=
False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=
10, # randomly rotate images in the range (degrees, 0 to 180)
zoom_range=0.1, # Randomly zoom image
width_shift_range=
0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=
0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=False, # randomly flip images
vertical_flip=False # randomly flip images
)
data_generator.fit(train.X)
history = model.fit_generator(
data_generator.flow(train.X, train.Y, batch_size=cli.batch_size),
epochs=cli.epochs,
validation_data=(validation.X, validation.Y),
verbose=2,
steps_per_epoch=train.X.shape[0] // cli.batch_size,
callbacks=[learning_rate_reduction])
# plot loss and accuracy
evaluation.plot_loss_and_accuracy(history)
# Predict digits for the validation dataset
prediction = DataSet()
prediction.Y = model.predict(validation.X)
prediction.X = validation.X
prediction.convert_one_hot_vectors_to_digits()
validation.convert_one_hot_vectors_to_digits()
confusion_mtx = confusion_matrix(validation.Y, prediction.Y)
evaluation.plot_confusion_matrix(confusion_mtx)
# Predict results
results = model.predict(test.X)
results = convert_one_hot_vectors_to_digits(results)
print(results)
# Generate Submission file
submission_file = Submission(results)
submission_file.save()
def mine_bloc(self):
global VOTE_WINDOW
"""Create a new bloc and add open submissions to it."""
# update your ip (only if your publickey is registered) so that mining can be shared with all nodes
if self.public_key is None:
return None
publickey = {"publickey": self.public_key}
requests.post('https://blocbit.net/kitty.php' ,params=publickey)
# Fetch the currently last bloc of the blocchain
last_bloc = self.__chain[-1]
#last_pf = last_bloc.proof
#window = self.load_window_data()
# Hash the last bloc (=> to be able to compare it to the stored hash
# value)
hashed_bloc = hash_bloc(last_bloc)
proof = self.proof_by_vote()
# Added to avoid blocchain startup error after genesis bloxk as it contains no submission i.e. no zero
# last_pf = last_bloc.proof
# if last_pf != 86400:
# zero = self.submission_zero()
# else:
# zero = 365.0
zero = self.submission_zero()
# Voters have the right to vote daily, so let's create a window submission
# reward_submission = {
# 'voter': 'STATION',
# 'candidate': owner,
# 'amount': 0 or 1
# }
Station_open = Submission(
'STATION', self.public_key, zero, '', 1)
Station_closed = Submission(
'STATION', self.public_key, zero, '', 0)
# Copy submission instead of manipulating the original
# open_submissions list
# This ensures that if for some reason the mining should fail,
# we don't have the reward submission stored in the open submissions
copied_submissions = self.__open_submissions[:]
for tx in copied_submissions:
if not Ballot.verify_submission(tx):
return None
# if global var is set to true award right and then set back to false
if VOTE_WINDOW is False:
copied_submissions.append(Station_closed)
else:
copied_submissions.append(Station_open)
VOTE_WINDOW = False
bloc = Bloc(len(self.__chain), hashed_bloc,
copied_submissions, proof)
self.__chain.append(bloc)
self.__open_submissions = []
self.save_data()
for node in self.__peer_nodes:
url = 'http://{}/broadcast-bloc'.format(node)
converted_bloc = bloc.__dict__.copy()
converted_bloc['submissions'] = [
tx.__dict__ for tx in converted_bloc['submissions']]
try:
response = requests.post(url, json={'bloc': converted_bloc})
if response.status_code == 400 or response.status_code == 500:
print('Bloc declined, needs resolving')
if response.status_code == 409:
self.resolve_conflicts = True
except requests.exceptions.ConnectionError:
continue
return bloc
def getTumblrUserLikedSubmissions(clientId,
clientSecret,
tokenId,
tokenSecret,
likeRequestLimit=100,
requestOnlyNewCache=None):
tumblrClient = pytumblr.TumblrRestClient(clientId, clientSecret, tokenId,
tokenSecret)
# This is an annoying limit the api seems to impose
POSTS_PER_PAGE = 50
oldestPageTimestamp = 0
totalRequests = 0
submissions = []
foundOldSubmission = False
while totalRequests < likeRequestLimit:
if oldestPageTimestamp:
tumblrLikes = tumblrClient.likes(**{
'limit': POSTS_PER_PAGE,
'offset': totalRequests
})
else:
tumblrLikes = tumblrClient.likes(**{'limit': POSTS_PER_PAGE})
numPostsThisPage = len(tumblrLikes['liked_posts'])
if not numPostsThisPage:
break
logger.log(
str(numPostsThisPage) + ' Tumblr likes requested. Total likes: ' +
str(tumblrLikes['liked_count']))
for postIndex, post in reversed(
list(enumerate(tumblrLikes['liked_posts']))):
if 'photos' in post:
for photoIndex, photo in enumerate(post['photos']):
newSubmission = Submission()
newSubmission.source = u'Tumblr'
# Tumblr submissions don't have titles, so make one
# This'll look ugly in the file browser, unfortunately
if len(post['photos']) > 1:
newSubmission.title = str(
signedCrc32(post['short_url'].encode()))
newSubmission.title += u'_'
newSubmission.title += str(photoIndex)
else:
newSubmission.title = str(
signedCrc32(post['short_url'].encode()))
"""logger.log(post)
return"""
newSubmission.author = post['blog_name']
newSubmission.subreddit = post['short_url']
newSubmission.subredditTitle = post['blog_name'] + '_Tumblr'
newSubmission.body = post['caption']
newSubmission.bodyUrl = photo['original_size']['url']
newSubmission.postUrl = post['short_url']
submissions.append(newSubmission)
if (requestOnlyNewCache and requestOnlyNewCache[0]
and newSubmission.postUrl
== requestOnlyNewCache[0].postUrl):
logger.log(
'Found early out point after ' +
str(len(submissions)) + ' new submissions.'
' If you e.g. changed your total requests value and want to go deeper, set'
' Tumblr_Try_Request_Only_New to False in your settings.txt'
)
foundOldSubmission = True
break
else:
logger.log('Skipped ' + post['short_url'] +
' (does not have images)')
if foundOldSubmission:
break
if foundOldSubmission:
break
oldestPageTimestamp = tumblrLikes['liked_posts'][-1]['liked_timestamp']
# If we didn't get a full page's worth of posts, we're on the last page
# Sometimes pages don't have POSTS_PER_PAGE, they're a little under
RANDOM_PAGE_TOLERANCE = 10
if numPostsThisPage < POSTS_PER_PAGE - RANDOM_PAGE_TOLERANCE:
break
totalRequests += numPostsThisPage
newEarlyOut = submissions[0] if len(submissions) else None
return submissions, newEarlyOut
def import_old(path, as_convention = False, random_rsvp_users = 0, submission_limit = sys.maxint, timeslot_limit = sys.maxint):
if as_convention == True:
convention = Convention()
convention.name = 'Penguicon 2013'
convention.url = '2013'
convention.description = 'Penguicon 2013 schedule imported from schedule2013.html'
convention.start_dt = datetime.datetime(year=2013, month=4, day=26, hour=16)
convention.end_dt = datetime.datetime(year=2013, month=4, day=28, hour=16)
convention.timeslot_duration = datetime.timedelta(hours=1)
penguicontrax.db.session.add(convention)
current_day = convention.start_dt.date()
current_time = None
existing_tags = {}
for tag in Tag.query.all():
existing_tags[tag.name] = tag
existing_people = {}
for person in Person.query.all():
existing_people[person.name] = person
existing_tracks = {}
for track in Track.query.all():
existing_tracks[track.name] = track
existing_rooms = {}
existing_submissions = []
submission_count = 0
with penguicontrax.app.open_resource(path, mode='r') as f:
tree = ET.fromstring(f.read())
events = tree.find('document')
for section in events:
if submission_count == submission_limit:
break
if as_convention == True and section.tag == 'time':
time_text= section.text.split(' ')
hour = int(time_text[0])
if time_text[1] == 'PM' and hour != 12:
hour += 12
elif time_text[1] == 'AM' and hour == 12:
hour = 0
new_time = datetime.time(hour = hour)
if not current_time is None and new_time.hour < current_time.hour:
current_day = current_day + datetime.timedelta(days=1)
current_time = new_time
elif section.tag == 'div' and section.attrib['class'] == 'section':
name = section[0].text
tag_list = section[1].text # Tag doesn't seem to be in the DB yet
room = section[2].text
person = section[3][0].text
description = section[3][0].tail
submission = Submission() if as_convention == False else Events()
submission.title = name
submission.description = description
submission.duration = 1
submission.setupTime = 0
submission.repetition = 0
submission.followUpState = 0
submission.eventType = 'talk'
#Load presenters
submission.personPresenters= []
for presenter in [presenter.strip() for presenter in person.split(',')]:
if presenter == 'Open':
continue #"Open" person will cause the schedule to become infesible
person = None
if not presenter in existing_people:
person = Person(presenter)
penguicontrax.db.session.add(person)
existing_people[presenter] = person
else:
person = existing_people[presenter]
submission.personPresenters.append(person)
#Load Tags
submission.tags = []
for tag in tag_list.split(','):
tag = normalize_tag_name(tag)
db_tag = None
if not tag in existing_tags:
db_tag = Tag(tag, tag, True)
penguicontrax.db.session.add(db_tag)
existing_tags[tag] = db_tag
else:
db_tag = existing_tags[tag]
# Set track -- pick any tag that is also a track
if submission.track is None:
if tag in existing_tracks:
submission.track = existing_tracks[tag]
submission.tags.append(db_tag)
#Load rooms
if as_convention == True:
submission.convention = convention
db_room = None
if not room in existing_rooms:
db_room = Rooms()
db_room.room_name = room
db_room.convention = convention
penguicontrax.db.session.add(db_room)
existing_rooms[room] = db_room
else:
db_room = existing_rooms[room]
if not current_day is None and not current_time is None:
submission.rooms.append(db_room)
submission.start_dt = datetime.datetime(year=current_day.year, month=current_day.month, day=current_day.day,\
hour = current_time.hour, minute=current_time.minute)
submission.duration = 4 #1 hour
existing_submissions.append(submission)
penguicontrax.db.session.add(submission)
submission_count = submission_count + 1
penguicontrax.db.session.commit()
if random_rsvp_users > 0:
for user_index in range(random_rsvp_users):
user = User()
user.name = 'Random User %d' % user_index
user.email = '*****@*****.**' % user_index
user.public_rsvps = True
user.staff = False
user.special_tag = None
user.superuser = False
generate_account_name(user)
gravatar_image_update(user)
for rsvp_index in range(user.points):
rand = random.randint(0, len(existing_submissions) - 1)
while user in existing_submissions[rand].rsvped_by:
rand = random.randint(0, len(existing_submissions) - 1)
existing_submissions[rand].rsvped_by.append(user)
user.points = 0
penguicontrax.db.session.add(user)
penguicontrax.db.session.commit()
if as_convention == True:
from event import generate_schedule, generate_timeslots
generate_timeslots(convention, timeslot_limit)
all_rooms = [room for room in existing_rooms.viewvalues()]
hackerspace = [existing_rooms['Hackerspace A'], existing_rooms['Hackerspace B']]
food = [existing_rooms['Food']]
from copy import copy
general_rooms = copy(all_rooms)
general_rooms.remove(hackerspace[0])
general_rooms.remove(hackerspace[1])
general_rooms.remove(food[0])
timeslots = [timeslot for timeslot in convention.timeslots]
for submission in existing_submissions:
if food[0] in submission.rooms:
submission.suitable_rooms = food
elif hackerspace[0] in submission.rooms or hackerspace[1] in submission.rooms:
submission.suitable_rooms = hackerspace
else:
submission.suitable_rooms = general_rooms
for room in all_rooms:
room.available_timeslots = timeslots
generate_schedule(convention)
def getPinterestUserPinnedSubmissions(email, username, password,
cacheFileName):
submissions = []
lastIds = {} if not cacheFileName else loadPinterestCache(cacheFileName)
updatedLastIds = lastIds
pinterest = Pinterest(email=email,
password=password,
username=username,
cred_root='pinterest_creds')
logger.log("Logging in to Pinterest...")
pinterest.login()
boards = pinterest.boards(username=username)
for board in boards:
# Get all pins for the board
board_pins = []
pin_batch = pinterest.board_feed(board_id=board['id'])
while len(pin_batch) > 0:
for pin in pin_batch:
if pin['id'] not in lastIds:
# Only using the dict for its key lookup
updatedLastIds[pin['id']] = 1
board_pins.append(pin)
pin_batch = pinterest.board_feed(board_id=board['id'])
for pin in board_pins:
# I'm not sure how important it is to support these
if pin['type'] == 'story':
continue
newSubmission = Submission()
newSubmission.source = u'Pinterest'
# While pins do have titles, 90% of the time they seem useless
newSubmission.title = pin['id']
# There is probably a way to figure out who the original pinner is, but oh well
newSubmission.author = 'N/A'
newSubmission.subreddit = board['url']
newSubmission.subredditTitle = board['name'] + '_Pinterest'
if 'rich_summary' in pin and pin['rich_summary']:
if 'display_description' in pin['rich_summary']:
newSubmission.body = pin['rich_summary'][
'display_description']
else:
newSubmission.body = 'N/A'
newSubmission.postUrl = pin['rich_summary']['url']
# What is actually downloaded
newSubmission.bodyUrl = pin['images']['orig']['url']
submissions.append(newSubmission)
if cacheFileName:
savePinterestCache(cacheFileName, updatedLastIds)
logger.log("Found {} new Pinterest submissions".format(len(submissions)))
return submissions
def get_submission_by_id(self, submission_id: str):
self.cursor.execute("SELECT * FROM submissions where id = ?",
(submission_id,))
if db_response := self.cursor.fetchone():
return Submission(*db_response)
def getSubmissionsFromRedditList(redditList,
source,
earlyOutPoint=None,
unlikeUnsave=False):
submissions = []
comments = []
numTotalSubmissions = len(redditList)
for currentSubmissionIndex, singleSubmission in enumerate(redditList):
if currentSubmissionIndex and currentSubmissionIndex % 100 == 0:
logger.log('Got {} submissions...'.format(currentSubmissionIndex))
if type(singleSubmission) is praw.models.Submission:
newSubmission = Submission()
newSubmission.source = u'reddit'
newSubmission.title = singleSubmission.title
newSubmission.author = singleSubmission.author.name if singleSubmission.author else u'no_author'
newSubmission.subreddit = singleSubmission.subreddit.url
newSubmission.subredditTitle = singleSubmission.subreddit.title
newSubmission.body = singleSubmission.selftext
newSubmission.bodyUrl = singleSubmission.url
newSubmission.postUrl = singleSubmission.permalink
submissions.append(newSubmission)
logger.log(
percentageComplete(currentSubmissionIndex,
numTotalSubmissions))
if unlikeUnsave:
if source == 'liked':
singleSubmission.clear_vote()
else:
singleSubmission.unsave()
logger.log('Unsaved/cleared vote on submission ' +
singleSubmission.permalink)
# Check to see if we've already downloaded this submission; if so, early out
if (earlyOutPoint and earlyOutPoint[0]
and newSubmission.postUrl == earlyOutPoint[0].postUrl):
logger.log(
'Found early out point after ' + str(len(submissions)) +
' new submissions.'
' If you e.g. changed your total requests value and want to go deeper, set'
' Reddit_Try_Request_Only_New to False in your settings.txt'
)
break
# The submission is actually a saved comment
else:
# I looked at https://praw.readthedocs.io/en/latest/getting_started/quick_start.html
# very bottom to learn how to enumerate what information a submission can provide
# logger.log(singleSubmission.body)
# pprint.plogger.log(vars(singleSubmission))
newSubmission = Submission()
newSubmission.source = u'reddit'
newSubmission.title = u'Comment on ' + singleSubmission.link_title
newSubmission.author = singleSubmission.author.name if singleSubmission.author else u'no_author'
newSubmission.subreddit = singleSubmission.subreddit.url
newSubmission.subredditTitle = singleSubmission.subreddit.title
newSubmission.body = singleSubmission.body
newSubmission.bodyUrl = singleSubmission.permalink
newSubmission.postUrl = singleSubmission.link_permalink
comments.append(newSubmission)
return submissions, comments
def get_recent_submissions(self, start_time_utc: int) -> list[Submission]:
self.cursor.execute("SELECT * FROM submissions WHERE created_utc >= ?",
(start_time_utc,))
if db_response := self.cursor.fetchall():
return [Submission(*row) for row in db_response]
def submit_submission(self, link, description, assignment, grades):
'''Method that creates instance of Submission class and adds this instance to list of all instance of
Submission class created by the given instance of Student class.'''
self.submissions.append(Submission(link, description, assignment, grades))
from submission import Submission
from tweet import Tweet
from datetime import datetime
from time import sleep
while True:
time = datetime.now()
# Check if current time is within 2 minutes of the hour
if time.minute <= 2 or time.minute >= 58:
submission = Submission(['aww'])
submission.generate()
permalink = submission.permalink()
url = submission.url()
permalink_short = submission.permalink_short()
tweet = Tweet(permalink, url, permalink_short)
tweet.post()
print('Trying again after {} minutes.\n'.format(60 - time.minute))
sleep(60 * (60 - time.minute))
