def main():
print("""
WELCOME TO STUDENTS DATABASE
############################
Select Operation :
(1) for create Table
(2) for Add Record
(3) for Show Records
(4) for Delete Records
(5) for Records Selection
""")
operation = input("Enter your Choice : ")
if operation == "2":
database.add_record()
elif operation == "3":
database.show_all()
elif operation == "4":
database.delete_one(id)
elif operation == "5":
database.select()
elif operation == "1":
print("Please Contact DataBase Admonistrator for this Operation")
else:
print("Try again !!")
def get_products_links_from_sellers(seller_link, seller_id):
req = requests.get(seller_link, headers=headers)
bsobj = BeautifulSoup(req.text, 'lxml')
counter = 0
for product in bsobj.findAll('div'):
if 'data-asin' in product.attrs:
counter += 1
product_link = 'https://www.amazon.ae/dp/' + product.attrs[
'data-asin']
check_book = check_if_book(product_link)
if check_book is True:
print('Book scrapper to be implemented')
else:
logger.info('Getting link for product number {}'.format(
product.attrs['data-index']))
# product_link=product.find('a',{'class':'a-link-normal s-access-detail-page s-color-twister-title-link a-text-normal'}).attrs['href']
logger.info('Product link is {}'.format(product_link))
print(product_link
) #do not comment ..... Important for error tracing
logger.info('Creating Product object from scrapper')
product, seller = a_scrapper.get_product_details(
product_link, 'From Sellers Scrapper')
logger.info('Got product , seller pair = {} , {} '.format(
product, seller))
# # print(product)
# # print(seller)
logger.info('Add product seller pair to products database')
database.add_record(product, seller)
logger.info('product seller pair added to products database')
time.sleep(3)
def add(args):
"""Invoked with the `add` command line argument. Attempts to add
an entry to a specified phonebook. Raises exceptions if the phonebook
doesn't exist or if the entry already exists in the phonebook."""
database.add_record((args.name, args.number), args.b, args.db)
print "Added the following entry to %s:" % args.b
print "%s\t%s" % (args.name, args.number)
def setUp(self):
if glob.glob('*.db'):
raise Exception(".db files exist on setUp!")
# Create test database and table
database.create_database(TEST_DB)
database.create_table(TEST_PB, TEST_DB)
# Add some test records
add_records()
# Add the test record
database.add_record(TEST_RECORD, TEST_PB, TEST_DB)
self.parser = phonebook.parse()
def _post(self):
(header, data) = self._get_payload()
current_time = time.time()
try:
_logger.debug("Assembling database record...")
record = {
'_id': header.get('uid') or uuid.uuid1().hex,
'keys': self._build_keys(header),
'physical': {
'family': header['physical'].get('family'),
'ctime': current_time,
'minRes': CONFIG.storage_minute_resolution,
'atime': int(current_time),
'format': self._build_format(header),
},
'policy': self._build_policy(header),
'stats': {
'accesses': 0,
},
'meta': header.get('meta') or {},
}
except (KeyError, TypeError, AttributeError) as e:
_logger.error("Request received did not adhere to expected structure: %(error)s" % {
'error': str(e),
})
self.send_error(409)
return
else:
_logger.info("Proceeding with storage request for '%(uid)s'..." % {
'uid': record['_id'],
})
_logger.debug("Evaluating compression requirements...")
target_compression = record['physical']['format'].get('comp')
if target_compression and self.request.headers.get('Media-Storage-Compress-On-Server') == 'yes':
_logger.info("Compressing file...")
data = compression.get_compressor(target_compression)(data)
_logger.debug("Storing entity...")
database.add_record(record)
fs = state.get_filesystem(record['physical']['family'])
fs.put(record, data)
return {
'uid': record['_id'],
'keys': record['keys'],
}
def search(Item):
start_search = time.time()
# getBazaarPrices
bazaar_url = 'http://api.torn.com/market/%s?selections=bazaar&key=%s' % (
Item.item_id, api_key)
bazaar_prices = json.loads(requests.get(bazaar_url).text)['bazaar']
#a --> instance of the active item
a = ActiveItem(Item.item_id, Item.item_name, bazaar_prices[0]['cost'],
bazaar_prices[0]['quantity'], bazaar_prices[1]['cost'])
if int(a.profit_percent.replace(
'%', '')) > 3 or a.profit > 200000 and a.sum < 200000000:
#if not database.check_row(a):
#add data to database
database.add_record(a)
match(a)
end = time.time()
#if the check was too fast, slow it down
if end - start_search < 0.8:
time.sleep(0.4)
#print a dot to console after each item check
print("-", end='')
def submission_thread():
"""
This is the main submission thread, it listens to all
new submissions in a subreddit, returns a post instance,
extract image and text, fills out the database. If SIGINT
or SIGTERM is received, the database connection will be
closed and the program will be gracefully killed.
"""
logging.warning("Sleeping for 10 seconds. Waiting for the database to turn on...")
time.sleep(10)
killer = signal_handler.GracefulKiller
conn = psycopg2.connect(dbname=config.db_name,
user=config.db_user,
password=config.db_pass,
host=config.db_host,
port=config.db_port)
database.init_database(conn)
for submission in subreddit.stream.submissions():
post = reddit.submission(submission)
post_ID = str(post.id)
logging.info("")
logging.info("------------------------")
logging.info(f"Starting new submission. {post_ID}")
latest_posts = database.get_latest(conn)
# If the post has been processed recently,
# skip it then
if (post_ID in latest_posts):
logging.info("The post already has been evaluated.")
continue
formatted_text = validate_text(post)
if (not formatted_text):
database.add_record(conn, post_ID)
continue
citation = search_quote(conn, formatted_text, post)
if (not citation):
database.add_record(conn, post_ID)
logging.info("Citation is not found.")
continue
database.add_record(conn, post_ID, citation)
logging.info("Record has been added to the database.")
show_out(conn)
safe_kill(killer, conn)
def get_products_links(link):
logger.info('Started inside get_products_links({})'.format(link))
req = requests.get(link, headers=headers)
bsobj = BeautifulSoup(req.text, 'lxml')
results_div = bsobj.find('div', {'id': 'search-results'})
# products=results_div.findAll('a',{'class':'a-link-normal s-access-detail-page s-color-twister-title-link a-text-normal'})
products = results_div.findAll('div', {'class': 's-item-container'})
logger.info('Found {} products'.format(len(products) - 1))
li_asin = results_div.findAll('li', id=re.compile(r'^result*'))
bulk_asin = []
for data in li_asin:
bulk_asin.append(data.attrs['data-asin'])
logger.info('Check bulk_asin for {}'.format(bulk_asin))
check_all = database.check_product_exists_bulk(bulk_asin)
# print(check_all)
if check_all is None or len(check_all) == 0:
logger.info('All records to be added')
for product in products[:-1]:
logger.info('Getting link for product number {}'.format(
products.index(product)))
product_link = product.find(
'a', {
'class':
'a-link-normal s-access-detail-page s-color-twister-title-link a-text-normal'
}).attrs['href']
logger.info('Product link is {}'.format(product_link))
print(product_link
) #do not comment ..... Important for error tracing
logger.info('Category link={}'.format(link))
logger.info('Creating Product object from scrapper')
product, seller = a_scrapper.get_product_details(
product_link, link)
logger.info('Got product , seller pair = {} , {} '.format(
product, seller))
# # print(product)
# # print(seller)
logger.info('Add product seller pair to products database')
database.add_record(product, seller)
logger.info('product seller pair added to products database')
time.sleep(3)
elif len(check_all) > 0:
diff_asins = list(set(bulk_asin) - set([x[0] for x in check_all]))
# print(diff_asins)
# print(check_all)
logger.info('Unique bulk ASINs={}'.format(len(diff_asins)))
print('Unique bulk ASINs={}'.format(len(diff_asins)))
for x in diff_asins:
product_link = 'https://www.amazon.ae/dp/' + x
logger.info('Getting link for product number {}'.format(
diff_asins.index(x)))
# product_link=product.find('a',{'class':'a-link-normal s-access-detail-page s-color-twister-title-link a-text-normal'}).attrs['href']
logger.info('Product link is {}'.format(product_link))
print(product_link
) #do not comment ..... Important for error tracing
logger.info('Creating Product object from scrapper')
product, seller = a_scrapper.get_product_details(
product_link, link)
logger.info('Got product , seller pair = {} , {} '.format(
product, seller))
# # print(product)
# # print(seller)
logger.info('Add product seller pair to products database')
database.add_record(product, seller)
logger.info('product seller pair added to products database')
time.sleep(3)
def add_records(pb=TEST_PB, db=TEST_DB):
"""Adds test records to the test database."""
records = read_records()
for record in records: database.add_record(record, pb, db)
def handle_mssg(mssg):
# Unexpected bit:
# When the bot sends a message, this function is called. If the sender is
# itself, quit, to avoid an infinite loop.
if mssg['sender_email'] == client.email:
return
print 'Message received!'
content = mssg['content'].strip()
first = content.split()[0]
strip_len = len(first)
while first.endswith(':'):
strip_len += 1
first = first[:-1]
content = mssg['content'][strip_len:]
content = content.lstrip()
# First, assume the message is a "ssh" request.
shh = first
if shh.lower() == 'shh' or shh.lower() == 'ssh': # Allow for the ssh pun...because it's fun
anon_shh(stream)
return
# Next, assume first is the recipient of an anonymous message
recipient_email = first
if recipient_email in emails:
sender_email = mssg['sender_email']
# Generate code, send it to person
code = gen_new_relay_code()
if not database.database_exists(filename):
print 'Database %s doesn\'t yet exist. Creating it now...'%(filename)
database.create_database(filename)
if not database.table_exists(table, filename):
# TODO: Put this block of code into the database module as the create_table function
# The current problem is that I need a way of passing a tuple of strings, and
# I'm not sure how to securely do this...
# database.create_table(cols, table, database) where cols = (col1, col2)
with sqlite3.connect(filename) as con:
c = con.cursor()
c.execute("CREATE TABLE %s (code TEXT, email TEXT)"%(table))
database.add_record((code, sender_email), table, filename)
end_content = '\n\nThis sender\'s key is %s. To reply, send a message in the following format:\n' %(code)
end_content += '%s: <your_message>' %(code)
content += end_content
send_anon_pm(recipient_email, content)
send_anon_pm(sender_email, 'Anon message sent to %s' %(recipient_email))
return
# Next, assume first is the code of a relax message
code = first
email_match = None
if database.database_exists(filename):
email_match = search_for_code(code)
if email_match:
sender_email = mssg['sender_email']
content = 'Response from %s:\n'%(sender_email) + content
end_content = '\nTo reply anonymously, send a message in the following format:\n'
end_content += '"%s: <your_message>" (without the quotes)' %(sender_email)
content += end_content
send_anon_pm(email_match, content)
send_anon_pm(sender_email, 'Your reply was sent to %s.' %(code))
return
# All assumptions exhausted, reply with help message.
sender_email = mssg['sender_email']
send_anon_pm(sender_email, get_help_response())
