def craigslist_searcher(locations, site=None):
for location in locations:
url = "https://{}.craigslist.org/search/{}/".format(
location, (site or CRAIGSLIST_SITE))
response = requests.get(url)
soup = BeautifulSoup(response.content, 'lxml')
db.insert_into_db(
build_craigslist_data_object(soup, url, location,
'gigs, computer'))
def read_csv_and_store_in_db():
with open('developer_score_5_criteria.csv', 'r') as fin:
dr = csv.DictReader(fin)
to_db = [(i['name'], i['LinkedIn content'],
i['Public coding activities'], i['Github Analysis'],
i['Stackoverflow Analysis'], i['Tech Keys involved'])
for i in dr]
for person in to_db:
dbs.insert_into_db(person[0], person[1], person[2], person[3],
person[4], person[5])
def post_uuids(): UUID_LENGTH = 16 print(request.content_type) assert request.content_type == 'application/octet-stream' data = request.get_data() assert len(data) % UUID_LENGTH == 0 uuids = (data[i:i + UUID_LENGTH] for i in range(0, len(data), UUID_LENGTH)) db.insert_into_db(get_db_connection(), uuids) return ""
def get(self, key):
#handle case we have a db
if self.conn:
value = lookup_table(self.conn, key)
if not value:
#the additional tuple is a bit dirty…
#but the fetchone method called in lookup_table
#return a tuple, and this way it works in both cases
value = (self.f(key), )
insert_into_db(self.conn, key, tuple(value[0]))
return value[0]
else:
#handle case we don't have a db (for instance, for tests)
if key not in self.cache:
value = self.f(key)
self.cache[key] = value
return self.cache[key]
def align_images(path_originals, path_scanned, output_path, phone=None, conn=None):
scanned = [s for s in os.listdir(path_scanned)]
total_mse = 0
total_ssim = 0
count = 0
for image in scanned:
if conn and check_if_image_exists(image, 2, phone, conn):
print("Image {} for phase {} and phone {} already exists in db".format(image, 2, phone))
continue
original_image_name = image.split('.')[0] + '.png'
result = align_image(os.path.join(path_originals, original_image_name), os.path.join(path_scanned, image),
os.path.join(output_path, image))
try:
status, ssim, mse = result
total_mse += mse
total_ssim += ssim
count += 1
except:
pass
if conn and result:
status, ssim = result
insert_into_db(image, status, 2, ssim, None, phone, conn)
print('Average ssim: {}; average mse: {}'.format(total_ssim/count, total_mse/count))
def insert_callback(session, response):
soup = BeautifulSoup(response.content, 'lxml')
location = urllib.parse.urlparse(response.url).hostname.split('.')[0]
print("Executing callback for:" + location)
data = build_craigslist_data_object(soup, response.url, location, 'gigs, computer')
db.insert_into_db(data)
def craigslist_searcher(locations, site=None):
for location in locations:
url = "https://{}.craigslist.org/search/{}/".format(location, (site or CRAIGSLIST_SITE))
response = requests.get(url)
soup = BeautifulSoup(response.content, 'lxml')
db.insert_into_db(build_craigslist_data_object(soup, url, location, 'gigs, computer'))
def login():
if request.method == 'POST':
data = request.data.decode("utf-8").split(",")
insert_into_db(data[0], data[1])
return render_template('login.html')
def insert_callback(session, response):
soup = BeautifulSoup(response.content, 'lxml')
location = urllib.parse.urlparse(response.url).hostname.split('.')[0]
data = build_craigslist_data_object(soup, response.url, location,
'gigs, computer')
db.insert_into_db(data)
