class ScrapeFreeProxies():
def __init__(self):
self.result_dir = os.path.join(os.getcwd(), "Result")
if not os.path.exists(self.result_dir):
os.makedirs(self.result_dir)
self.collector = create_collector('my-collector', 'http')
self.DB_Name = "proxies.db"
self.tbl_name = "http"
self.db_proxy = DB(dst_path=self.result_dir, dbname=self.DB_Name)
self.fields = [
"anonymous", "code", "country", "host", "port", "type", "date", "status"
]
self.db_proxy.createTable(tbl_name=self.tbl_name, fields=self.fields)
def get_proxies(self):
# Retrieve only anonymous 'uk' or 'us' proxies
proxies = self.collector.get_proxies({'anonymous': True})
for proxy in proxies:
result_row = {
'anonymous': proxy.anonymous,
'code': proxy.code,
'country': proxy.country,
'host': proxy.host,
'port': proxy.port,
'type': proxy.type,
'date': get_now_str()
}
self.db_proxy.insert_row(tbl_name=self.tbl_name, result_row=result_row)
print(result_row)
def test_03_check_population_total_less_2billions(self):
db = DB()
expected_total_population = 2e9
with db:
populations = db.get_population_list_of_all_country()
actual_total_population = reduce(lambda x, y: (x[0] + y[0],), populations)
self.assertLess(actual_total_population[0], expected_total_population,
"Population total is greater than 2billions. "
"Actual total population: {}".format(actual_total_population[0]))
def test_02_country_where_population_density_less_50(self):
db = DB()
population_density = 50
expected_country = "USA"
with db:
actual_country = db.get_country_where_population_density_less_given_value(population_density)
for country in actual_country:
self.assertEqual(country[0], expected_country,
"Incorrect country name with population density less {}. Actual country: {}. "
"Expected country: {}.".format(population_density, country[0], expected_country))
def search_and_extract(manager):
manager = DB()
videodir = os.listdir(config.produced_data_dir)
multithreads = [
] # appedning compressing threads here to be executed parallel
ecount = 1
try:
for videoname in videodir:
# videoname == invite_id
videofolder = os.path.join(config.produced_data_dir, videoname)
videofiles = [
filename for filename in os.listdir(videofolder)
if filename.__contains__('.mp4')
]
matadata_path = os.path.join(videofolder, 'metadata.json')
with open(matadata_path, 'r') as jF:
s = jF.read()
s = s.replace("'", '"')
metadata = json.loads(s)
print(metadata)
for video in videofiles:
videopath = os.path.join(videofolder, video)
tags = 'null'
try:
if (int(metadata['infoExtracted']) == 0):
tags = extract_info(videopath)
with open(matadata_path, 'w') as jF:
metadata['infoExtracted'] = 1
jF.write(str(metadata))
except Exception as err:
print(err)
with open(matadata_path, 'w') as jF:
metadata['infoExtracted'] = 1
jF.write(str(metadata))
tags = extract_info(videopath)
data = {
'status': 1,
'path': videopath,
'tags': tags, #'male, person'
'id': metadata['video_id']
}
manager.update_video(data)
except Exception as err:
print(err)
return
def init():
global manager
data_dir = os.path.join(os.curdir, 'db')
if not os.path.exists(data_dir):
os.mkdir(data_dir)
manager = DB(app.config['DATABASE_PATH'])
if not os.path.exists(app.config['DATABASE_PATH']):
manager.create_table(db_manager.CREATE_TABLE_STATEMENT)
print("REQUIRED TABLES CREATED..")
print("VECTORIZER LOADED")
def __init__(self):
self.result_dir = os.path.join(os.getcwd(), "Result")
if not os.path.exists(self.result_dir):
os.makedirs(self.result_dir)
self.collector = create_collector('my-collector', 'http')
self.DB_Name = "proxies.db"
self.tbl_name = "http"
self.db_proxy = DB(dst_path=self.result_dir, dbname=self.DB_Name)
self.fields = [
"anonymous", "code", "country", "host", "port", "type", "date", "status"
]
self.db_proxy.createTable(tbl_name=self.tbl_name, fields=self.fields)
def run():
file_loader = FileLoader()
students = file_loader.load(args.path_students)
rooms = file_loader.load(args.path_rooms)
data_merged = solution._merge_rooms_students(students, rooms)
if args.output_format == 'json':
saver = SaverJSON()
saver.save(data=data_merged, path='output', output_format='json')
elif args.output_format == 'xml':
xml_serializer = XMLSerializer()
xml_serialized_data = xml_serializer.serialize(data_merged)
saver = SaverXML()
saver.save(data=xml_serialized_data,
path='output',
output_format='xml')
db = DB()
db._insert_rooms(rooms)
db._insert_students(students)
__query_list = [
db._rooms_and_students_amount_query,
db._top5_with_smallest_avg_age_query,
db._top5_with_biggest_age_diff_query,
db._rooms_with_diff_sex_students_query
]
for query in __query_list:
query()
def compress(videofolder, producedvideofolder, videoname, metadata):
try:
logging.debug('Compressing Started => ' + videoname)
save_path = os.path.join(producedvideofolder,
secure_filename(videoname + '.mp4'))
mSave_path = save_path
chunks = sorted(os.listdir(videofolder))
for video in chunks:
if video.__contains__('.json'):
continue
raw_video = os.path.join(videofolder, video)
# cmd = 'ffmpeg -i ' + raw_video + ' -s ' + config.VID_RESOLUTION + ' -strict -2 '
cmd = 'ffmpeg -i ' + raw_video + ' -s ' + config.VID_RESOLUTION \
+ ' -strict -2 -filter:v fps=fps=' + str(config.FPS) \
+ ' -threads ' + str(config.NUMBER_OF_CORES) + ' ' + save_path
logging.debug(cmd)
# saving metadata to check worker to compressing in process or done then upload the video
save_path = os.path.join(producedvideofolder, 'metadata.json')
metadata['infoExtracted'] = 0
with open(save_path, 'w') as f:
metadata['isCompressingDone'] = 0
f.write(str(metadata))
manager = DB()
if os.system(cmd) == 0: # success
data = {
'status': 1,
'path': mSave_path,
'tags': "null", # 'male, person'
'id': metadata['video_id']
}
manager.update_video(data)
with open(save_path, 'w') as f:
metadata['isCompressingDone'] = 1
f.write(str(metadata))
shutil.rmtree(videofolder) # removing whole raw dir
logging.debug('Compressing Complete => ' + producedvideofolder)
# upload(producedvideofolder, bucket, producedvideofolder.replace(config.produced_data_dir, ''))
else:
data = {
'status': 1,
'path': os.path.join(videofolder, video),
'tags': "null", # 'male, person'
'id': metadata['video_id']
}
manager.update_video(data)
logging.debug(
'Error in ffmpeg compressing so uploading raw video ' +
videofolder)
# upload(videofolder, bucket, videofolder.replace(config.data_dir, ''))
with open(save_path, 'w') as f:
metadata['isCompressingDone'] = 99
f.write(str(metadata))
except Exception as err:
logging.debug(err)
from intel import extract_info from db_manager import DB extract_info.search_and_extract(DB())
except Exception as err:
resultDf = manager.get_invitations()
data = []
for index, rows in resultDf.iterrows():
print(rows)
data.append({
'id': rows['invite_id'],
'room_token': rows['room_token'],
'contact_info': rows['contact_info'],
'isconnected': rows['isconnected']
})
result = {'data': data}
if len(data) > 0:
result['success'] = 1
else:
result['success'] = 0
return make_response(json.dumps(result), 200)
# start the server with the 'run()' method
if __name__ == '__main__':
manager = DB()
if not os.path.exists(app.config['DATABASE_PATH']):
manager.create_table(db_manager.CREATE_TABLE_STATEMENT)
print("REQUIRED TABLES CREATED..")
app.run(debug=True)