class ScrapeFreeProxies(): def __init__(self): self.result_dir = os.path.join(os.getcwd(), "Result") if not os.path.exists(self.result_dir): os.makedirs(self.result_dir) self.collector = create_collector('my-collector', 'http') self.DB_Name = "proxies.db" self.tbl_name = "http" self.db_proxy = DB(dst_path=self.result_dir, dbname=self.DB_Name) self.fields = [ "anonymous", "code", "country", "host", "port", "type", "date", "status" ] self.db_proxy.createTable(tbl_name=self.tbl_name, fields=self.fields) def get_proxies(self): # Retrieve only anonymous 'uk' or 'us' proxies proxies = self.collector.get_proxies({'anonymous': True}) for proxy in proxies: result_row = { 'anonymous': proxy.anonymous, 'code': proxy.code, 'country': proxy.country, 'host': proxy.host, 'port': proxy.port, 'type': proxy.type, 'date': get_now_str() } self.db_proxy.insert_row(tbl_name=self.tbl_name, result_row=result_row) print(result_row)
def test_03_check_population_total_less_2billions(self): db = DB() expected_total_population = 2e9 with db: populations = db.get_population_list_of_all_country() actual_total_population = reduce(lambda x, y: (x[0] + y[0],), populations) self.assertLess(actual_total_population[0], expected_total_population, "Population total is greater than 2billions. " "Actual total population: {}".format(actual_total_population[0]))
def test_02_country_where_population_density_less_50(self): db = DB() population_density = 50 expected_country = "USA" with db: actual_country = db.get_country_where_population_density_less_given_value(population_density) for country in actual_country: self.assertEqual(country[0], expected_country, "Incorrect country name with population density less {}. Actual country: {}. " "Expected country: {}.".format(population_density, country[0], expected_country))
def search_and_extract(manager): manager = DB() videodir = os.listdir(config.produced_data_dir) multithreads = [ ] # appedning compressing threads here to be executed parallel ecount = 1 try: for videoname in videodir: # videoname == invite_id videofolder = os.path.join(config.produced_data_dir, videoname) videofiles = [ filename for filename in os.listdir(videofolder) if filename.__contains__('.mp4') ] matadata_path = os.path.join(videofolder, 'metadata.json') with open(matadata_path, 'r') as jF: s = jF.read() s = s.replace("'", '"') metadata = json.loads(s) print(metadata) for video in videofiles: videopath = os.path.join(videofolder, video) tags = 'null' try: if (int(metadata['infoExtracted']) == 0): tags = extract_info(videopath) with open(matadata_path, 'w') as jF: metadata['infoExtracted'] = 1 jF.write(str(metadata)) except Exception as err: print(err) with open(matadata_path, 'w') as jF: metadata['infoExtracted'] = 1 jF.write(str(metadata)) tags = extract_info(videopath) data = { 'status': 1, 'path': videopath, 'tags': tags, #'male, person' 'id': metadata['video_id'] } manager.update_video(data) except Exception as err: print(err) return
def init(): global manager data_dir = os.path.join(os.curdir, 'db') if not os.path.exists(data_dir): os.mkdir(data_dir) manager = DB(app.config['DATABASE_PATH']) if not os.path.exists(app.config['DATABASE_PATH']): manager.create_table(db_manager.CREATE_TABLE_STATEMENT) print("REQUIRED TABLES CREATED..") print("VECTORIZER LOADED")
def __init__(self): self.result_dir = os.path.join(os.getcwd(), "Result") if not os.path.exists(self.result_dir): os.makedirs(self.result_dir) self.collector = create_collector('my-collector', 'http') self.DB_Name = "proxies.db" self.tbl_name = "http" self.db_proxy = DB(dst_path=self.result_dir, dbname=self.DB_Name) self.fields = [ "anonymous", "code", "country", "host", "port", "type", "date", "status" ] self.db_proxy.createTable(tbl_name=self.tbl_name, fields=self.fields)
def run(): file_loader = FileLoader() students = file_loader.load(args.path_students) rooms = file_loader.load(args.path_rooms) data_merged = solution._merge_rooms_students(students, rooms) if args.output_format == 'json': saver = SaverJSON() saver.save(data=data_merged, path='output', output_format='json') elif args.output_format == 'xml': xml_serializer = XMLSerializer() xml_serialized_data = xml_serializer.serialize(data_merged) saver = SaverXML() saver.save(data=xml_serialized_data, path='output', output_format='xml') db = DB() db._insert_rooms(rooms) db._insert_students(students) __query_list = [ db._rooms_and_students_amount_query, db._top5_with_smallest_avg_age_query, db._top5_with_biggest_age_diff_query, db._rooms_with_diff_sex_students_query ] for query in __query_list: query()
def compress(videofolder, producedvideofolder, videoname, metadata): try: logging.debug('Compressing Started => ' + videoname) save_path = os.path.join(producedvideofolder, secure_filename(videoname + '.mp4')) mSave_path = save_path chunks = sorted(os.listdir(videofolder)) for video in chunks: if video.__contains__('.json'): continue raw_video = os.path.join(videofolder, video) # cmd = 'ffmpeg -i ' + raw_video + ' -s ' + config.VID_RESOLUTION + ' -strict -2 ' cmd = 'ffmpeg -i ' + raw_video + ' -s ' + config.VID_RESOLUTION \ + ' -strict -2 -filter:v fps=fps=' + str(config.FPS) \ + ' -threads ' + str(config.NUMBER_OF_CORES) + ' ' + save_path logging.debug(cmd) # saving metadata to check worker to compressing in process or done then upload the video save_path = os.path.join(producedvideofolder, 'metadata.json') metadata['infoExtracted'] = 0 with open(save_path, 'w') as f: metadata['isCompressingDone'] = 0 f.write(str(metadata)) manager = DB() if os.system(cmd) == 0: # success data = { 'status': 1, 'path': mSave_path, 'tags': "null", # 'male, person' 'id': metadata['video_id'] } manager.update_video(data) with open(save_path, 'w') as f: metadata['isCompressingDone'] = 1 f.write(str(metadata)) shutil.rmtree(videofolder) # removing whole raw dir logging.debug('Compressing Complete => ' + producedvideofolder) # upload(producedvideofolder, bucket, producedvideofolder.replace(config.produced_data_dir, '')) else: data = { 'status': 1, 'path': os.path.join(videofolder, video), 'tags': "null", # 'male, person' 'id': metadata['video_id'] } manager.update_video(data) logging.debug( 'Error in ffmpeg compressing so uploading raw video ' + videofolder) # upload(videofolder, bucket, videofolder.replace(config.data_dir, '')) with open(save_path, 'w') as f: metadata['isCompressingDone'] = 99 f.write(str(metadata)) except Exception as err: logging.debug(err)
from intel import extract_info from db_manager import DB extract_info.search_and_extract(DB())
except Exception as err: resultDf = manager.get_invitations() data = [] for index, rows in resultDf.iterrows(): print(rows) data.append({ 'id': rows['invite_id'], 'room_token': rows['room_token'], 'contact_info': rows['contact_info'], 'isconnected': rows['isconnected'] }) result = {'data': data} if len(data) > 0: result['success'] = 1 else: result['success'] = 0 return make_response(json.dumps(result), 200) # start the server with the 'run()' method if __name__ == '__main__': manager = DB() if not os.path.exists(app.config['DATABASE_PATH']): manager.create_table(db_manager.CREATE_TABLE_STATEMENT) print("REQUIRED TABLES CREATED..") app.run(debug=True)