def __init__(self,config_parser):
# Connect to engine
database_path = get_from_config_parser(config_parser,'Database','path','database')
database_debug = get_boolean_from_config_parser(config_parser,'Database','debug',False)
dir = os.path.dirname(database_path)
if not os.path.exists(dir):
mkdir(dir)
sys.stderr.write('Connecting to database at "%s"\n' % database_path)
self._engine = create_engine('sqlite:///%s' % database_path,echo=database_debug)
# Start session
Session = sessionmaker(bind=self._engine)
self._session = Session()
# Initialize feed storage
self._feed_storage = FeedStorage(self._engine,self._session)
# Initialize item storage
self._item_storage = ItemStorage(self._engine,self._session)
# A list of subprocess.Popen processes that will be maintained
# by the Coffer object.
self._external_processes = []
# File storage (data dump)
file_storage_path = get_from_config_parser(config_parser,'FileStorage','path','datadump')
max_block_size = get_int_from_config_parser(config_parser,'FileStorage','max-block-size',
file_storage.DEFAULT_MAX_BLOCK_SIZE)
bzip2_path = get_from_config_parser(config_parser,'FileStorage','bzip2-path','/usr/bin/bzip2')
self._file_storage = FileStorage(self._external_processes,file_storage_path,
max_block_size,bzip2_path)
# Content fetcher configuration
self._fetcher = Fetcher(config_parser)
def get_storage_obj(dir_name=""):
global OBJECT_STORAGE_OBJ
if OBJECT_STORAGE_OBJ is None:
if settings.DATA_STORAGE_USE_S3:
OBJECT_STORAGE_OBJ = ObjectStorage()
else:
OBJECT_STORAGE_OBJ = FileStorage(dir_name)
return OBJECT_STORAGE_OBJ
def factory(type, conf):
"""Creates an instance of AbstractStorage class
:raises: RuntimeError
"""
if type == 'file':
return FileStorage(conf)
raise RuntimeError('Unable to instantiate storage with type %s' % type)
def post(self, request):
serializer = UserRegisterSerializer(data=request.data)
if serializer.is_valid():
#serializer.validated_data['password'] = make_password(serializer.validated_data['password'])
serializer.save()
# create personal folder
email_info = serializer.validated_data['email']
file_storage = FileStorage()
file_storage.create_folder(email_info)
file_storage.create_folder_in_specific_folder(
email_info + "@" + "data", email_info)
data_map = {
"folders": {
"000000": {
"name": "/",
"parent": "",
"create_date": "17/03/2020"
}
},
"files": {},
"last_submission":
str(int(datetime.timestamp(datetime.now()) * 1000))
}
base_dir = ".temp/"
try:
os.mkdir(os.path.join(base_dir, email_info))
except:
print("folder exists")
file_path = base_dir + email_info + "/map.json"
map_file = open(file_path, 'w')
json.dump(data_map, map_file)
map_file.close()
file_storage.upload_file_with_path_in_specific_folder(
file_path, email_info)
requests.post('http://127.0.0.1:5000/api/gen_key',
data={
"email": serializer.validated_data['email'],
"secret_password": "******"
})
return JsonResponse(
{
'message': 'Register successful!',
'data_map': data_map
},
status=status.HTTP_201_CREATED)
return JsonResponse(
{
'error_messages': serializer.errors,
'errors_code': 400,
},
status=status.HTTP_400_BAD_REQUEST)
def put(self, request, *args, **kwargs):
base_dir = ".temp/"
try:
serializer = FileUploadSerializer(data=request.data)
if serializer.is_valid():
file_name = request.data['name']
file_data = request.data['data']['data']
file_path = base_dir + file_name
file = open(file_path, 'wb')
file.write(bytes(bytearray(file_data)))
file.close()
file_storage = FileStorage()
files = file_storage.get_file_list_in_specific_folder(
request.user.email)
remove_file = ""
for f in files:
if f['title'] == file_name:
remove_file = f
break
if remove_file != "":
print(remove_file['id'])
file_storage.delete_file_by_id(remove_file['id'])
file_storage.upload_file_with_path_in_specific_folder(
file_path, request.user.email)
if os.path.exists(file_path):
os.remove(file_path)
return JsonResponse({'message': request.data},
status=status.HTTP_200_OK)
else:
return JsonResponse(
{'message': "Service unavaible"},
status=status.HTTP_503_SERVICE_UNAVAILABLE)
else:
return JsonResponse(
{'message': "Service unavaible"},
status=status.HTTP_503_SERVICE_UNAVAILABLE)
return JsonResponse({'message': "Request's data is wrong"},
status=status.HTTP_401_UNAUTHORIZED)
except:
JsonResponse(
{
'error_message': "Somethings Error",
'errors_code': 400,
},
status=status.HTTP_402_PAYMENT_REQUIRED)
def setup(cls, channel, pool_size):
cls.pool_size = pool_size
cls.file_storage = FileStorage()
cls.file_storage.setup(channel)
cls.ret = cls.file_storage.read_file()
for one_name in cls.ret.splitlines():
if one_name not in cls.names:
cls.names.append(one_name)
print("INIT")
print("READ " + cls.ret)
print("PARSED " + str(cls.names))
def get(self, request, *args, **kwargs):
base_dir = ".temp/"
file_id = kwargs['file_id']
file_id = '0' * (6 - len(str(file_id))) + str(file_id)
file_storage = FileStorage()
files = file_storage.get_files_by_title_in_specific_folder(
"map.json", request.user.email)
if len(files) == 0:
return JsonResponse({'message': "Service unavaible"},
status=status.HTTP_503_SERVICE_UNAVAILABLE)
map_json_content = json.loads(file_storage.get_content_file(files[0]))
if file_id not in map_json_content['files']:
return JsonResponse({'message': "File is not found"},
status=status.HTTP_400_BAD_REQUEST)
file_name = map_json_content['files'][file_id]['name']
file_name = file_id + file_name + '.aes'
files = file_storage.get_files_by_title(file_name)
if len(files) == 0:
return JsonResponse({'message': "Service unavaible"},
status=status.HTTP_503_SERVICE_UNAVAILABLE)
file_storage.download_file(files[0], base_dir + file_name)
f = open(base_dir + file_name, 'rb')
data = f.read()
f.close()
return JsonResponse(
{
'message': {
'type': 'Buffer',
'data': list(bytearray(data)),
'name': file_name,
'map_data': map_json_content['files'][file_id]
}
},
status=status.HTTP_200_OK)
def __init__(self, file, folder, vol=5):
print('sounds starts')
# get path
# self.cwd = os.getcwd() + folder
# self.cwd = file + '/' + folder
# get sound clips
self.cwd = folder
fs = FileStorage()
fs.readJson(file)
self.db = fs.db
print('Found {} sounds clips'.format(len(self.db)))
self.audio_player = AudioPlayer()
# self.audio_player.set_volume(vol) # volume is 0-100%
time.sleep(0.1)
print('AudioPlayer found:', self.audio_player.audio_player)
self.r2 = TTAstromech()
def setUp(self):
# Mock values to be passed to File class constructor
file_name = "file_name"
file_path = "C:/Desktop/my_dir/file_name"
file_extension = "txt"
absolute_path = "C:/Desktop/my_dir/file_name.txt"
# An object of type DetectionIndicators
detection_indicators = DetectionIndicators()
"""Initialise File class
args:
file_name (str) : file name
file_path (str) : file path
file_extension (str) : file extension
absolute_path (str) : absolute path
detection_indicators(DetectionIndicators): object of type DetectionIndicators
"""
self.file = File(file_name, file_path, file_extension, absolute_path,
detection_indicators)
"""Initialise FileStorage class
args:
[] (list) : an empty list to store files
"""
self.file_storage = FileStorage([])
def get(self, request, *args, **kwargs):
base_dir = ".temp/"
file_name = "map.json"
file_storage = FileStorage()
files = file_storage.get_files_by_title_in_specific_folder(
file_name, request.user.email)
if len(files) == 0:
return JsonResponse({'message': "Service unavaible"},
status=status.HTTP_503_SERVICE_UNAVAILABLE)
file_storage.download_file(files[0], base_dir + file_name)
f = open(base_dir + file_name, 'rb')
data = f.read()
f.close()
return JsonResponse(
{'message': {
'type': 'Buffer',
'data': list(bytearray(data)),
}},
status=status.HTTP_200_OK)
def post(self, request):
base_dir = ".temp/"
try:
serializer = FileUploadSerializer(data=request.data)
if serializer.is_valid():
file_name = request.data['name']
file_data = request.data['data']['data']
file_path = base_dir + file_name
file = open(file_path, 'wb')
file.write(bytes(bytearray(file_data)))
file.close()
file_storage = FileStorage()
file_storage.upload_file_with_path_in_specific_folder(
file_path, request.user.email + "@" + "data")
if os.path.exists(file_path):
os.remove(file_path)
return JsonResponse({'message': request.data},
status=status.HTTP_200_OK)
else:
return JsonResponse(
{'message': "Service unavaible"},
status=status.HTTP_503_SERVICE_UNAVAILABLE)
return JsonResponse({'message': "Request's data is wrong"},
status=status.HTTP_400_BAD_REQUEST)
except:
JsonResponse(
{
'error_message': "Somethings Error",
'errors_code': 400,
},
status=status.HTTP_400_BAD_REQUEST)
def delete(self, request, *args, **kwargs):
try:
serializer = FileRemoveSerializer(data=request.data)
if serializer.is_valid():
file_name = request.data['name']
file_storage = FileStorage()
files = file_storage.get_file_list_in_specific_folder(
request.user.email + "@" + "data")
remove_file = ""
for f in files:
if f['title'] == file_name:
remove_file = f
break
if remove_file != "":
file_storage.delete_file_by_id(remove_file['id'])
return JsonResponse({'message': request.data},
status=status.HTTP_200_OK)
else:
return JsonResponse(
{'message': "Service unavaible"},
status=status.HTTP_503_SERVICE_UNAVAILABLE)
return JsonResponse({'message': "Request's data is wrong"},
status=status.HTTP_401_UNAUTHORIZED)
except:
JsonResponse(
{
'error_message': "Somethings Error",
'errors_code': 400,
},
status=status.HTTP_402_PAYMENT_REQUIRED)
def post(self, request, *args, **kwargs):
try:
serializer = FileUploadSerializer(data=request.data)
if serializer.is_valid():
file_name = request.data['name']
file_storage = FileStorage()
files = file_storage.get_file_list_in_specific_folder(
request.user.email)
map_file_name = "map.json"
map_file = ""
for f in files:
if f['title'] == map_file_name:
map_file = f
break
if map_file == "":
return JsonResponse(
{'message': "Internal server error"},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
map_file = json.loads(file_storage.get_content_file(map_file))
file_data = request.data['data']['data']
file_data = bytes(bytearray(file_data))
file_data = json.loads(file_data)
if int(map_file['last_submission']) == int(
file_data['last_submission']):
return JsonResponse(
{
'message': "There is nothing to sync",
'code': 0
},
status=status.HTTP_200_OK)
files_1 = map_file['files']
files_2 = file_data['files']
file_add = list(set(files_2.keys()) - set(files_1.keys()))
file_modified = []
file_delete = list(set(files_1.keys()) - set(files_2.keys()))
intersec_files = list(
set(files_2.keys()) & set(files_1.keys()))
for file_id in intersec_files:
if files_2[file_id]['check_sum'] != files_1[file_id][
'check_sum']:
file_modified.append(file_id)
print(file_add)
print(file_modified)
print(file_delete)
if int(map_file['last_submission']) < int(
file_data['last_submission']):
# sync to server
return JsonResponse(
{
'message': "Sync data to server",
'data': {
'upload': file_add,
'update': file_modified,
'remove': file_delete
},
'code': 1
},
status=status.HTTP_200_OK)
elif int(map_file['last_submission']) > int(
file_data['last_submission']):
#sync to client
return JsonResponse(
{
'message': "Sync data to client",
'data': {
'upload': file_delete,
'update': file_modified,
'remove': file_add
},
'code': 2
},
status=status.HTTP_200_OK)
return JsonResponse({'message': "Request's data is wrong"},
status=status.HTTP_200_OK)
except:
JsonResponse(
{
'error_message': "Somethings Error",
'errors_code': 400,
},
status=status.HTTP_400_BAD_REQUEST)
import logging
import os
import tempfile
from file_storage import FileStorage
from thegoogs import TheGoogs
logger = logging.getLogger("gmusic")
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())
STORAGE_PATH = "/Users/jd/ws/gmusic/data"
thegoogs = TheGoogs()
storage = FileStorage(STORAGE_PATH)
libdata = storage.read_libdata()
for index, song in enumerate(libdata.uploaded_songs):
if not storage.song_exists(song):
logger.info("Downloading song {}/{}: {}".format(
index, len(libdata.uploaded_songs), song["title"]))
data = thegoogs.get_uploaded_song(song["id"])
storage.write_song(data, song)
else:
logger.info("Skipping downloaded song: {}".format(song["title"]))
for index, song in enumerate(libdata.all_songs):
if not storage.song_exists(song):
logger.info("Downloading unowned song {}/{}: {}".format(
import os
import redis
from file_storage import FileStorage
from s3_storage import S3Storage
from elasticsearch import Elasticsearch
from elasticsearch_dsl import connections
from pathlib import Path
import yaml
app = Flask(__name__)
app_settings = os.getenv('APP_SETTINGS', 'server.config.DevelopmentConfig')
app.config.from_object(app_settings)
if app.config.get('UPLOAD_TYPE') == 'file':
storage = FileStorage(app.config.get('UPLOAD_FOLDER'))
elif app.config.get('UPLOAD_TYPE') == 'aws':
storage = S3Storage(app.config.get('AWS_BUCKET'))
tm = tus_manager(app,
upload_url='/uploads',
upload_folder=app.config.get('UPLOAD_FOLDER'),
overwrite=True,
upload_finish_cb=None,
storage=storage)
db = SQLAlchemy(app)
redis_db = redis.StrictRedis(host=app.config.get('REDIS_SERVERNAME'),
port=6379,
db=0,
password='******')
def setUp(self):
self.subject = FileStorage('test_storage.json')
info = r.json()
print('+', end='')
# pprint(data)
# print("Found:", data["hostname"], data["network"]["IPv4"]["address"])
key = info["network"]["IPv4"]["mac"]
data = {
'timestamp': getTimeStamp(),
'changed': False,
'json': info
}
if key in rpis:
if rpis[key]["json"]["hostname"] != data["json"]["hostname"]:
data["changed"] = True
rpis[key] = data
except KeyboardInterrupt:
break
except requests.ConnectTimeout:
# print("Invalid:", url)
print('.', end='')
except requests.ConnectionError as e:
# print("Strange:", url, e)
print('/', end='')
# time.sleep(0.1)
sys.stdout.flush()
print("\nFinished scan")
if len(rpis) > 0:
fs = FileStorage()
fs.writeJson('network.json', rpis)
