def validate_fields(required_fields, request_json):
try:
for field in required_fields:
if request_json.get(field) is None:
raise APIBadRequest("{} is required".format(field))
except (ValueError, AttributeError):
raise APIBadRequest(f"{' '.join(required_fields)} is/are required")
async def get_chats(request):
skip = [request.args.get("skip"), 0][request.args.get("skip") == None]
limit = [request.args.get("limit"), 10][request.args.get("limit") == None]
start_date = request.args.get("start_date")
end_date = request.args.get("end_date")
username = request.args.get("username")
if not username:
raise APIBadRequest("username is required")
search_text = request.args.get("search_text")
if search_text:
search_text = search_text.lower()
logger.info(request.args)
if not username:
raise APIBadRequest("Username for this datasource is required")
if start_date:
start_date = dateparser.parse(start_date)
if end_date:
end_date = dateparser.parse(end_date)
if start_date and end_date:
if end_date < start_date:
raise APIBadRequest("Start date should be less than End date")
_chats, count = await filter_chats(
request.app.config[DATASOURCE_NAME]["tables"]["chat_table"], username,
start_date, end_date, int(skip), int(limit), search_text)
for chat in _chats:
chat.pop("message_content")
chat.pop("chat_path")
if chat.get("messages"):
messages = json.loads(chat.get("messages"))
chat.update({"messages": messages})
##we need to give one message to frontend, We need to loop over messages till the time
## we get a message which is not None, then break the loop
for message in messages:
if message.get("content"):
chat.update({"last_message": message.get("content")})
break
if chat.get("participants"):
participants = json.loads(chat.get("participants"))
chat.update({"participants": participants})
return response.json({
'error': False,
'success': True,
"message": None,
"data": {
"chats": _chats,
"count": count
}
})
def extract(src_path: str, dst_path_prefix: str, config: Dict[str, Any],
datasource_name: str, username: str) -> Tuple[str, str]:
"""
src_path : where the user has downloaded their ZIP file,
temp_directory = tempfile.TemporaryDirectory()
"""
# temp_directory = tempfile.mkdtemp()
logger.info("Entered into the extract function")
if not os.path.exists(src_path):
raise APIBadRequest("This path doesnt exists")
# try:
# the_zip_file = zipfile.ZipFile(src_path)
# except:
# raise APIBadRequest("Invalid zip file")
# logger.info(f"Testing zip {src_path} file")
# ret = the_zip_file.testzip()
# if ret is not None:
# raise APIBadRequest(f"Invalid zip datasource_name file")
_checksum = checksum(src_path)
archives_present = get_archives(
config[datasource_name]["tables"]["archives_table"], _checksum)
if archives_present:
raise APIBadRequest("Zip file already have been uploaded")
utc_timestamp = datetime.datetime.utcnow().strftime("%d-%m-%Y")
dst_path_suffix = f"{utc_timestamp}-{_checksum}"
logger.info(f"This is the new destination suffix {dst_path_suffix}")
dst_path = os.path.join(dst_path_prefix, username, dst_path_suffix)
try:
with zipfile.ZipFile(src_path) as zf:
zf.extractall(dst_path)
#shutil.unpack_archive(src_path, extract_dir=dst_path, format=None)
except MemoryError:
logger.error(
f"We ran out of memory while processing {datasource_name}, Please try again"
)
raise Exception(
f"We ran out of memory while processing {datasource_name}, Please try again"
)
except:
raise APIBadRequest(f"Invalid zip {datasource_name} file")
logger.info(f"Setting new archival for {datasource_name} ")
set_archives(config[datasource_name]["tables"]["archives_table"], dst_path,
username, _checksum)
logger.info(f"This is the dst_path for {datasource_name} is {dst_path}")
return _checksum, dst_path
async def aws_temp_creds(self):
creds = await get_credentials(
self.config[USER_DATASOURCE_NAME]["tables"]["creds_table"])
if not creds:
raise APIBadRequest("User is not logged in")
creds = list(creds)[0]
r = requests.post(self.config.LOGIN,
data=json.dumps({
"username": creds["username"],
"password": creds["password"]
}))
result = r.json()
if result.get("error"):
logger.error(result["message"])
raise APIBadRequest(result["message"])
r = requests.post(
self.config.TEMPORARY_S3_CREDS,
data=json.dumps({"id_token": result["data"]["id_token"]}),
headers={"Authorization": result["data"]["id_token"]})
result = r.json()
if result.get("error"):
logger.error(result["message"])
raise APIBadRequest(result["message"])
return result["data"]["identity_id"], result["data"][
"access_key"], result["data"]["secret_key"], result["data"][
"session_token"]
async def github_re_backup_whole(request):
"""
"""
username = request.args.get("username")
if not username:
raise APIBadRequest("Username for this datasource is required")
try:
username, password = await get_creds(
request.app.config[DATASOURCE_NAME]["tables"]["creds_table"],
request.json["username"])
except:
raise APIBadRequest("Credentials aren't present")
#update_datasources_status(request.app.config.DATASOURCES_TBL , "CODEREPOS/Github",request.json["username"] , request.app.config.DATASOURCES_CODE["REPOSITORY"]["GITHUB"], "IN_PROGRESS", "PROGRESS")
# else:
# raise APIBadRequest("Unknown format")
# logger.info(f"THe request was successful with github path {request.json['path']}")
request.app.add_task(
background_github_parse(request.app.config,
username,
password,
re_backup=True))
return response.json({
'error': False,
'success': True,
"message":
"Your whole github backup has started, Once done it will start reflecting on your github Dashboard",
"data": None,
})
async def store_permissions(request):
"""
{'backup': ['stats_table', 'status_table', 'backup_table', 'backup_list_table'],
'coderepos': ['creds_table', 'stats_table', 'status_table', 'archives_table', 'repos_table'],
'facebook': ['creds_table', 'image_table', 'archives_table', 'yourposts_table', 'other_posts',
'content', 'chat_table', 'chat_content', 'stats_table', 'address_table', 'status_table'],
'google': ['creds_table', 'image_table', 'email_table', 'email_attachment_table', 'email_content_table',
'purchase_table', 'archives_table', 'reservation_table', 'location_table', 'location_approximate_table',
'stats_table', 'status_table'],
'twitter': ['creds_table', 'archives_table', 'stats_table', 'status_table', 'account_table',
'tweet_table', 'indexed_tweet_table'], 'users': ['creds_table']}
"""
request.app.config.VALIDATE_FIELDS(["plugin_name", "permissions"], request.json)
logger.debug(request.json["permissions"])
if not isinstance(request.json['permissions'], list):
raise APIBadRequest("Permissions should be instance of list")
if not request.json["plugin_name"] in request.app.config.plugins:
raise APIBadRequest("Plugin is unknown")
for permission in request.json["permissions"]:
permission.update({"tbl_object": permission_table, "plugin_name": request.json["plugin_name"] })
await store_permission(**permission)
return response.json({
'error': False,
'success': True,
"message": None,
"data": None})
async def check_mnemonic(request):
"""
API to be used when user has reinstalled datapod
since he/she already has menmonic intialized somewhere in the past ,
this mnemonic has to be checked against the hash of the Mnemonic
"""
request.app.config.VALIDATE_FIELDS(["mnemonic"], request.json)
creds = await get_credentials(
request.app.config[USER_DATASOURCE_NAME]["tables"]["creds_table"])
if not creds:
raise APIBadRequest("User is not logged in")
creds = list(creds)[0]
r = requests.post(request.app.config.LOGIN,
data=json.dumps({
"username": creds["username"],
"password": creds["password"]
}))
login_result = r.json()["data"]
##converting mnemonic list of words into a string of 24 words of mnemonic
mnemonic = " ".join(request.json["mnemonic"])
logger.debug(f"The mnemonic is {mnemonic}")
sha3_256 = hashlib.sha3_256(mnemonic.encode()).hexdigest()
logger.debug(f"Sha 256 of mnemonic is {sha3_256}")
r = requests.post(request.app.config.CHECK_MNEMONIC,
data=json.dumps({
"username": login_result["username"],
"sha3_256": sha3_256,
}),
headers={"Authorization": login_result["id_token"]})
check_mnemonic_result = r.json()
if check_mnemonic_result["error"]:
raise APIBadRequest(check_mnemonic_result["message"])
##TODO Store mnemonic in the local db
mnemonic_keys = child_keys(mnemonic, 0)
await update_mnemonic_n_address(
request.app.config[USER_DATASOURCE_NAME]["tables"]["creds_table"],
login_result["username"], mnemonic, mnemonic_keys["address"],
mnemonic_keys["private_key"])
return response.json({
"error": False,
"success": True,
"message": "Mnemonic has been saved on your machine",
"data": None
})
async def tweets(request):
logger.info("Number is ", request.args.get("limit"))
skip = [request.args.get("skip"), 0][request.args.get("skip") == None]
limit = [request.args.get("limit"), 10][request.args.get("limit") == None]
matching_string = request.args.get("match_string")
start_date = request.args.get("start_date")
end_date = request.args.get("end_date")
username = request.args.get("username")
if not username:
raise APIBadRequest("Username for this datasource is required")
logger.info(f"Params are {request.args}")
if start_date:
start_date = dateparser.parse(start_date)
if end_date:
end_date = dateparser.parse(end_date)
if start_date and end_date:
if end_date < start_date:
raise APIBadRequest("Start date should be less than End date")
logger.info(f"This is the start_date {start_date}")
logger.info(f"This is the end_date {end_date}")
if matching_string:
logger.info(f"THis is the matchiing_String {matching_string}")
result, count = await match_text(request.app.config[DATASOURCE_NAME]["tables"]["tweet_table"],
username,
request.app.config[DATASOURCE_NAME]["tables"]["indexed_tweet_table"], \
matching_string , start_date, end_date, int(skip), int(limit))
else:
result, count = await filter_tweet(
request.app.config[DATASOURCE_NAME]["tables"]["tweet_table"],
username, start_date, end_date, int(skip), int(limit))
# [repo.update({
# "created_at":repo.get("created_at").strftime("%d, %b %Y"),
# }) for repo in result]
return response.json({
'error': False,
'success': True,
'data': {
"tweets": result,
"count": count
},
'message': None
})
async def attachment_filter(request):
"""
To get all the assets created by the requester
"""
logger.debug(f"Args are {request.args.items()}")
skip = [request.args.get("skip"), 0][request.args.get("skip") == None]
limit = [request.args.get("limit"), 50][request.args.get("limit") == None]
start_date = request.args.get("start_date")
end_date = request.args.get("end_date")
message_type = request.args.get("message_type")
matching_string = request.args.get("match_string")
username = request.args.get("username")
if not username:
raise APIBadRequest("Username for this datasource is required")
# logger.debug(f"Skip type is {skip}")
# logger.debug(f"limit type is {limit}")
# logger.debug(f"start date type is {start_date}, and type is {type(start_date)}")
if start_date:
start_date = dateparser.parse(start_date)
if end_date:
end_date = dateparser.parse(end_date)
if start_date and end_date:
if end_date < start_date:
raise APIBadRequest("Start date should be less than End date")
if not matching_string:
attachments, count = await filter_attachments(
request.app.config[DATASOURCE_NAME]["tables"]
["email_attachment_table"], username, message_type, start_date,
end_date, int(skip), int(limit))
else:
attachments, count = await filter_attachments_on_text(
request.app.config[DATASOURCE_NAME]["tables"]
["email_attachment_table"], username, message_type, start_date,
end_date, int(skip), int(limit), matching_string)
# for iage in images:
# creation_time = image.pop("creation_time")
# #data:image/png;base64
# image.update({"creation_time": creation_time.strftime("%Y-%m-%d")})
return response.json({
'error': False,
'success': True,
"data": {
"result": attachments,
"count": count
},
"message": None
})
async def image_filter(request):
logger.debug("Number is ", request.args.get("limit"))
skip = [request.args.get("skip"), 0][request.args.get("skip") == None]
limit = [request.args.get("limit"), 10][request.args.get("limit") == None]
start_date = request.args.get("start_date")
end_date = request.args.get("end_date")
username = request.args.get("username")
if not username:
raise APIBadRequest("Username for this datasource is required")
logger.debug(f"Params are {request.args}")
if start_date:
start_date = dateparser.parse(start_date)
if end_date:
end_date = dateparser.parse(end_date)
if start_date and end_date:
if end_date < start_date:
raise APIBadRequest("Start date should be less than End date")
images, count = await filter_images(
request.app.config[DATASOURCE_NAME]["tables"]["image_table"], username,
start_date, end_date, int(skip), int(limit))
logger.debug(images)
for image in images:
b64_data = await image_base64(image['image_path'])
creation_time = image.pop("creation_time")
encoded_string = "data:image/jpeg;base64," + b64_data
#data:image/png;base64
image.update({
"creation_time": creation_time.strftime("%Y-%m-%d"),
"uri": encoded_string
})
# [repo.update({
# "created_at":repo.get("created_at").strftime("%d, %b %Y"),
# }) for repo in result]
return response.json({
'error': False,
'success': True,
'data': {
"images": images,
"count": count
},
'message': None
})
async def parse(request):
"""
To get all the assets created by the requester
"""
request.app.config.VALIDATE_FIELDS(["path", "username"], request.json)
res = await get_status(
request.app.config[DATASOURCE_NAME]["tables"]["status_table"])
res = list(res)
logger.info(res)
if res:
for element in res:
if element.get("status") == "PROGRESS":
raise APIBadRequest(
"Already processing a twitter for the user")
username = request.json["username"].lower()
config = request.app.config
dst_path_prefix = os.path.join(config.RAW_DATA_PATH, DATASOURCE_NAME)
logger.info(
f"The dst_path_prefix fo rthis datasource is {dst_path_prefix}")
try:
checksum, dest_path = await extract(request.json["path"],
dst_path_prefix, config,
DATASOURCE_NAME, username)
await update_status(config[DATASOURCE_NAME]["tables"]["status_table"],
DATASOURCE_NAME, username, "PROGRESS", checksum,
dest_path, request.json["path"])
except Exception as e:
logger.error(e)
await delete_status(config[DATASOURCE_NAME]["tables"]["status_table"],
DATASOURCE_NAME, username)
raise APIBadRequest(e)
request.app.add_task(
_parse(request.app.config, dest_path, username, checksum))
return response.json({
'error': False,
'success': True,
"message":
"Twitter parsing has been Started and you will be notified once it is complete",
"data": None
})
async def decorated_function(request, *args, **kwargs):
# run some method that checks the request
# for the client's authorization status
#is_authorized = check_request_for_authorization_status(request)
result = await get_credentials(request.app.config["Users"]["tables"]["creds_table"])
#logger.info(f"Data from the credential table in id_token_validity decorator {result}")
if not result:
logger.error("Credentials aren't present, Please Login again")
raise APIBadRequest("Credentials aren't present, Please Login again")
result = list(result)[0]
try:
id_token = convert_type(result["id_token"])
access_token = convert_type(result["access_token"])
refresh_token = convert_type(result["refresh_token"])
username = result["username"]
##this is because all the token are byte object, we need to upate user object in request object
# with str type of tokens
result.update({"id_token": id_token, "access_token": access_token, "refresh_token": refresh_token})
request["user"] = result
except Exception as e:
logger.error(f"User must have signed out, Please Login again with an error {e.__str__()}")
raise APIBadRequest("Please Login again")
payload = jwt.get_unverified_claims(id_token)
time_now = datetime.datetime.fromtimestamp(revoke_time_stamp(timezone=request.app.config.TIMEZONE))
time_expiry = datetime.datetime.fromtimestamp(payload["exp"])
rd = dateutil.relativedelta.relativedelta (time_expiry, time_now)
logger.warning("Difference between now and expiry of id_token")
logger.warning(f"{rd.years} years, {rd.months} months, {rd.days} days, {rd.hours} hours, {rd.minutes} minutes and {rd.seconds} seconds")
if rd.minutes < 20:
logger.error("Renewing id_token, as it will expire soon")
id_token = update_tokens(request.app.config, username, refresh_token)
if isinstance(id_token, bytes):
id_token = id_token.decode()
response = await f(request, *args, **kwargs)
return response
async def create(self, src_path, dst_path, datasource_name):
#temp = tempfile.NamedTemporaryFile('wb', suffix='.tar.lzma', delete=False)
temp = tempfile.NamedTemporaryFile('wb',
suffix='.tar.gz',
delete=False)
#temp = tempfile.TemporaryFile()
# backup_path = f"{self.backup_path}/{archival_name}/backup.tar.lzma"
##this is the file under ~/.datapod/user_indexes for a corresponding datasource
## which wil keep track of all the files which have been backed up previously
user_index_file = os.path.join(self.user_index_dir,
f"{datasource_name.lower()}.index")
logger.debug(
f"{datasource_name} This is the user_index_file {user_index_file}, used to create a compressed file at {temp.name} from a directory at {src_path} "
)
if platform.system() == "Linux":
if self.full_backup:
backup_command = f"tar --create --gzip --no-check-device --verbose -f {temp.name} {src_path}"
else:
backup_command = f"tar --create --gzip --no-check-device --verbose --listed-incremental={user_index_file} -f {temp.name} {src_path}"
elif platform.system() == "Darwin":
if self.full_backup:
backup_command = f"gtar --create --lzma --no-check-device --verbose -f {temp.name} {src_path}"
else:
backup_command = f"gtar --create --lzma --no-check-device --verbose --listed-incremental={user_index_file} -f {temp.name} {src_path}"
else:
raise APIBadRequest(
"The platform is not available for this os distribution")
#backup_command = f"tar --create --verbose --listed-incremental={user_index_file} --lzma {backup_path} {self.raw_data_path}"
initial_time = int(time.time())
next_time = initial_time + 15
for out in self.config.OS_COMMAND_OUTPUT(backup_command, "Backup"):
if int(time.time()) >= next_time:
# await self.send_sse_message(f"Archiving {out.split('/')[-1]} for {datasource_name}")
logger.debug(
f"Archiving {out.split('/')[-1]} for {datasource_name}")
next_time += 10
split_backup_dir = tempfile.mkdtemp()
logger.debug(
f"Now, splitting the single compressed file {temp.name} in a temporary directory {split_backup_dir}"
)
async for msg in self.split(split_backup_dir, temp.name):
# await self.send_sse_message(msg)
logger.debug(msg)
##because temp.name will automatically be removed
logger.debug(f"Now removing single comporessed file at {temp.name}")
self.remove_temporary_archive(temp.name)
return split_backup_dir
async def dashboard(request):
username = request.args.get("username")
if not username:
raise APIBadRequest("Username for this datasource is required")
res = await get_account(
request.app.config[DATASOURCE_NAME]["tables"]["account_table"],
username)
result = res[0]
result.update({
"common_hashtags":
json.loads(result["common_hashtags"]),
"common_user_mentions":
json.loads(result["common_user_mentions"])
})
return response.json({
'error': False,
'success': True,
'data': result,
"message": None
})
async def __init__(self, config):
"""
number is the percentage number which will be sent in sse message,
Then number has already been incremented by the backup scripts above,
"""
self.config = config
self.bucket_name = config.AWS_S3['bucket_name']
#self.credentials = get_credentials(config.CREDENTIALS_TBL)
self.credentials = await get_credentials(
self.config[USER_DATASOURCE_NAME]["tables"]["creds_table"])
if not self.credentials:
raise APIBadRequest("User is not logged in")
self.credentials = list(self.credentials)[0]
##in this temporary file, private key is now written
if not self.credentials["encryption_key"]:
raise MnemonicRequiredError()
self.encryption_key = binascii.unhexlify(
self.credentials["encryption_key"].encode())
self.identity_id, self.access_key, self.secret_key, self.session_token = await self.aws_temp_creds(
)
os.environ[
'AWS_ACCESS_KEY_ID'] = self.access_key # visible in this process + all children
os.environ[
'AWS_SECRET_ACCESS_KEY'] = self.secret_key # visible in this process + all children
os.environ[
'AWS_SESSION_TOKEN'] = self.session_token # visible in this process + all children
os.environ["AWS_DEFAULT_REGION"] = self.config.AWS_S3["default_region"]
def store_credentials(credential_table, username, password, password_hash, id_token, access_token, refresh_token, name, email):
query = credential_table.select()
usernames = [user.username for user in query]
print (usernames)
if usernames:
if username not in usernames:
raise Exception("Different usernames arent allowed on the same machine")
try:
credential_table.insert(username=username,
password_hash=password_hash,
password=password,
id_token=id_token,
access_token= access_token,
refresh_token=refresh_token,
name = name,
email=email
).execute()
except IntegrityError:
logger.info(f"Credentials for the user already exists, updating it now")
credential_table.update(
id_token=id_token,
access_token= access_token,
refresh_token=refresh_token)\
.where(credential_table.username==username).\
execute()
except Exception as e:
logger.error("Saving credentials of the users failed {e}")
raise APIBadRequest("Could save credentials because of {e.__str__()}")
return
async def start_fresh_backup(request):
"""
##TODO ADD entries to BACKUP_TBL
"""
##This has a rare chance of happening, that users reach here and doesnt have mnemonic in the database but have mnemonic in the cloud
def str2bool(v):
return v.lower() in ("yes", "true", "t", "1")
full_backup = request.args.get("full_backup")
if not full_backup:
raise APIBadRequest(
"Please specify which kind of backup is required, for full backup send True, for incremental_backup send False"
)
full_backup = str2bool(full_backup)
if not isinstance(full_backup, bool):
raise APIBadRequest("full_backup should only be a bool type")
await update_status(
request.app.config[DATASOURCE_NAME]["tables"]["status_table"],
DATASOURCE_NAME, "backup", "PROGRESS")
##this is being called here, because this will try to fetch temporary creds from AWS from the id_token stored in the creds user
##table, if they are stale, it will send an error to the user directly to login again,
##TODO, find a better way to to this
backup_instance = await S3Backup(request.app.config)
try:
request.app.add_task(
backup_upload(request.app.config, full_backup, backup_instance))
# new_log_entry = request.app.config.LOGS_TBL.create(timestamp=archival_object, message=f"Archival was successful on {archival_name}", error=0, success=1)
# new_log_entry.save()
except Exception as e:
logger.error(e.__str__())
# new_log_entry = request.app.config.LOGS_TBL.create(timestamp=archival_object, message=f"Archival failed because of {e.__str__()} on {archival_name}", error=1, success=0)
# new_log_entry.save()
return response.json({
'error': False,
'success': True,
})
async def decorated_function(request, *args, **kwargs):
# run some method that checks the request
# for the client's authorization status
#is_authorized = check_request_for_authorization_status(request)
if not request.app.config.TESTING_MODE:
raise APIBadRequest("This API cant be executed in Production environment")
response = await f(request, *args, **kwargs)
return response
async def delete_original_path(request):
"""
After the processing of the whole data source, this api can be used to delete the original zip
correspoding to a particular username
"""
username = request.args.get("username")
if not username:
raise APIBadRequest("Username for this datasource is required")
res = await get_status(
request.app.config[DATASOURCE_NAME]["tables"]["status_table"],
username)
result = list(res)
logger.debug(result[0].get("username"))
if not result:
raise APIBadRequest(
f"No status present for {DATASOURCE_NAME} for username {username}")
result = result[0]
logger.debug(result)
path_to_be_deleted = result.get("original_path")
logger.warning(f"Path to be deleted is {path_to_be_deleted}")
try:
os.remove(path_to_be_deleted)
logger.success(f"{path_to_be_deleted} is deleted now")
except Exception as e:
return response.json({
'error': False,
'success': True,
"message":
f"Original path at {path_to_be_deleted} couldnt be delete because of {e.__str__()}",
"data": None
})
return response.json({
'error': False,
'success': True,
"message": f"Original path at {path_to_be_deleted} is deleted",
"data": None
})
async def purchases_filter(request):
skip = [request.args.get("skip"), 0][request.args.get("skip") == None]
limit = [request.args.get("limit"), 10][request.args.get("limit") == None]
start_date = request.args.get("start_date")
end_date = request.args.get("end_date")
merchant_name = request.args.get("match_string")
username = request.args.get("username")
if not username:
raise APIBadRequest("Username for this datasource is required")
logger.debug(f"Params are {request.args}")
if start_date:
start_date = dateparser.parse(start_date)
if end_date:
end_date = dateparser.parse(end_date)
if start_date and end_date:
if end_date < start_date:
raise APIBadRequest("Start date should be less than End date")
# logger.debug(f"This is the start_date {start_date}")
# logger.debug(f"This is the end_date {end_date}")
purchases, count = await filter_purchases(
request.app.config[DATASOURCE_NAME]["tables"]["purchase_table"],
username, start_date, end_date, int(skip), int(limit), merchant_name)
result = [
format_purchase(request.app.config, purchase) for purchase in purchases
]
return response.json({
'error': False,
'success': True,
'data': {
"purchases": result,
"count": count
},
'message': None
})
async def make_backup(self):
"""
--level=0, for fullbackup
--level=1, for incremental backup
--listed_incremental is equivalent to -g
--atime-preserve=system
brew install gnu-tar
#tar --create --lzma --verbose --multi-volume --tape-length 102400 --file=MyArchive.tgz raw -g user.index
With --newer you're simply updating/creating the archive with the files that have changed since the date you pass it.
tar --create --lzma --verbose --file=MyArchive raw/facebook/facebook-sauravverma14473426
"""
datasources = os.listdir(self.raw_data_path)
logger.debug(datasources)
if not datasources:
raise APIBadRequest(
"The directory whose backup needs to be made is empty")
archival_object = datetime.datetime.utcnow()
archival_name = archival_object.strftime("%B-%d-%Y_%H-%M-%S")
parent_destination_path = os.path.join(self.backup_path, archival_name)
s3_backup_instance = await BotoBackup(self.config)
step = int(90 / len(datasources))
for (index, datasource_name) in enumerate(datasources):
s3_folder_name = archival_name + "/" + datasource_name
dst_path = os.path.join(self.backup_path, archival_name,
datasource_name)
src_path = os.path.join(self.raw_data_path, datasource_name)
# if not os.path.exists(dst_path):
# os.makedirs(dst_path)
backup_archival_temporary_path = await self.create(
src_path, dst_path, datasource_name)
# # res = {"message": "Progress", "percentage": int(i*step)}
# # await self.config["send_sse_message"](config, DATASOURCE_NAME, res)
await s3_backup_instance.sync_backup(
datasource_name, backup_archival_temporary_path, archival_name)
##the split archival for a datasource in a temporary folder hasnt been removed yet, removing it now
self.remove_split_archival_dir(backup_archival_temporary_path)
logger.debug(
f"Now removing the split files present {backup_archival_temporary_path} "
)
self.percentage = (index + 1) * step
await self.send_sse_message(
f"Archiving of {datasource_name} completed")
self.percentage = 100
await self.send_sse_message(f"Backup completed")
return parent_destination_path, archival_name
async def single_chat(request):
"""
To get all the chats created by the user
thread_path = 'inbox/KapilDevGarg_lapjbN90Hw'
"""
thread_path = request.args.get("thread_path")
if not thread_path:
raise APIBadRequest("thread_path is required")
logger.info(f'This is the chat id {request.args.get("chat_id")}')
ds_path = os.path.join(request.app.config.RAW_DATA_PATH,
f"facebook/messages/{thread_path}")
if not os.path.exists(ds_path):
raise APIBadRequest("This thread_path doesnt exists")
logger.info(ds_path)
chats = []
chat_files = [(os.path.join(ds_path, file))
for file in os.listdir(ds_path)]
for _file in chat_files:
with open(_file, "r") as json_file:
data = json.load(json_file)
logger.info(data)
chats.extend(data.get("messages"))
# if request.json.get("message_type") not in chat_types:
# raise APIBadRequest("This message type is not available")
# chat_path = os.path.join(ds_path, request.json.get("message_type"))
# all_chats = os.listdir(chat_path)
# result = [{"name": e.split("_")[0], "chat_id": e} for e in all_chats]
return response.json({
'error': False,
'success': True,
"message":
"Facebook data parsing has been Started and you will be notified once it is complete",
"data": chats
})
async def locations_filter(request):
logger.debug(f"Args are {request.args.items()}")
start_date = request.args.get("start_date")
end_date = request.args.get("end_date")
username = request.args.get("username")
if not username:
raise APIBadRequest("Username for this datasource is required")
logger.debug(f"Params are {request.args}")
if start_date:
start_date = dateparser.parse(start_date)
start_date = calendar.timegm(start_date.timetuple())
if end_date:
end_date = dateparser.parse(end_date)
end_date = calendar.timegm(end_date.timetuple())
if start_date and end_date:
if end_date < start_date:
raise APIBadRequest("Start date should be less than End date")
locations, count = await filter_locations(
request.app.config[DATASOURCE_NAME]["tables"]
["location_approximate_table"], username, start_date, end_date)
logger.debug(locations)
#result = [format_purchase(request.app.config, purchase) for purchase in purchases]
result = list(locations)
return response.json({
'error': False,
'success': True,
'data': {
"locations": result,
"count": count
},
'message': None
})
def revoke_time_stamp(days=0, hours=0, minutes=0, timezone=None):
if not timezone:
logger.error("Please specify valid timezone for your servers")
raise APIBadRequest("Please specify valid timezone for your servers")
tz_kolkata = pytz.timezone(timezone)
time_format = "%Y-%m-%d %H:%M:%S"
naive_timestamp = datetime.datetime.now()
aware_timestamp = tz_kolkata.localize(naive_timestamp)
##This actually creates a new instance od datetime with Days and hours
_future = datetime.timedelta(days=days, hours=hours, minutes=minutes)
result = aware_timestamp + _future
return result.timestamp()
async def login(config):
creds = await get_credentials(
config[USER_DATASOURCE_NAME]["tables"]["creds_table"])
if not creds:
raise APIBadRequest("User is not logged in")
creds = list(creds)[0]
#r = requests.post(request.app.config.LOGIN, data=json.dumps({"username": creds["username"], "password": creds["password"]}))
if creds["mnemonic"]:
raise APIBadRequest("The mnemonic is already present")
##renew tokens
r = requests.post(config.LOGIN,
data=json.dumps({
"username": creds["username"],
"password": creds["password"]
}))
login_result = r.json()
return login_result["data"]
async def aws_temp_creds(self):
creds = await get_credentials(
self.config[USER_DATASOURCE_NAME]["tables"]["creds_table"])
if not creds:
raise APIBadRequest("User is not logged in")
creds = list(creds)[0]
# r = requests.post(self.config.LOGIN, data=json.dumps({"username": creds["username"], "password": creds["password"]}))
# result = r.json()
# if result.get("error"):
# logger.error(result["message"])
# raise APIBadRequest(result["message"])
r = requests.post(
self.config.TEMPORARY_S3_CREDS,
data=json.dumps({"id_token": creds["id_token"].decode()}),
headers={"Authorization": creds["id_token"].decode()})
result = r.json()
if result.get("message") == 'The incoming token has expired':
return response.json(
{
"error": True,
"sucess": False,
"message": "The id token has expired, Please login again",
"data": None
},
status=401)
if result.get("error"):
logger.error(result["message"])
raise APIBadRequest(result["message"])
return result["data"]["identity_id"], result["data"][
"access_key"], result["data"]["secret_key"], result["data"][
"session_token"]
async def github_backup_single_repo(request):
"""
"""
if not request.args.get("name"):
raise APIBadRequest("Name of the repository is required")
username = request.args.get("username")
if not username:
raise APIBadRequest("Username for this datasource is required")
logger.info(request.args)
result = await get_single_repository(
request.app.config[DATASOURCE_NAME]["tables"]["repos_table"], username,
request.args.get("name"))
if not result:
raise APIBadRequest("No repo exists")
for repository in result:
logger.info(repository)
owner = json.loads(repository["owner"])
repository.update({"owner": owner})
request.app.add_task(
per_repository(username, repository["path"], repository,
request.app.config, None))
##await per_repository(repository["path"], repository, request.app.config, None)
return response.json({
'error': False,
'success': True,
'message':
f"Backup the repository {request.args.get('name')} has been started",
'data': None
})
async def restart_parse(request):
request.app.config.VALIDATE_FIELDS(["username"], request.json)
res = await get_status(
request.app.config[DATASOURCE_NAME]["tables"]["status_table"],
request.json["username"])
result = list(res)
if not result:
raise APIBadRequest(
f"No status present for {DATASOURCE_NAME} for username {request.json['username']}"
)
result = result[0]
original_path = result.get("original_path")
if not original_path:
raise APIBadRequest(
f"No Path is present for {DATASOURCE_NAME} for username {request.json['username']}, Please cancel this processing"
)
if not os.path.exists(original_path):
raise APIBadRequest(
f"This path {original_path} doesnts exists anymore, Please cancel this processing"
)
request.app.add_task(
start_parse(request.app.config, original_path,
request.json["username"]))
return response.json({
'error': False,
'success': True,
"message":
"Takeout data parsing for {request.json['username']} has been restarted and you will be notified once it is complete",
"data": None
})
async def dashboard(request):
username = request.args.get("username")
if not username:
raise APIBadRequest("username is required")
res = await dashboard_data(
username, request.app.config[DATASOURCE_NAME]["tables"]["image_table"],
request.app.config[DATASOURCE_NAME]["tables"]["chat_table"],
request.app.config[DATASOURCE_NAME]["tables"]["address_table"])
return response.json({
'error': False,
'success': True,
"message": None,
"data": res
})
async def decorated_function(request, *args, **kwargs):
# run some method that checks the request
# for the client's authorization status
#is_authorized = check_request_for_authorization_status(request)
result = get_credentials(request.app.config.CREDENTIALS_TBL)
logger.info(f"Data from the credential table in id_token_validity decorator {result}")
if not result:
logger.error("Credentials aren't present, Please Login again")
raise APIBadRequest("Credentials aren't present, Please Login again")
username = result["username"]
if isinstance(username, bytes):
username = username.decode()
response = await f(request, username, *args, **kwargs)
return response