-
Notifications
You must be signed in to change notification settings - Fork 0
/
client.py
148 lines (139 loc) · 5.19 KB
/
client.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
"""Client side CLI Program
3 basic tasks, similar to git add, git pull and git push
In the backend, there's a SQLiteDB"""
import click
import os
from os.path import isfile, join
import json
import time
import hashlib
import dataset
import pprint
import requests
from subprocess import Popen
def calculate_checksum(filename):
""" Uses Hashlib library to create a checksum """
hasher = hashlib.md5()
with open(filename,'rb') as afile:
buf = afile.read()
hasher.update(buf)
return hasher.hexdigest()
def create_dict(filename):
""" Creates a dict for each file """
(mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) = os.stat(filename)
filedict = {}
filedict["size"] = size
filedict["accesstime"] = atime
filedict["modtime"] = mtime
filedict["Ctime"] = ctime
filedict["checksum"]=calculate_checksum(filename)
filedict["filepath"] = filename
filedict["timestamp"] = time.time()
return filedict
def add_file_to_tracked(filename):
""" Helper function to add files to the sqlite db """
db = dataset.connect('sqlite:///mydatabase.db')
table = db['files']
check = table.find_one(filepath=filename)
if not check:
try:
filedict = create_dict(filename)
pprint.pprint(filedict)
except IOError:
print "Error in getting file stats"
table.insert(filedict)
db.commit()
return
else:
print("already added")
return
@click.command()
@click.argument('filepath')
@click.option('--isdirectory','-id',is_flag=True,help = "Flag denoting whether directory path",default=False)
def add(filepath,isdirectory):
"""Point a file, it should be tracked. Point to a directory with the id flag and all files in the directory get added"""
if isdirectory:
result = [os.path.join(dp, f) for dp, dn, filenames in os.walk(filepath) for f in filenames]
click.echo(result)
map(add_file_to_tracked,result)
else:
add_file_to_tracked(filepath)
click.echo("Add a single file")
return
@click.command()
def pull():
""" Sync your local working directory with remote server directory WILL OVERWRITE ALL CHANGES TO FILES LOCALLY"""
# serverurl = "http://localhost:5000/v1/replytopull"
serverurl = "http://139.59.90.147:5000/v1/replytopull"
db = dataset.connect('sqlite:///mydatabase.db')
table = db['files']
try:
r = requests.get(serverurl)
filedata = r.json()
args = ["-avz","karm@139.59.90.147:/home/karm/datafiles/","."]
p = Popen(['rsync'] + args, shell=False)
print p.wait()
table.delete()
for filed in filedata:
table.insert(create_dict(filed['filepath']))
db.commit()
except Exception as e:
print e
finally:
return
def check_if_changed(filed):
""" Helper to check if file has changed locally """
filepath = filed['filepath']
# First member of Tuple being True means file exists.
# Second Member of Tuple being True means file has changed
if os.path.exists(filepath):
(mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) = os.stat(filepath)
if mtime != filed['modtime'] or ctime != filed['Ctime'] or size != filed['size']:
if calculate_checksum(filepath)!=filed['checksum']:
return (True,True)
else:
return (True,False)
else:
return (True,False)
else:
return (False,False)
@click.command()
@click.option('--force','-f',is_flag=True,help = "Flag if present will force a push",default=False)
def push(force):
serverurl = "http://139.59.90.147:5000/v1/sendfilelist"
""" Sync your local directory with server will always overwrite server"""
# 1. check local head and see if there are any changes
# 2. if yes, update filetable
# 3 Transmit updated filetable
# 4 use rsync to transfer files themselves.
db = dataset.connect('sqlite:///mydatabase.db')
table = db['files']
flag = False #Boolean flag to check if a push even needs to occur
for filed in db['files']:
if check_if_changed(filed) == (True,True) or force:
table.delete(filepath=filed['filepath'])
table.insert(create_dict(filed['filepath']))
flag = True
elif check_if_changed(filed) == (False,False):
# File has been deleted locally
table.delete(filepath=filed['filepath'])
flag = True
db.commit()
if flag or force:
result = db['files'].all()
dataset.freeze(result, format='json', filename='files.json')
with open("files.json","rb") as filed:
json_data = json.load(filed)
click.echo(json_data)
try:
r = requests.get(serverurl,json=json_data,headers={'Content-Type': 'application/json'})
except Exception as e:
print e
for filed in db['files']:
f = filed['filepath']
args = ["-avz",f,"karm@139.59.90.147:/home/karm/datafiles/","--relative"]
p = Popen(['rsync'] + args, shell=False)
print p.wait()
else:
click.echo("No files have been changed to push")
return